如何抓取 Alibaba 搜索结果产品、价格与供应商

mkdir alibaba-serp-scraper
cd alibaba-serp-scraper

npm init -y
npm install crawlbase cheerio

const { CrawlingAPI } = require('crawlbase');

// Replace with your token from the Crawlbase dashboard
const api = new CrawlingAPI({ token: 'YOUR_CRAWLBASE_TOKEN' });

const alibabaSerpURL =
  'https://www.alibaba.com/trade/search?SearchText=samsung+s24+ultra';

api
  .get(alibabaSerpURL)
  .then((response) => {
    // original_status is the status Alibaba itself returned
    console.log('Status:', response.originalStatus);
    console.log(response.body.slice(0, 500));
  })
  .catch((error) => {
    console.error('Request failed:', error);
  });

const cheerio = require('cheerio');

// Alibaba serves protocol-relative URLs (//...); normalise them to https
function toHttps(href) {
  if (!href) return null;
  return href.includes('http') ? href : `https:${href}`;
}

function parseSerp(html) {
  const $ = cheerio.load(html);
  const results = [];

  const numberOfResults = $('.seb-refine-result_all').text().trim();

  $('.offer-list-wrapper .J-search-card-wrapper').each((index, element) => {
    const card = $(element);

    const title = card.find("[data-spm='d_title']").text().trim();
    const url = card.find("[data-spm='d_title']").attr('href');
    const price = card.find('.search-card-e-price-main').text().trim();
    const minItem = card.find('.search-card-m-sale-features__item').text().trim();
    const storeName = card.find('.search-card-e-company').text().trim();
    const storeLink = card.find('.search-card-e-company').attr('href');
    const image = card.find('.search-card-e-slider__img').attr('src');
    const reviews = card.find('.search-card-e-review').text().trim();

    if (!title) return;

    results.push({
      position: index + 1,
      title,
      price,
      minItem,
      storeName,
      reviews,
      url: toHttps(url),
      storeLink: toHttps(storeLink),
      image: toHttps(image),
    });
  });

  return { numberOfResults, results };
}

module.exports = { parseSerp };

const { CrawlingAPI } = require('crawlbase');
const cheerio = require('cheerio');
const fs = require('fs');

const api = new CrawlingAPI({ token: 'YOUR_CRAWLBASE_TOKEN' });

function toHttps(href) {
  if (!href) return null;
  return href.includes('http') ? href : `https:${href}`;
}

function parseSerp(html) {
  const $ = cheerio.load(html);
  const results = [];
  const numberOfResults = $('.seb-refine-result_all').text().trim();

  $('.offer-list-wrapper .J-search-card-wrapper').each((index, element) => {
    const card = $(element);
    const title = card.find("[data-spm='d_title']").text().trim();
    if (!title) return;

    results.push({
      position: index + 1,
      title,
      price: card.find('.search-card-e-price-main').text().trim(),
      minItem: card.find('.search-card-m-sale-features__item').text().trim(),
      storeName: card.find('.search-card-e-company').text().trim(),
      reviews: card.find('.search-card-e-review').text().trim(),
      url: toHttps(card.find("[data-spm='d_title']").attr('href')),
      storeLink: toHttps(card.find('.search-card-e-company').attr('href')),
      image: toHttps(card.find('.search-card-e-slider__img').attr('src')),
    });
  });

  return { numberOfResults, results };
}

function toCsv(results) {
  const headers = ['position', 'title', 'price', 'minItem', 'storeName', 'url'];
  const escape = (v) => `"${(v || '').toString().replace(/"/g, '""')}"`;
  const rows = results.map((r) =>
    headers.map((h) => escape(r[h])).join(',')
  );
  return [headers.join(','), ...rows].join('\n');
}

async function main() {
  const url =
    'https://www.alibaba.com/trade/search?SearchText=samsung+s24+ultra';
  const response = await api.get(url);

  if (response.originalStatus !== 200) {
    throw new Error(`Unable to crawl, status ${response.originalStatus}`);
  }

  const data = parseSerp(response.body);
  fs.writeFileSync('alibaba-serp.json', JSON.stringify(data, null, 2));
  fs.writeFileSync('alibaba-serp.csv', toCsv(data.results));
  console.log(`Saved ${data.results.length} products`);
}

main().catch((error) => console.error(error));

{
  "numberOfResults": "3,000+ products found",
  "results": [
    {
      "position": 1,
      "title": "Mobile Phone Case For Samsung Galaxy S24 Ultra Plus Tpu Pc Shockproof Covers",
      "price": "US$1.29 - US$1.69",
      "minItem": "Min. order: 50 pieces",
      "storeName": "Guangzhou Junbo Electronic Co., Ltd.",
      "reviews": "4.9/5.0 (68)",
      "url": "https://www.alibaba.com/product-detail/Mobile-Phone-Case_1600969904884.html",
      "storeLink": "https://gzjunbo.en.alibaba.com/company_profile.html",
      "image": "https://s.alicdn.com/@sc04/kf/Hcdcc7db446e9420f9378c0ec3482037bk.png_300x300.png"
    },
    {
      "position": 2,
      "title": "Cellphone Original S24 Ultra 16GB+512GB Smartphone 7inch Unlocked 5G",
      "price": "US$43.42 - US$54.47",
      "minItem": "Min. order: 1 piece",
      "storeName": "Dongguan Zhongfu Electronic Technology Co., Ltd.",
      "reviews": "3.3/5.0 (197)",
      "url": "https://www.alibaba.com/product-detail/Hot-selling-S24-Ultra_1600969407142.html",
      "storeLink": "https://fukadi.en.alibaba.com/company_profile.html",
      "image": "https://s.alicdn.com/@sc04/kf/H771126c0475c4a3d9ee7842740b0cf4an.jpg_300x300.jpg"
    }
  ]
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

async function scrapeQuery(searchText, pages = 3) {
  const encoded = encodeURIComponent(searchText).replace(/%20/g, '+');
  const allResults = [];

  for (let page = 1; page <= pages; page++) {
    const url =
      `https://www.alibaba.com/trade/search?SearchText=${encoded}&page=${page}`;
    const response = await api.get(url);
    if (response.originalStatus === 200) {
      allResults.push(...parseSerp(response.body).results);
    }
    await sleep(3000);
  }

  console.log(`Collected ${allResults.length} products across ${pages} pages`);
  return allResults;
}