如何抓取三星产品使用 JavaScript 获取规格和价格

mkdir scrape-samsung-products && cd scrape-samsung-products
npm init -y

npm install crawlbase cheerio

const { CrawlingAPI } = require("crawlbase");

const api = new CrawlingAPI({ token: "YOUR_CRAWLBASE_JS_TOKEN" });
const samsungProductsURL =
  "https://www.samsung.com/levant/smartphones/all-smartphones/";

api
  .get(samsungProductsURL, { ajax_wait: true, page_wait: 10000 })
  .then((response) => {
    if (response.statusCode === 200) {
      console.log(response.body.slice(0, 500));
    } else {
      throw new Error(`Failed to fetch HTML. Status: ${response.statusCode}`);
    }
  })
  .catch(console.error);

const cheerio = require("cheerio");

function scrapeProducts(html) {
  const $ = cheerio.load(html);
  const products = [];

  $(".js-pfv2-content-wrap .js-pfv2-product-card").each((_, element) => {
    const card = $(element);

    const name = card
      .find(".pd03-product-card__product-name-text")
      .text()
      .trim();

    const color = card
      .find(".option-selector-v2__color-name-text-in")
      .text()
      .trim();

    const variants = card
      .find(".option-selector-v2__size-text")
      .map((_, el) => $(el).text().trim())
      .get();

    const rating = card
      .find(".rating__point span:last-child")
      .text()
      .trim();

    const reviewCount = card
      .find(".rating__count")
      .text()
      .replace(/[^0-9]/g, "");

    const specifications = card
      .find(".pd03-product-card__spec-list .pd03-product-card__spec-item")
      .map((_, el) => $(el).text().trim())
      .get();

    const url = card
      .find(".pd03-product-card__product-image-link")
      .attr("href");

    products.push({
      name,
      model: [color, ...variants].filter(Boolean).join(", "),
      rating,
      reviewCount,
      specifications: specifications.join(", "),
      url: url && (url.startsWith("http") ? url : `https://www.samsung.com${url}`),
    });
  });

  return products;
}

const { CrawlingAPI } = require("crawlbase");
const cheerio = require("cheerio");
const fs = require("fs");

const api = new CrawlingAPI({ token: "YOUR_CRAWLBASE_JS_TOKEN" });
const samsungProductsURL =
  "https://www.samsung.com/levant/smartphones/all-smartphones/";

function scrapeProducts(html) {
  const $ = cheerio.load(html);
  const products = [];
  const totalResults = $(".js-pfv2-result-total-count").text().trim();

  $(".js-pfv2-content-wrap .js-pfv2-product-card").each((_, element) => {
    const card = $(element);
    const color = card
      .find(".option-selector-v2__color-name-text-in")
      .text()
      .trim();
    const variants = card
      .find(".option-selector-v2__size-text")
      .map((_, el) => $(el).text().trim())
      .get();
    const url = card
      .find(".pd03-product-card__product-image-link")
      .attr("href");

    products.push({
      name: card.find(".pd03-product-card__product-name-text").text().trim(),
      model: [color, ...variants].filter(Boolean).join(", "),
      rating: card.find(".rating__point span:last-child").text().trim(),
      reviewCount: card.find(".rating__count").text().replace(/[^0-9]/g, ""),
      specifications: card
        .find(".pd03-product-card__spec-list .pd03-product-card__spec-item")
        .map((_, el) => $(el).text().trim())
        .get()
        .join(", "),
      url: url && (url.startsWith("http") ? url : `https://www.samsung.com${url}`),
    });
  });

  return { totalResults, products };
}

api
  .get(samsungProductsURL, { ajax_wait: true, page_wait: 10000 })
  .then((response) => {
    if (response.statusCode !== 200) {
      throw new Error(`Failed to fetch HTML. Status: ${response.statusCode}`);
    }
    const data = scrapeProducts(response.body);
    console.log(data);
    fs.writeFileSync("samsung-scraped.json", JSON.stringify(data, null, 2));
  })
  .catch(console.error);

{
  "totalResults": "42 results",
  "products": [
    {
      "name": "Galaxy S24 Ultra",
      "model": "Titanium Yellow, 256 GB, 512 GB, 1 TB",
      "rating": "4.3",
      "reviewCount": "128",
      "specifications": "Industry-leading hardware meets world-changing AI, Made with titanium. Built to last, 200MP high-resolution photography",
      "url": "https://www.samsung.com/levant/smartphones/galaxy-s24-ultra/"
    },
    {
      "name": "Galaxy A04s",
      "model": "Black, 64 GB, 128 GB",
      "rating": "5.0",
      "reviewCount": "17",
      "specifications": "6.5\" Infinity-V Display, 50MP Camera, 5000mAh Battery",
      "url": "https://www.samsung.com/levant/smartphones/galaxy-a/galaxy-a04s-black-64gb-sm-a047fzkgmeb/"
    }
  ]
}

const listingURLs = [
  "https://www.samsung.com/levant/smartphones/all-smartphones/",
  "https://www.samsung.com/levant/smartphones/galaxy-z/",
];

async function scrapeAll(urls) {
  const all = [];
  for (const url of urls) {
    const response = await api.get(url, { ajax_wait: true, page_wait: 10000 });
    if (response.statusCode === 200) {
      all.push(...scrapeProducts(response.body).products);
    }
    await new Promise((r) => setTimeout(r, 2000));
  }
  return all;
}