如何抓取 Product Hunt产品、点赞数与创作者

node --version

mkdir producthunt-scraper && cd producthunt-scraper
npm init -y

npm install crawlbase cheerio

const { CrawlingAPI } = require('crawlbase');
const fs = require('fs');

const api = new CrawlingAPI({ token: 'YOUR_CRAWLBASE_TOKEN' });

const producthuntPageURL =
  'https://www.producthunt.com/categories/engineering-development';

api
  .get(producthuntPageURL)
  .then((response) => {
    if (response.statusCode === 200) {
      fs.writeFileSync('response.html', response.body);
      console.log('HTML saved to response.html');
    }
  })
  .catch((error) => console.error('API request error:', error));

const fs = require('fs');
const cheerio = require('cheerio');

function parseProducts(html) {
  const $ = cheerio.load(html);
  const products = [];

  const containers = $(
    'div.flex.direction-column.mb-mobile-10.mb-tablet-15.mb-desktop-15.mb-widescreen-15'
  );

  containers.each((index, element) => {
    const card = $(element);

    const name = card
      .find('div.color-blue.fontSize-18.fontWeight-600')
      .text()
      .trim();

    // Each filled star is one label element inside the rating row
    const upvotes = card.find(
      'div.flex.direction-row.align-center label'
    ).length;

    const reviews = card
      .find('div.ml-3.color-lighter-grey.fontSize-14.fontWeight-400')
      .text()
      .trim();

    const tagline = card
      .find(
        'div.color-lighter-grey.fontSize-mobile-14.fontSize-tablet-16.fontSize-desktop-16.fontSize-widescreen-16.fontWeight-400'
      )
      .text()
      .trim();

    const href = card.find('a').first().attr('href');
    const link = href
      ? new URL(href, 'https://www.producthunt.com').href
      : '';

    if (name) {
      products.push({ rank: index + 1, name, tagline, upvotes, reviews, link });
    }
  });

  return products;
}

const fs = require('fs');
const { CrawlingAPI } = require('crawlbase');
const cheerio = require('cheerio');

const api = new CrawlingAPI({ token: 'YOUR_CRAWLBASE_TOKEN' });

async function crawl(pageUrl) {
  const response = await api.get(pageUrl);
  if (response.statusCode === 200) return response.body;
  console.error(`Request failed: ${response.statusCode}`);
  return null;
}

function toCsv(rows) {
  const headers = ['rank', 'name', 'tagline', 'upvotes', 'reviews', 'link'];
  const escape = (value) =>
    `"${String(value).replace(/"/g, '""')}"`;
  const lines = [headers.join(',')];
  for (const row of rows) {
    lines.push(headers.map((h) => escape(row[h])).join(','));
  }
  return lines.join('\n');
}

async function main() {
  const url =
    'https://www.producthunt.com/categories/engineering-development';
  const html = await crawl(url);
  if (!html) return;

  const products = parseProducts(html);
  fs.writeFileSync('products.json', JSON.stringify(products, null, 2));
  fs.writeFileSync('products.csv', toCsv(products));
  console.log(`Saved ${products.length} products to JSON and CSV`);
}

main();

[
  {
    "rank": 1,
    "name": "The Free Website Guys",
    "tagline": "A free website program that has helped over 10,000 entrepreneurs.",
    "upvotes": 5,
    "reviews": "151 reviews",
    "link": "https://www.producthunt.com/products/the-free-website-guys"
  },
  {
    "rank": 2,
    "name": "Zipy",
    "tagline": "A debugging platform with session replay and network monitoring.",
    "upvotes": 5,
    "reviews": "132 reviews",
    "link": "https://www.producthunt.com/products/zipy"
  }
]

rank,name,tagline,upvotes,reviews,link
"1","The Free Website Guys","A free website program that has helped over 10,000 entrepreneurs.","5","151 reviews","https://www.producthunt.com/products/the-free-website-guys"
"2","Zipy","A debugging platform with session replay and network monitoring.","5","132 reviews","https://www.producthunt.com/products/zipy"

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

async function scrapeCategory(slug, maxPages = 5) {
  const all = [];
  for (let page = 1; page <= maxPages; page++) {
    const url =
      `https://www.producthunt.com/categories/${slug}?page=${page}`;
    const html = await crawl(url);
    if (!html) break;

    const rows = parseProducts(html);
    if (rows.length === 0) break;

    all.push(...rows.map((p) => ({ category: slug, ...p })));
    await sleep(2000);
  }
  return all;
}

scrapeCategory('engineering-development').then((rows) => {
  console.log(`Collected ${rows.length} products`);
});

function parseProfile(html, handle) {
  const $ = cheerio.load(html);
  const profile = {};

  profile.name = $(
    'h1.color-darker-grey.fontSize-24.fontWeight-600'
  ).text().trim();
  profile.headline = $(
    'div.color-lighter-grey.fontSize-18.fontWeight-300'
  ).text().trim();
  profile.followers = $(
    `a[href="/@${handle}/followers"]`
  ).text().trim();
  profile.following = $(
    `a[href="/@${handle}/following"]`
  ).text().replace(/\n\s+/g, ' ').trim();
  profile.points = $(
    'span.color-lighter-grey.fontSize-14.fontWeight-400:contains("points")'
  ).text().trim();

  // Public list of products the maker has shipped
  profile.products = [];
  $('.styles_even__Qeyum, .styles_odd__wazk7').each((i, el) => {
    profile.products.push({
      name: $(el).find('img.styles_thumbnail__Y9ZpZ').attr('alt'),
    });
  });

  return profile;
}

async function scrapeProfile(handle) {
  const html = await crawl(`https://www.producthunt.com/@${handle}`);
  return html ? parseProfile(html, handle) : null;
}