如何抓取 Stack Overflow 问题问题、标签与票数

node --version

mkdir stackoverflow-scraper && cd stackoverflow-scraper
npm init -y

npm install crawlbase cheerio

const { CrawlingAPI } = require('crawlbase');

const api = new CrawlingAPI({ token: 'YOUR_CRAWLBASE_TOKEN' });

async function crawl(pageUrl) {
  const response = await api.get(pageUrl);
  if (response.statusCode === 200) {
    return response.body;
  }
  console.error(`Request failed: ${response.statusCode}`);
  return null;
}

const tagUrl = 'https://stackoverflow.com/questions/tagged/javascript';
crawl(tagUrl).then((html) => {
  console.log(html ? html.slice(0, 500) : 'No HTML returned');
});

const cheerio = require('cheerio');

const clean = (text) => text.replace(/\s+/g, ' ').trim();

function parseQuestions(html) {
  const $ = cheerio.load(html);
  const questions = [];

  $('#questions .js-post-summary').each((_, element) => {
    const el = $(element);
    const title = clean(el.find('.s-post-summary--content-title').text());
    const link = el.find('.s-link').attr('href') || '';
    const votes = clean(
      el.find('.js-post-summary-stats .s-post-summary--stats-item:first-child').text()
    );
    const answers =
      clean(el.find('.js-post-summary-stats .has-answers').text()) || '0 answers';
    const views = clean(
      el.find('.js-post-summary-stats .s-post-summary--stats-item:last-child').text()
    );
    const tags = el
      .find('.js-post-tag-list-item')
      .map((__, tag) => clean($(tag).text()))
      .get()
      .filter(Boolean);

    questions.push({
      title,
      tags,
      votes,
      answers,
      views,
      link: link.includes('https://') ? link : `https://stackoverflow.com${link}`,
    });
  });

  return questions;
}

const { CrawlingAPI } = require('crawlbase');
const cheerio = require('cheerio');
const fs = require('fs');

const api = new CrawlingAPI({ token: 'YOUR_CRAWLBASE_TOKEN' });
const clean = (text) => text.replace(/\s+/g, ' ').trim();

async function crawl(pageUrl) {
  const response = await api.get(pageUrl);
  if (response.statusCode === 200) return response.body;
  console.error(`Request failed: ${response.statusCode}`);
  return null;
}

function parseQuestions(html) {
  const $ = cheerio.load(html);
  const questions = [];
  $('#questions .js-post-summary').each((_, element) => {
    const el = $(element);
    const link = el.find('.s-link').attr('href') || '';
    questions.push({
      title: clean(el.find('.s-post-summary--content-title').text()),
      tags: el
        .find('.js-post-tag-list-item')
        .map((__, tag) => clean($(tag).text()))
        .get()
        .filter(Boolean),
      votes: clean(
        el.find('.js-post-summary-stats .s-post-summary--stats-item:first-child').text()
      ),
      answers:
        clean(el.find('.js-post-summary-stats .has-answers').text()) || '0 answers',
      views: clean(
        el.find('.js-post-summary-stats .s-post-summary--stats-item:last-child').text()
      ),
      link: link.includes('https://') ? link : `https://stackoverflow.com${link}`,
    });
  });
  return questions;
}

function toCsv(rows) {
  const headers = ['title', 'tags', 'votes', 'answers', 'views', 'link'];
  const escape = (value) => `"${String(value).replace(/"/g, '""')}"`;
  const lines = [headers.join(',')];
  for (const row of rows) {
    lines.push(
      [
        escape(row.title),
        escape(row.tags.join('|')),
        escape(row.votes),
        escape(row.answers),
        escape(row.views),
        escape(row.link),
      ].join(',')
    );
  }
  return lines.join('\n');
}

async function main() {
  const tagUrl = 'https://stackoverflow.com/questions/tagged/javascript';
  const html = await crawl(tagUrl);
  if (!html) return;
  const questions = parseQuestions(html);
  fs.writeFileSync('questions.json', JSON.stringify(questions, null, 2));
  fs.writeFileSync('questions.csv', toCsv(questions));
  console.log(`Saved ${questions.length} questions to questions.json and questions.csv`);
}

main();

[
  {
    "title": "How do I return the response from an asynchronous call?",
    "tags": ["javascript", "ajax", "asynchronous", "promise"],
    "votes": "8632 votes",
    "answers": "42 answers",
    "views": "2.1m views",
    "link": "https://stackoverflow.com/questions/14220321/how-do-i-return-the-response-from-an-asynchronous-call"
  },
  {
    "title": "What does \"use strict\" do in JavaScript?",
    "tags": ["javascript", "syntax", "jslint", "use-strict"],
    "votes": "9201 votes",
    "answers": "32 answers",
    "views": "1.0m views",
    "link": "https://stackoverflow.com/questions/1335851/what-does-use-strict-do-in-javascript"
  }
]

title,tags,votes,answers,views,link
"How do I return the response from an asynchronous call?","javascript|ajax|asynchronous|promise","8632 votes","42 answers","2.1m views","https://stackoverflow.com/questions/14220321/..."
"What does ""use strict"" do in JavaScript?","javascript|syntax|jslint|use-strict","9201 votes","32 answers","1.0m views","https://stackoverflow.com/questions/1335851/..."

async function scrapeTag(tag, totalPages) {
  const all = [];
  for (let page = 1; page <= totalPages; page++) {
    const url =
      `https://stackoverflow.com/questions/tagged/${tag}?tab=newest&page=${page}`;
    const html = await crawl(url);
    if (html) all.push(...parseQuestions(html));
  }
  return all;
}

scrapeTag('javascript', 3).then((rows) => {
  console.log(`Collected ${rows.length} questions`);
});