如何从 Google 提取并分析 SEO 数据将 SERP 转化为 SEO 洞见

python --version

python -m venv seo_env
source seo_env/bin/activate

pip install requests beautifulsoup4 pandas

import json
import requests
from urllib.parse import quote_plus

API_TOKEN = "YOUR_CRAWLBASE_TOKEN"  # replace with your token
API_ENDPOINT = "https://api.crawlbase.com/"

def search_url(keyword):
    return f"https://www.google.com/search?q={quote_plus(keyword)}&hl=en&gl=us"

def crawl(url):
    params = {"token": API_TOKEN, "url": url, "javascript": "true"}
    response = requests.get(API_ENDPOINT, params=params)
    response.raise_for_status()

    data = json.loads(response.text)
    if data["original_status"] != 200:
        raise Exception(f"Unable to crawl '{url}'")

    return data["body"]

if __name__ == "__main__":
    html = crawl(search_url("web scraping api"))
    print(html[:500])

from urllib.parse import urlparse
from bs4 import BeautifulSoup

def scrape_serp(html, keyword):
    soup = BeautifulSoup(html, "html.parser")

    results = []
    for position, block in enumerate(soup.select("div.g"), start=1):
        heading = block.select_one("h3")
        link = block.select_one("a[href]")
        snippet = block.select_one("div.VwiC3b")
        if not heading or not link:
            continue
        url = link["href"]
        results.append({
            "keyword": keyword,
            "position": position,
            "title": heading.get_text(strip=True),
            "url": url,
            "domain": urlparse(url).netloc,
            "snippet": snippet.get_text(strip=True) if snippet else None,
        })

    paa = [
        q.get_text(strip=True)
        for q in soup.select("div.related-question-pair")
        if q.get_text(strip=True)
    ]

    related = [
        r.get_text(strip=True)
        for r in soup.select("div.y6Uyqe div.b2Rnsc, a.k8XOCe")
        if r.get_text(strip=True)
    ]

    return {
        "keyword": keyword,
        "results": results,
        "people_also_ask": paa,
        "related_searches": related,
    }

import json
import time
import requests
from urllib.parse import quote_plus, urlparse
from bs4 import BeautifulSoup

API_TOKEN = "YOUR_CRAWLBASE_TOKEN"
API_ENDPOINT = "https://api.crawlbase.com/"

KEYWORDS = [
    "web scraping api",
    "how to scrape google",
    "rotating proxy service",
]

def search_url(keyword):
    return f"https://www.google.com/search?q={quote_plus(keyword)}&hl=en&gl=us"

def crawl(url):
    params = {"token": API_TOKEN, "url": url, "javascript": "true"}
    response = requests.get(API_ENDPOINT, params=params)
    response.raise_for_status()
    data = json.loads(response.text)
    if data["original_status"] != 200:
        raise Exception(f"Unable to crawl '{url}'")
    return data["body"]

def scrape_serp(html, keyword):
    soup = BeautifulSoup(html, "html.parser")
    results = []
    for position, block in enumerate(soup.select("div.g"), start=1):
        heading = block.select_one("h3")
        link = block.select_one("a[href]")
        snippet = block.select_one("div.VwiC3b")
        if not heading or not link:
            continue
        url = link["href"]
        results.append({
            "keyword": keyword,
            "position": position,
            "title": heading.get_text(strip=True),
            "url": url,
            "domain": urlparse(url).netloc,
            "snippet": snippet.get_text(strip=True) if snippet else None,
        })
    paa = [q.get_text(strip=True) for q in soup.select("div.related-question-pair") if q.get_text(strip=True)]
    related = [r.get_text(strip=True) for r in soup.select("div.y6Uyqe div.b2Rnsc, a.k8XOCe") if r.get_text(strip=True)]
    return {"keyword": keyword, "results": results, "people_also_ask": paa, "related_searches": related}

def main():
    serps = []
    for keyword in KEYWORDS:
        html = crawl(search_url(keyword))
        serps.append(scrape_serp(html, keyword))
        print(f"Scraped '{keyword}': {len(serps[-1]['results'])} results")
        time.sleep(2)
    with open("google_seo_data.json", "w", encoding="utf-8") as f:
        json.dump(serps, f, ensure_ascii=False, indent=2)

if __name__ == "__main__":
    main()

{
  "keyword": "web scraping api",
  "results": [
    {
      "keyword": "web scraping api",
      "position": 1,
      "title": "Crawling API - Crawlbase",
      "url": "https://crawlbase.com/crawling-api-avoid-captchas-blocks",
      "domain": "crawlbase.com",
      "snippet": "Crawl any website with a single API call and skip blocks and CAPTCHAs."
    },
    {
      "keyword": "web scraping api",
      "position": 2,
      "title": "Best Web Scraping APIs Compared",
      "url": "https://example-blog.com/web-scraping-apis",
      "domain": "example-blog.com",
      "snippet": "A side-by-side look at popular scraping APIs and how they handle blocks."
    }
  ],
  "people_also_ask": [
    "What is a web scraping API?",
    "Is using a scraping API legal?"
  ],
  "related_searches": [
    "free web scraping api",
    "web scraping api python"
  ]
}

import json
import pandas as pd

with open("google_seo_data.json", encoding="utf-8") as f:
    serps = json.load(f)

# Flatten every organic result across all keywords into one table
rows = [r for serp in serps for r in serp["results"]]
df = pd.DataFrame(rows)

# 1. Where do I rank? Find your own domain's position per keyword
MY_DOMAIN = "crawlbase.com"
mine = df[df["domain"] == MY_DOMAIN][["keyword", "position", "title"]]
print("Your rankings:")
print(mine.to_string(index=False))

# 2. Who am I competing with? Domains that appear most across SERPs
top_domains = df["domain"].value_counts().head(10)
print("\nTop competing domains:")
print(top_domains)

# 3. Content gaps: PAA and related searches you have not covered yet
ideas = []
for serp in serps:
    for q in serp["people_also_ask"] + serp["related_searches"]:
        ideas.append({"keyword": serp["keyword"], "idea": q})
gaps = pd.DataFrame(ideas).drop_duplicates()
print(f"\n{len(gaps)} content ideas from PAA and related searches")
print(gaps.head(10).to_string(index=False))

df.to_csv("google_seo_results.csv", index=False)