如何抓取 Just Eat 数据餐厅、菜单与评分

python --version

python -m venv just_eat_env
source just_eat_env/bin/activate

pip install crawlbase beautifulsoup4

touch just_eat_scraper.py

from crawlbase import CrawlingAPI

api = CrawlingAPI({"token": "YOUR_CRAWLBASE_TOKEN"})

def fetch_listings(url):
    options = {"ajax_wait": "true", "page_wait": 3000}
    response = api.get(url, options)
    if response["status_code"] == 200:
        return response["body"].decode("utf-8")
    print(f"Failed to fetch the page. Status: {response['status_code']}")
    return None

if __name__ == "__main__":
    area_url = "https://www.just-eat.co.uk/area/ec4r3tn"
    html = fetch_listings(area_url)
    print(html[:500] if html else "No HTML returned")

from bs4 import BeautifulSoup

BASE = "https://www.just-eat.co.uk"

def text_of(card, selector):
    el = card.select_one(selector)
    return el.get_text(strip=True) if el else ""

def parse_restaurants(html):
    soup = BeautifulSoup(html, "html.parser")
    restaurants = []
    cards = soup.select('div[data-qa="restaurant-card"]')
    for card in cards:
        try:
            anchor = card.select_one("a[href]")
            link = BASE + anchor["href"] if anchor else ""
            restaurants.append({
                "name": text_of(card, 'div[data-qa="restaurant-info-name"]'),
                "cuisine": text_of(card, 'div[data-qa="restaurant-cuisine"]'),
                "rating": text_of(card, 'div[data-qa="restaurant-ratings"]'),
                "link": link,
            })
        except Exception as e:
            print(f"Skipped a card: {e}")
    return restaurants

def fetch_listings(url):
    options = {"scroll": "true", "scroll_interval": "20"}
    response = api.get(url, options)
    if response["status_code"] == 200:
        return response["body"].decode("utf-8")
    print(f"Failed to fetch the page. Status: {response['status_code']}")
    return None

import csv
import json
from crawlbase import CrawlingAPI
from bs4 import BeautifulSoup

api = CrawlingAPI({"token": "YOUR_CRAWLBASE_TOKEN"})
BASE = "https://www.just-eat.co.uk"
FIELDS = ["name", "cuisine", "rating", "link"]

def fetch_listings(url):
    options = {"scroll": "true", "scroll_interval": "20"}
    response = api.get(url, options)
    if response["status_code"] == 200:
        return response["body"].decode("utf-8")
    print(f"Failed to fetch the page. Status: {response['status_code']}")
    return None

def text_of(card, selector):
    el = card.select_one(selector)
    return el.get_text(strip=True) if el else ""

def parse_restaurants(html):
    soup = BeautifulSoup(html, "html.parser")
    restaurants = []
    cards = soup.select('div[data-qa="restaurant-card"]')
    for card in cards:
        try:
            anchor = card.select_one("a[href]")
            link = BASE + anchor["href"] if anchor else ""
            restaurants.append({
                "name": text_of(card, 'div[data-qa="restaurant-info-name"]'),
                "cuisine": text_of(card, 'div[data-qa="restaurant-cuisine"]'),
                "rating": text_of(card, 'div[data-qa="restaurant-ratings"]'),
                "link": link,
            })
        except Exception as e:
            print(f"Skipped a card: {e}")
    return restaurants

def export(rows, name="just_eat_restaurants"):
    with open(f"{name}.json", "w", encoding="utf-8") as f:
        json.dump(rows, f, indent=4, ensure_ascii=False)
    with open(f"{name}.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=FIELDS)
        writer.writeheader()
        writer.writerows(rows)
    print(f"Saved {len(rows)} restaurants to {name}.json and {name}.csv")

def main():
    url = "https://www.just-eat.co.uk/area/ec4r3tn"
    html = fetch_listings(url)
    if not html:
        return
    rows = parse_restaurants(html)
    export(rows)

if __name__ == "__main__":
    main()

[
  {
    "name": "Tower Mangal",
    "cuisine": "Turkish, Mediterranean",
    "rating": "4.5(26)",
    "link": "https://www.just-eat.co.uk/restaurants-tower-mangal-southwark/menu"
  },
  {
    "name": "Sud Italia",
    "cuisine": "Pizza, Italian",
    "rating": "3(2)",
    "link": "https://www.just-eat.co.uk/restaurants-sud-italia-aldgate/menu"
  }
]

import csv
import json
import re
from crawlbase import CrawlingAPI
from bs4 import BeautifulSoup

api = CrawlingAPI({"token": "YOUR_CRAWLBASE_TOKEN"})
MENU_FIELDS = ["category", "name", "price", "description"]

def fetch_menu_page(url):
    options = {"scroll": "true", "scroll_interval": "15"}
    response = api.get(url, options)
    if response["status_code"] == 200:
        return response["body"].decode("utf-8")
    print(f"Failed to fetch the menu page. Status: {response['status_code']}")
    return None

def text_of(node, selector, default=""):
    el = node.select_one(selector)
    return el.get_text(strip=True) if el else default

def parse_menu(html):
    soup = BeautifulSoup(html, "html.parser")
    menu = []
    categories = soup.select('section[data-qa="item-category"]')
    for category in categories:
        category_name = text_of(category, 'h2[data-qa="heading"]', "Uncategorized")
        items = category.select('div[data-qa="item-category-list"] div[data-qa="item"]')
        for item in items:
            description = text_of(item, 'div[class^="new-item-style_item-description"]')
            menu.append({
                "category": category_name,
                "name": text_of(item, 'h2[data-qa="heading"]'),
                "price": text_of(item, 'span[class^="formatted-currency-style"]'),
                "description": re.sub(r"\s+", " ", description),
            })
    return menu

def export_menu(rows, name="just_eat_menu"):
    with open(f"{name}.json", "w", encoding="utf-8") as f:
        json.dump(rows, f, indent=4, ensure_ascii=False)
    with open(f"{name}.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=MENU_FIELDS)
        writer.writeheader()
        writer.writerows(rows)
    print(f"Saved {len(rows)} menu items to {name}.json and {name}.csv")

def main():
    menu_url = "https://www.just-eat.co.uk/restaurants-tower-mangal-southwark/menu"
    html = fetch_menu_page(menu_url)
    if not html:
        return
    rows = parse_menu(html)
    export_menu(rows)

if __name__ == "__main__":
    main()

[
  {
    "category": "What's New?",
    "name": "Terry's Chocolate Orange Pie",
    "price": "£2.49",
    "description": "Crispy chocolate pastry filled with a chocolate orange ganache."
  },
  {
    "category": "What's New?",
    "name": "Large Grimace Shake",
    "price": "£3.99",
    "description": "Milkshake base blended with blueberry-flavour syrup."
  }
]