验证 LLM 引用用实时网页核查 AI 答案

# final/citation_verifier/fetch.py
def crawl_options(readability: bool) -> dict[str, str]:
    """Query parameters for one Crawling API call.

    ``md_readability`` is omitted on the fallback request. The docs define it
    only as a switch on ``format=md``, not as a guarantee about thin pages.
    """
    options = {
        "format": "md",
        "store": "true",
        "custom_success_codes": "404,410",
    }
    if readability:
        options["md_readability"] = "true"
    return options

# final/citation_verifier/fetch.py
def _normalized_path(url: str) -> str:
    path = urlsplit(url).path or "/"
    if len(path) > 1 and path.endswith("/"):
        path = path[:-1]
    return path or "/"


def is_homepage_redirect(requested: str, final: str | None) -> bool:
    """True when a deep link landed on ``/``.

    The Crawling API returns the post-redirect URL in the ``url`` header.
    A citation that survives only as the site homepage did not survive.
    """
    if not final:
        return False
    return _normalized_path(requested) != "/" and _normalized_path(final) == "/"

# Production shape, not from the repository: when to retry on the JavaScript token.
def needs_javascript(response: dict, markdown: str, min_chars: int) -> bool:
    if response.get("status_code") == 520:
        return True  # the crawl failed; failed requests are not billed
    headers = response.get("headers") or {}
    if str(headers.get("pc_status")) == "207":
        return True  # the origin answered 200 with an empty page
    return len("".join(markdown.split())) < min_chars  # still thin after the readability retry

# final/citation_verifier/passages.py (excerpt)
def split_passages(markdown: str, max_words: int = DEFAULT_MAX_WORDS) -> list[str]:
    """Split on blank lines, then pack up to ``max_words`` words.

    A paragraph longer than the cap is cut on word boundaries. Shorter
    paragraphs share a passage until the next one would overflow. Blank
    blocks are dropped.
    """
    if max_words < 1:
        raise ValueError("max_words must be at least 1")
    text = markdown.replace("\r\n", "\n").replace("\r", "\n")
    pieces: list[list[str]] = []
    for paragraph in re.split(r"\n\s*\n", text):
        words = paragraph.split()
        if not words:
            continue
        if len(words) > max_words:
            for start in range(0, len(words), max_words):
                pieces.append(words[start : start + max_words])
        else:
            pieces.append(words)

# final/citation_verifier/retrieval.py
def top_k_passages(
    claim: str,
    passages: list[str],
    embedder: Embedder,
    k: int = DEFAULT_TOP_K,
) -> list[str]:
    """Return up to ``k`` passages, highest cosine similarity first."""
    if k < 1 or not passages:
        return []
    vectors = embedder.embed([claim, *passages])
    query = vectors[0]
    ranked = [
        (cosine(query, vectors[index + 1]), index, passage)
        for index, passage in enumerate(passages)
    ]
    ranked.sort(key=lambda item: (-item[0], item[1]))
    return [passage for _, _, passage in ranked[:k]]

# final/citation_verifier/judge.py
SYSTEM_PROMPT = """You verify one claim against the passages in the user message. The passages are the only evidence you may use.

Return JSON with these fields:
- label: supported, contradicted, or not_found
- evidence_quote: a contiguous quote copied from the passages, or null when the label is not_found
- confidence: a number from 0 to 1

supported means the passages state the claim. contradicted means the passages state the opposite. not_found means the passages do not settle the claim. If you cannot copy a quote that appears in the passages, return not_found and null. Do not add words, numbers, or sources that are not in the passages."""


class RawJudgement(BaseModel):
    label: Label
    evidence_quote: str | None = None
    confidence: float = Field(ge=0, le=1)

# final/citation_verifier/judge.py
def guard_judgement(
    raw: RawJudgement,
    passages: list[str],
) -> tuple[Label, str | None, float, str | None]:
    """Drop a quote the passages do not contain.

    Supported and contradicted both need a real quote. A quote that fails the
    check downgrades the citation to not_found with confidence 0.
    """
    quote = raw.evidence_quote
    if raw.label in {Label.supported, Label.contradicted}:
        if quote and quote_in_passages(quote, passages):
            return raw.label, quote, raw.confidence, None
        return Label.not_found, None, 0.0, REASON_QUOTE
    if quote and not quote_in_passages(quote, passages):
        return Label.not_found, None, 0.0, REASON_QUOTE
    return Label.not_found, None, raw.confidence, None

# final/citation_verifier/schema.py
class UrlVerdict(BaseModel):
    url: str
    label: Label
    evidence_quote: str | None = None
    confidence: float = Field(ge=0, le=1)
    final_url: str | None = None
    original_status: int | None = None
    rid: str | None = None
    reason: str | None = None


class VerifyRequest(BaseModel):
    claim: str = Field(min_length=1)
    urls: list[str] = Field(min_length=1)

# final/citation_verifier/aggregate.py
def aggregate(verdicts: list[UrlVerdict]) -> Label:
    """One contradiction outweighs every supporting citation.

    Otherwise any supported citation carries the claim. The claim is
    unreachable only when every citation is unreachable. Every remaining mix,
    including an empty list, is not_found.
    """
    labels = [verdict.label for verdict in verdicts]
    if any(label == Label.contradicted for label in labels):
        return Label.contradicted
    if any(label == Label.supported for label in labels):
        return Label.supported
    if labels and all(label == Label.unreachable for label in labels):
        return Label.unreachable
    return Label.not_found

{
  "claim": "Water boils at 100 degrees Celsius at sea level.",
  "overall": "supported",
  "citations": [
    {
      "url": "https://example.com/articles/water-boiling-point",
      "label": "supported",
      "evidence_quote": "Water boils at 100 degrees Celsius at sea level.",
      "confidence": 0.96,
      "final_url": "https://example.com/articles/water-boiling-point",
      "original_status": 200,
      "rid": "rid-boil",
      "reason": null
    },
    {
      "url": "https://example.com/old-report",
      "label": "unreachable",
      "evidence_quote": null,
      "confidence": 1.0,
      "final_url": "https://example.com/old-report",
      "original_status": 404,
      "rid": "rid-404",
      "reason": "http_404"
    }
  ]
}

# final/citation_verifier/batch.py
@dataclass(frozen=True)
class Precision:
    """Citation precision over claim-URL pairs.

    citation precision is supported citations divided by every citation.
    Reachable-only precision uses the same numerator and drops unreachable
    citations from the denominator. It is None when nothing was reachable.
    """

    supported: int
    total: int
    reachable: int

citation_precision=0.2222 (2/9)
reachable_precision=0.3333 (2/6)

# final/citation_verifier/audit.py (docstring omitted)
def replay_rid(rid: str, token: str | None = None) -> str:
    if not rid:
        raise ValueError("rid is required")
    settings = get_settings()
    used = token if token else settings.crawlbase_token
    if not used:
        raise MissingTokenError(
            "CRAWLBASE_TOKEN is not set. Replay needs the token that stored the page."
        )
    client = StorageAPI({"token": used})
    response = _storage_get(client, rid)