skills/astro-han__karpathy-llm-wiki
1
0
Fork 0
mirror of https://github.com/Astro-Han/karpathy-llm-wiki.git synced 2026-09-14 19:09:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: close verification escapes and false-suspect sources

Browse Source 
- Sentence-final punctuation in the raw no longer blocks matching;
  the boundary only rejects continuations that form a different value.
  This was a regression from the boundary fix itself and would have
  flooded reports with verbatim facts.
- Raw links must resolve inside raw/; anything else is an evidence
  error, since the invariant's permanence rests on raw immutability.
- No-material suppression is anchored to ingest entry headings, so
  prose mentions elsewhere in the log cannot hide backlog.
- Metadata semantics are scoped to the header block after the H1:
  an Archived line in the body no longer grants exemption, and
  metadata-shaped lines in a raw file's body stay evidence.
- Status blocks are skipped wholesale, including their explanation
  lines, which legitimately cite newer sources outside the article's
  own Raw links.
- Fence stripping handles tilde and 3+-backtick fences via a state
  machine; candidate tokens are whitespace-normalized before dedup;
  index/log args are skipped even when passed explicitly; articles
  are checked once, not twice.
This commit is contained in:
Yuhan Lei
2026-07-23 23:15:06 +08:00
parent f8e592f979
commit e3b7c68fee
2 changed files with 282 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/check_evidence.py
+98 -42
View File
@@ -5,12 +5,13 @@ Report-only; never modifies files. Three sweeps:
1. Fidelity — extract candidate literals (specific numbers, ISO dates,
direct quotes) from each wiki article and verify that each candidate
appears verbatim in the raw files linked by that article's Raw field.
Misses are listed as suspects. Derived values, product names, and
deliberate paraphrases will show up as suspects; judging them is the
reader's job, not this script's.
appears verbatim in the body of the raw files linked by that
article's Raw field. Misses are listed as suspects. Derived values,
product names, and deliberate paraphrases will show up as suspects;
judging them is the reader's job, not this script's.
2. Evidence errors — articles that cannot be verified at all: a missing
Raw field on a non-archive article, or Raw links that do not resolve.
Raw field on a non-archive article, Raw links that do not resolve,
or Raw links that escape raw/ (evidence must live in immutable raw/).
3. Inventory — raw files that no article's Raw field references,
excluding files whose ingest was logged as "no material".
@@ -30,7 +31,7 @@ import re
import sys
from pathlib import Path
NUMBER_TOKEN_RE = re.compile(r"\d[\d,]*(?:\.\d+)?\s?[KMB%]?")
NUMBER_TOKEN_RE = re.compile(r"\d[\d,]*(?:\.\d+)?\s*[KMB%]?")
SUFFIX_RE = re.compile(r"[KMB%]$")
DATE_RE = re.compile(r"\d{4}-\d{2}(?:-\d{2})?")
QUOTE_RES = [re.compile(r'"([^"\n]{15,})"'), re.compile(r"“([^”\n]{15,})”")]
@@ -38,10 +39,12 @@ METADATA_RE = re.compile(r"^>\s*(Sources?|Raw|Collected|Published|Updated|Archiv
STATUS_LINE_RE = re.compile(r"^>\s*\*\*Status:")
LINK_RE = re.compile(r"\[([^\]]*)\]\(([^)]*)\)")
INLINE_CODE_RE = re.compile(r"`[^`\n]*`")
FENCED_CODE_RE = re.compile(r"```.*?```", re.DOTALL)
RAW_LINK_RE = re.compile(r"\(([^)]+\.md)[^)]*\)")
NO_MATERIAL_RE = re.compile(r"no material:\s*(\S+)", re.IGNORECASE)
ARCHIVED_RE = re.compile(r"^>\s*Archived:", re.MULTILINE)
NO_MATERIAL_HEADING_RE = re.compile(
r"^## \[[^\]]*\]\s*ingest\s*\|\s*no material:\s*(\S+)", re.IGNORECASE
)
ARCHIVED_RE = re.compile(r"^>\s*Archived:")
FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})")
WS_RE = re.compile(r"\s+")
SKIP_FILES = {"index.md", "log.md"}
@@ -51,6 +54,45 @@ def normalize(text: str) -> str:
return WS_RE.sub(" ", text).strip()
def split_header(lines: list[str]) -> tuple[list[str], list[str]]:
"""Split lines into the metadata header (the contiguous blockquote
block right after the H1) and everything else. Only the header may
carry metadata semantics; identical lines in the body are content."""
i = 0
while i < len(lines) and not lines[i].startswith("# "):
i += 1
if i >= len(lines):
return [], lines
i += 1
while i < len(lines) and not lines[i].strip():
i += 1
header = []
while i < len(lines) and lines[i].strip().startswith(">"):
header.append(lines[i])
i += 1
return header, lines[i:]
def strip_fences(text: str) -> str:
"""Remove fenced code blocks (3+ backticks or tildes, closed by a
fence of the same character and at least the same length)."""
out = []
fence_char = None
fence_len = 0
for line in text.splitlines():
m = FENCE_RE.match(line)
if fence_char:
if m and m.group(1)[0] == fence_char and len(m.group(1)) >= fence_len:
fence_char = None
continue
if m:
fence_char = m.group(1)[0]
fence_len = len(m.group(1))
continue
out.append(line)
return "\n".join(out)
def strip_noise(text: str) -> str:
text = INLINE_CODE_RE.sub(" ", text)
text = LINK_RE.sub(r"\1", text)
@@ -65,12 +107,19 @@ def keep_number(token: str) -> bool:
def extract_candidates(text: str) -> list[str]:
text = FENCED_CODE_RE.sub(" ", text)
text = strip_fences(text)
header, body = split_header(text.splitlines())
candidates = []
for line in text.splitlines():
skip_status_block = False
for line in [l for l in header if not METADATA_RE.match(l.strip())] + body:
stripped = line.strip()
if METADATA_RE.match(stripped) or STATUS_LINE_RE.match(stripped):
if STATUS_LINE_RE.match(stripped):
skip_status_block = True
continue
if skip_status_block:
if stripped.startswith(">"):
continue
skip_status_block = False
line = strip_noise(line)
candidates.extend(m.group(0) for m in DATE_RE.finditer(line))
candidates.extend(
@@ -81,7 +130,7 @@ def extract_candidates(text: str) -> list[str]:
seen = set()
unique = []
for cand in candidates:
cand = cand.strip(".,;:()[]")
cand = cand.strip().strip(".,;:()[]")
if cand and cand not in seen:
seen.add(cand)
unique.append(cand)
@@ -99,41 +148,47 @@ def raw_links_of(article_text: str) -> list[str]:
def contains(haystack: str, needle: str, is_quote: bool) -> bool:
if is_quote:
return needle in haystack
pattern = r"(?<![\d.,])" + re.escape(needle) + r"(?![\d.,KMB%])"
# The value must stand on its own: not part of a longer number
# (142K must not pass for 42K), but sentence-final punctuation
# after it is fine (raw "hit 42K." must pass for 42K).
pattern = r"(?<![\d.,])" + re.escape(needle) + r"(?!\d|[.,]\d|[KMB%])"
return re.search(pattern, haystack) is not None
def source_content(path: Path) -> str:
"""Raw file content minus its metadata header. Collection metadata
(Source/Collected/Published) is bookkeeping, not evidence; letting it
match candidates would false-pass dates and years."""
"""Raw file body with the metadata header removed. Collection
metadata (Source/Collected/Published) is bookkeeping, not evidence;
letting it match candidates would false-pass dates and years."""
lines = path.read_text(encoding="utf-8").splitlines()
body = [line for line in lines if not METADATA_RE.match(line.strip())]
_, body = split_header(lines)
return normalize("\n".join(body))
def check_article(article: Path) -> tuple[list[str], list[str]]:
def check_article(article: Path, root: Path) -> tuple[list[str], list[str]]:
"""Return (fidelity suspects, evidence errors) for one article."""
text = article.read_text(encoding="utf-8")
links = raw_links_of(text)
if not links:
if ARCHIVED_RE.search(text):
header, _ = split_header(text.splitlines())
if any(ARCHIVED_RE.match(line.strip()) for line in header):
return [], []
return [], ["article has no Raw field"]
raw_root = (root / "raw").resolve()
raws = []
errors = []
for link in links:
target = (article.parent / link).resolve()
if target.is_file():
raws.append(source_content(target))
else:
if not target.is_relative_to(raw_root):
errors.append(f"Raw link escapes raw/: {link}")
elif not target.is_file():
errors.append(f"unresolvable Raw link: {link}")
else:
raws.append(source_content(target))
misses = []
if raws:
quote_min_len = 15
for cand in extract_candidates(text):
needle = normalize(cand)
is_quote = len(cand) >= quote_min_len and not any(ch.isdigit() for ch in cand[:2])
is_quote = len(cand) >= 15 and not cand[:1].isdigit()
if not any(contains(raw, needle, is_quote) for raw in raws):
misses.append(cand)
return misses, errors
@@ -150,7 +205,7 @@ def no_material_paths(log_file: Path) -> set[str]:
return set()
paths = set()
for line in log_file.read_text(encoding="utf-8").splitlines():
m = NO_MATERIAL_RE.search(line)
m = NO_MATERIAL_HEADING_RE.match(line)
if m:
paths.add(m.group(1).rstrip("`.,;"))
return paths
@@ -190,6 +245,9 @@ def main(argv: list[str]) -> int:
path = Path(arg)
if not path.is_absolute():
path = root / path
if path.name in SKIP_FILES:
print(f"warning: {arg} is an index/log file, skipping", file=sys.stderr)
continue
if not path.is_file():
print(f"warning: article not found: {arg}", file=sys.stderr)
continue
@@ -197,18 +255,22 @@ def main(argv: list[str]) -> int:
if len(argv) <= 2:
articles = list(iter_articles(wiki_dir))
results = {}
for article in articles:
results[article] = check_article(article, root)
def label(article: Path) -> Path:
try:
return article.resolve().relative_to(root)
except ValueError:
return article
print("# Evidence check\n")
print("## Fidelity suspects")
suspect_count = 0
for article in articles:
misses, _ = check_article(article)
for article, (misses, _) in results.items():
if misses:
label = article
try:
label = article.resolve().relative_to(root)
except ValueError:
pass
print(f"\n{label}")
print(f"\n{label(article)}")
for miss in misses:
print(f"- {miss}")
suspect_count += 1
@@ -217,15 +279,9 @@ def main(argv: list[str]) -> int:
print("\n## Evidence errors")
error_count = 0
for article in articles:
_, errors = check_article(article)
for article, (_, errors) in results.items():
if errors:
label = article
try:
label = article.resolve().relative_to(root)
except ValueError:
pass
print(f"\n{label}")
print(f"\n{label(article)}")
for error in errors:
print(f"- {error}")
error_count += 1
tests/test_check_evidence.py
+184 -5
View File
@@ -40,13 +40,91 @@ example 78K and 9,999
> The forks situation changed after this was written.
"""
SECOND_RAW = """# Unrelated Notes
SENTENCE_FINAL_RAW = """# Numbers
> Source: https://example.com/notes
> Collected: 2026-05-01
> Source: https://example.com/numbers
> Collected: 2026-06-01
> Published: Unknown
Nothing here is compiled anywhere.
Revenue hit 42K. Uptime was 99.9%. The round closed on 2026-06-15.
"""
SENTENCE_FINAL_ARTICLE = """# Sentence final
> Sources: Example, 2026-06-01
> Raw: [numbers](../../raw/t/numbers.md)
Revenue hit 42K and uptime was 99.9%. The round closed on 2026-06-15.
"""
STATUS_EXPLANATION_ARTICLE = """# Status article
> Sources: Example, 2026-04-16
> Raw: [ghostty](../../raw/ai-research/2026-04-17-ghostty.md)
Ghostty has 42K stars.
> **Status: Outdated** (2026-07-23)
> Superseded by the 2026-07-18 report, which restated the count as 55K.
"""
FENCE_VARIANTS_ARTICLE = """# Fences
> Sources: Example, 2026-04-16
> Raw: [ghostty](../../raw/ai-research/2026-04-17-ghostty.md)
~~~text
not a factual claim: 777K
~~~
````text
also not a claim: 888K
````
"""
BODY_ARCHIVED_ARTICLE = """# Ordinary article
> Sources: Example, 2026-01-01
Some paragraph first.
> Archived: 2025-01-01
The release reached 321K users.
"""
RAW_WITH_BODY_METADATA_LINE = """# Source
> Source: https://example.com/x
> Collected: 2026-06-01
> Published: Unknown
First paragraph.
> Updated: The release reached 999K users.
"""
BODY_METADATA_ARTICLE = """# Body metadata
> Sources: Example, 2026-06-01
> Raw: [src](../../raw/t/src.md)
The release reached 999K users.
"""
ESCAPE_ARTICLE = """# Escape
> Sources: Example, 2026-01-01
> Raw: [support](../../notes/support.md)
The release reached 654K users.
"""
DEDUP_ARTICLE = """# Dedup
> Sources: Example, 2026-04-16
> Raw: [ghostty](../../raw/ai-research/2026-04-17-ghostty.md)
Missing value 88,123 appears here and again as 88,123 elsewhere.
"""
BOUNDARY_RAW = """# Numbers
@@ -133,7 +211,7 @@ class WikiTestCase(unittest.TestCase):
self.root = Path(self.tmp.name)
def tearDown(self):
self.tmp.cleanup
self.tmp.cleanup()
class FidelityCheckTest(WikiTestCase):
@@ -242,6 +320,107 @@ class CliTest(WikiTestCase):
self.assertNotIn("Traceback", result.stderr)
SECOND_RAW = """# Unrelated Notes
> Source: https://example.com/notes
> Collected: 2026-05-01
> Published: Unknown
Nothing here is compiled anywhere.
"""
def plain_wiki(root: Path, article_name: str, article: str, raw: str = PLAIN_RAW):
(root / "raw" / "t").mkdir(parents=True)
(root / "raw" / "t" / "src.md").write_text(raw)
(root / "wiki" / "t").mkdir(parents=True)
(root / "wiki" / "t" / article_name).write_text(article)
(root / "wiki" / "index.md").write_text("# Knowledge Base Index\n")
(root / "wiki" / "log.md").write_text("# Wiki Log\n")
class SentenceFinalTest(WikiTestCase):
def test_sentence_final_values_pass(self):
plain_wiki(self.root, "a.md", SENTENCE_FINAL_ARTICLE, raw=SENTENCE_FINAL_RAW)
(self.root / "raw" / "t" / "src.md").unlink()
(self.root / "raw" / "t" / "numbers.md").write_text(SENTENCE_FINAL_RAW)
result = run_checker(self.root)
self.assertIn("0 fidelity suspect(s)", result.stdout)
class StatusBlockTest(WikiTestCase):
def test_status_block_explanation_is_not_checked(self):
make_wiki(self.root)
(self.root / "wiki" / "ai-research" / "statused.md").write_text(STATUS_EXPLANATION_ARTICLE)
result = run_checker(self.root)
self.assertNotIn("2026-07-18", result.stdout)
self.assertNotIn("55K", result.stdout)
class FenceVariantsTest(WikiTestCase):
def test_tilde_and_long_backtick_fences_are_stripped(self):
make_wiki(self.root)
(self.root / "wiki" / "ai-research" / "fences.md").write_text(FENCE_VARIANTS_ARTICLE)
result = run_checker(self.root)
self.assertNotIn("777K", result.stdout)
self.assertNotIn("888K", result.stdout)
class HeaderScopeTest(WikiTestCase):
def test_archived_marker_in_body_does_not_exempt(self):
plain_wiki(self.root, "a.md", BODY_ARCHIVED_ARTICLE)
result = run_checker(self.root)
self.assertIn("no Raw field", result.stdout)
def test_raw_body_metadata_line_remains_evidence(self):
plain_wiki(self.root, "a.md", BODY_METADATA_ARTICLE, raw=RAW_WITH_BODY_METADATA_LINE)
result = run_checker(self.root)
self.assertNotIn("999K", result.stdout)
class RawEscapeTest(WikiTestCase):
def test_raw_link_outside_raw_dir_is_an_evidence_error(self):
(self.root / "raw").mkdir()
(self.root / "notes").mkdir()
(self.root / "notes" / "support.md").write_text("The release reached 654K users.\n")
(self.root / "wiki" / "t").mkdir(parents=True)
(self.root / "wiki" / "t" / "a.md").write_text(ESCAPE_ARTICLE)
(self.root / "wiki" / "index.md").write_text("# Knowledge Base Index\n")
(self.root / "wiki" / "log.md").write_text("# Wiki Log\n")
result = run_checker(self.root)
self.assertIn("escapes raw/", result.stdout)
class NoMaterialParsingTest(WikiTestCase):
def test_prose_mention_in_lint_entry_does_not_suppress(self):
(self.root / "raw" / "t").mkdir(parents=True)
(self.root / "raw" / "t" / "orphan.md").write_text("# Orphan\n")
(self.root / "wiki").mkdir()
(self.root / "wiki" / "index.md").write_text("# Knowledge Base Index\n")
(self.root / "wiki" / "log.md").write_text(
"# Wiki Log\n\n"
"## [2026-01-01] lint | 1 issues found, 0 auto-fixed\n"
"- Note: investigate no material: raw/t/orphan.md\n"
)
result = run_checker(self.root)
self.assertIn("raw/t/orphan.md", result.stdout)
class DedupTest(WikiTestCase):
def test_candidate_reported_once_without_trailing_space(self):
make_wiki(self.root)
(self.root / "wiki" / "ai-research" / "dedup.md").write_text(DEDUP_ARTICLE)
result = run_checker(self.root)
self.assertEqual(result.stdout.count("- 88,123"), 1)
class ExplicitArgsTest(WikiTestCase):
def test_index_and_log_are_skipped_even_when_passed_explicitly(self):
make_wiki(self.root)
result = run_checker(self.root, "wiki/log.md")
self.assertNotIn("no Raw field", result.stdout)
class RawInventoryTest(WikiTestCase):
def test_reports_raw_file_never_compiled(self):
make_wiki(self.root)