fix: make the evidence checker actually check

- Boundary matching: '42K' no longer passes against '142K' in the raw,
  closing the substring false-pass on the exact transcription-error
  class the checker exists for.
- Coverage: decimals and 4+ digit numbers (incl. bare years) are now
  candidates; small plain integers stay out of scope by design and the
  boundary is documented in the docstring.
- Fenced code is stripped before extraction (line-by-line processing
  made the multiline regex useless), and Status/Archived metadata
  lines no longer produce marker-date false suspects.
- Raw files are matched without their collection-metadata header, so a
  Collected date cannot false-pass an article's claim about that year.
- Evidence errors are reported instead of silently passing: ordinary
  articles without a Raw field, and Raw links that do not resolve.
- Article path arguments resolve against the project root, and missing
  files are warnings, not tracebacks.

scripts/check_evidence.py

@@ -1,36 +1,47 @@
 #!/usr/bin/env python3
 """Mechanical evidence check for a Karpathy-style LLM wiki.
 
-Report-only; never modifies files. Two sweeps:
+Report-only; never modifies files. Three sweeps:
 
-1. Fidelity — extract candidate literals (numbers, ISO dates, direct
-   quotes) from each wiki article and verify that each candidate appears
-   verbatim in the raw files linked by that article's Raw field. Misses
-   are listed as suspects. Derived values, product names, and deliberate
-   paraphrases will show up as suspects; judging them is the reader's
-   job, not this script's.
-2. Inventory — list raw files that no article's Raw field references,
+1. Fidelity — extract candidate literals (specific numbers, ISO dates,
+   direct quotes) from each wiki article and verify that each candidate
+   appears verbatim in the raw files linked by that article's Raw field.
+   Misses are listed as suspects. Derived values, product names, and
+   deliberate paraphrases will show up as suspects; judging them is the
+   reader's job, not this script's.
+2. Evidence errors — articles that cannot be verified at all: a missing
+   Raw field on a non-archive article, or Raw links that do not resolve.
+3. Inventory — raw files that no article's Raw field references,
    excluding files whose ingest was logged as "no material".
 
-Usage: check_evidence.py [wiki-root] [article.md ...]
-Defaults: wiki-root is the current directory; all wiki/**/*.md articles
-except index.md and log.md are checked.
+Coverage boundary: only specific-enough literals are candidates —
+numbers with a K/M/B/% suffix, a comma, a decimal point, or 4+ digits,
+plus ISO dates and quotes of 15+ characters. Small plain integers
+("42", "500") are deliberately not checked; they are too common in
+prose to keep the report worth reading.
+
+Usage: check_evidence.py [project-root] [article.md ...]
+Defaults: project-root is the current directory; all wiki/**/*.md
+articles except index.md and log.md are checked. Article paths may be
+absolute or relative to the project root.
 """
 
 import re
 import sys
 from pathlib import Path
 
-NUMBER_RE = re.compile(r"\d[\d,]*(?:\.\d+)?\s?[KMB%]")
-PLAIN_NUMBER_RE = re.compile(r"\d[\d,]{1,}")
+NUMBER_TOKEN_RE = re.compile(r"\d[\d,]*(?:\.\d+)?\s?[KMB%]?")
+SUFFIX_RE = re.compile(r"[KMB%]$")
 DATE_RE = re.compile(r"\d{4}-\d{2}(?:-\d{2})?")
 QUOTE_RES = [re.compile(r'"([^"\n]{15,})"'), re.compile(r"“([^”\n]{15,})”")]
-METADATA_RE = re.compile(r"^>\s*(Sources?|Raw|Collected|Published|Updated):")
+METADATA_RE = re.compile(r"^>\s*(Sources?|Raw|Collected|Published|Updated|Archived):")
+STATUS_LINE_RE = re.compile(r"^>\s*\*\*Status:")
 LINK_RE = re.compile(r"\[([^\]]*)\]\(([^)]*)\)")
 INLINE_CODE_RE = re.compile(r"`[^`\n]*`")
 FENCED_CODE_RE = re.compile(r"```.*?```", re.DOTALL)
 RAW_LINK_RE = re.compile(r"\(([^)]+\.md)[^)]*\)")
 NO_MATERIAL_RE = re.compile(r"no material:\s*(\S+)")
+ARCHIVED_RE = re.compile(r"^>\s*Archived:", re.MULTILINE)
 WS_RE = re.compile(r"\s+")
 
 SKIP_FILES = {"index.md", "log.md"}
@@ -41,22 +52,29 @@ def normalize(text: str) -> str:
 
 
 def strip_noise(text: str) -> str:
-    text = FENCED_CODE_RE.sub(" ", text)
     text = INLINE_CODE_RE.sub(" ", text)
     text = LINK_RE.sub(r"\1", text)
     return text
 
 
+def keep_number(token: str) -> bool:
+    token = token.strip()
+    if SUFFIX_RE.search(token) or "," in token or "." in token:
+        return True
+    return len(token) >= 4
+
+
 def extract_candidates(text: str) -> list[str]:
+    text = FENCED_CODE_RE.sub(" ", text)
     candidates = []
     for line in text.splitlines():
-        if METADATA_RE.match(line.strip()):
+        stripped = line.strip()
+        if METADATA_RE.match(stripped) or STATUS_LINE_RE.match(stripped):
             continue
         line = strip_noise(line)
         candidates.extend(m.group(0) for m in DATE_RE.finditer(line))
-        candidates.extend(m.group(0) for m in NUMBER_RE.finditer(line))
         candidates.extend(
-            m.group(0) for m in PLAIN_NUMBER_RE.finditer(line) if "," in m.group(0)
+            m.group(0) for m in NUMBER_TOKEN_RE.finditer(line) if keep_number(m.group(0))
         )
         for quote_re in QUOTE_RES:
             candidates.extend(m.group(1) for m in quote_re.finditer(line))
@@ -78,29 +96,55 @@ def raw_links_of(article_text: str) -> list[str]:
     return links
 
 
+def contains(haystack: str, needle: str, is_quote: bool) -> bool:
+    if is_quote:
+        return needle in haystack
+    pattern = r"(?<![\d.,])" + re.escape(needle) + r"(?![\d.,KMB%])"
+    return re.search(pattern, haystack) is not None
+
+
+def source_content(path: Path) -> str:
+    """Raw file content minus its metadata header. Collection metadata
+    (Source/Collected/Published) is bookkeeping, not evidence; letting it
+    match candidates would false-pass dates and years."""
+    lines = path.read_text(encoding="utf-8").splitlines()
+    body = [line for line in lines if not METADATA_RE.match(line.strip())]
+    return normalize("\n".join(body))
+
+
+def check_article(article: Path) -> tuple[list[str], list[str]]:
+    """Return (fidelity suspects, evidence errors) for one article."""
+    text = article.read_text(encoding="utf-8")
+    links = raw_links_of(text)
+    if not links:
+        if ARCHIVED_RE.search(text):
+            return [], []
+        return [], ["article has no Raw field"]
+    raws = []
+    errors = []
+    for link in links:
+        target = (article.parent / link).resolve()
+        if target.is_file():
+            raws.append(source_content(target))
+        else:
+            errors.append(f"unresolvable Raw link: {link}")
+    misses = []
+    if raws:
+        quote_min_len = 15
+        for cand in extract_candidates(text):
+            needle = normalize(cand)
+            is_quote = len(cand) >= quote_min_len and not any(ch.isdigit() for ch in cand[:2])
+            if not any(contains(raw, needle, is_quote) for raw in raws):
+                misses.append(cand)
+    return misses, errors
+
+
 def iter_articles(wiki_dir: Path):
     for path in sorted(wiki_dir.rglob("*.md")):
         if path.name not in SKIP_FILES:
             yield path
 
 
-def check_article(article: Path) -> list[str]:
-    text = article.read_text(encoding="utf-8")
-    raws = []
-    for link in raw_links_of(text):
-        target = (article.parent / link).resolve()
-        if target.is_file():
-            raws.append(normalize(target.read_text(encoding="utf-8")))
-    if not raws:
-        return []
-    misses = []
-    for cand in extract_candidates(text):
-        needle = normalize(cand)
-        if not any(needle in raw for raw in raws):
-            misses.append(cand)
-    return misses
-
-
 def no_material_names(log_file: Path) -> set[str]:
     if not log_file.is_file():
         return set()
@@ -141,16 +185,23 @@ def main(argv: list[str]) -> int:
         print(f"no wiki/ directory under {root}")
         return 1
 
-    if len(argv) > 2:
-        articles = [Path(a) for a in argv[2:]]
-    else:
+    articles = []
+    for arg in argv[2:]:
+        path = Path(arg)
+        if not path.is_absolute():
+            path = root / path
+        if not path.is_file():
+            print(f"warning: article not found: {arg}", file=sys.stderr)
+            continue
+        articles.append(path)
+    if len(argv) <= 2:
         articles = list(iter_articles(wiki_dir))
 
     print("# Evidence check\n")
     print("## Fidelity suspects")
     suspect_count = 0
     for article in articles:
-        misses = check_article(article)
+        misses, _ = check_article(article)
         if misses:
             label = article
             try:
@@ -164,6 +215,23 @@ def main(argv: list[str]) -> int:
     if suspect_count == 0:
         print("\n(none)")
 
+    print("\n## Evidence errors")
+    error_count = 0
+    for article in articles:
+        _, errors = check_article(article)
+        if errors:
+            label = article
+            try:
+                label = article.resolve().relative_to(root)
+            except ValueError:
+                pass
+            print(f"\n{label}")
+            for error in errors:
+                print(f"- {error}")
+                error_count += 1
+    if error_count == 0:
+        print("(none)")
+
     print("\n## Unreferenced raw files")
     orphans = unreferenced_raws(root)
     for path in orphans:
@@ -171,7 +239,10 @@ def main(argv: list[str]) -> int:
     if not orphans:
         print("(none)")
 
-    print(f"\n## Summary\n{suspect_count} fidelity suspect(s), {len(orphans)} unreferenced raw file(s)")
+    print(
+        f"\n## Summary\n{suspect_count} fidelity suspect(s), "
+        f"{error_count} evidence error(s), {len(orphans)} unreferenced raw file(s)"
+    )
     return 0
 
 

tests/test_check_evidence.py

@@ -30,11 +30,14 @@ The maintainer said "the terminal should feel invisible to users".
 ## Growth
 
 Forks grew to 3,020 last week.
-Install with `--limit 500` after downloading.
+Install with `--limit 9000` after downloading.
 
 ```
-ignore this 8888 number
+example 78K and 9,999
 ```
+
+> **Status: Outdated** (2026-07-23)
+> The forks situation changed after this was written.
 """
 
 SECOND_RAW = """# Unrelated Notes
@@ -46,6 +49,63 @@ SECOND_RAW = """# Unrelated Notes
 Nothing here is compiled anywhere.
 """
 
+BOUNDARY_RAW = """# Numbers
+
+> Source: https://example.com/numbers
+> Collected: 2026-06-01
+> Published: Unknown
+
+Ghostty reached 142K stars. Uptime was 95.5%. Forks: 13,020.
+"""
+
+BOUNDARY_ARTICLE = """# Boundary
+
+> Sources: Example, 2026-06-01
+> Raw: [numbers](../../raw/t/numbers.md)
+
+Ghostty has 42K stars and uptime of 5.5%. Forks grew to 3,020.
+"""
+
+PLAIN_RAW = """# Plain
+
+> Source: https://example.com/plain
+> Collected: 2026-06-01
+> Published: Unknown
+
+No numeric facts here at all.
+"""
+
+PLAIN_ARTICLE = """# Plain numbers
+
+> Sources: Example, 2026-06-01
+> Raw: [plain](../../raw/t/plain.md)
+
+There were 42 users; the ratio was 3.14; founded in 2026.
+"""
+
+ARCHIVE_ARTICLE = """# Old answer
+
+> Sources: [Ghostty](ghostty.md)
+> Archived: 2026-07-01
+
+At the time, Ghostty had 999K stars and the maintainer said "totally made up quote here".
+"""
+
+NO_RAW_ARTICLE = """# No raw
+
+> Sources: Example, 2026-06-01
+
+This ordinary article forgot its Raw field and claims 999K users.
+"""
+
+BROKEN_RAW_ARTICLE = """# Broken
+
+> Sources: Example, 2026-06-01
+> Raw: [gone](../../raw/t/nonexistent.md)
+
+Claims 999K users with no evidence anywhere.
+"""
+
 
 def make_wiki(root: Path, log: str = ""):
     (root / "raw" / "ai-research").mkdir(parents=True)
@@ -58,22 +118,28 @@ def make_wiki(root: Path, log: str = ""):
     (root / "wiki" / "log.md").write_text(log or "# Wiki Log\n")
 
 
-def run_checker(root: Path, *args: str) -> subprocess.CompletedProcess:
+def run_checker(root: Path, *args: str, cwd: str | None = None) -> subprocess.CompletedProcess:
     return subprocess.run(
         [sys.executable, str(SCRIPT), str(root), *args],
         capture_output=True,
         text=True,
+        cwd=cwd,
     )
 
 
-class FidelityCheckTest(unittest.TestCase):
+class WikiTestCase(unittest.TestCase):
     def setUp(self):
         self.tmp = tempfile.TemporaryDirectory()
         self.root = Path(self.tmp.name)
-        make_wiki(self.root)
 
     def tearDown(self):
-        self.tmp.cleanup()
+        self.tmp.cleanup
+
+
+class FidelityCheckTest(WikiTestCase):
+    def setUp(self):
+        super().setUp()
+        make_wiki(self.root)
 
     def test_flags_value_absent_from_raw(self):
         result = run_checker(self.root)
@@ -87,18 +153,96 @@ class FidelityCheckTest(unittest.TestCase):
 
     def test_ignores_numbers_in_code(self):
         result = run_checker(self.root)
-        self.assertNotIn("500", result.stdout)
-        self.assertNotIn("8888", result.stdout)
+        self.assertNotIn("9000", result.stdout)
+        self.assertNotIn("78K", result.stdout)
+        self.assertNotIn("9,999", result.stdout)
+
+    def test_ignores_status_block_marker_date(self):
+        result = run_checker(self.root)
+        self.assertNotIn("2026-07-23", result.stdout)
 
 
-class RawInventoryTest(unittest.TestCase):
+class BoundaryMatchingTest(WikiTestCase):
     def setUp(self):
-        self.tmp = tempfile.TemporaryDirectory()
-        self.root = Path(self.tmp.name)
+        super().setUp()
+        (self.root / "raw" / "t").mkdir(parents=True)
+        (self.root / "raw" / "t" / "numbers.md").write_text(BOUNDARY_RAW)
+        (self.root / "wiki" / "t").mkdir(parents=True)
+        (self.root / "wiki" / "t" / "a.md").write_text(BOUNDARY_ARTICLE)
+        (self.root / "wiki" / "index.md").write_text("# Knowledge Base Index\n")
+        (self.root / "wiki" / "log.md").write_text("# Wiki Log\n")
 
-    def tearDown(self):
-        self.tmp.cleanup()
+    def test_substring_of_larger_number_does_not_pass(self):
+        result = run_checker(self.root)
+        self.assertIn("42K", result.stdout)
+        self.assertIn("5.5%", result.stdout)
+        self.assertIn("3,020", result.stdout)
 
+
+class NumberCoverageTest(WikiTestCase):
+    def setUp(self):
+        super().setUp()
+        (self.root / "raw" / "t").mkdir(parents=True)
+        (self.root / "raw" / "t" / "plain.md").write_text(PLAIN_RAW)
+        (self.root / "wiki" / "t").mkdir(parents=True)
+        (self.root / "wiki" / "t" / "a.md").write_text(PLAIN_ARTICLE)
+        (self.root / "wiki" / "index.md").write_text("# Knowledge Base Index\n")
+        (self.root / "wiki" / "log.md").write_text("# Wiki Log\n")
+
+    def test_flags_decimals_and_long_numbers(self):
+        result = run_checker(self.root)
+        self.assertIn("3.14", result.stdout)
+        self.assertIn("2026", result.stdout)
+
+    def test_small_plain_integers_are_out_of_scope(self):
+        result = run_checker(self.root)
+        self.assertNotIn("42 users", result.stdout)
+        for line in result.stdout.splitlines():
+            self.assertFalse(line.strip() == "- 42", f"small int flagged: {line}")
+
+
+class EvidenceErrorTest(WikiTestCase):
+    def setUp(self):
+        super().setUp()
+        make_wiki(self.root)
+
+    def test_archive_page_without_raw_is_legitimate(self):
+        (self.root / "wiki" / "ai-research" / "old-answer.md").write_text(ARCHIVE_ARTICLE)
+        result = run_checker(self.root)
+        self.assertNotIn("999K", result.stdout)
+        self.assertNotIn("old-answer", result.stdout)
+
+    def test_ordinary_article_without_raw_is_an_evidence_error(self):
+        (self.root / "wiki" / "ai-research" / "no-raw.md").write_text(NO_RAW_ARTICLE)
+        result = run_checker(self.root)
+        self.assertIn("no-raw", result.stdout)
+        self.assertIn("no Raw field", result.stdout)
+
+    def test_broken_raw_link_is_an_evidence_error(self):
+        (self.root / "wiki" / "ai-research" / "broken.md").write_text(BROKEN_RAW_ARTICLE)
+        result = run_checker(self.root)
+        self.assertIn("broken", result.stdout)
+        self.assertIn("unresolvable Raw link", result.stdout)
+
+
+class CliTest(WikiTestCase):
+    def setUp(self):
+        super().setUp()
+        make_wiki(self.root)
+
+    def test_article_args_resolve_against_root_not_cwd(self):
+        result = run_checker(self.root, "wiki/ai-research/ghostty.md", cwd="/")
+        self.assertEqual(result.returncode, 0)
+        self.assertIn("3,020", result.stdout)
+
+    def test_missing_article_is_a_warning_not_a_traceback(self):
+        result = run_checker(self.root, "wiki/nope.md")
+        self.assertEqual(result.returncode, 0)
+        self.assertIn("wiki/nope.md", result.stderr)
+        self.assertNotIn("Traceback", result.stderr)
+
+
+class RawInventoryTest(WikiTestCase):
     def test_reports_raw_file_never_compiled(self):
         make_wiki(self.root)
         result = run_checker(self.root)