From 1175bfe7a9f2468eea4136e7aafc3725f50cfb11 Mon Sep 17 00:00:00 2001 From: RIC <80094642+ddggkkcc@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:01:51 +0800 Subject: [PATCH] bench(wiki-rag): reproducible 60-question doc-vs-chunk evaluation harness Three scripts + 60 hand-annotated questions over the public countbot.cn/docs corpus (52 pages, fetched by fetch_corpus.py): - run_g0.py: doc-level BM25 baseline (pre-change behaviour) - run_g1.py: chunk-level BM25 (this PR) on the same questions - fetch_corpus.py: downloads the public corpus into rag-bench/corpus/ Pure programmatic scoring (recall@k / MRR@10 / NDCG@10 / top1 / injected-token volume), no LLM judge, no private data. The headline numbers quoted in the PR description come from running these on the same corpus/questions before and after the chunk change. Refs: #107 --- rag-bench/questions.jsonl | 60 +++++++ rag-bench/scripts/fetch_corpus.py | 280 ++++++++++++++++++++++++++++++ rag-bench/scripts/run_g0.py | 223 ++++++++++++++++++++++++ rag-bench/scripts/run_g1.py | 150 ++++++++++++++++ 4 files changed, 713 insertions(+) create mode 100644 rag-bench/questions.jsonl create mode 100644 rag-bench/scripts/fetch_corpus.py create mode 100644 rag-bench/scripts/run_g0.py create mode 100644 rag-bench/scripts/run_g1.py diff --git a/rag-bench/questions.jsonl b/rag-bench/questions.jsonl new file mode 100644 index 0000000..22d9247 --- /dev/null +++ b/rag-bench/questions.jsonl @@ -0,0 +1,60 @@ +{"id": "S1-01", "type": "single_doc", "semantic": false, "question": "CountBot 的长期记忆存储在哪里？是什么格式的文件？", "expected_points": ["工作空间中的单文件存储", "JSON 格式，可直接查看编辑"], "source_pages": ["core/memory.md"]} +{"id": "S1-02", "type": "single_doc", "semantic": false, "question": "CountBot 记忆条目的格式长什么样？包含哪些字段？", "expected_points": ["条目字段结构（id/时间戳/内容/来源等）"], "source_pages": ["core/memory.md"]} +{"id": "S1-03", "type": "single_doc", "semantic": false, "question": "什么是 auto-overflow 记忆？什么情况下会产生？", "expected_points": ["会话历史超出窗口限制时", "旧消息被压缩写入记忆，来源标记为 auto-overflow"], "source_pages": ["core/memory.md"]} +{"id": "S1-04", "type": "single_doc", "semantic": false, "question": "AI 模型是怎么读写长期记忆的？需要直接操作文件吗？", "expected_points": ["通过工具调用读写", "模型不直接操作文件"], "source_pages": ["core/memory.md"]} +{"id": "S1-05", "type": "single_doc", "semantic": false, "question": "CountBot 有哪几种启动方式？分别用什么命令？", "expected_points": ["start_app.py / start_desktop.py / start_dev.py 等启动脚本及区别"], "source_pages": ["getting-started/installation.md"]} +{"id": "S1-06", "type": "single_doc", "semantic": false, "question": "CountBot 对运行环境有什么要求？需要什么版本的 Python？", "expected_points": ["Python 版本要求", "操作系统支持情况"], "source_pages": ["getting-started/installation.md"]} +{"id": "S1-07", "type": "single_doc", "semantic": false, "question": "怎么让局域网内其他设备访问 CountBot 的 Web 界面？", "expected_points": ["COUNTBOT_HOST=0.0.0.0 环境变量", "通过本机 IP+端口访问"], "source_pages": ["getting-started/installation.md"]} +{"id": "S1-08", "type": "single_doc", "semantic": false, "question": "桌面版相比 Web 版有什么限制？", "expected_points": ["桌面版功能边界", "系统依赖差异（libwebkit2gtk）"], "source_pages": ["getting-started/installation.md"]} +{"id": "S1-09", "type": "single_doc", "semantic": false, "question": "requirements.txt 里的依赖是怎么分类的？", "expected_points": ["依赖分组结构（核心/渠道/桌面等分类方式）"], "source_pages": ["getting-started/installation.md"]} +{"id": "S1-10", "type": "single_doc", "semantic": false, "question": "CountBot 启动后的自检清单有哪些检查项？", "expected_points": ["启动自检项列表（配置/网络/渠道等）"], "source_pages": ["getting-started/installation.md"]} +{"id": "S1-11", "type": "single_doc", "semantic": false, "question": "workspace 工作空间目录里都有哪些内容？", "expected_points": ["skills/、memory/、temp/、.skills_config.json 等结构"], "source_pages": ["getting-started/configuration-manual.md"]} +{"id": "S1-12", "type": "single_doc", "semantic": false, "question": "前端构建产物放在哪个目录？", "expected_points": ["frontend/dist/"], "source_pages": ["advanced/deployment.md"]} +{"id": "S1-13", "type": "single_doc", "semantic": false, "question": "后端 lifespan 的初始化顺序是什么？", "expected_points": ["数据库初始化→加载配置→共享组件→消息队列→渠道→消息处理→Cron→心跳等顺序"], "source_pages": ["advanced/deployment.md"]} +{"id": "S1-14", "type": "single_doc", "semantic": false, "question": "CountBot 内置支持多少个模型 provider？", "expected_points": ["22 个"], "source_pages": ["core/providers.md"]} +{"id": "S1-15", "type": "single_doc", "semantic": false, "question": "Cron 定时任务底层用什么调度库实现？", "expected_points": ["APScheduler"], "source_pages": ["core/cron.md"]} +{"id": "S1-16", "type": "single_doc", "semantic": true, "question": "我想让 CountBot 到点自动干活，不用我发消息触发，能做到吗？", "expected_points": ["Cron 定时任务功能", "APScheduler 调度"], "source_pages": ["core/cron.md"]} +{"id": "S1-17", "type": "single_doc", "semantic": true, "question": "CountBot 能在新的一次对话里想起我上次跟它说过的话吗？靠什么机制？", "expected_points": ["长期记忆机制", "跨会话记忆存储"], "source_pages": ["core/memory.md"]} +{"id": "S1-18", "type": "single_doc", "semantic": true, "question": "一件复杂的事能让几个 AI 角色接力完成吗？", "expected_points": ["Agent 团队功能", "多角色协作"], "source_pages": ["core/agent-teams.md"]} +{"id": "S1-19", "type": "single_doc", "semantic": true, "question": "我平时在终端里用 AI 编码工具写代码，CountBot 能直接调用它吗？", "expected_points": ["外部编码工具集成（Claude Code/Codex 等）"], "source_pages": ["core/external-coding-tools.md"]} +{"id": "S1-20", "type": "single_doc", "semantic": true, "question": "同事想在他自己的电脑上打开我的 CountBot 页面来用，行得通吗？", "expected_points": ["远程访问方案", "局域网/公网暴露与认证"], "source_pages": ["advanced/remote-access.md"]} +{"id": "S1-21", "type": "single_doc", "semantic": true, "question": "知识库文档越存越多，查询会越来越慢吗？", "expected_points": ["Wiki LRU 缓存机制", "缓存容量 128 个文档，重复查询性能提升"], "source_pages": ["releases/v0.9.0.md"]} +{"id": "S1-22", "type": "single_doc", "semantic": true, "question": "我有好几个 API Key，一个额度用完了能自动换下一个吗？", "expected_points": ["API Key 轮换机制", "401/429 触发自动切换"], "source_pages": ["releases/v0.9.0.md"]} +{"id": "S1-23", "type": "single_doc", "semantic": true, "question": "我在两个不同的群里分别跟 CountBot 聊，它会把两边的话题记混吗？", "expected_points": ["会话按渠道/聊天隔离", "channels 会话作用域"], "source_pages": ["core/channels.md"]} +{"id": "S1-24", "type": "single_doc", "semantic": true, "question": "聊到一半想抛开前面的话题重新开始，有什么办法？", "expected_points": ["/new 等斜杠命令重开会话"], "source_pages": ["core/im-commands.md"]} +{"id": "S1-25", "type": "single_doc", "semantic": true, "question": "升级到新版本的时候，我之前的会话和记忆还能保住吗？", "expected_points": ["升级保留数据", "backup/数据兼容说明"], "source_pages": ["getting-started/update-guide.md"]} +{"id": "S2-01", "type": "cross_doc", "semantic": false, "question": "怎么配置 CountBot 每天早上 8 点自动搜索新闻并发送到 Telegram？", "expected_points": ["Cron 定时任务配置", "Telegram 渠道配置", "搜索技能与消息发送联动"], "source_pages": ["core/cron.md", "core/channels.md", "scenarios/每天早上一条消息搞定全天：AI早报自动生成的场景与调度技巧.md"]} +{"id": "S2-02", "type": "cross_doc", "semantic": false, "question": "我在家用桌面版运行 CountBot，出差在外能用手机浏览器访问吗？", "expected_points": ["远程访问方案", "桌面版运行特点", "认证与安全"], "source_pages": ["getting-started/installation.md", "advanced/remote-access.md"]} +{"id": "S2-03", "type": "cross_doc", "semantic": false, "question": "钉钉上配置完成后机器人不回复消息，应该怎么排查？", "expected_points": ["渠道配置检查", "troubleshooting 排查步骤"], "source_pages": ["getting-started/ChannelConfig.md", "getting-started/troubleshooting.md", "core/channels.md"]} +{"id": "S2-04", "type": "cross_doc", "semantic": false, "question": "配置好模型 provider 之后，怎么验证连接是正常的？", "expected_points": ["测试模型连接的方式", "provider 状态检查"], "source_pages": ["core/providers.md", "getting-started/quick-reference.md"]} +{"id": "S2-05", "type": "cross_doc", "semantic": false, "question": "子代理执行完任务之后，结果是怎么回到主对话里的？", "expected_points": ["子代理结果回传机制", "agent loop 中的汇合点"], "source_pages": ["core/subagent.md", "core/agent-loop.md"]} +{"id": "S2-06", "type": "cross_doc", "semantic": false, "question": "Agent 团队里不同成员想用不同模型，怎么指定优先级？", "expected_points": ["团队成员模型配置", "provider 优先级"], "source_pages": ["core/agent-teams.md", "core/providers.md"]} +{"id": "S2-07", "type": "cross_doc", "semantic": false, "question": "CountBot 执行 Shell 命令有什么安全限制？怎么防止危险命令？", "expected_points": ["Shell 工具限制", "安全机制（确认/黑名单等）"], "source_pages": ["core/tools.md", "advanced/security.md"]} +{"id": "S2-08", "type": "cross_doc", "semantic": false, "question": "MCP 工具和外部编码工具（比如 Claude Code）有什么区别？分别什么场景用？", "expected_points": ["MCP 集成方式", "外部编码工具定位", "两者适用场景差异"], "source_pages": ["core/tools.md", "core/external-coding-tools.md"]} +{"id": "S2-09", "type": "cross_doc", "semantic": false, "question": "什么前提下才能把消息路由到 direct 模式直接给编码工具处理？", "expected_points": ["/route direct 命令", "使用前提（需配置编码工具）"], "source_pages": ["core/im-commands.md", "core/external-coding-tools.md"]} +{"id": "S2-10", "type": "cross_doc", "semantic": false, "question": "一个会话聊得特别长，系统会发生什么？", "expected_points": ["历史窗口限制", "自动总结触发（30 条/15000 字符）", "auto-overflow 记忆"], "source_pages": ["core/agent-loop.md", "advanced/http-api-deep-dive.md", "core/memory.md"]} +{"id": "S2-11", "type": "cross_doc", "semantic": false, "question": "把 CountBot 部署在局域网给全组使用，有哪些安全注意事项？", "expected_points": ["认证机制", "API Key 保护", "网络安全配置"], "source_pages": ["advanced/deployment.md", "advanced/security.md", "advanced/remote-access.md"]} +{"id": "S2-12", "type": "cross_doc", "semantic": false, "question": "升级到新版本之前需要做哪些准备工作？", "expected_points": ["备份", "依赖更新", "数据迁移注意事项"], "source_pages": ["getting-started/update-guide.md", "getting-started/installation.md"]} +{"id": "S2-13", "type": "cross_doc", "semantic": false, "question": "能在 Web 界面上创建定时任务吗，还是只能改配置文件？", "expected_points": ["Web 界面任务管理", "cron 任务创建方式"], "source_pages": ["core/cron.md", "getting-started/quick-reference.md"]} +{"id": "S2-14", "type": "cross_doc", "semantic": false, "question": "不想用云端的 API，能不能完全在本地跑模型来用 CountBot？", "expected_points": ["本地模型支持（Ollama 等）", "provider 配置方式", "性能取舍"], "source_pages": ["core/providers.md", "getting-started/installation.md"]} +{"id": "S2-15", "type": "cross_doc", "semantic": false, "question": "用 /route 和 /coder 切换了路由之后，这些切换会记录进会话历史吗？", "expected_points": ["命令是否写入历史", "会话状态保持方式"], "source_pages": ["core/im-commands.md", "core/agent-loop.md"]} +{"id": "S3-01", "type": "needle", "semantic": false, "question": "会话达到多少条消息会触发自动总结？", "expected_points": ["30 条"], "source_pages": ["advanced/http-api-deep-dive.md"]} +{"id": "S3-02", "type": "needle", "semantic": false, "question": "自动总结的字符数阈值是多少？", "expected_points": ["15000 字符"], "source_pages": ["advanced/http-api-deep-dive.md"]} +{"id": "S3-03", "type": "needle", "semantic": false, "question": "子代理最多允许迭代多少轮？", "expected_points": ["15"], "source_pages": ["core/subagent.md"]} +{"id": "S3-04", "type": "needle", "semantic": false, "question": "Wiki 知识库的 LRU 缓存容量是多少个文档？", "expected_points": ["128"], "source_pages": ["releases/v0.9.0.md"]} +{"id": "S3-05", "type": "needle", "semantic": false, "question": "v0.9.0 的心跳问候优化把每次的 LLM 调用次数从几次降到了几次？", "expected_points": ["从 2 次降到 1 次"], "source_pages": ["releases/v0.9.0.md"]} +{"id": "S3-06", "type": "needle", "semantic": false, "question": "API Key 轮换是在收到哪些 HTTP 状态码时触发的？", "expected_points": ["401 和 429"], "source_pages": ["releases/v0.9.0.md"]} +{"id": "S3-07", "type": "needle", "semantic": false, "question": "CountBot 内置支持多少个模型 provider？", "expected_points": ["22 个"], "source_pages": ["core/providers.md"]} +{"id": "S3-08", "type": "needle", "semantic": false, "question": "SQLite 数据库文件叫什么名字？定时任务存在哪张表里？", "expected_points": ["countbot.db", "cron_jobs 表"], "source_pages": ["advanced/deployment.md"]} +{"id": "S3-09", "type": "needle", "semantic": false, "question": "Telegram 的代理设置支持什么协议？", "expected_points": ["socks5://"], "source_pages": ["getting-started/troubleshooting.md"]} +{"id": "S3-10", "type": "needle", "semantic": false, "question": "Linux 上安装桌面版需要额外装哪个系统依赖包？", "expected_points": ["libwebkit2gtk-4.0-37"], "source_pages": ["getting-started/installation.md"]} +{"id": "S4-01", "type": "negative", "semantic": false, "question": "CountBot 怎么部署到 Kubernetes 集群上？", "expected_points": ["应拒答：官方不支持 K8s 部署"], "source_pages": []} +{"id": "S4-02", "type": "negative", "semantic": false, "question": "CountBot 有官方手机 App 吗？在哪里下载？", "expected_points": ["应拒答：没有官方移动端 App"], "source_pages": []} +{"id": "S4-03", "type": "negative", "semantic": false, "question": "CountBot 支持语音通话吗？", "expected_points": ["应拒答：不支持语音通话"], "source_pages": []} +{"id": "S4-04", "type": "negative", "semantic": false, "question": "CountBot 能接入 Home Assistant 控制智能家居设备吗？", "expected_points": ["应拒答：无智能家居集成"], "source_pages": []} +{"id": "S4-05", "type": "negative", "semantic": false, "question": "CountBot 商业版是怎么收费的？", "expected_points": ["应拒答：开源项目无商业定价"], "source_pages": []} +{"id": "S4-06", "type": "negative", "semantic": false, "question": "可以把 CountBot 的 SQLite 换成 PostgreSQL 吗？官方支持吗？", "expected_points": ["应拒答：仅支持 SQLite，无 PostgreSQL 支持"], "source_pages": []} +{"id": "S4-07", "type": "negative", "semantic": false, "question": "哪里可以下载 CountBot 的第三方插件？", "expected_points": ["应拒答：无插件市场"], "source_pages": []} +{"id": "S4-08", "type": "negative", "semantic": false, "question": "CountBot 能运行在树莓派或 ARM 设备上吗？", "expected_points": ["应拒答：官方未支持 ARM/树莓派"], "source_pages": []} +{"id": "S4-09", "type": "negative", "semantic": false, "question": "CountBot 支持 SSO 单点登录吗？", "expected_points": ["应拒答：无 SSO 支持"], "source_pages": []} +{"id": "S4-10", "type": "negative", "semantic": false, "question": "CountBot 支持视频通话或者屏幕共享吗？", "expected_points": ["应拒答：不支持"], "source_pages": []} diff --git a/rag-bench/scripts/fetch_corpus.py b/rag-bench/scripts/fetch_corpus.py new file mode 100644 index 0000000..1d9e688 --- /dev/null +++ b/rag-bench/scripts/fetch_corpus.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""抓取 countbot.cn 官方文档站全量页面，转为 Markdown 语料。 + +用途：RAG 基线测试（G0）的 C1/C2 语料构建。 +来源全部为官方公开页面，保证可复现。 + +输出： + rag-bench/corpus/.md —— 每页一份 Markdown（首行为 H1 标题） + rag-bench/corpus/manifest.json —— 抓取清单（slug/title/url/chars/est_tokens） + +用法： + python fetch_corpus.py [--out ../corpus] +""" + +import json +import re +import sys +import time +import urllib.request +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import quote, unquote, urlparse + +BASE = "https://countbot.cn" +UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36") + +# 额外页面（llms.txt 列出但 sitemap 缺失：scenarios / 新版 releases / update-guide） +EXTRA_PATHS = [ + "/docs/scenarios/一句话清空收件箱：AI邮件分拣的场景拆解与提示词技巧/", + "/docs/scenarios/帮我做个网页然后上线：从需求描述到生产部署的完整话术/", + "/docs/scenarios/帮我搜一下最近AI有什么新动态：多源信息聚合的场景与检索技巧/", + "/docs/scenarios/帮我盯着这件事：定时监控与主动提醒的场景设计与自动化技巧/", + "/docs/scenarios/帮我规划一趟旅行：多技能联动的复合任务拆解与编排技巧/", + "/docs/scenarios/每天早上一条消息搞定全天：AI早报自动生成的场景与调度技巧/", + "/docs/releases/v0.7.0/", + "/docs/releases/v0.8.0/", + "/docs/releases/v0.9.0/", + "/docs/getting-started/update-guide/", +] + + +class ArticleToMarkdown(HTMLParser): + """把
内的 HTML 转为 Markdown 文本（容忍不完整标签）。 + + 保留结构：h1-h6 标题、列表项、代码块（围栏）、表格（管道行）、引用。 + 行内标签只保留文本。script/style/svg/nav 忽略。 + """ + + BLOCK_TAGS = {"p", "div", "section", "header", "footer", "br", "hr", + "ul", "ol", "table", "thead", "tbody", "blockquote", "details", "summary"} + SKIP_TAGS = {"script", "style", "svg", "nav", "img", "button", "input"} + + def __init__(self): + super().__init__(convert_charrefs=True) + self.lines: list[str] = [] + self._buf: list[str] = [] + self._skip_depth = 0 + self._pre_depth = 0 + self._pre_buf: list[str] = [] + self._in_cell = False + self._cell_buf: list[str] = [] + self._row_cells: list[str] | None = None + self._is_header_row = False + self._list_depth = 0 + self._li_open = False + self._quote = False + + # ---------- 基础 ---------- + def _text(self, s: str): + if self._skip_depth > 0: + return + if self._pre_depth > 0: + self._pre_buf.append(s) + elif self._in_cell: + self._cell_buf.append(s) + else: + self._buf.append(s) + if self._li_open: + self._li_open = False + + def _flush_para(self): + text = re.sub(r"\s+", " ", "".join(self._buf)).strip() + self._buf = [] + if not text: + return + prefix = "> " if self._quote else "" + indent = " " * max(self._list_depth - 1, 0) + bullet = "- " if self._list_depth > 0 else "" + self.lines.append(prefix + indent + bullet + text) + + def _flush_cell(self): + text = re.sub(r"\s+", " ", "".join(self._cell_buf)).strip() + self._cell_buf = [] + if self._row_cells is not None: + self._row_cells.append(text) + + # ---------- 标签处理 ---------- + def handle_starttag(self, tag, attrs): + if tag in self.SKIP_TAGS: + self._skip_depth += 1 + return + if self._skip_depth > 0: + return + if tag == "pre": + self._flush_para() + self._pre_depth += 1 + if self._pre_depth == 1: + self._pre_buf = [] + elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"): + self._flush_para() + self._buf.append("\x00H%d\x00" % int(tag[1])) # 标记，flush 时替换 + elif tag in ("p", "div", "section", "header", "footer", "blockquote", + "details", "summary", "ul", "ol", "table", "thead", "tbody"): + self._flush_para() + if tag == "blockquote": + self._quote = True + elif tag in ("ul", "ol"): + self._list_depth += 1 + elif tag == "li": + self._flush_para() + self._li_open = True + elif tag == "tr": + self._row_cells = [] + elif tag in ("td", "th"): + self._in_cell = True + if tag == "th": + self._is_header_row = True + elif tag == "br": + self._text(" ") + elif tag == "hr": + self._flush_para() + self.lines.append("---") + + def handle_endtag(self, tag): + if tag in self.SKIP_TAGS: + self._skip_depth = max(0, self._skip_depth - 1) + return + if self._skip_depth > 0: + return + if tag == "pre": + self._pre_depth -= 1 + if self._pre_depth == 0: + code = "".join(self._pre_buf).rstrip("\n") + self.lines.append("```\n" + code + "\n```") + elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"): + level = int(tag[1]) + raw = "".join(self._buf) + self._buf = [] + text = re.sub(r"\s+", " ", raw.replace("\x00H%d\x00" % level, "")).strip() + prefix = "> " if self._quote else "" + self.lines.append(prefix + "#" * level + " " + text) + elif tag in ("p", "div", "section", "header", "footer", "details", "summary"): + self._flush_para() + elif tag == "blockquote": + self._quote = False + self._flush_para() + elif tag in ("ul", "ol"): + self._list_depth = max(0, self._list_depth - 1) + self._flush_para() + elif tag in ("td", "th"): + self._in_cell = False + self._flush_cell() + elif tag == "tr": + if self._row_cells is not None: + cells = self._row_cells + self._row_cells = None + if cells: + self.lines.append("| " + " | ".join(cells) + " |") + if self._is_header_row: + self.lines.append("|" + "---|" * len(cells)) + self._is_header_row = False + elif tag == "table": + self._flush_para() + + def handle_data(self, data): + self._text(data) + + def result(self) -> str: + self._flush_para() + out = [] + prev_blank = True + for ln in self.lines: + if ln.strip(): + out.append(ln) + prev_blank = False + else: + if not prev_blank: + out.append("") + prev_blank = True + return "\n".join(out).strip() + "\n" + + +def extract_article(html: str) -> tuple[str, str]: + """返回 (title, markdown)。title 取

或 。""" + m = re.search(r"<article[^>]*>(.*?)</article>", html, re.S | re.I) + body_html = m.group(1) if m else html + parser = ArticleToMarkdown() + parser.feed(body_html) + md = parser.result() + h1 = re.search(r"^#\s+(.+)$", md, re.M) + title = h1.group(1).strip() if h1 else "" + if not title: + t = re.search(r"<title>(.*?)", html, re.S) + title = t.group(1).strip() if t else "" + return title, md + + +def slug_of(path: str) -> str: + p = path[len("/docs/"):] if path.startswith("/docs/") else path + p = p.strip("/") + if not p: + p = "index" + return p + + +def fetch(url: str) -> str: + req = urllib.request.Request(url, headers={"User-Agent": UA}) + with urllib.request.urlopen(req, timeout=25) as resp: + return resp.read().decode("utf-8", errors="ignore") + + +def main(): + out_dir = Path(sys.argv[sys.argv.index("--out") + 1]) if "--out" in sys.argv \ + else Path(__file__).resolve().parent.parent / "corpus" + out_dir.mkdir(parents=True, exist_ok=True) + + # 1) URL 全集 = sitemap + EXTRA（按 unquote 后路径去重，中文与百分号编码视为同页） + sitemap_url = BASE + "/sitemap.xml" + xml = fetch(sitemap_url) + paths = [urlparse(u).path for u in re.findall(r"(.*?)", xml) + if "/docs/" in u] + paths += EXTRA_PATHS + # 归一化：unquote 去重，再统一以 / 结尾 + norm = {} + for p in paths: + key = unquote(p).rstrip("/") + norm[key] = p if p.endswith("/") else p + "/" + paths = sorted(norm.values()) + # sitemap 中的 articles/* 在 cn 站已 404（死链，指向另一域名的 SPA 兜底页），跳过 + paths = [p for p in paths if "/docs/articles/" not in unquote(p)] + + manifest = [] + failed = [] + for i, path in enumerate(paths, 1): + slug = slug_of(unquote(path)) + # 统一 URL 编码（中文路径必须 quote，否则 ascii 编码错误） + url = BASE + quote(unquote(path), safe="/:") + try: + html = fetch(url) + title, md = extract_article(html) + if len(md.strip()) < 100: + raise ValueError(f"content too short: {len(md)} chars") + out_file = out_dir / f"{slug}.md" + out_file.parent.mkdir(parents=True, exist_ok=True) + out_file.write_text(md, encoding="utf-8") + chars = len(md) + manifest.append({ + "slug": slug, "title": title, "url": url, + "chars": chars, "est_tokens": int(chars / 1.5), + "section": slug.split("/")[0] if "/" in slug else slug, + }) + print(f"[{i}/{len(paths)}] {slug} ({chars} chars)") + except Exception as e: + failed.append({"slug": slug, "url": url, "error": str(e)}) + print(f"[{i}/{len(paths)}] FAIL {slug}: {e}") + time.sleep(0.25) + + (out_dir / "manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + total_chars = sum(m["chars"] for m in manifest) + print(f"\nOK: {len(manifest)} pages, {total_chars} chars " + f"(~{total_chars // 1500}K est tokens); failed: {len(failed)}") + if failed: + (out_dir / "failed.json").write_text( + json.dumps(failed, ensure_ascii=False, indent=2), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/rag-bench/scripts/run_g0.py b/rag-bench/scripts/run_g0.py new file mode 100644 index 0000000..ec91ab2 --- /dev/null +++ b/rag-bench/scripts/run_g0.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""G0 基线：测量 CountBot 现状 BM25 文档级检索在 60 题问题集上的表现。 + +口径：纯程序化打分（recall@k / MRR@10 / NDCG@10 / top1 / 注入 token 量），无 LLM 判分，可直接复现。 +指标： + - S1/S3（单文档）: recall@5/10, MRR@10, NDCG@10 + - S2（跨文档）: 全源 recall@10, primary MRR@10, 平均源覆盖率 + - S4（负样本）: 得分分布（top1 score），与非负样本对比 + - token 注入（镜像生产 top_k=3, _handle_ask 整篇拼接）: est tokens 注入量 +est_tokens = chars / 1.5（CJK 近似）。 +""" +import importlib.util +import json +import math +import sys +from pathlib import Path + +BENCH = Path(__file__).resolve().parent.parent # rag-bench/ +REPO = Path(__file__).resolve().parents[2] # 仓库根（含 backend/，从任意克隆位置可复现） +INDEX_PY = REPO / "backend/modules/wiki/index.py" + +NO_JIEBA = "--no-jieba" in sys.argv # 模拟生产默认环境（requirements.txt 中 jieba 被注释） + +# ---- 加载 worktree 的 BM25Index（不 import 整个 backend）---- +spec = importlib.util.spec_from_file_location("cb_bm25_index", INDEX_PY) +mod = importlib.util.module_from_spec(spec) +sys.modules["cb_bm25_index"] = mod +spec.loader.exec_module(mod) +BM25Index = mod.BM25Index + +if NO_JIEBA: + # 强制走降级路径（单字分词）：_jieba 为 falsy 且非 None 时 _load_jieba 直接 return + BM25Index._jieba = False + assert BM25Index.tokenize("测试分词") == list("测试分词"), "no-jieba mode failed" + +# ---- 建索引：corpus manifest + md 文件 ---- +manifest = json.loads((BENCH / "corpus/manifest.json").read_text(encoding="utf-8")) +index = BM25Index() +docs = {} # slug -> {title, content, chars, est_tokens} +for entry in manifest: + slug = entry["slug"] + p = BENCH / "corpus" / f"{slug}.md" + content = p.read_text(encoding="utf-8") + docs[slug] = { + "title": entry["title"], + "content": content, + "chars": entry["chars"], + "est_tokens": entry["est_tokens"], + } + index.add_document(slug, entry["title"], content) + +print(f"[index] {len(docs)} docs loaded") + +questions = [json.loads(l) for l in (BENCH / "questions.jsonl").read_text(encoding="utf-8").splitlines() if l.strip()] +print(f"[questions] {len(questions)} questions") + +# manifest slug 不含 .md 后缀（如 core/memory）；questions.jsonl 的 source_pages 带 .md，统一去掉 +def to_slug(page: str) -> str: + return page[:-3] if page.endswith(".md") else page + +for q in questions: + q["source_pages"] = [to_slug(p) for p in q["source_pages"]] + +TOP_K_EVAL = 10 # 评估口径 +TOP_K_PROD = 3 # 生产口径（wiki/tool.py _handle_ask） + + +def est_tokens(text: str) -> int: + return int(len(text) / 1.5) + + +def prod_injection(results) -> int: + """镜像生产行为：每个命中文档全文 `### {title}\\n{content}` 拼接进 prompt。""" + total = 0 + for slug, _score in results: + d = docs.get(slug) + if d: + total += est_tokens(f"### {d['title']}\n{d['content']}") + return total + + +def dcg(rels): + return sum(r / math.log2(i + 2) for i, r in enumerate(rels)) + + +detail = [] +agg = { + "single": {"n": 0, "rec5": 0, "rec10": 0, "mrr10": 0, "ndcg10": 0, "top1": 0}, + "semantic": {"n": 0, "rec5": 0, "rec10": 0, "mrr10": 0, "ndcg10": 0, "top1": 0}, + "needle": {"n": 0, "rec5": 0, "rec10": 0, "mrr10": 0, "ndcg10": 0, "top1": 0}, + "cross": {"n": 0, "all_source_rec10": 0, "primary_mrr10": 0, "avg_coverage": 0.0, + "avg_sources": 0, "primary_top1": 0}, + "negative": {"n": 0, "scores": [], "returned": 0}, +} + +for q in questions: + qid, qtype = q["id"], q["type"] + eval_results = index.search(q["question"], top_k=TOP_K_EVAL) + prod_results = index.search(q["question"], top_k=TOP_K_PROD) + eval_slugs = [s for s, _ in eval_results] + + rec = {"id": qid, "type": qtype, "semantic": q["semantic"], + "question": q["question"], "expected_pages": q["source_pages"], + "top10": [{"slug": s, "score": round(sc, 3)} for s, sc in eval_results], + "prod_top3_slugs": [s for s, _ in prod_results], + "prod_injection_tokens": prod_injection(prod_results)} + + if qtype in ("single_doc", "needle"): + src = q["source_pages"][0] + if src in eval_slugs: + rank = eval_slugs.index(src) + 1 + rec["rank"] = rank + rec["hit"] = True + else: + rank = None + rec["hit"] = False + bucket = agg["semantic" if q["semantic"] else ("needle" if qtype == "needle" else "single")] + bucket["n"] += 1 + if rank: + bucket["rec5"] += 1 if rank <= 5 else 0 + bucket["rec10"] += 1 + bucket["mrr10"] += 1.0 / rank + bucket["ndcg10"] += 1.0 / (1 + math.log2(rank)) # 单相关文档的二值 NDCG + bucket["top1"] += 1 if rank == 1 else 0 + + elif qtype == "cross_doc": + srcs = q["source_pages"] + hits = [s for s in srcs if s in eval_slugs] + primary = srcs[0] + p_rank = (eval_slugs.index(primary) + 1) if primary in eval_slugs else None + rec["hit_sources"] = hits + rec["missed_sources"] = [s for s in srcs if s not in hits] + rec["primary_rank"] = p_rank + b = agg["cross"] + b["n"] += 1 + b["all_source_rec10"] += 1 if len(hits) == len(srcs) else 0 + b["primary_mrr10"] += (1.0 / p_rank) if p_rank else 0.0 + b["avg_coverage"] += len(hits) / len(srcs) + b["avg_sources"] += len(srcs) + b["primary_top1"] += 1 if p_rank == 1 else 0 + + elif qtype == "negative": + b = agg["negative"] + b["n"] += 1 + top1_score = eval_results[0][1] if eval_results else 0.0 + b["scores"].append(round(top1_score, 3)) + b["returned"] += 1 if eval_results else 0 + rec["top1_score"] = round(top1_score, 3) + rec["returned_any"] = bool(eval_results) + + detail.append(rec) + +# ---- 负样本 vs 非负样本 top1 分数对比（S4 的核心观察）---- +nonneg_top1 = [] +for q, r in zip(questions, detail): + if q["type"] != "negative" and r["top10"]: + nonneg_top1.append(r["top10"][0]["score"]) + +neg_scores = agg["negative"]["scores"] + +def stats(xs): + if not xs: + return {"n": 0} + xs2 = sorted(xs) + return {"n": len(xs), "mean": round(sum(xs)/len(xs), 2), + "median": round(xs2[len(xs2)//2], 2), + "min": round(xs2[0], 2), "max": round(xs2[-1], 2)} + +# S1/S3 生产 token 注入统计（正样本部分） +inj_pos = [r["prod_injection_tokens"] for r in detail if r["type"] != "negative"] +inj_neg = [r["prod_injection_tokens"] for r in detail if r["type"] == "negative"] + +summary = {} +for k in ("single", "semantic", "needle"): + b = agg[k] + n = b["n"] or 1 + summary[k] = { + "n": b["n"], + "recall@5": round(b["rec5"]/n, 3), + "recall@10": round(b["rec10"]/n, 3), + "MRR@10": round(b["mrr10"]/n, 3), + "NDCG@10": round(b["ndcg10"]/n, 3), + "top1_acc": round(b["top1"]/n, 3), + } +b = agg["cross"] +n = b["n"] or 1 +summary["cross"] = { + "n": b["n"], + "avg_sources_per_q": round(b["avg_sources"]/n, 2), + "all_source_recall@10": round(b["all_source_rec10"]/n, 3), + "primary_MRR@10": round(b["primary_mrr10"]/n, 3), + "primary_top1": round(b["primary_top1"]/n, 3), + "avg_source_coverage@10": round(b["avg_coverage"]/n, 3), +} +summary["negative"] = { + "n": agg["negative"]["n"], + "returned_any": agg["negative"]["returned"], + "top1_score_stats": stats(neg_scores), + "nonneg_top1_score_stats": stats(nonneg_top1), +} +summary["prod_injection"] = { + "note": "镜像 wiki/tool.py _handle_ask: top_k=3 整篇文档拼接, est_tokens=chars/1.5", + "positive_qs": stats(inj_pos), + "negative_qs": stats(inj_neg), + "corpus_total_est_tokens": sum(d["est_tokens"] for d in docs.values()), + "top3_injection_ratio_of_corpus": round((sum(inj_pos)/len(inj_pos)) / sum(d["est_tokens"] for d in docs.values()), 4) if inj_pos else None, +} + +out = {"config": {"top_k_eval": TOP_K_EVAL, "top_k_prod": TOP_K_PROD, + "docs": len(docs), "questions": len(questions), "no_jieba": NO_JIEBA}, + "summary": summary, "detail": detail} +(BENCH / "results").mkdir(exist_ok=True) +suffix = "-nojieba" if NO_JIEBA else "" +(BENCH / f"results/g0{suffix}-detail.json").write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8") +print(json.dumps(summary, ensure_ascii=False, indent=1)) + +# ---- 控制台快速诊断：未命中的题 ---- +print("\n== misses ==") +for r in detail: + if r["type"] in ("single_doc", "needle") and not r["hit"]: + print(f" MISS {r['id']}: {r['question'][:40]}... expected={r['expected_pages']}") + elif r["type"] == "cross_doc" and r.get("missed_sources"): + print(f" PART {r['id']}: missed {r['missed_sources']}") diff --git a/rag-bench/scripts/run_g1.py b/rag-bench/scripts/run_g1.py new file mode 100644 index 0000000..ecbb08f --- /dev/null +++ b/rag-bench/scripts/run_g1.py @@ -0,0 +1,150 @@ +"""G1 实验：分块索引（本 PR）在同一批 60 题上与 G0 的 before/after 对比 + +与 run_g0.py 同一把尺子（同题、同语料、同指标口径），仅检索单元不同： +G0 = 文档级 BM25（整篇检索，改前），G1 = 分块级 BM25（backend/modules/rag，改后）。 + +输出：results/g1-detail.json + 控制台对比摘要。 +""" + +import json +import math +import sys +from pathlib import Path + +BENCH = Path(__file__).resolve().parent.parent +REPO = Path(__file__).resolve().parents[2] # 仓库根（含 backend/，从任意克隆位置可复现） + +sys.path.insert(0, str(REPO)) + +from backend.modules.rag.stores import ChunkedBM25Index # noqa: E402 + +TOP_K_EVAL = 10 # 评估口径 +TOP_K_PROD = 6 # 生产镜像：ask 注入的块数（G0 生产是 top3 整篇） + +def est_tokens(text: str) -> int: + return int(len(text) / 1.5) + +def load_questions(): + qs = [json.loads(l) for l in (BENCH / "questions.jsonl").read_text(encoding="utf-8").splitlines() if l.strip()] + for q in qs: + q["expected_pages"] = [p.replace(".md", "") for p in q["source_pages"]] + return qs + +def build_index(): + store = ChunkedBM25Index() + manifest = json.load(open(BENCH / "corpus/manifest.json", encoding="utf-8")) + for e in manifest: + content = (BENCH / "corpus" / f"{e['slug']}.md").read_text(encoding="utf-8") + store.add_document(e["slug"], e["title"], content, [e.get("section", "")]) + return store, len(manifest) + +def main(): + questions = load_questions() + print(f"[questions] {len(questions)}") + store, n_docs = build_index() + n_chunks = store.stats()["total_chunks"] + print(f"[index] {n_docs} docs -> {n_chunks} chunks") + + detail = [] + for q in questions: + results = store.search_chunks(q["question"], top_k=TOP_K_EVAL) + top_slugs = [r["slug"] for r in results] + + # 生产镜像：top6 块注入（口径与 G0 相同的 header+content） + prod = store.search_chunks(q["question"], top_k=TOP_K_PROD) + prod_injection = sum(est_tokens(f"### {r['doc_title']} › {r['section']}\n{r['content']}") for r in prod) + prod_slugs = [r["slug"] for r in prod] + + rec = { + "id": q["id"], + "type": q["type"], + "semantic": q.get("semantic", False), + "question": q["question"], + "expected_pages": q["expected_pages"], + "top10": [{"chunk_id": r["chunk_id"], "slug": r["slug"], "score": r["score"]} for r in results], + "prod_top6_slugs": prod_slugs, + "prod_top6_chunks": [r["chunk_id"] for r in prod], + "prod_injection_tokens": prod_injection, + } + + if q["type"] == "negative": + rec["top1_score"] = results[0]["score"] if results else 0.0 + else: + exp = q["expected_pages"] + # 文档级命中（去重后按首次出现位置） + first_rank = None + for i, s in enumerate(top_slugs): + if s in exp: + first_rank = i + 1 + break + rec["first_rank"] = first_rank + rec["hit"] = first_rank is not None + rec["all_source_hit"] = all(s in top_slugs for s in exp) + # 生产口径命中：期望文档出现在注入的 top6 块中 + rec["prod_hit"] = any(s in prod_slugs for s in exp) + # 跨文档：生产覆盖的源比例 + rec["prod_source_coverage"] = len([s for s in exp if s in prod_slugs]) / len(exp) + detail.append(rec) + + # ---------- 汇总 ---------- + def group(pred): + return [r for r in detail if pred(r)] + + summary = {} + for name, rs in [ + ("S1 直接", lambda: group(lambda r: r["type"] == "single_doc" and not r["semantic"])), + ("S1 语义", lambda: group(lambda r: r["type"] == "single_doc" and r["semantic"])), + ("S3 针", lambda: group(lambda r: r["type"] == "needle")), + ("S2 跨文档", lambda: group(lambda r: r["type"] == "cross_doc")), + ]: + rows = rs() + n = len(rows) + r5 = sum(1 for r in rows if r.get("first_rank") and r["first_rank"] <= 5) + r10 = sum(1 for r in rows if r.get("first_rank") and r["first_rank"] <= 10) + mrr = sum(1.0 / r["first_rank"] for r in rows if r.get("first_rank")) / n + top1 = sum(1 for r in rows if r.get("first_rank") == 1) / n + # NDCG@10（文档级，单相关文档：1/log2(rank+1)） + ndcg = sum(1.0 / math.log2(r["first_rank"] + 1) for r in rows if r.get("first_rank")) / n + inj = [r["prod_injection_tokens"] for r in rows] + prod_hit = sum(1 for r in rows if r.get("prod_hit")) + summary[name] = { + "n": n, "recall@5": r5 / n, "recall@10": r10 / n, "MRR@10": mrr, + "NDCG@10": ndcg, "top1": top1, + "prod_hit@6": prod_hit / n, + "prod_injection_mean": sum(inj) / n, "prod_injection_max": max(inj), + } + if name == "S2 跨文档": + summary[name]["all_source_hit@10"] = sum(1 for r in rows if r.get("all_source_hit")) / n + summary[name]["prod_source_coverage"] = sum(r["prod_source_coverage"] for r in rows) / n + + neg = group(lambda r: r["type"] == "negative") + summary["S4 负样本"] = { + "n": len(neg), + "returned": sum(1 for r in neg if r["top10"]) / len(neg), + "prod_injection_mean": sum(r["prod_injection_tokens"] for r in neg) / len(neg), + "top1_score_max": max((r["top1_score"] for r in neg), default=0.0), + } + + # 正样本总体注入（M1 验收：token 下降 >= 70%） + pos = [r for r in detail if r["type"] != "negative"] + summary["正样本总体"] = { + "n": len(pos), + "prod_injection_mean": sum(r["prod_injection_tokens"] for r in pos) / len(pos), + "prod_injection_max": max(r["prod_injection_tokens"] for r in pos), + "prod_hit@6": sum(1 for r in pos if r["prod_hit"]) / len(pos), + } + + out = {"config": {"top_k_eval": TOP_K_EVAL, "top_k_prod": TOP_K_PROD, + "docs": n_docs, "chunks": n_chunks, "questions": len(questions)}, + "summary": summary, "detail": detail} + (BENCH / "results/g1-detail.json").write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8") + + print("\n===== G1 分块索引摘要 =====") + for name, s in summary.items(): + print(f"\n{name} (n={s['n']})") + for k, v in s.items(): + if k != "n": + print(f" {k}: {v:.3f}" if isinstance(v, float) else f" {k}: {v}") + +if __name__ == "__main__": + main()