mirror of
https://github.com/countbot-ai/CountBot.git
synced 2026-09-14 20:46:47 +08:00
bench(wiki-rag): reproducible 60-question doc-vs-chunk evaluation harness
Three scripts + 60 hand-annotated questions over the public countbot.cn/docs corpus (52 pages, fetched by fetch_corpus.py): - run_g0.py: doc-level BM25 baseline (pre-change behaviour) - run_g1.py: chunk-level BM25 (this PR) on the same questions - fetch_corpus.py: downloads the public corpus into rag-bench/corpus/ Pure programmatic scoring (recall@k / MRR@10 / NDCG@10 / top1 / injected-token volume), no LLM judge, no private data. The headline numbers quoted in the PR description come from running these on the same corpus/questions before and after the chunk change. Refs: #107
This commit is contained in:
RIC
@@ -0,0 +1,60 @@
|
||||
{"id": "S1-01", "type": "single_doc", "semantic": false, "question": "CountBot 的长期记忆存储在哪里？是什么格式的文件？", "expected_points": ["工作空间中的单文件存储", "JSON 格式，可直接查看编辑"], "source_pages": ["core/memory.md"]}
|
||||
{"id": "S1-02", "type": "single_doc", "semantic": false, "question": "CountBot 记忆条目的格式长什么样？包含哪些字段？", "expected_points": ["条目字段结构（id/时间戳/内容/来源等）"], "source_pages": ["core/memory.md"]}
|
||||
{"id": "S1-03", "type": "single_doc", "semantic": false, "question": "什么是 auto-overflow 记忆？什么情况下会产生？", "expected_points": ["会话历史超出窗口限制时", "旧消息被压缩写入记忆，来源标记为 auto-overflow"], "source_pages": ["core/memory.md"]}
|
||||
{"id": "S1-04", "type": "single_doc", "semantic": false, "question": "AI 模型是怎么读写长期记忆的？需要直接操作文件吗？", "expected_points": ["通过工具调用读写", "模型不直接操作文件"], "source_pages": ["core/memory.md"]}
|
||||
{"id": "S1-05", "type": "single_doc", "semantic": false, "question": "CountBot 有哪几种启动方式？分别用什么命令？", "expected_points": ["start_app.py / start_desktop.py / start_dev.py 等启动脚本及区别"], "source_pages": ["getting-started/installation.md"]}
|
||||
{"id": "S1-06", "type": "single_doc", "semantic": false, "question": "CountBot 对运行环境有什么要求？需要什么版本的 Python？", "expected_points": ["Python 版本要求", "操作系统支持情况"], "source_pages": ["getting-started/installation.md"]}
|
||||
{"id": "S1-07", "type": "single_doc", "semantic": false, "question": "怎么让局域网内其他设备访问 CountBot 的 Web 界面？", "expected_points": ["COUNTBOT_HOST=0.0.0.0 环境变量", "通过本机 IP+端口访问"], "source_pages": ["getting-started/installation.md"]}
|
||||
{"id": "S1-08", "type": "single_doc", "semantic": false, "question": "桌面版相比 Web 版有什么限制？", "expected_points": ["桌面版功能边界", "系统依赖差异（libwebkit2gtk）"], "source_pages": ["getting-started/installation.md"]}
|
||||
{"id": "S1-09", "type": "single_doc", "semantic": false, "question": "requirements.txt 里的依赖是怎么分类的？", "expected_points": ["依赖分组结构（核心/渠道/桌面等分类方式）"], "source_pages": ["getting-started/installation.md"]}
|
||||
{"id": "S1-10", "type": "single_doc", "semantic": false, "question": "CountBot 启动后的自检清单有哪些检查项？", "expected_points": ["启动自检项列表（配置/网络/渠道等）"], "source_pages": ["getting-started/installation.md"]}
|
||||
{"id": "S1-11", "type": "single_doc", "semantic": false, "question": "workspace 工作空间目录里都有哪些内容？", "expected_points": ["skills/、memory/、temp/、.skills_config.json 等结构"], "source_pages": ["getting-started/configuration-manual.md"]}
|
||||
{"id": "S1-12", "type": "single_doc", "semantic": false, "question": "前端构建产物放在哪个目录？", "expected_points": ["frontend/dist/"], "source_pages": ["advanced/deployment.md"]}
|
||||
{"id": "S1-13", "type": "single_doc", "semantic": false, "question": "后端 lifespan 的初始化顺序是什么？", "expected_points": ["数据库初始化→加载配置→共享组件→消息队列→渠道→消息处理→Cron→心跳等顺序"], "source_pages": ["advanced/deployment.md"]}
|
||||
{"id": "S1-14", "type": "single_doc", "semantic": false, "question": "CountBot 内置支持多少个模型 provider？", "expected_points": ["22 个"], "source_pages": ["core/providers.md"]}
|
||||
{"id": "S1-15", "type": "single_doc", "semantic": false, "question": "Cron 定时任务底层用什么调度库实现？", "expected_points": ["APScheduler"], "source_pages": ["core/cron.md"]}
|
||||
{"id": "S1-16", "type": "single_doc", "semantic": true, "question": "我想让 CountBot 到点自动干活，不用我发消息触发，能做到吗？", "expected_points": ["Cron 定时任务功能", "APScheduler 调度"], "source_pages": ["core/cron.md"]}
|
||||
{"id": "S1-17", "type": "single_doc", "semantic": true, "question": "CountBot 能在新的一次对话里想起我上次跟它说过的话吗？靠什么机制？", "expected_points": ["长期记忆机制", "跨会话记忆存储"], "source_pages": ["core/memory.md"]}
|
||||
{"id": "S1-18", "type": "single_doc", "semantic": true, "question": "一件复杂的事能让几个 AI 角色接力完成吗？", "expected_points": ["Agent 团队功能", "多角色协作"], "source_pages": ["core/agent-teams.md"]}
|
||||
{"id": "S1-19", "type": "single_doc", "semantic": true, "question": "我平时在终端里用 AI 编码工具写代码，CountBot 能直接调用它吗？", "expected_points": ["外部编码工具集成（Claude Code/Codex 等）"], "source_pages": ["core/external-coding-tools.md"]}
|
||||
{"id": "S1-20", "type": "single_doc", "semantic": true, "question": "同事想在他自己的电脑上打开我的 CountBot 页面来用，行得通吗？", "expected_points": ["远程访问方案", "局域网/公网暴露与认证"], "source_pages": ["advanced/remote-access.md"]}
|
||||
{"id": "S1-21", "type": "single_doc", "semantic": true, "question": "知识库文档越存越多，查询会越来越慢吗？", "expected_points": ["Wiki LRU 缓存机制", "缓存容量 128 个文档，重复查询性能提升"], "source_pages": ["releases/v0.9.0.md"]}
|
||||
{"id": "S1-22", "type": "single_doc", "semantic": true, "question": "我有好几个 API Key，一个额度用完了能自动换下一个吗？", "expected_points": ["API Key 轮换机制", "401/429 触发自动切换"], "source_pages": ["releases/v0.9.0.md"]}
|
||||
{"id": "S1-23", "type": "single_doc", "semantic": true, "question": "我在两个不同的群里分别跟 CountBot 聊，它会把两边的话题记混吗？", "expected_points": ["会话按渠道/聊天隔离", "channels 会话作用域"], "source_pages": ["core/channels.md"]}
|
||||
{"id": "S1-24", "type": "single_doc", "semantic": true, "question": "聊到一半想抛开前面的话题重新开始，有什么办法？", "expected_points": ["/new 等斜杠命令重开会话"], "source_pages": ["core/im-commands.md"]}
|
||||
{"id": "S1-25", "type": "single_doc", "semantic": true, "question": "升级到新版本的时候，我之前的会话和记忆还能保住吗？", "expected_points": ["升级保留数据", "backup/数据兼容说明"], "source_pages": ["getting-started/update-guide.md"]}
|
||||
{"id": "S2-01", "type": "cross_doc", "semantic": false, "question": "怎么配置 CountBot 每天早上 8 点自动搜索新闻并发送到 Telegram？", "expected_points": ["Cron 定时任务配置", "Telegram 渠道配置", "搜索技能与消息发送联动"], "source_pages": ["core/cron.md", "core/channels.md", "scenarios/每天早上一条消息搞定全天：AI早报自动生成的场景与调度技巧.md"]}
|
||||
{"id": "S2-02", "type": "cross_doc", "semantic": false, "question": "我在家用桌面版运行 CountBot，出差在外能用手机浏览器访问吗？", "expected_points": ["远程访问方案", "桌面版运行特点", "认证与安全"], "source_pages": ["getting-started/installation.md", "advanced/remote-access.md"]}
|
||||
{"id": "S2-03", "type": "cross_doc", "semantic": false, "question": "钉钉上配置完成后机器人不回复消息，应该怎么排查？", "expected_points": ["渠道配置检查", "troubleshooting 排查步骤"], "source_pages": ["getting-started/ChannelConfig.md", "getting-started/troubleshooting.md", "core/channels.md"]}
|
||||
{"id": "S2-04", "type": "cross_doc", "semantic": false, "question": "配置好模型 provider 之后，怎么验证连接是正常的？", "expected_points": ["测试模型连接的方式", "provider 状态检查"], "source_pages": ["core/providers.md", "getting-started/quick-reference.md"]}
|
||||
{"id": "S2-05", "type": "cross_doc", "semantic": false, "question": "子代理执行完任务之后，结果是怎么回到主对话里的？", "expected_points": ["子代理结果回传机制", "agent loop 中的汇合点"], "source_pages": ["core/subagent.md", "core/agent-loop.md"]}
|
||||
{"id": "S2-06", "type": "cross_doc", "semantic": false, "question": "Agent 团队里不同成员想用不同模型，怎么指定优先级？", "expected_points": ["团队成员模型配置", "provider 优先级"], "source_pages": ["core/agent-teams.md", "core/providers.md"]}
|
||||
{"id": "S2-07", "type": "cross_doc", "semantic": false, "question": "CountBot 执行 Shell 命令有什么安全限制？怎么防止危险命令？", "expected_points": ["Shell 工具限制", "安全机制（确认/黑名单等）"], "source_pages": ["core/tools.md", "advanced/security.md"]}
|
||||
{"id": "S2-08", "type": "cross_doc", "semantic": false, "question": "MCP 工具和外部编码工具（比如 Claude Code）有什么区别？分别什么场景用？", "expected_points": ["MCP 集成方式", "外部编码工具定位", "两者适用场景差异"], "source_pages": ["core/tools.md", "core/external-coding-tools.md"]}
|
||||
{"id": "S2-09", "type": "cross_doc", "semantic": false, "question": "什么前提下才能把消息路由到 direct 模式直接给编码工具处理？", "expected_points": ["/route direct 命令", "使用前提（需配置编码工具）"], "source_pages": ["core/im-commands.md", "core/external-coding-tools.md"]}
|
||||
{"id": "S2-10", "type": "cross_doc", "semantic": false, "question": "一个会话聊得特别长，系统会发生什么？", "expected_points": ["历史窗口限制", "自动总结触发（30 条/15000 字符）", "auto-overflow 记忆"], "source_pages": ["core/agent-loop.md", "advanced/http-api-deep-dive.md", "core/memory.md"]}
|
||||
{"id": "S2-11", "type": "cross_doc", "semantic": false, "question": "把 CountBot 部署在局域网给全组使用，有哪些安全注意事项？", "expected_points": ["认证机制", "API Key 保护", "网络安全配置"], "source_pages": ["advanced/deployment.md", "advanced/security.md", "advanced/remote-access.md"]}
|
||||
{"id": "S2-12", "type": "cross_doc", "semantic": false, "question": "升级到新版本之前需要做哪些准备工作？", "expected_points": ["备份", "依赖更新", "数据迁移注意事项"], "source_pages": ["getting-started/update-guide.md", "getting-started/installation.md"]}
|
||||
{"id": "S2-13", "type": "cross_doc", "semantic": false, "question": "能在 Web 界面上创建定时任务吗，还是只能改配置文件？", "expected_points": ["Web 界面任务管理", "cron 任务创建方式"], "source_pages": ["core/cron.md", "getting-started/quick-reference.md"]}
|
||||
{"id": "S2-14", "type": "cross_doc", "semantic": false, "question": "不想用云端的 API，能不能完全在本地跑模型来用 CountBot？", "expected_points": ["本地模型支持（Ollama 等）", "provider 配置方式", "性能取舍"], "source_pages": ["core/providers.md", "getting-started/installation.md"]}
|
||||
{"id": "S2-15", "type": "cross_doc", "semantic": false, "question": "用 /route 和 /coder 切换了路由之后，这些切换会记录进会话历史吗？", "expected_points": ["命令是否写入历史", "会话状态保持方式"], "source_pages": ["core/im-commands.md", "core/agent-loop.md"]}
|
||||
{"id": "S3-01", "type": "needle", "semantic": false, "question": "会话达到多少条消息会触发自动总结？", "expected_points": ["30 条"], "source_pages": ["advanced/http-api-deep-dive.md"]}
|
||||
{"id": "S3-02", "type": "needle", "semantic": false, "question": "自动总结的字符数阈值是多少？", "expected_points": ["15000 字符"], "source_pages": ["advanced/http-api-deep-dive.md"]}
|
||||
{"id": "S3-03", "type": "needle", "semantic": false, "question": "子代理最多允许迭代多少轮？", "expected_points": ["15"], "source_pages": ["core/subagent.md"]}
|
||||
{"id": "S3-04", "type": "needle", "semantic": false, "question": "Wiki 知识库的 LRU 缓存容量是多少个文档？", "expected_points": ["128"], "source_pages": ["releases/v0.9.0.md"]}
|
||||
{"id": "S3-05", "type": "needle", "semantic": false, "question": "v0.9.0 的心跳问候优化把每次的 LLM 调用次数从几次降到了几次？", "expected_points": ["从 2 次降到 1 次"], "source_pages": ["releases/v0.9.0.md"]}
|
||||
{"id": "S3-06", "type": "needle", "semantic": false, "question": "API Key 轮换是在收到哪些 HTTP 状态码时触发的？", "expected_points": ["401 和 429"], "source_pages": ["releases/v0.9.0.md"]}
|
||||
{"id": "S3-07", "type": "needle", "semantic": false, "question": "CountBot 内置支持多少个模型 provider？", "expected_points": ["22 个"], "source_pages": ["core/providers.md"]}
|
||||
{"id": "S3-08", "type": "needle", "semantic": false, "question": "SQLite 数据库文件叫什么名字？定时任务存在哪张表里？", "expected_points": ["countbot.db", "cron_jobs 表"], "source_pages": ["advanced/deployment.md"]}
|
||||
{"id": "S3-09", "type": "needle", "semantic": false, "question": "Telegram 的代理设置支持什么协议？", "expected_points": ["socks5://"], "source_pages": ["getting-started/troubleshooting.md"]}
|
||||
{"id": "S3-10", "type": "needle", "semantic": false, "question": "Linux 上安装桌面版需要额外装哪个系统依赖包？", "expected_points": ["libwebkit2gtk-4.0-37"], "source_pages": ["getting-started/installation.md"]}
|
||||
{"id": "S4-01", "type": "negative", "semantic": false, "question": "CountBot 怎么部署到 Kubernetes 集群上？", "expected_points": ["应拒答：官方不支持 K8s 部署"], "source_pages": []}
|
||||
{"id": "S4-02", "type": "negative", "semantic": false, "question": "CountBot 有官方手机 App 吗？在哪里下载？", "expected_points": ["应拒答：没有官方移动端 App"], "source_pages": []}
|
||||
{"id": "S4-03", "type": "negative", "semantic": false, "question": "CountBot 支持语音通话吗？", "expected_points": ["应拒答：不支持语音通话"], "source_pages": []}
|
||||
{"id": "S4-04", "type": "negative", "semantic": false, "question": "CountBot 能接入 Home Assistant 控制智能家居设备吗？", "expected_points": ["应拒答：无智能家居集成"], "source_pages": []}
|
||||
{"id": "S4-05", "type": "negative", "semantic": false, "question": "CountBot 商业版是怎么收费的？", "expected_points": ["应拒答：开源项目无商业定价"], "source_pages": []}
|
||||
{"id": "S4-06", "type": "negative", "semantic": false, "question": "可以把 CountBot 的 SQLite 换成 PostgreSQL 吗？官方支持吗？", "expected_points": ["应拒答：仅支持 SQLite，无 PostgreSQL 支持"], "source_pages": []}
|
||||
{"id": "S4-07", "type": "negative", "semantic": false, "question": "哪里可以下载 CountBot 的第三方插件？", "expected_points": ["应拒答：无插件市场"], "source_pages": []}
|
||||
{"id": "S4-08", "type": "negative", "semantic": false, "question": "CountBot 能运行在树莓派或 ARM 设备上吗？", "expected_points": ["应拒答：官方未支持 ARM/树莓派"], "source_pages": []}
|
||||
{"id": "S4-09", "type": "negative", "semantic": false, "question": "CountBot 支持 SSO 单点登录吗？", "expected_points": ["应拒答：无 SSO 支持"], "source_pages": []}
|
||||
{"id": "S4-10", "type": "negative", "semantic": false, "question": "CountBot 支持视频通话或者屏幕共享吗？", "expected_points": ["应拒答：不支持"], "source_pages": []}
|
||||
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""抓取 countbot.cn 官方文档站全量页面，转为 Markdown 语料。
|
||||
|
||||
用途：RAG 基线测试（G0）的 C1/C2 语料构建。
|
||||
来源全部为官方公开页面，保证可复现。
|
||||
|
||||
输出：
|
||||
rag-bench/corpus/<slug>.md —— 每页一份 Markdown（首行为 H1 标题）
|
||||
rag-bench/corpus/manifest.json —— 抓取清单（slug/title/url/chars/est_tokens）
|
||||
|
||||
用法：
|
||||
python fetch_corpus.py [--out ../corpus]
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote, unquote, urlparse
|
||||
|
||||
BASE = "https://countbot.cn"
|
||||
UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36")
|
||||
|
||||
# 额外页面（llms.txt 列出但 sitemap 缺失：scenarios / 新版 releases / update-guide）
|
||||
EXTRA_PATHS = [
|
||||
"/docs/scenarios/一句话清空收件箱：AI邮件分拣的场景拆解与提示词技巧/",
|
||||
"/docs/scenarios/帮我做个网页然后上线：从需求描述到生产部署的完整话术/",
|
||||
"/docs/scenarios/帮我搜一下最近AI有什么新动态：多源信息聚合的场景与检索技巧/",
|
||||
"/docs/scenarios/帮我盯着这件事：定时监控与主动提醒的场景设计与自动化技巧/",
|
||||
"/docs/scenarios/帮我规划一趟旅行：多技能联动的复合任务拆解与编排技巧/",
|
||||
"/docs/scenarios/每天早上一条消息搞定全天：AI早报自动生成的场景与调度技巧/",
|
||||
"/docs/releases/v0.7.0/",
|
||||
"/docs/releases/v0.8.0/",
|
||||
"/docs/releases/v0.9.0/",
|
||||
"/docs/getting-started/update-guide/",
|
||||
]
|
||||
|
||||
|
||||
class ArticleToMarkdown(HTMLParser):
|
||||
"""把 <article> 内的 HTML 转为 Markdown 文本（容忍不完整标签）。
|
||||
|
||||
保留结构：h1-h6 标题、列表项、代码块（围栏）、表格（管道行）、引用。
|
||||
行内标签只保留文本。script/style/svg/nav 忽略。
|
||||
"""
|
||||
|
||||
BLOCK_TAGS = {"p", "div", "section", "header", "footer", "br", "hr",
|
||||
"ul", "ol", "table", "thead", "tbody", "blockquote", "details", "summary"}
|
||||
SKIP_TAGS = {"script", "style", "svg", "nav", "img", "button", "input"}
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.lines: list[str] = []
|
||||
self._buf: list[str] = []
|
||||
self._skip_depth = 0
|
||||
self._pre_depth = 0
|
||||
self._pre_buf: list[str] = []
|
||||
self._in_cell = False
|
||||
self._cell_buf: list[str] = []
|
||||
self._row_cells: list[str] | None = None
|
||||
self._is_header_row = False
|
||||
self._list_depth = 0
|
||||
self._li_open = False
|
||||
self._quote = False
|
||||
|
||||
# ---------- 基础 ----------
|
||||
def _text(self, s: str):
|
||||
if self._skip_depth > 0:
|
||||
return
|
||||
if self._pre_depth > 0:
|
||||
self._pre_buf.append(s)
|
||||
elif self._in_cell:
|
||||
self._cell_buf.append(s)
|
||||
else:
|
||||
self._buf.append(s)
|
||||
if self._li_open:
|
||||
self._li_open = False
|
||||
|
||||
def _flush_para(self):
|
||||
text = re.sub(r"\s+", " ", "".join(self._buf)).strip()
|
||||
self._buf = []
|
||||
if not text:
|
||||
return
|
||||
prefix = "> " if self._quote else ""
|
||||
indent = " " * max(self._list_depth - 1, 0)
|
||||
bullet = "- " if self._list_depth > 0 else ""
|
||||
self.lines.append(prefix + indent + bullet + text)
|
||||
|
||||
def _flush_cell(self):
|
||||
text = re.sub(r"\s+", " ", "".join(self._cell_buf)).strip()
|
||||
self._cell_buf = []
|
||||
if self._row_cells is not None:
|
||||
self._row_cells.append(text)
|
||||
|
||||
# ---------- 标签处理 ----------
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag in self.SKIP_TAGS:
|
||||
self._skip_depth += 1
|
||||
return
|
||||
if self._skip_depth > 0:
|
||||
return
|
||||
if tag == "pre":
|
||||
self._flush_para()
|
||||
self._pre_depth += 1
|
||||
if self._pre_depth == 1:
|
||||
self._pre_buf = []
|
||||
elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
|
||||
self._flush_para()
|
||||
self._buf.append("\x00H%d\x00" % int(tag[1])) # 标记，flush 时替换
|
||||
elif tag in ("p", "div", "section", "header", "footer", "blockquote",
|
||||
"details", "summary", "ul", "ol", "table", "thead", "tbody"):
|
||||
self._flush_para()
|
||||
if tag == "blockquote":
|
||||
self._quote = True
|
||||
elif tag in ("ul", "ol"):
|
||||
self._list_depth += 1
|
||||
elif tag == "li":
|
||||
self._flush_para()
|
||||
self._li_open = True
|
||||
elif tag == "tr":
|
||||
self._row_cells = []
|
||||
elif tag in ("td", "th"):
|
||||
self._in_cell = True
|
||||
if tag == "th":
|
||||
self._is_header_row = True
|
||||
elif tag == "br":
|
||||
self._text(" ")
|
||||
elif tag == "hr":
|
||||
self._flush_para()
|
||||
self.lines.append("---")
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag in self.SKIP_TAGS:
|
||||
self._skip_depth = max(0, self._skip_depth - 1)
|
||||
return
|
||||
if self._skip_depth > 0:
|
||||
return
|
||||
if tag == "pre":
|
||||
self._pre_depth -= 1
|
||||
if self._pre_depth == 0:
|
||||
code = "".join(self._pre_buf).rstrip("\n")
|
||||
self.lines.append("```\n" + code + "\n```")
|
||||
elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
|
||||
level = int(tag[1])
|
||||
raw = "".join(self._buf)
|
||||
self._buf = []
|
||||
text = re.sub(r"\s+", " ", raw.replace("\x00H%d\x00" % level, "")).strip()
|
||||
prefix = "> " if self._quote else ""
|
||||
self.lines.append(prefix + "#" * level + " " + text)
|
||||
elif tag in ("p", "div", "section", "header", "footer", "details", "summary"):
|
||||
self._flush_para()
|
||||
elif tag == "blockquote":
|
||||
self._quote = False
|
||||
self._flush_para()
|
||||
elif tag in ("ul", "ol"):
|
||||
self._list_depth = max(0, self._list_depth - 1)
|
||||
self._flush_para()
|
||||
elif tag in ("td", "th"):
|
||||
self._in_cell = False
|
||||
self._flush_cell()
|
||||
elif tag == "tr":
|
||||
if self._row_cells is not None:
|
||||
cells = self._row_cells
|
||||
self._row_cells = None
|
||||
if cells:
|
||||
self.lines.append("| " + " | ".join(cells) + " |")
|
||||
if self._is_header_row:
|
||||
self.lines.append("|" + "---|" * len(cells))
|
||||
self._is_header_row = False
|
||||
elif tag == "table":
|
||||
self._flush_para()
|
||||
|
||||
def handle_data(self, data):
|
||||
self._text(data)
|
||||
|
||||
def result(self) -> str:
|
||||
self._flush_para()
|
||||
out = []
|
||||
prev_blank = True
|
||||
for ln in self.lines:
|
||||
if ln.strip():
|
||||
out.append(ln)
|
||||
prev_blank = False
|
||||
else:
|
||||
if not prev_blank:
|
||||
out.append("")
|
||||
prev_blank = True
|
||||
return "\n".join(out).strip() + "\n"
|
||||
|
||||
|
||||
def extract_article(html: str) -> tuple[str, str]:
|
||||
"""返回 (title, markdown)。title 取 <h1> 或 <title>。"""
|
||||
m = re.search(r"<article[^>]*>(.*?)</article>", html, re.S | re.I)
|
||||
body_html = m.group(1) if m else html
|
||||
parser = ArticleToMarkdown()
|
||||
parser.feed(body_html)
|
||||
md = parser.result()
|
||||
h1 = re.search(r"^#\s+(.+)$", md, re.M)
|
||||
title = h1.group(1).strip() if h1 else ""
|
||||
if not title:
|
||||
t = re.search(r"<title>(.*?)</title>", html, re.S)
|
||||
title = t.group(1).strip() if t else ""
|
||||
return title, md
|
||||
|
||||
|
||||
def slug_of(path: str) -> str:
|
||||
p = path[len("/docs/"):] if path.startswith("/docs/") else path
|
||||
p = p.strip("/")
|
||||
if not p:
|
||||
p = "index"
|
||||
return p
|
||||
|
||||
|
||||
def fetch(url: str) -> str:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
||||
with urllib.request.urlopen(req, timeout=25) as resp:
|
||||
return resp.read().decode("utf-8", errors="ignore")
|
||||
|
||||
|
||||
def main():
|
||||
out_dir = Path(sys.argv[sys.argv.index("--out") + 1]) if "--out" in sys.argv \
|
||||
else Path(__file__).resolve().parent.parent / "corpus"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 1) URL 全集 = sitemap + EXTRA（按 unquote 后路径去重，中文与百分号编码视为同页）
|
||||
sitemap_url = BASE + "/sitemap.xml"
|
||||
xml = fetch(sitemap_url)
|
||||
paths = [urlparse(u).path for u in re.findall(r"<loc>(.*?)</loc>", xml)
|
||||
if "/docs/" in u]
|
||||
paths += EXTRA_PATHS
|
||||
# 归一化：unquote 去重，再统一以 / 结尾
|
||||
norm = {}
|
||||
for p in paths:
|
||||
key = unquote(p).rstrip("/")
|
||||
norm[key] = p if p.endswith("/") else p + "/"
|
||||
paths = sorted(norm.values())
|
||||
# sitemap 中的 articles/* 在 cn 站已 404（死链，指向另一域名的 SPA 兜底页），跳过
|
||||
paths = [p for p in paths if "/docs/articles/" not in unquote(p)]
|
||||
|
||||
manifest = []
|
||||
failed = []
|
||||
for i, path in enumerate(paths, 1):
|
||||
slug = slug_of(unquote(path))
|
||||
# 统一 URL 编码（中文路径必须 quote，否则 ascii 编码错误）
|
||||
url = BASE + quote(unquote(path), safe="/:")
|
||||
try:
|
||||
html = fetch(url)
|
||||
title, md = extract_article(html)
|
||||
if len(md.strip()) < 100:
|
||||
raise ValueError(f"content too short: {len(md)} chars")
|
||||
out_file = out_dir / f"{slug}.md"
|
||||
out_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_file.write_text(md, encoding="utf-8")
|
||||
chars = len(md)
|
||||
manifest.append({
|
||||
"slug": slug, "title": title, "url": url,
|
||||
"chars": chars, "est_tokens": int(chars / 1.5),
|
||||
"section": slug.split("/")[0] if "/" in slug else slug,
|
||||
})
|
||||
print(f"[{i}/{len(paths)}] {slug} ({chars} chars)")
|
||||
except Exception as e:
|
||||
failed.append({"slug": slug, "url": url, "error": str(e)})
|
||||
print(f"[{i}/{len(paths)}] FAIL {slug}: {e}")
|
||||
time.sleep(0.25)
|
||||
|
||||
(out_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
total_chars = sum(m["chars"] for m in manifest)
|
||||
print(f"\nOK: {len(manifest)} pages, {total_chars} chars "
|
||||
f"(~{total_chars // 1500}K est tokens); failed: {len(failed)}")
|
||||
if failed:
|
||||
(out_dir / "failed.json").write_text(
|
||||
json.dumps(failed, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
"""G0 基线：测量 CountBot 现状 BM25 文档级检索在 60 题问题集上的表现。
|
||||
|
||||
口径：纯程序化打分（recall@k / MRR@10 / NDCG@10 / top1 / 注入 token 量），无 LLM 判分，可直接复现。
|
||||
指标：
|
||||
- S1/S3（单文档）: recall@5/10, MRR@10, NDCG@10
|
||||
- S2（跨文档）: 全源 recall@10, primary MRR@10, 平均源覆盖率
|
||||
- S4（负样本）: 得分分布（top1 score），与非负样本对比
|
||||
- token 注入（镜像生产 top_k=3, _handle_ask 整篇拼接）: est tokens 注入量
|
||||
est_tokens = chars / 1.5（CJK 近似）。
|
||||
"""
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
BENCH = Path(__file__).resolve().parent.parent # rag-bench/
|
||||
REPO = Path(__file__).resolve().parents[2] # 仓库根（含 backend/，从任意克隆位置可复现）
|
||||
INDEX_PY = REPO / "backend/modules/wiki/index.py"
|
||||
|
||||
NO_JIEBA = "--no-jieba" in sys.argv # 模拟生产默认环境（requirements.txt 中 jieba 被注释）
|
||||
|
||||
# ---- 加载 worktree 的 BM25Index（不 import 整个 backend）----
|
||||
spec = importlib.util.spec_from_file_location("cb_bm25_index", INDEX_PY)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules["cb_bm25_index"] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
BM25Index = mod.BM25Index
|
||||
|
||||
if NO_JIEBA:
|
||||
# 强制走降级路径（单字分词）：_jieba 为 falsy 且非 None 时 _load_jieba 直接 return
|
||||
BM25Index._jieba = False
|
||||
assert BM25Index.tokenize("测试分词") == list("测试分词"), "no-jieba mode failed"
|
||||
|
||||
# ---- 建索引：corpus manifest + md 文件 ----
|
||||
manifest = json.loads((BENCH / "corpus/manifest.json").read_text(encoding="utf-8"))
|
||||
index = BM25Index()
|
||||
docs = {} # slug -> {title, content, chars, est_tokens}
|
||||
for entry in manifest:
|
||||
slug = entry["slug"]
|
||||
p = BENCH / "corpus" / f"{slug}.md"
|
||||
content = p.read_text(encoding="utf-8")
|
||||
docs[slug] = {
|
||||
"title": entry["title"],
|
||||
"content": content,
|
||||
"chars": entry["chars"],
|
||||
"est_tokens": entry["est_tokens"],
|
||||
}
|
||||
index.add_document(slug, entry["title"], content)
|
||||
|
||||
print(f"[index] {len(docs)} docs loaded")
|
||||
|
||||
questions = [json.loads(l) for l in (BENCH / "questions.jsonl").read_text(encoding="utf-8").splitlines() if l.strip()]
|
||||
print(f"[questions] {len(questions)} questions")
|
||||
|
||||
# manifest slug 不含 .md 后缀（如 core/memory）；questions.jsonl 的 source_pages 带 .md，统一去掉
|
||||
def to_slug(page: str) -> str:
|
||||
return page[:-3] if page.endswith(".md") else page
|
||||
|
||||
for q in questions:
|
||||
q["source_pages"] = [to_slug(p) for p in q["source_pages"]]
|
||||
|
||||
TOP_K_EVAL = 10 # 评估口径
|
||||
TOP_K_PROD = 3 # 生产口径（wiki/tool.py _handle_ask）
|
||||
|
||||
|
||||
def est_tokens(text: str) -> int:
|
||||
return int(len(text) / 1.5)
|
||||
|
||||
|
||||
def prod_injection(results) -> int:
|
||||
"""镜像生产行为：每个命中文档全文 `### {title}\\n{content}` 拼接进 prompt。"""
|
||||
total = 0
|
||||
for slug, _score in results:
|
||||
d = docs.get(slug)
|
||||
if d:
|
||||
total += est_tokens(f"### {d['title']}\n{d['content']}")
|
||||
return total
|
||||
|
||||
|
||||
def dcg(rels):
|
||||
return sum(r / math.log2(i + 2) for i, r in enumerate(rels))
|
||||
|
||||
|
||||
detail = []
|
||||
agg = {
|
||||
"single": {"n": 0, "rec5": 0, "rec10": 0, "mrr10": 0, "ndcg10": 0, "top1": 0},
|
||||
"semantic": {"n": 0, "rec5": 0, "rec10": 0, "mrr10": 0, "ndcg10": 0, "top1": 0},
|
||||
"needle": {"n": 0, "rec5": 0, "rec10": 0, "mrr10": 0, "ndcg10": 0, "top1": 0},
|
||||
"cross": {"n": 0, "all_source_rec10": 0, "primary_mrr10": 0, "avg_coverage": 0.0,
|
||||
"avg_sources": 0, "primary_top1": 0},
|
||||
"negative": {"n": 0, "scores": [], "returned": 0},
|
||||
}
|
||||
|
||||
for q in questions:
|
||||
qid, qtype = q["id"], q["type"]
|
||||
eval_results = index.search(q["question"], top_k=TOP_K_EVAL)
|
||||
prod_results = index.search(q["question"], top_k=TOP_K_PROD)
|
||||
eval_slugs = [s for s, _ in eval_results]
|
||||
|
||||
rec = {"id": qid, "type": qtype, "semantic": q["semantic"],
|
||||
"question": q["question"], "expected_pages": q["source_pages"],
|
||||
"top10": [{"slug": s, "score": round(sc, 3)} for s, sc in eval_results],
|
||||
"prod_top3_slugs": [s for s, _ in prod_results],
|
||||
"prod_injection_tokens": prod_injection(prod_results)}
|
||||
|
||||
if qtype in ("single_doc", "needle"):
|
||||
src = q["source_pages"][0]
|
||||
if src in eval_slugs:
|
||||
rank = eval_slugs.index(src) + 1
|
||||
rec["rank"] = rank
|
||||
rec["hit"] = True
|
||||
else:
|
||||
rank = None
|
||||
rec["hit"] = False
|
||||
bucket = agg["semantic" if q["semantic"] else ("needle" if qtype == "needle" else "single")]
|
||||
bucket["n"] += 1
|
||||
if rank:
|
||||
bucket["rec5"] += 1 if rank <= 5 else 0
|
||||
bucket["rec10"] += 1
|
||||
bucket["mrr10"] += 1.0 / rank
|
||||
bucket["ndcg10"] += 1.0 / (1 + math.log2(rank)) # 单相关文档的二值 NDCG
|
||||
bucket["top1"] += 1 if rank == 1 else 0
|
||||
|
||||
elif qtype == "cross_doc":
|
||||
srcs = q["source_pages"]
|
||||
hits = [s for s in srcs if s in eval_slugs]
|
||||
primary = srcs[0]
|
||||
p_rank = (eval_slugs.index(primary) + 1) if primary in eval_slugs else None
|
||||
rec["hit_sources"] = hits
|
||||
rec["missed_sources"] = [s for s in srcs if s not in hits]
|
||||
rec["primary_rank"] = p_rank
|
||||
b = agg["cross"]
|
||||
b["n"] += 1
|
||||
b["all_source_rec10"] += 1 if len(hits) == len(srcs) else 0
|
||||
b["primary_mrr10"] += (1.0 / p_rank) if p_rank else 0.0
|
||||
b["avg_coverage"] += len(hits) / len(srcs)
|
||||
b["avg_sources"] += len(srcs)
|
||||
b["primary_top1"] += 1 if p_rank == 1 else 0
|
||||
|
||||
elif qtype == "negative":
|
||||
b = agg["negative"]
|
||||
b["n"] += 1
|
||||
top1_score = eval_results[0][1] if eval_results else 0.0
|
||||
b["scores"].append(round(top1_score, 3))
|
||||
b["returned"] += 1 if eval_results else 0
|
||||
rec["top1_score"] = round(top1_score, 3)
|
||||
rec["returned_any"] = bool(eval_results)
|
||||
|
||||
detail.append(rec)
|
||||
|
||||
# ---- 负样本 vs 非负样本 top1 分数对比（S4 的核心观察）----
|
||||
nonneg_top1 = []
|
||||
for q, r in zip(questions, detail):
|
||||
if q["type"] != "negative" and r["top10"]:
|
||||
nonneg_top1.append(r["top10"][0]["score"])
|
||||
|
||||
neg_scores = agg["negative"]["scores"]
|
||||
|
||||
def stats(xs):
|
||||
if not xs:
|
||||
return {"n": 0}
|
||||
xs2 = sorted(xs)
|
||||
return {"n": len(xs), "mean": round(sum(xs)/len(xs), 2),
|
||||
"median": round(xs2[len(xs2)//2], 2),
|
||||
"min": round(xs2[0], 2), "max": round(xs2[-1], 2)}
|
||||
|
||||
# S1/S3 生产 token 注入统计（正样本部分）
|
||||
inj_pos = [r["prod_injection_tokens"] for r in detail if r["type"] != "negative"]
|
||||
inj_neg = [r["prod_injection_tokens"] for r in detail if r["type"] == "negative"]
|
||||
|
||||
summary = {}
|
||||
for k in ("single", "semantic", "needle"):
|
||||
b = agg[k]
|
||||
n = b["n"] or 1
|
||||
summary[k] = {
|
||||
"n": b["n"],
|
||||
"recall@5": round(b["rec5"]/n, 3),
|
||||
"recall@10": round(b["rec10"]/n, 3),
|
||||
"MRR@10": round(b["mrr10"]/n, 3),
|
||||
"NDCG@10": round(b["ndcg10"]/n, 3),
|
||||
"top1_acc": round(b["top1"]/n, 3),
|
||||
}
|
||||
b = agg["cross"]
|
||||
n = b["n"] or 1
|
||||
summary["cross"] = {
|
||||
"n": b["n"],
|
||||
"avg_sources_per_q": round(b["avg_sources"]/n, 2),
|
||||
"all_source_recall@10": round(b["all_source_rec10"]/n, 3),
|
||||
"primary_MRR@10": round(b["primary_mrr10"]/n, 3),
|
||||
"primary_top1": round(b["primary_top1"]/n, 3),
|
||||
"avg_source_coverage@10": round(b["avg_coverage"]/n, 3),
|
||||
}
|
||||
summary["negative"] = {
|
||||
"n": agg["negative"]["n"],
|
||||
"returned_any": agg["negative"]["returned"],
|
||||
"top1_score_stats": stats(neg_scores),
|
||||
"nonneg_top1_score_stats": stats(nonneg_top1),
|
||||
}
|
||||
summary["prod_injection"] = {
|
||||
"note": "镜像 wiki/tool.py _handle_ask: top_k=3 整篇文档拼接, est_tokens=chars/1.5",
|
||||
"positive_qs": stats(inj_pos),
|
||||
"negative_qs": stats(inj_neg),
|
||||
"corpus_total_est_tokens": sum(d["est_tokens"] for d in docs.values()),
|
||||
"top3_injection_ratio_of_corpus": round((sum(inj_pos)/len(inj_pos)) / sum(d["est_tokens"] for d in docs.values()), 4) if inj_pos else None,
|
||||
}
|
||||
|
||||
out = {"config": {"top_k_eval": TOP_K_EVAL, "top_k_prod": TOP_K_PROD,
|
||||
"docs": len(docs), "questions": len(questions), "no_jieba": NO_JIEBA},
|
||||
"summary": summary, "detail": detail}
|
||||
(BENCH / "results").mkdir(exist_ok=True)
|
||||
suffix = "-nojieba" if NO_JIEBA else ""
|
||||
(BENCH / f"results/g0{suffix}-detail.json").write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=1))
|
||||
|
||||
# ---- 控制台快速诊断：未命中的题 ----
|
||||
print("\n== misses ==")
|
||||
for r in detail:
|
||||
if r["type"] in ("single_doc", "needle") and not r["hit"]:
|
||||
print(f" MISS {r['id']}: {r['question'][:40]}... expected={r['expected_pages']}")
|
||||
elif r["type"] == "cross_doc" and r.get("missed_sources"):
|
||||
print(f" PART {r['id']}: missed {r['missed_sources']}")
|
||||
@@ -0,0 +1,150 @@
|
||||
"""G1 实验：分块索引（本 PR）在同一批 60 题上与 G0 的 before/after 对比
|
||||
|
||||
与 run_g0.py 同一把尺子（同题、同语料、同指标口径），仅检索单元不同：
|
||||
G0 = 文档级 BM25（整篇检索，改前），G1 = 分块级 BM25（backend/modules/rag，改后）。
|
||||
|
||||
输出：results/g1-detail.json + 控制台对比摘要。
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
BENCH = Path(__file__).resolve().parent.parent
|
||||
REPO = Path(__file__).resolve().parents[2] # 仓库根（含 backend/，从任意克隆位置可复现）
|
||||
|
||||
sys.path.insert(0, str(REPO))
|
||||
|
||||
from backend.modules.rag.stores import ChunkedBM25Index # noqa: E402
|
||||
|
||||
TOP_K_EVAL = 10 # 评估口径
|
||||
TOP_K_PROD = 6 # 生产镜像：ask 注入的块数（G0 生产是 top3 整篇）
|
||||
|
||||
def est_tokens(text: str) -> int:
|
||||
return int(len(text) / 1.5)
|
||||
|
||||
def load_questions():
|
||||
qs = [json.loads(l) for l in (BENCH / "questions.jsonl").read_text(encoding="utf-8").splitlines() if l.strip()]
|
||||
for q in qs:
|
||||
q["expected_pages"] = [p.replace(".md", "") for p in q["source_pages"]]
|
||||
return qs
|
||||
|
||||
def build_index():
|
||||
store = ChunkedBM25Index()
|
||||
manifest = json.load(open(BENCH / "corpus/manifest.json", encoding="utf-8"))
|
||||
for e in manifest:
|
||||
content = (BENCH / "corpus" / f"{e['slug']}.md").read_text(encoding="utf-8")
|
||||
store.add_document(e["slug"], e["title"], content, [e.get("section", "")])
|
||||
return store, len(manifest)
|
||||
|
||||
def main():
|
||||
questions = load_questions()
|
||||
print(f"[questions] {len(questions)}")
|
||||
store, n_docs = build_index()
|
||||
n_chunks = store.stats()["total_chunks"]
|
||||
print(f"[index] {n_docs} docs -> {n_chunks} chunks")
|
||||
|
||||
detail = []
|
||||
for q in questions:
|
||||
results = store.search_chunks(q["question"], top_k=TOP_K_EVAL)
|
||||
top_slugs = [r["slug"] for r in results]
|
||||
|
||||
# 生产镜像：top6 块注入（口径与 G0 相同的 header+content）
|
||||
prod = store.search_chunks(q["question"], top_k=TOP_K_PROD)
|
||||
prod_injection = sum(est_tokens(f"### {r['doc_title']} › {r['section']}\n{r['content']}") for r in prod)
|
||||
prod_slugs = [r["slug"] for r in prod]
|
||||
|
||||
rec = {
|
||||
"id": q["id"],
|
||||
"type": q["type"],
|
||||
"semantic": q.get("semantic", False),
|
||||
"question": q["question"],
|
||||
"expected_pages": q["expected_pages"],
|
||||
"top10": [{"chunk_id": r["chunk_id"], "slug": r["slug"], "score": r["score"]} for r in results],
|
||||
"prod_top6_slugs": prod_slugs,
|
||||
"prod_top6_chunks": [r["chunk_id"] for r in prod],
|
||||
"prod_injection_tokens": prod_injection,
|
||||
}
|
||||
|
||||
if q["type"] == "negative":
|
||||
rec["top1_score"] = results[0]["score"] if results else 0.0
|
||||
else:
|
||||
exp = q["expected_pages"]
|
||||
# 文档级命中（去重后按首次出现位置）
|
||||
first_rank = None
|
||||
for i, s in enumerate(top_slugs):
|
||||
if s in exp:
|
||||
first_rank = i + 1
|
||||
break
|
||||
rec["first_rank"] = first_rank
|
||||
rec["hit"] = first_rank is not None
|
||||
rec["all_source_hit"] = all(s in top_slugs for s in exp)
|
||||
# 生产口径命中：期望文档出现在注入的 top6 块中
|
||||
rec["prod_hit"] = any(s in prod_slugs for s in exp)
|
||||
# 跨文档：生产覆盖的源比例
|
||||
rec["prod_source_coverage"] = len([s for s in exp if s in prod_slugs]) / len(exp)
|
||||
detail.append(rec)
|
||||
|
||||
# ---------- 汇总 ----------
|
||||
def group(pred):
|
||||
return [r for r in detail if pred(r)]
|
||||
|
||||
summary = {}
|
||||
for name, rs in [
|
||||
("S1 直接", lambda: group(lambda r: r["type"] == "single_doc" and not r["semantic"])),
|
||||
("S1 语义", lambda: group(lambda r: r["type"] == "single_doc" and r["semantic"])),
|
||||
("S3 针", lambda: group(lambda r: r["type"] == "needle")),
|
||||
("S2 跨文档", lambda: group(lambda r: r["type"] == "cross_doc")),
|
||||
]:
|
||||
rows = rs()
|
||||
n = len(rows)
|
||||
r5 = sum(1 for r in rows if r.get("first_rank") and r["first_rank"] <= 5)
|
||||
r10 = sum(1 for r in rows if r.get("first_rank") and r["first_rank"] <= 10)
|
||||
mrr = sum(1.0 / r["first_rank"] for r in rows if r.get("first_rank")) / n
|
||||
top1 = sum(1 for r in rows if r.get("first_rank") == 1) / n
|
||||
# NDCG@10（文档级，单相关文档：1/log2(rank+1)）
|
||||
ndcg = sum(1.0 / math.log2(r["first_rank"] + 1) for r in rows if r.get("first_rank")) / n
|
||||
inj = [r["prod_injection_tokens"] for r in rows]
|
||||
prod_hit = sum(1 for r in rows if r.get("prod_hit"))
|
||||
summary[name] = {
|
||||
"n": n, "recall@5": r5 / n, "recall@10": r10 / n, "MRR@10": mrr,
|
||||
"NDCG@10": ndcg, "top1": top1,
|
||||
"prod_hit@6": prod_hit / n,
|
||||
"prod_injection_mean": sum(inj) / n, "prod_injection_max": max(inj),
|
||||
}
|
||||
if name == "S2 跨文档":
|
||||
summary[name]["all_source_hit@10"] = sum(1 for r in rows if r.get("all_source_hit")) / n
|
||||
summary[name]["prod_source_coverage"] = sum(r["prod_source_coverage"] for r in rows) / n
|
||||
|
||||
neg = group(lambda r: r["type"] == "negative")
|
||||
summary["S4 负样本"] = {
|
||||
"n": len(neg),
|
||||
"returned": sum(1 for r in neg if r["top10"]) / len(neg),
|
||||
"prod_injection_mean": sum(r["prod_injection_tokens"] for r in neg) / len(neg),
|
||||
"top1_score_max": max((r["top1_score"] for r in neg), default=0.0),
|
||||
}
|
||||
|
||||
# 正样本总体注入（M1 验收：token 下降 >= 70%）
|
||||
pos = [r for r in detail if r["type"] != "negative"]
|
||||
summary["正样本总体"] = {
|
||||
"n": len(pos),
|
||||
"prod_injection_mean": sum(r["prod_injection_tokens"] for r in pos) / len(pos),
|
||||
"prod_injection_max": max(r["prod_injection_tokens"] for r in pos),
|
||||
"prod_hit@6": sum(1 for r in pos if r["prod_hit"]) / len(pos),
|
||||
}
|
||||
|
||||
out = {"config": {"top_k_eval": TOP_K_EVAL, "top_k_prod": TOP_K_PROD,
|
||||
"docs": n_docs, "chunks": n_chunks, "questions": len(questions)},
|
||||
"summary": summary, "detail": detail}
|
||||
(BENCH / "results/g1-detail.json").write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
print("\n===== G1 分块索引摘要 =====")
|
||||
for name, s in summary.items():
|
||||
print(f"\n{name} (n={s['n']})")
|
||||
for k, v in s.items():
|
||||
if k != "n":
|
||||
print(f" {k}: {v:.3f}" if isinstance(v, float) else f" {k}: {v}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user