From 00c78b38a6d8242eaac7c8a045dc623b58df6fe3 Mon Sep 17 00:00:00 2001 From: dafang Date: Sat, 11 Jul 2026 10:39:45 +0800 Subject: [PATCH] release: CodeStable 1.0.3 --- .agents/plugins/marketplace.json | 2 +- .claude-plugin/marketplace.json | 2 +- .../scripts/promote_feedback_fixture.py | 483 ++++++ .codestable/.gitignore | 4 + .codestable/attention.md | 2 + .../approval-report.md | 78 + ...s-feedback-evidence-pipeline-acceptance.md | 154 ++ ...-feedback-evidence-pipeline-checklist.yaml | 117 ++ ...eedback-evidence-pipeline-design-review.md | 94 + .../cs-feedback-evidence-pipeline-design.md | 297 ++++ ...eedback-evidence-pipeline-dod-results.json | 66 + ...idence-pipeline-evidence-pack-results.json | 25 + ...eedback-evidence-pipeline-evidence-pack.md | 222 +++ ...edback-evidence-pipeline-gate-results.json | 102 ++ ...ce-pipeline-implementation-review-fixes.md | 56 + ...edback-evidence-pipeline-implementation.md | 297 ++++ .../cs-feedback-evidence-pipeline-qa.md | 110 ++ ...edback-evidence-pipeline-review-history.md | 112 ++ .../cs-feedback-evidence-pipeline-review.md | 90 + .../goal-plan.md | 68 + .../goal-protocol.md | 65 + .../goal-state.yaml | 102 ++ .../reference/execution-conventions.md | 5 + .codestable/reference/shared-conventions.md | 11 +- .codestable/reference/system-overview.md | 2 +- .codestable/requirements/VISION.md | 14 + .../feedback-evidence-pipeline.md | 41 + .codestable/runtime-manifest.json | 4 +- CHANGELOG.md | 6 + README.en.md | 94 +- README.md | 4 +- SKILL_CATALOG.en.md | 2 +- SKILL_CATALOG.md | 2 +- VERSION | 2 +- WORKFLOW.en.md | 3 +- WORKFLOW.md | 3 +- docs/adr/003-cs-skill-evaluation-loop.md | 4 +- .../fixtures/routing/rt-c17.json | 16 + plugins/codestable/.claude-plugin/plugin.json | 2 +- plugins/codestable/.codex-plugin/plugin.json | 2 +- .../codestable/skills/cs-feedback/SKILL.md | 175 +- .../cs-feedback/references/report-template.md | 129 +- .../scripts/collect_feedback_context.py | 690 +++----- .../cs-feedback/scripts/feedback_incidents.py | 507 ++++++ .../cs-feedback/scripts/feedback_models.py | 98 ++ .../cs-feedback/scripts/feedback_privacy.py | 298 ++++ .../scripts/feedback_repo_context.py | 172 ++ .../scripts/feedback_to_fixture.py | 271 ++- .../scripts/feedback_transcripts.py | 547 ++++++ .../cs-feedback/scripts/feedback_triage.py | 576 +++++++ .../scripts/report_feedback_issue.py | 73 +- .../skills/cs-onboard/codestable.gitignore | 4 + .../references/execution-conventions.md | 5 + .../references/shared-conventions.md | 11 +- .../cs-onboard/references/system-overview.md | 2 +- tests/test_cs_feedback.py | 85 +- tests/test_cs_feedback_candidate.py | 322 ++++ tests/test_cs_feedback_evidence_pipeline.py | 1525 +++++++++++++++++ tests/test_cs_feedback_fixture_promotion.py | 401 +++++ tests/test_cs_feedback_reporting.py | 500 ++++++ tests/test_cs_skill_bootstrap.py | 23 +- tests/test_skill_contracts.py | 6 +- tests/test_skill_entry_simplification.py | 9 +- 63 files changed, 8384 insertions(+), 810 deletions(-) create mode 100644 .claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/approval-report.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-acceptance.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-checklist.yaml create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design-review.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-dod-results.json create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack-results.json create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-gate-results.json create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation-review-fixes.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-qa.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review-history.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-plan.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-protocol.md create mode 100644 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-state.yaml create mode 100644 .codestable/requirements/VISION.md create mode 100644 .codestable/requirements/feedback-evidence-pipeline.md create mode 100644 experiments/cs-routing-001/fixtures/routing/rt-c17.json create mode 100644 plugins/codestable/skills/cs-feedback/scripts/feedback_incidents.py create mode 100644 plugins/codestable/skills/cs-feedback/scripts/feedback_models.py create mode 100644 plugins/codestable/skills/cs-feedback/scripts/feedback_privacy.py create mode 100644 plugins/codestable/skills/cs-feedback/scripts/feedback_repo_context.py create mode 100644 plugins/codestable/skills/cs-feedback/scripts/feedback_transcripts.py create mode 100644 plugins/codestable/skills/cs-feedback/scripts/feedback_triage.py create mode 100644 tests/test_cs_feedback_candidate.py create mode 100644 tests/test_cs_feedback_evidence_pipeline.py create mode 100644 tests/test_cs_feedback_fixture_promotion.py create mode 100644 tests/test_cs_feedback_reporting.py diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index e336a4e..f34d12c 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -3,7 +3,7 @@ "plugins": [ { "name": "codestable", - "version": "1.0.2", + "version": "1.0.3", "description": "CodeStable AI coding workflow skills.", "source": { "source": "local", diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c682c9b..5baf2b0 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -7,7 +7,7 @@ "plugins": [ { "name": "codestable", - "version": "1.0.2", + "version": "1.0.3", "description": "CodeStable AI coding workflow skills.", "source": "./plugins/codestable" } diff --git a/.claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py b/.claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py new file mode 100644 index 0000000..7eb7dd5 --- /dev/null +++ b/.claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py @@ -0,0 +1,483 @@ +#!/usr/bin/env python3 +"""Promote a cs-feedback candidate into a repo-local experiment fixture.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +sys.dont_write_bytecode = True +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from _model import Fixture # noqa: E402 +from buildprompt import build_prompt # noqa: E402 +from config import ExperimentConfig # noqa: E402 +from fixtures import validate_fixture_dict # noqa: E402 + + +ROUTING_INCIDENT_KINDS = { + "wrong-route", + "skipped-gate", + "missing-artifact", + "goal-driver", + "unnecessary-detour", +} +FINDINGS_INCIDENT_KINDS = { + "tool-failure", + "install-version", + "privacy-reporting", + "unclear-rule", +} +TASK_KIND_BY_TARGET = { + "cs-code-review": "review", + "cs-issue": "fix", + "cs-audit": "audit", + "cs-feat": "design", + "cs-refactor": "design", + "cs-epic": "design", + "cs-req": "design", + "cs-domain": "design", + "cs-docs": "docs", + "cs-docs-neat": "docs", +} +MOCK_HARNESSES = {"mock", "mock-weak"} +PLACEHOLDER_PATTERN = re.compile(r"\b(?:TODO|TBD|unknown)\b", re.IGNORECASE) +SAFE_FIXTURE_ID_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z") +SECRET_KEY_PATTERN = re.compile( + r"""(?imx)['"]?(?Papi[_-]?key|token|secret|password|authorization|bearer)['"]? + \s*[:=：＝]\s*""" +) +AUTHORIZATION_HEADER_PATTERN = re.compile( + r"(?im)(?]+|/(?!(?i:goal)(?:\b|/))[^\s`'\"<>/]+" + r"(?:/[^\s`'\"<>/]+)*|[A-Za-z]:\\[^\s`'\"<>]+)" +) +REMOTE_PATTERN = re.compile( + r"(?:https?|ssh|git)://[^\s`'\"<>]+|[\w.+-]+@[\w.-]+:[^\s`'\"<>]+" +) +ENV_PATTERN = re.compile(r"\b[A-Z][A-Z0-9_]{2,}\s*=\s*[^\s`'\"<>]+") +ENV_NAME_PATTERN = re.compile(r"\b[A-Z][A-Z0-9_]{2,}\b") +PRIVATE_MARKER_PATTERN = re.compile(r"\blocal-private\b", re.IGNORECASE) +ROUTING_TASK_FIELDS = {"kind", "state", "intent", "utterance"} +FINDINGS_TASK_FIELDS = {"kind", "spec", "diff", "context", "audience"} + + +def _quoted_segment_end(text: str, start: int) -> tuple[int, int]: + quote = text[start] + index = start + 1 + logical_length = 0 + while index < len(text): + char = text[index] + if char == "\\" and index + 1 < len(text): + if text[index + 1] == "\r" and index + 2 < len(text) and text[index + 2] == "\n": + index += 3 + elif text[index + 1] in "\r\n": + index += 2 + else: + logical_length += 1 + index += 2 + continue + if char == quote: + return index + 1, logical_length + if char not in "\r\n": + logical_length += 1 + index += 1 + return len(text), logical_length + + +def _shell_expansion_end(text: str, start: int) -> int: + stack = [")" if text.startswith("$(", start) else "}"] + index = start + 2 + while index < len(text) and stack: + char = text[index] + if char == "\\" and index + 1 < len(text): + if text[index + 1] == "\r" and index + 2 < len(text) and text[index + 2] == "\n": + index += 3 + else: + index += 2 + continue + if char in "'\"`": + index, _length = _quoted_segment_end(text, index) + continue + if text.startswith("$(", index): + stack.append(")") + index += 2 + continue + if text.startswith("${", index): + stack.append("}") + index += 2 + continue + opener = "(" if stack[-1] == ")" else "{" + if char == opener: + stack.append(stack[-1]) + elif char == stack[-1]: + stack.pop() + index += 1 + return index + + +def _secret_value_end(text: str, start: int) -> int | None: + placeholder_end = start + len("") + if text.startswith("", start) and ( + placeholder_end == len(text) or text[placeholder_end].isspace() + ): + return None + + index = start + logical_length = 0 + has_expansion = False + starts_quoted = ( + index < len(text) and text[index] in "'\"`" + ) or text.startswith(("$'", '$"'), index) + while index < len(text) and not text[index].isspace(): + char = text[index] + if text.startswith(("$(", "${"), index): + has_expansion = True + index = _shell_expansion_end(text, index) + continue + if char == "$" and index + 1 < len(text) and text[index + 1] in "'\"": + index += 1 + continue + if char in "'\"`": + index, segment_length = _quoted_segment_end(text, index) + logical_length += segment_length + continue + if char == "\\" and index + 1 < len(text): + if text[index + 1] == "\r" and index + 2 < len(text) and text[index + 2] == "\n": + index += 3 + elif text[index + 1] == "\n": + index += 2 + else: + logical_length += 1 + index += 2 + continue + logical_length += 1 + index += 1 + + minimum = 4 if starts_quoted else 6 + return index if has_expansion or logical_length >= minimum else None + + +def _contains_secret_assignment(text: str) -> bool: + cursor = 0 + while match := SECRET_KEY_PATTERN.search(text, cursor): + end = _secret_value_end(text, match.end()) + if end is not None: + return True + cursor = match.end() + return False + + +def _load_config(experiment: Path) -> ExperimentConfig: + config_path = experiment / "config.json" + if not config_path.is_file(): + raise ValueError(f"missing experiment config: {config_path}") + data = json.loads(config_path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("experiment config must be a JSON object") + return ExperimentConfig.from_dict(data) + + +def _fixture_from_candidate(candidate: dict) -> dict: + fixture = { + "id": candidate.get("id"), + "incident_id": candidate.get("incident_id"), + "answerType": candidate.get("answerType"), + "task": candidate.get("task"), + } + if candidate.get("answerType") == "routing-decision": + fixture["expect"] = candidate.get("expect") + elif candidate.get("answerType") == "findings-recall": + fixture["answer"] = candidate.get("answer") + return fixture + + +def _string_fields(value, path: str = "fixture") -> list[tuple[str, str]]: + if isinstance(value, str): + return [(path, value)] + if isinstance(value, list): + fields: list[tuple[str, str]] = [] + for index, item in enumerate(value): + fields.extend(_string_fields(item, f"{path}[{index}]")) + return fields + if isinstance(value, dict): + fields = [] + for index, (key, item) in enumerate(value.items()): + fields.append((f"{path}.key[{index}]", str(key))) + fields.extend(_string_fields(item, f"{path}.value[{index}]")) + return fields + return [] + + +def _commit_safe_issues(fixture: dict) -> list[str]: + problems: list[str] = [] + for field_path, text in _string_fields(fixture): + reasons: list[str] = [] + if not text.strip(): + reasons.append("blank") + if PLACEHOLDER_PATTERN.search(text): + reasons.append("placeholder") + if ( + _contains_secret_assignment(text) + or TOKEN_PATTERN.search(text) + or any(pattern.search(text) for pattern in CREDENTIAL_PATTERNS) + ): + reasons.append("secret") + if ABSOLUTE_PATH_PATTERN.search(text): + reasons.append("absolute-path") + if REMOTE_PATTERN.search(text): + reasons.append("remote") + if ENV_PATTERN.search(text): + reasons.append("environment") + if ENV_NAME_PATTERN.search(text): + reasons.append("environment-name") + if PRIVATE_MARKER_PATTERN.search(text): + reasons.append("private-marker") + if reasons: + problems.append( + f"not commit-safe: {field_path} ({','.join(dict.fromkeys(reasons))})" + ) + return problems + + +def _real_judge_issues(cfg: ExperimentConfig) -> list[str]: + judge_model = (cfg.judge_model or "").strip() + if not judge_model: + return ["recall_judge requires non-empty judge_model"] + if "mock" in judge_model.lower(): + return ["recall_judge judge_model must not be mock"] + if judge_model in set(cfg.model_list): + return ["recall_judge judge_model must be independent from model_list"] + return [] + + +def _nonblank_text(value) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _validate_candidate(candidate: dict, cfg: ExperimentConfig) -> list[str]: + problems: list[str] = [] + fixture_id = candidate.get("id") + if not isinstance(fixture_id, str) or not SAFE_FIXTURE_ID_PATTERN.fullmatch( + fixture_id + ): + problems.append("id must be a safe fixture basename") + if candidate.get("_source") != "cs-feedback": + problems.append("_source must be cs-feedback") + if candidate.get("_status") != "candidate": + problems.append("_status must be candidate") + if candidate.get("privacy") != "local-private": + problems.append("privacy must be local-private") + privacy_review = candidate.get("privacy_review") + if not isinstance(privacy_review, dict): + problems.append("privacy_review must be an object") + elif privacy_review.get("status") != "approved": + problems.append("privacy_review.status must be approved") + promotion_blockers = candidate.get("promotion_blockers") + if not isinstance(promotion_blockers, list): + problems.append("promotion_blockers must be an array") + elif not all(isinstance(item, str) for item in promotion_blockers): + problems.append("promotion_blockers entries must be strings") + elif promotion_blockers: + problems.append("promotion_blockers must be empty") + quality = candidate.get("quality") + if not isinstance(quality, dict): + problems.append("quality must be an object") + else: + if quality.get("triage_ready") is not True: + problems.append("quality.triage_ready must be true") + if quality.get("regression_ready") is not True: + problems.append("quality.regression_ready must be true") + missing_fields = quality.get("missing_fields") + if not isinstance(missing_fields, list) or not all( + isinstance(item, str) for item in missing_fields + ): + problems.append("quality.missing_fields must be an array of strings") + incident_id = candidate.get("incident_id") + if not _nonblank_text(incident_id): + problems.append("incident_id must be selected") + if "regression" not in cfg.fixture_classes: + problems.append("experiment config must enable the regression fixture class") + + target_skill_value = candidate.get("target_skill") + if not _nonblank_text(target_skill_value): + problems.append("target_skill must be a non-empty string") + target_skill = "unknown" + else: + target_skill = target_skill_value + if cfg.skill_under_test != target_skill: + problems.append( + f"candidate target_skill={target_skill} does not match config skill_under_test={cfg.skill_under_test}" + ) + profile = candidate.get("_profile") + answer_type = candidate.get("answerType") + if not isinstance(profile, str): + problems.append("_profile must be a string") + if not isinstance(answer_type, str): + problems.append("answerType must be a string") + if profile != answer_type: + problems.append("_profile must match answerType") + incident_kind_value = candidate.get("incident_kind") + if not _nonblank_text(incident_kind_value): + problems.append("incident_kind must be a non-empty string") + incident_kind = "unknown" + else: + incident_kind = incident_kind_value + task_value = candidate.get("task") + if not isinstance(task_value, dict): + problems.append("task must be an object") + task = {} + else: + task = task_value + + if profile == "routing-decision": + if set(task) - ROUTING_TASK_FIELDS: + problems.append("routing task contains unsupported fields") + if incident_kind not in ROUTING_INCIDENT_KINDS: + problems.append("routing-decision incident_kind is incompatible") + if "routing_decision" not in cfg.scorers: + problems.append("routing-decision candidate requires routing_decision scorer") + if task.get("kind") != "routing": + problems.append("routing candidate task.kind must be routing") + for key in ("state", "intent"): + if key in task and not isinstance(task[key], dict): + problems.append(f"routing task.{key} must be an object") + if "utterance" in task and not isinstance(task["utterance"], str): + problems.append("routing task.utterance must be a string") + if not ( + (isinstance(task.get("state"), dict) and bool(task["state"])) + or (isinstance(task.get("intent"), dict) and bool(task["intent"])) + or _nonblank_text(task.get("utterance")) + ): + problems.append("routing candidate needs state, intent, or utterance") + expect = candidate.get("expect") + if not isinstance(expect, dict) or not _nonblank_text(expect.get("result_type")): + problems.append("routing candidate requires expect.result_type") + elif profile == "findings-recall": + if set(task) - FINDINGS_TASK_FIELDS: + problems.append("findings-recall task contains unsupported fields") + if incident_kind not in FINDINGS_INCIDENT_KINDS: + problems.append("findings-recall incident_kind is incompatible") + if "recall_judge" not in cfg.scorers: + problems.append("findings-recall candidate requires recall_judge scorer") + else: + problems.extend(_real_judge_issues(cfg)) + kind_value = task.get("kind") + if not isinstance(kind_value, str): + problems.append("findings-recall task.kind must be a string") + kind = "" + else: + kind = kind_value + expected_kind = TASK_KIND_BY_TARGET.get(target_skill) + if not expected_kind or kind != expected_kind: + problems.append("findings-recall task.kind is incompatible with target_skill") + for key in ("spec", "diff", "context", "audience"): + if key in task and not isinstance(task[key], str): + problems.append(f"findings-recall task.{key} must be a string") + if kind in {"review", "audit"} and not _nonblank_text(task.get("diff")): + problems.append(f"{kind} candidate requires diff") + if kind == "fix" and ( + not _nonblank_text(task.get("spec")) or not _nonblank_text(task.get("diff")) + ): + problems.append("fix candidate requires spec and diff") + if kind == "design" and not _nonblank_text(task.get("spec")): + problems.append("design candidate requires spec") + if kind == "docs" and ( + not _nonblank_text(task.get("spec")) or not _nonblank_text(task.get("diff")) + ): + problems.append("docs candidate requires spec and diff") + if kind in {"design", "docs"} and not any( + harness not in MOCK_HARNESSES for harness in cfg.harnesses + ): + problems.append(f"{kind} candidate requires a non-mock harness") + answer = candidate.get("answer") + if not isinstance(answer, list) or not answer or not all( + isinstance(item, str) and item.strip() for item in answer + ): + problems.append("findings-recall candidate requires non-empty answer") + else: + problems.append(f"unsupported profile: {profile}") + + fixture = _fixture_from_candidate(candidate) + problems.extend(validate_fixture_dict(fixture)) + if not problems: + try: + fixture_model = Fixture.from_dict(fixture) + prompt = build_prompt(fixture_model, "# validation skill snapshot") + if not prompt.strip(): + problems.append("buildPrompt returned empty output") + except (AttributeError, TypeError, ValueError) as exc: + problems.append(f"fixture/buildPrompt validation failed: {exc}") + problems.extend(_commit_safe_issues(fixture)) + return list(dict.fromkeys(problems)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="promote cs-feedback candidate to experiment fixture" + ) + parser.add_argument("--candidate", required=True) + parser.add_argument("--experiment", required=True) + args = parser.parse_args(argv) + + candidate_path = Path(args.candidate).expanduser() + experiment = Path(args.experiment).expanduser() + try: + candidate = json.loads(candidate_path.read_text(encoding="utf-8")) + if not isinstance(candidate, dict): + raise ValueError("candidate must be a JSON object") + cfg = _load_config(experiment) + problems = _validate_candidate(candidate, cfg) + except (OSError, ValueError, TypeError, json.JSONDecodeError) as exc: + print(f"promotion blocked: {exc}", file=sys.stderr) + return 2 + if problems: + for problem in problems: + print(f"promotion blocked: {problem}", file=sys.stderr) + return 2 + + fixture = _fixture_from_candidate(candidate) + target_dir = experiment / "fixtures/regression" + target = target_dir / f"{fixture['id']}.json" + if target.resolve().parent != target_dir.resolve(): + print("promotion blocked: fixture target escapes regression directory", file=sys.stderr) + return 2 + text = json.dumps(fixture, ensure_ascii=False, indent=2) + "\n" + if target.is_file(): + if target.read_text(encoding="utf-8") == text: + print(f"feedback fixture already promoted -> {target}") + return 0 + print(f"promotion blocked: refusing to overwrite different fixture {target}", file=sys.stderr) + return 2 + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_suffix(".json.tmp") + temporary.write_text(text, encoding="utf-8") + temporary.replace(target) + print(f"promoted feedback fixture -> {target}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.codestable/.gitignore b/.codestable/.gitignore index 7c3a8d3..6e971db 100644 --- a/.codestable/.gitignore +++ b/.codestable/.gitignore @@ -1,2 +1,6 @@ **/__pycache__/ **/*.pyc +feedback/*/*-report.md +feedback/*/evidence.json +feedback/*/triage.json +feedback/*/regression-candidate.json diff --git a/.codestable/attention.md b/.codestable/attention.md index 42cf84f..85300ec 100644 --- a/.codestable/attention.md +++ b/.codestable/attention.md @@ -23,3 +23,5 @@ CodeStable 所有落盘产出的正文用**中文**：plan / design、plan revie ### 环境变量与凭证 ### 其他 + +- 本仓库所有 CodeStable review gate（含 design review、code review 和修复后复审）统一使用 Paseo `provider=claude`、`model=claude-fable-5`、`thinkingOptionId=high`；不可用时停下报告，不静默降级 diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/approval-report.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/approval-report.md new file mode 100644 index 0000000..a4397a9 --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/approval-report.md @@ -0,0 +1,78 @@ +--- +doc_type: approval-report +unit: 2026-07-10-cs-feedback-evidence-pipeline +status: approved +reason: blocker +created_at: 2026-07-11 +decision: option-a +answered_at: 2026-07-11 +--- + +# Approval Report + +## Decision History + +- 2026-07-11：owner 选择 Option A，批准为反馈证据能力新建 requirement backfill；后续仍按 + `cs-req` 的 ReviewDraft checkpoint 审核完整正文。 +- 2026-07-11：owner 确认完整 `feedback-evidence-pipeline` requirement 初稿“可以了”；允许 + 落盘 current requirement、刷新 `VISION.md` 并恢复 feature acceptance。 + +## Decision Needed + +决定本 feature 的用户可感能力如何进入长期 requirement 层。当前实现、Round 17 code review 和 +Round 2 QA 均已通过，但 acceptance 不能在 requirement 归属为空时直接标记完成。 + +## Why Now + +Design frontmatter 的 `requirement` 为空；仓库现有 `.codestable/requirements/` 只有 +`plugin-market-distribution.md`，不覆盖反馈证据能力。本 feature 改变了用户调用方式、反馈 +产物、公开确认边界和 regression candidate 交接，命中 acceptance L3 的“新增用户可感能力” +分支，必须先有 owner-approved backfill/delta。 + +## Context + +建议 backfill 的能力边界： + +- 用户显式调用 `cs-feedback` 后，系统安全定位当前会话并生成可追溯 evidence/triage。 +- 公开 preview 只含 allowlist 字段，GitHub 上传必须逐次确认。 +- 未就绪反馈可保存和分诊，但不能进入正式 regression fixture。 +- shipped skill 只产 local-private candidate，维护仓库 promotion 工具负责正式 fail-closed 提升。 +- 不包含后台遥测、自动上传、默认全历史扫描或自动修改目标 skill。 + +## Options + +### A. 新建反馈证据能力 requirement（推荐） + +授权后续通过 `cs-req` backfill 新建一份 current capability requirement，以上述能力边界为 +愿景与用户故事，并把本 feature 记录为首个实现变更。随后回到 acceptance，机械关联 requirement、 +复核 21 checks 并完成最终审计。 + +### B. 暂缓 requirement 决策 + +保留当前实现、passed review 和 passed QA，但 goal 继续停在 handoff/blocked；不创建长期 +requirement，也不把 feature 标记 complete。后续 owner 准备好能力命名/边界后再恢复。 + +## Recommendation + +选择 A。现有代码已经形成稳定的用户入口与跨 skill artifact 边界；不落 requirement 会让后续 +feature 无法从能力愿景层发现这套约束，容易重新引入自动上传、私有产物越界或 shipped/eval +运行时耦合。 + +## Risks And Tradeoffs + +- A 会新增一份长期 requirement，并需要 `cs-req` 流程落盘；但不会改变已经批准的实现行为。 +- B 不引入文档范围，但 feature 不能完成，后续恢复仍需同一决策。 +- 不能选择“直接忽略 requirement 影响并完成”：这与 acceptance 的 Global Route Governance + 冲突，也不符合当前用户可见改动事实。 + +## Non-Automatic Actions + +- 不会自动 commit、merge、push、创建 GitHub issue或上传任何反馈。 +- 不会在 owner 回答前创建/改写长期 requirement。 +- 不会借 requirement backfill 改动已批准 design 的功能范围或重新修改实现代码。 + +## After You Answer + +- 选择 A：将本报告标为 approved，记录回答日期，加载 `cs-req` 完成 backfill，再恢复 + acceptance 的 21 checks 与 final audit。 +- 选择 B：将本报告记录为 deferred，保持 goal handoff/blocked，等待后续恢复。 diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-acceptance.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-acceptance.md new file mode 100644 index 0000000..26be798 --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-acceptance.md @@ -0,0 +1,154 @@ +--- +doc_type: feature-acceptance +feature: 2026-07-10-cs-feedback-evidence-pipeline +status: passed +accepted: 2026-07-11 +round: 1 +--- + +# CS Feedback Evidence Pipeline 验收报告 + +> 阶段：阶段 3（验收闭环） +> 验收日期：2026-07-11 +> 关联方案：`cs-feedback-evidence-pipeline-design.md` + +## 1. 接口契约核对 + +**接口示例逐项核对**： + +- [x] collector：current/显式 session 输入生成 local-private evidence、triage 与条件性 public + projection；真实 Codex/Claude CLI 与结构化 integration tests 一致。 +- [x] triage：单一 incident、assessment source/ref/confidence、双 readiness 与缺口字段均按 + design schema 输出；非法 identity/ref fail-closed。 +- [x] candidate：canonical triage 只在同目录生成 local-private candidate；v1 public context + 只生成未就绪 candidate，旧直写入口非零。 +- [x] promotion：repo-local 工具消费 candidate/config，完成 profile/input/privacy/judge/harness/ + commit-safe 校验后才落正式 fixture。 + +**名词层与流程图核对**： + +- [x] `NormalizedRecord`、`FeedbackIncident` 有明确类型；Observation/Assessment/Quality 作为 + schema 与机械计算边界落在 incidents/triage 模块。 +- [x] schema v2 incidents 为 canonical，v1 matched_events/public events 保持兼容投影。 +- [x] “显式调用 → metadata-only 定位 → normalize/incident → repo enrich → evidence/triage → + quality → public/candidate”各节点均有代码与测试落点。 + +## 2. 行为与决策核对 + +**需求与关键决策**： + +- [x] 仅显式调用采集；current 优先且歧义时只让用户选，不扩大扫描。 +- [x] snapshot 与 trigger cutoff 分离；无 user anchor 未就绪，post-anchor 记录排除。 +- [x] provider/adjacency/unpaired 三态配对，跨 user-turn incident 不误合并。 +- [x] Observation 与 Assessment 分层，推断必须带来源、置信度和 evidence refs。 +- [x] public projection 只从 8+7 allowlist 构建；local-private 文件永不作为公开正文。 +- [x] quality gap 驱动单问；triage readiness 不等于 regression readiness。 +- [x] shipped candidate 与 repo-local promotion 只通过 JSON artifact 交接，无运行时 import。 +- [x] 共享提示集中在 execution conventions，未复制到每个 skill。 + +**明确不做与流程级约束**： + +- [x] 无后台 telemetry、自动上传、默认全历史扫描、自动修改目标 skill 或自动付费评测。 +- [x] 重采集保留用户补充；incident/fingerprint 漂移进入 pending/accept 状态，不静默覆盖。 +- [x] evidence/triage/public 三文件 staged+rollback；candidate/promotion 失败均诊断且 no-write。 + +**挂载点与可卸载性**： + +- [x] M1：`cs-feedback/SKILL.md` + report template 承载用户协议与报告格式。 +- [x] M2：collector + feedback modules 承载 schema v2、current、incident、triage 与 privacy。 +- [x] M3：shipped converter + repo-local promotion 承载评测交接。 +- [x] M4：onboard execution/shared/system templates 与 runtime copy 承载共享提示和布局。 +- [x] M5：tests、`rt-c17`、ADR-003 与公开 docs 承载 gate 和对外投影。 +- [x] 反向核查：scope-gate changed files 除 workflow/requirement/验收产物外均落在 M1-M5； + 无清单外运行入口。按 M1-M5 逆序拔除后只剩 feature/requirement 历史，不留可执行路径。 + +## 3. 验收场景核对 + +- [x] S1：current 唯一候选自动选择；skill 不传 since-days，collector 报 ignored。 +- [x] S2：真实 current 产生 5 个 metadata-only 候选，候选无 message/tool/content 字段。 +- [x] S3：provider id、唯一相邻 fallback、歧义 unpaired 与 source order 均通过。 +- [x] S4：两个不重叠 user-turn 生成两个 incident，不因相同 skill 名合并。 +- [x] S5：用户纠正进入 expected source=user，actual 引用此前 observation。 +- [x] S6：expected unknown 时保存反馈、quality 只优先追问该缺口。 +- [x] S7：runtime/artifact/git file-level context 存在时记录，缺失为 unknown 而不失败。 +- [x] S8：secret/path/remote/env/raw JSON/code 双层负向矩阵及三类私有文件真实拒绝通过。 +- [x] S9：assessment 缺 ref、source 或 inferred confidence 时 triage_ready=false。 +- [x] S10：triage-ready 但缺 reproduction/oracle 时可报告，不可形成正式 fixture。 +- [x] S11：routing/findings 正向 promotion 与 config/profile/privacy/judge/harness/no-write 负向通过。 +- [x] S12：v1 context 仍可读；events 精确 8 字段/6 值域，v2 incident_kind 只进 incidents。 +- [x] S13：共享约定只提示显式调用，不采集、不上传。 +- [x] S14：真实未确认 reporter 返回 manual；confirmed body/title 与私有文件边界由集成测试锁定。 +- [x] S15：无 anchor capture_cutoff unknown 且未就绪；有 anchor 时后续记录不进 evidence。 +- [x] S16：Codex JSONL 与 Claude JSON/JSONL 合成同构，真实两 provider snapshot 均解析成功。 + +**review/QA/gate 复核**： + +- [x] Round 17 Fable 5/high review `passed`，无 blocking/important。 +- [x] Round 2 QA `passed`，功能性核心路径均有 unit/integration/真实 CLI 证据。 +- [x] QA residual 仅为隐私侧过脱敏、窄路径边界和需用户授权的真实上传，不承载核心缺口。 +- [x] evidence pack、scope gate、DoD 均 passed；CMD-004 仅既有根 `cs-onboard/` baseline。 + +## 4. 术语一致性 + +- `Feedback Incident`：design、model、incident builder、skill 文案与 schema 使用一致。 +- `Observation` / `Assessment`：分别归 evidence 与 triage；无反向混写。 +- `Feedback Quality Gate`：实现为 `triage_ready/regression_ready/missing_fields/reasons` 机械结果。 +- `Optimization Handoff`：shipped candidate 与 repo-local promotion 的 artifact 边界一致。 +- canonical `incident_kind` 与 v1 `failure_type` 分区明确，无同名异义。 + +## 5. 领域影响盘点 + +- 新术语候选：Feedback Incident、Observation/Assessment 分层、Feedback Quality Gate。仓库目前 + 无 CONTEXT.md；建议后续通过 `cs-domain` 将三者纳入领域术语，不在 acceptance 代写。 +- 结构性决策：candidate artifact 边界已机械回写现有 ADR-003 applies-to/Decision/ + Consequences/lint；design 明确“不新增 ADR”，实现与之相符。 +- 流程约束：显式触发、public 逐次确认、skill 运行时独立性已进入 shared/execution conventions + 与测试，不需另开 ADR。 + +## 6. requirement 回写 + +- [x] Owner 在 `approval-report.md` 选择 Option A，并确认完整 backfill 初稿“可以了”。 +- [x] 新建 `.codestable/requirements/feedback-evidence-pipeline.md`，`status: current`，用户故事、 + pitch 与边界来自 approved design 和实际 QA，无新增实现范围。 +- [x] 新建 `VISION.md`，按 Current/Draft/Outdated 分组索引现有两份 requirement。 +- [x] design frontmatter 机械关联 `requirement: feedback-evidence-pipeline`。 + +## 7. roadmap 回写 + +- design frontmatter 的 `roadmap` / `roadmap_item` 均为空；本 feature 非 roadmap 起头，按协议跳过。 + +## 8. attention.md 候选盘点 + +- 本 feature 未暴露新的全局启动硬约束；Fable 5/high review 约束已在 attention.md。 +- compound 候选：隐私 matcher 必须同时测试“漏脱敏”和“过脱敏”，并覆盖词位、定界符后的 + 语言环境与真实 transcript 形态。建议收尾时询问是否走 `cs-keep`。 +- 用户指南/API 变化已在本 feature 内同步 README/WORKFLOW/catalog/SKILL，无额外 docs 缺口。 + +## 9. 遗留 + +- CJK/假名粘连、混合脚本路径、伪盘符、未闭合引号存在隐私侧过脱敏或窄尾段残留；公开前 + 人工 preview 仍是最终防线。 +- ENV_NAME 对普通全大写词可能过脱敏；方向 fail-closed。 +- converter/candidate 单文件写原子性、session id fallback、promotion temp/fsync 等 review nits + 未扩成核心正确性问题，后续按 issue/refactor 处理。 +- recall_judge `[soft]`、k=1 variance 与 candidate 语义真实性属于 eval 有效性残余；正式 campaign + 仍需分模型手读原始输出。 +- 未真实创建 GitHub issue；实际上传必须由用户逐次确认，不能为验收越过授权。 + +## 10. 最终审计 + +- 验证证据来源：Round 2 `cs-feedback-evidence-pipeline-qa.md`（functional / passed）。 +- Evidence sources：acceptance 阶段 fresh evidence pack、DoD results 与 scope gate，均 `passed`。 +- 聚合命令：targeted `171 passed`、full `341 passed`、eval/ADR lint `65 passed`；runtime + `status=ok`；`git diff --check` exit 0；package 仅既有根 `cs-onboard/` baseline。 +- 场景复核：re-verified 16 / trust-prior-verify 0；全部 design 场景由最终 targeted/integration + suite 重跑，真实 provider/reporter CLI 证据由同一最终代码状态的 QA 补强。 +- 交付物复核：collector/modules、candidate converter、repo-local promotion、reporter、schema/fixture、 + tests、skill/runtime templates、ADR/docs、current requirement 与 VISION 均存在；roadmap 不适用。 +- 完整工作区复核：tracked、untracked、machine artifacts 全部纳入 scope-gate；新增 requirement + 路径来自 owner-approved Option A，无未授权文件。 +- diff 清洁度：无新增 debug、cache、注释死代码或方案外文件；两条 TODO warning 仅为 placeholder + 拒绝规则/负向测试字面量；所有 Markdown ≤300 行。 +- 知识沉淀出口：无新 attention 候选；双向隐私 matcher 测试经验列为可选 `cs-keep`；领域术语 + 列为可选 `cs-domain`；用户指南/API 投影已在本 feature 内同步。 +- 结论：通过。21 checks 全部 `passed`，review/QA/DoD 无 failed/blocked，required artifacts 完整。 diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-checklist.yaml b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-checklist.yaml new file mode 100644 index 0000000..e8923fd --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-checklist.yaml @@ -0,0 +1,117 @@ +feature: 2026-07-10-cs-feedback-evidence-pipeline +created: 2026-07-10 + +steps: + - action: "行为等价微重构：按 transcript、privacy/model 和 reporter 职责拆分采集器与测试" + exit_signal: "现有 CLI/schema v1 行为不变，既有 9 个 feedback tests 全绿，diff 仅含移动、import 和测试重排" + status: done + - action: "证据编排：引入 metadata-only 会话选择、trigger cutoff、normalized record、反馈事件包和环境/仓库上下文" + exit_signal: "JSONL EOF/Claude JSON meta snapshot、correlation/unpaired 均锁定；metadata reader 禁止 read_records 且输出无 body；既有 3 个 current 用例+stale-mtime 新例证明只绕过 time_cutoff" + status: done + - action: "分诊计算：生成 observation/assessment 分层的 triage、字段来源、质量门和 public allowlist 投影" + exit_signal: "triage_ready/regression_ready 由机械规则决定；缺口驱动追问；隐私对抗测试证明 public preview 无禁止内容" + status: done + - action: "评测交接：shipped converter 只产 candidate，repo-local eval 工具负责正式 promotion" + exit_signal: "无 eval/config、profile/input/privacy 或真实 judge gate 失败均不落盘；repo-local 测试覆盖 candidate 正向+旧 skeleton 非零无文件，且只在 test 中连接两单元" + status: done + - action: "使用入口：更新 cs-feedback 协议、report template、execution conventions 模板/runtime copy及中英文公开文档" + exit_signal: "SKILL/template 使用 incident/triage/quality 与 v1 映射；shared/execution conventions 模板/runtime 布局同步；所有文档保持 local-first 和确认后上传" + status: done + - action: "验证闭环：补 unit/contract/decision fixtures，运行全量 gate 并完成独立代码评审" + exit_signal: "targeted/full pytest、runtime sync、diff check 通过；package findings 不新增；review 无 unresolved blocking/important" + status: done + +checks: + - item: "反馈以有序 incident 为 canonical 单元，保留 role、tool pairing、用户纠正和 evidence refs" + source: 名词契约 + status: passed + - item: "trigger_cutoff 锚定冻结边界前最后 user record；无 anchor 未就绪，anchor 后记录被排除" + source: 编排骨架 + status: passed + - item: "tool 配对按 provider id 优先、无 id 仅无歧义相邻 fallback；incident 按 user-turn 窗口分离" + source: 名词契约 + status: passed + - item: "skill 默认 current 不传 since-days；显式 since-days 不传 current；collector current 只绕过 time_cutoff 并报告 ignored，始终保留 trigger_cutoff" + source: 编排骨架 + status: passed + - item: "evidence observations 与 triage assessments 分层，推断带 source/confidence/evidence refs" + source: 名词契约 + status: passed + - item: "runtime 版本、相关 artifact/status 和 git 文件级状态进入 local-private repo context" + source: 验收场景 + status: passed + - item: "triage_ready 与 regression_ready 分开判定，缺口可观察且不编造" + source: 流程级约束 + status: passed + - item: "public preview 仅从 allowlist 结构生成，不反向抓报告或原始 transcript" + source: 流程级约束 + status: passed + - item: "未就绪 feedback 不得写入正式 experiment fixture 目录" + source: 范围守护 + status: passed + - item: "shipped converter 只以 local-private triage 为 canonical 输入并只产 candidate；v1 evidence 仅生成未就绪 candidate" + source: 名词契约 + status: passed + - item: "candidate 固定写 feedback 目录；正式 promotion 只在 repo-local eval skill，读取 experiment config 且不形成运行时跨 skill 依赖" + source: 名词契约 + status: passed + - item: "profile/最小 input/config 自校成立；空白/占位/敏感字段不落盘；commit-safe 扫描要求合成复现，不静默改写" + source: 名词契约 + status: passed + - item: "schema v2 保留 v1 matched_events；public events.failure_type 只用 v1 映射且精确 8 字段，v2 incident_kind 只进 incidents" + source: 名词契约 + status: passed + - item: "Codex JSONL 与 Claude JSON/JSONL 同语义产生字段、顺序和配对均同构的 incident" + source: 验收场景 + status: passed + - item: "secret、路径、remote、env、原始工具参数、代码块不会进入 public preview" + source: 验收场景 + status: passed + - item: "reporter 按文件名硬拒 evidence、triage、regression candidate，并按 local-private 字段二次拒绝" + source: 验收场景 + status: passed + - item: "cs-feedback 仍需用户确认 public preview 后才允许 GitHub 上报" + source: 范围守护 + status: passed + - item: "不引入后台 telemetry、自动上传、默认全历史扫描或自动修改目标 skill" + source: 范围守护 + status: passed + - item: "共享反馈提示与 feedback 产物布局只在 execution/shared conventions 维护并同步模板/runtime copy" + source: 挂载点 + status: passed + - item: "ADR-003、README、WORKFLOW、catalog 和 system overview 对反馈闭环表述一致" + source: 挂载点 + status: passed + - item: "ADR-003 repo-local tests 同时验证 candidate→promotion 兼容，但 shipped 脚本运行时不得 import eval 工具" + source: 挂载点 + status: passed + +dod: + commands: + - id: CMD-001 + command: "PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests/test_cs_feedback*.py tests/test_cs_skill_bootstrap.py tests/test_skill_entry_simplification.py" + core: true + failure_handling: fix-or-block + - id: CMD-002 + command: "PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests" + core: true + failure_handling: fix-or-block + - id: CMD-003 + command: "PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests/test_cs_skill_eval.py tests/test_cs_skill_convergence.py tests/test_cs_skill_release.py tests/test_cs_skill_bootstrap.py tests/test_cs_skill_selfref.py" + core: true + failure_handling: fix-or-block + - id: CMD-004 + command: "python3 tools/check-plugin-package.py --root . --json" + core: false + failure_handling: document-baseline + - id: CMD-005 + command: "python3 plugins/codestable/skills/cs-onboard/tools/codestable-runtime-sync.py --root . --source-skill-dir plugins/codestable/skills/cs-onboard --check --json" + core: true + failure_handling: fix-or-block + - id: CMD-006 + command: "git diff --check" + core: true + failure_handling: fix-or-block + evidence_required: [command_output, diff_summary, independent_review, privacy_negative_tests] + cleanliness: + debug_output: forbidden diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design-review.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design-review.md new file mode 100644 index 0000000..14c569c --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design-review.md @@ -0,0 +1,94 @@ +--- +doc_type: feature-design-review +feature: 2026-07-10-cs-feedback-evidence-pipeline +status: passed +reviewed: 2026-07-10 +round: 8 +--- + +# cs-feedback-evidence-pipeline feature design 审查报告 + +## 1. Scope And Inputs + +- Design: `.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design.md` +- Checklist: `.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-checklist.yaml` +- Intent / brainstorm / roadmap / requirement: none +- Related docs: `docs/adr/003-cs-skill-evaluation-loop.md`、`.codestable/reference/shared-conventions.md`、execution conventions +- Code facts checked: cs-feedback collector/reporter/candidate converter、feedback/bootstrap/eval tests、eval config/fixtures/buildPrompt/scorers + +### Independent Review + +- Status: completed +- Detection: paseo +- Provider / agent: `providers.audit=claude/opus` / `54019950-31f5-4544-93b2-3bf1b9ce96a3` +- Raw output: 第八轮最终审查明确“建议通过”，无 blocking/important；前七轮 reviewer 发现的契约缺口均已修订并重审 +- Merge policy: 主 agent 已逐条用 design、checklist、ADR、代码和测试事实核验；所有完成 reviewer 均已归档 +- Gate effect: none；进入用户整体 design review checkpoint + +## 2. Design Summary + +- Goal: 把一次 CS skill 使用问题整理成可分诊、可复现、可安全进入优化评测的本地证据包。 +- Key contracts: incident canonical model、Observation/Assessment 分层、`time_cutoff`/`trigger_cutoff`、private/public 投影、candidate artifact 与 repo-local promotion 边界。 +- Steps: 6；先行为等价拆分，再完成采集、分诊、评测交接、入口同步和验证闭环。 +- Checks: 21；覆盖跨 provider、隐私、v1/v2 兼容、profile/config gate、运行时 skill 独立性和 ADR-003 消费侧。 +- Baseline / validation: targeted/full pytest、ADR-003 lint、package baseline、runtime sync、YAML 和 diff check。 + +## 3. Findings + +### blocking + +none + +### important + +none + +### nit + +none + +### suggestion + +none + +### learning + +- feedback-to-fixture 的稳定边界是 artifact handoff：shipped skill 只产 local-private candidate，repo-local eval skill 才拥有 experiment config、validator、scorer 和正式 promotion。 +- `validate_fixture_dict` 只覆盖部分结构；profile 最小 input、task kind、scorer、harness、judge 和 commit-safe 必须由 promotion 工具独立 fail closed。 + +### praise + +- Observation/Assessment 分层、`cause_status=unclassified` 与 evidence refs 把认知诚实落实到数据结构。 +- current-session metadata-only reader、public allowlist、candidate/promotion 分离共同形成清晰的隐私边界。 + +## 4. User Review Focus + +- 用户需要重点拍板：显式触发且无遥测；默认 current session；不唯一时让用户选择。 +- 用户需要重点拍板：candidate 默认只留 feedback 目录；正式 fixture 仅由 repo-local eval 工具显式 promotion。 +- implement 需要遵守：shipped 脚本运行时不得 import eval 工具；两者只通过 candidate artifact 交接。 +- code review / QA / acceptance 需要复核：cutoff、provider 同构、commit-safe、v1 8 字段/6 值域、config/judge/harness gate。 + +## 5. Evidence Confidence Ledger + +| Check | Verdict | Evidence Class | Basis | Follow-up | +|---|---|---|---|---| +| Acceptance Coverage Matrix | pass | E | 16 个场景逐项映射 S2-S6 和证据动作 | acceptance 逐项核对 | +| DoD Contract | pass | E | Design/Implementation/Review/QA/Acceptance DoD 与 required artifacts 完整 | none | +| Steps and checks traceability | pass | E | 6 steps、21 checks 均能回到名词/编排/范围/挂载点/场景 | implementation 留 step 证据 | +| Roadmap contract compliance | n/a | E | 本 feature 非 roadmap 起头 | none | +| Module interface design | pass | C | 代码证实 metadata reader、candidate artifact、repo-local promotion 是真实 seam | code review 查运行时 import | +| Validation and artifacts | pass | C | 命令路径、ADR lint、runtime sync、package baseline 与仓库事实一致 | QA 运行完整命令 | + +Summary: E=4, C=2, H=0, H-only core checks=none。 + +## 6. Residual Risk + +- current-session 仍依赖 provider 的 cwd/session metadata；弱匹配必须让用户选择，不能静默自动选。 +- commit-safe 扫描只能覆盖已知敏感模式；正式 promotion 仍需人工 privacy approval 和合成复现纪律。 +- `recall_judge` 结论是 `[soft]` 且有 k=1 variance；acceptance 必须记录模型分布并人工读原始输出。 +- candidate→promotion 兼容测试只允许在 repo-local ADR lint 测试中连接两个安装单元，不能演化为 shipped runtime import。 + +## 7. Verdict + +- Status: passed +- Next: 交给用户整体 review；用户确认前 design 保持 `draft`，不得进入 goal package 或 implementation。 diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design.md new file mode 100644 index 0000000..abea856 --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design.md @@ -0,0 +1,297 @@ +--- +doc_type: feature-design +feature: 2026-07-10-cs-feedback-evidence-pipeline +requirement: feedback-evidence-pipeline +status: approved +summary: 把一次 CS skill 使用问题整理成高价值、可分诊、可复现并可转回归样本的反馈证据包 +tags: [codestable, feedback, evaluation, privacy] +--- + +# CS Feedback Evidence Pipeline + +## 0. 术语约定 + +| 术语 | 定义 | 防冲突结论 | +|---|---|---| +| 反馈事件包（Feedback Incident） | 围绕一次问题，把 agent 动作、工具结果、用户纠正和相关仓库事实按时间顺序聚合的证据单元 | 替代当前按单条 transcript record 排序的 `Event`，但保留旧 event 投影兼容期 | +| 客观观察（Observation） | 可指向 transcript record、工具结果或仓库产物的事实 | 不包含根因和修复建议 | +| 分析判断（Assessment） | agent 基于观察形成的 expected / actual、疑似归因和建议 | 必须标来源；仅 `source=inferred` 时必须有置信度，不能伪装成观察 | +| 反馈质量门（Feedback Quality Gate） | 机械检查反馈是否足够用于分诊或制作 regression fixture | 区分 `triage_ready` 与更严格的 `regression_ready` | +| 优化交接（Optimization Handoff） | 面向 CodeStable 维护者和 eval 闭环的结构化 `triage.json` | 不等同于 GitHub issue，也不直接修改被反馈 skill | + +## 1. 决策与约束 + +### 1.1 需求摘要 + +用户目标：在使用 CS skill 过程中遇到问题后，通过显式调用 `cs-feedback`，尽量少补充信息就能留下对后续优化真正有用的证据。 + +核心行为： + +- 默认只定位当前 cwd 对应的当前会话；无法唯一定位时让用户选，不静默扩大到最近三天全部历史。 +- 从离散命中升级为因果事件包，保留角色、顺序、工具名、用户纠正和证据指针。 +- 用当前仓库的 runtime 版本、相关 `.codestable` 产物状态和 git 文件级状态补足使用环境，但不复制业务代码。 +- 自动生成客观 evidence、结构化 triage、public preview；只针对质量门缺失项追问用户。 +- `triage_ready` 的反馈可直接进入维护分诊；只有 `regression_ready` 才允许提升为正式 regression fixture。 + +成功标准： + +- 常见“agent 走错阶段 / 跳过 gate / 工具失败 / 用户纠正”场景能恢复 target skill、可能阶段、expected、actual、关键时间线、模型/宿主、runtime 版本和相关产物。 +- 所有推断字段可追溯到观察或用户补充；未知字段明确为 unknown，不编造。 +- public preview 不含绝对路径、remote、环境变量、secret、原始工具参数、大段业务代码或完整 transcript。 +- 不完整反馈仍可保存，但质量门列出缺口；fixture promotion 对未就绪输入 fail closed。 +- Codex JSONL 与 Claude JSON/JSONL 的相同语义得到同构事件包。 + +明确不做： + +- 不做后台遥测、常驻监听、自动上传或默认扫描全部历史。 +- 不因捕获反馈而自动修改目标 skill、创建 GitHub issue或启动付费模型评测。 +- 不把 LLM 猜测写进客观 evidence；不把“模型偶发”直接归因成“skill 缺陷”。 +- 不把完整仓库 diff、业务文件内容、原始 session 路径写入公开产物。 +- 不保证每条反馈都能转为 fixture；不可复现反馈仍可用于聚类和人工分诊。 + +### 1.2 方案深度与复杂度档位 + +候选 A 是只扩充关键词和上下文窗口；它改动小，但仍无法表达因果关系，继续把核心价值押在文本命中率上。候选 B 是本方案的本地反馈证据管线；它保留显式触发和无网络边界，同时把数据结构做到可持续演进。候选 C 是 opt-in 遥测服务；它能提高数量，但引入服务端、身份、授权和合规体系，超出本轮目标。 + +选择候选 B。该能力是长期维护、直接处理私人 transcript、并承担后续评测输入质量的核心路径，不能用“多抓几行文本”的占位方案代替。 + +偏离内部工具默认档位： + +- Robustness = L3：外部 transcript、JSON schema 和文件路径全部验证，失败路径可诊断。 +- Structure = modules：当前 520 行采集器已混合来源解析、筛选、脱敏和输出，先按职责拆分。 +- Testability = verified：隐私边界、事件聚合和 readiness 都需要负向不变量测试。 +- Security = hardened：按 transcript 可能含 secret、私有路径和业务代码的对抗性输入设计。 +- Compatibility = backward-compatible：schema v2 以 `incidents` 为权威，v1 投影至少保留至下一 minor release；删除只能进入 major/schema migration。 + +### 1.3 关键决策 + +1. **显式调用才采集。** CS skill 遇到自身规则、工具或流程异常时可提示用户调用 `cs-feedback`；不得后台执行或上传。 +2. **当前会话优先落在 skill 编排层。** 默认调用只传 `--session current --cwd "$(pwd)"`；用户显式 `--since-days` 时走跨会话且不传 current。collector 的 `session=None + since_days=3` 保持 v1 兼容；若直接同时传 current/since-days，则 current 在应用 `time_cutoff` 前分支并输出 `since_days_ignored=true`。只有 cwd 精确匹配且候选唯一才自动选择，弱匹配或多候选都让用户选。 +3. **事件包是 canonical 单元。** collector 先冻结输入：JSONL 记完整 record 的 EOF offset，单 JSON 一次读成不可变 byte snapshot；`trigger_cutoff` 只认 snapshot 内最后一条 user record，是 `triage_ready` 前置，缺失时任何 failure signal 都不能替代。anchor 后记录排除。call/result 优先按 provider id 配对；无 id 只配忽略 metadata 后紧邻且无竞争 call 的 call→result；`correlation_source=provider/adjacency/unpaired`。unpaired 记录按 source order 进 timeline，result 可凭 failure 成为 incident signal，但不满足 cutoff、不挂猜测 call；窗口只合并重叠或显式 correlation edge。 +4. **观察与判断分层。** `evidence.json` 保存脱敏观察和本机上下文；`triage.json` 保存 expected / actual、疑似范围、建议和 readiness，并给每个判断挂 evidence ids / user-supplied 标记。 +5. **双层隐私投影。** public preview 只从 allowlist 字段构建，绝不从人写报告反向抓文本；`evidence.json` 与 `triage.json` 默认 local-private。 +6. **质量门驱动追问。** 优先补 target skill、expected、actual、影响、最小复现输入和 oracle；已能从用户纠正或仓库事实确定的内容不重复问。 +7. **fixture 提升 fail closed 且不跨 skill 依赖。** shipped `feedback_to_fixture.py` 只把 canonical `triage.json` 转为同目录 local-private candidate；v1 `--evidence` 也只能产未就绪 candidate，旧 `--failure --experiment` 及其他 shipped `--experiment` 直写入口返回迁移错误。正式 fixture 仅由 repo-local `eval-cs-skill` promotion 工具消费 candidate；普通用户仓库无 eval 工具时保留 candidate，不降级、不写 `experiments/`。 +8. **共享提示集中维护。** “遇到 CS 自身问题可调用 cs-feedback”写入 execution conventions 模板及 runtime copy，不向每个 skill 重复塞一段规则。 + +### 1.4 风险、依赖与假设 + +Top 3 风险： + +1. **隐私泄露。** 缓解：local/private 与 public projection 分离、allowlist 构建、敏感输入负向测试、上传前人工确认。 +2. **错误归因或无信号 fixture 污染优化。** 缓解：Observation/Assessment 分层，cause=`unclassified`，promotion profile/config gate；acceptance 仍记录 recall_judge `[soft]`、k=1 variance 与人工读输出的残余风险。 +3. **跨 provider transcript 漂移。** 缓解：provider adapter 只负责规范化为统一 record；未知记录保留但不猜角色或工具语义。 + +非显然依赖：当前 session 定位仍依赖 transcript 中的 cwd/mtime metadata；`.codestable/runtime-manifest.json` 可能不存在或落后，必须把 unknown / mismatch 当环境事实而非采集失败。 + +关键假设：本轮默认用户愿意在明确调用 `cs-feedback` 后让工具读取当前会话的本机历史，但任何公开或上传仍需逐次确认。 + +基线与必跑验证： + +- `PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests/test_cs_feedback*.py tests/test_cs_skill_bootstrap.py tests/test_skill_entry_simplification.py`：当前 45 passed。 +- `PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests`：当前 215 passed。 +- `python3 tools/check-plugin-package.py --root . --json`：当前被 ignored 的根 `cs-onboard/` legacy 目录触发既有失败；本 feature 不删除该用户资产，验收时要求 findings 不新增。 +- `python3 plugins/codestable/skills/cs-onboard/tools/codestable-runtime-sync.py --root . --source-skill-dir plugins/codestable/skills/cs-onboard --check --json`：当前 status=ok。 +- `git diff --check`：当前通过。 + +交付物：cs-feedback 协议与模板、shared-conventions 模板/runtime 布局、schema v2 本地反馈包、事件聚合与仓库上下文采集、质量门、eval-compatible fixture 安全提升、测试、decision fixtures、README/WORKFLOW/catalog 投影和完整 gate 证据。 + +清洁度：不新增调试输出、临时 TODO/FIXME、原始 transcript fixture、真实 secret、注释掉代码或未使用 compatibility 分支。 + +## 2. 名词与编排 + +### 2.1 名词层 + +**现状**：`collect_feedback_context.py` 将 provider 记录 flatten 成字符串，产出按分数排序的 `Event`；`public_summary_for()` 只输出 provider、粗粒度 failure type、tool、skill 和 excerpt。公开 allowlist 虽列出 expected / actual / proposed_fix，实际采集器没有生成。`feedback_to_fixture.py` 直接从 event 摘要生成带 TODO 的 skeleton。 + +**变化**： + +- 新增 `NormalizedRecord`：`id / provider / timestamp / role / record_type / tool_name / correlation_id / correlation_source / text / source_index`。 +- 新增 `FeedbackIncident`：`id / target_skill / stage_hint / incident_kind / observations[] / timeline[] / environment_context / repo_context / user_correction / capture_cutoff`。 +- 新增 `FeedbackTriage`：`target / assessment / reproduction / quality / privacy_review`；每个值对象内联 `source / evidence_refs`；它是 shipped candidate converter 的唯一权威输入。 +- 新增 `FeedbackQuality`：`triage_ready / regression_ready / missing_fields / reasons`。 +- `match_types` 仅表示观测信号。canonical `incident_kind` 取值为 `wrong-route / skipped-gate / missing-artifact / tool-failure / goal-driver / unnecessary-detour / install-version / privacy-reporting / unclear-rule / unknown`；v1 `failure_type` 只作兼容映射。 +- v1 10→6 映射固定为：`wrong-route/skipped-gate/missing-artifact/unnecessary-detour→agent-detour`、`tool-failure→tool-failure`、`goal-driver→goal-driver`、`install-version→install-distribution`、`unclear-rule→unclear-rule`、`privacy-reporting/unknown→unknown`。public `events[].failure_type` 值域不扩张；v2 `incident_kind` 只进 `incidents`。 +- `evidence.json` schema v2 以 `incidents` 为权威；旧 `matched_events` 保持 v1 字段。public `events` 冻结现有 8 字段；新 `incidents` 只允许 `incident_kind/target_skill/stage_hint/expected_behavior/actual_behavior/impact/proposed_fix`，GitHub body 只从该投影渲染，所有字符串再次 public redaction。 + +`evidence.json` 只含 observation： + +```json +{"schema_version":2,"privacy":"local-private","incidents":[{"id":"incident-01","target_skill":"cs-feat","observations":[{"id":"obs-04","role":"assistant","record_type":"message"}],"capture_cutoff":"record-18"}]} +``` + +`triage.json` 是 Assessment 与 promotion 的最小消费契约： + +```json +{ + "schema_version": 2, + "privacy": "local-private", + "incident_id": "incident-01", + "target": {"skill": "cs-feat", "stage_hint": "design-review", "suspected_area": "SKILL.md"}, + "incident_kind": "skipped-gate", + "assessment": { + "expected_behavior": {"value": "第二轮仍派独立 reviewer", "source": "user", "evidence_refs": ["obs-05"]}, + "actual_behavior": {"value": "主 agent 本地自审后继续", "source": "transcript", "evidence_refs": ["obs-04"]}, + "impact": {"value": "独立审查 gate 被跳过", "source": "inferred", "confidence": "high", "evidence_refs": ["obs-04", "obs-05"]}, + "cause_status": "unclassified" + }, + "reproduction": {"eval_profile": "routing-decision", "task_kind": "routing", "input": null, "oracle": null, "evidence_refs": []}, + "quality": {"triage_ready": true, "regression_ready": false, "missing_fields": ["reproduction.input", "reproduction.oracle"]}, + "privacy_review": {"status": "pending"} +} +``` + +每个 `triage.json` 只指向一个包含 `trigger_cutoff` 的 primary incident；无法唯一关联时先让用户选择，其他 incidents 留在 evidence 中另行分诊，converter 拒绝空或歧义 `incident_id`。 + +`triage_ready` 要求 session/incident 已唯一、target skill、expected、actual 和 observation ref；`regression_ready` 还要求下表 profile、可重放 input 与 oracle。candidate converter 按白名单标记缺口；repo promotion 工具拒绝错配，`unknown` 一律未就绪。 + +| Profile | Allowed `incident_kind` | Candidate → eval fixture | Repo promotion checks | +|---|---|---|---| +| `routing-decision` | wrong-route, skipped-gate, missing-artifact, goal-driver, unnecessary-detour | `input.intent/state/utterance → task.*`; `oracle.expect → expect`; `answerType=routing-decision`; `task.kind=routing` | 至少一个非空 state/intent/utterance；校验 `expect.result_type`、config 含 `routing_decision`，再跑同 skill 内 validator/buildPrompt | +| `findings-recall` | tool-failure, install-version, privacy-reporting, unclear-rule | `input.spec/diff/context/audience → task.*`; `oracle.coverage_points → answer`; `answerType=findings-recall`; `task.kind=review/fix/audit/design/docs` | review/audit 要 diff；fix 要 spec+diff；design 要 spec；docs 要 spec+diff；promotion 硬查 recall_judge、非空且非 mock judge_model，design/docs 非 mock harness，不复用 warning-only `judge_issues()` | + +`findings-recall.task_kind` 由 target 派生：`cs-code-review→review`、`cs-issue→fix`、`cs-audit→audit`、`cs-feat/cs-refactor/cs-epic/cs-req/cs-domain→design`、`cs-docs/cs-docs-neat→docs`；其他 target 仅保留 candidate，并在 `quality.reasons` 写 `unsupported_target`。 + +默认 candidate 为同目录 `regression-candidate.json`，`_status=candidate`、`privacy=local-private`。 + +shipped CLI 固定为 `feedback_to_fixture.py --triage ` 或兼容 `--evidence `，只写同目录 candidate。repo-local `eval-cs-skill/scripts/promote_feedback_fixture.py --candidate --experiment ` 读取目标 `config.json`，在自身单元内完成表中校验；缺工具/config、空白或 `TODO/TBD/unknown` 都非零且不落盘，现有 validator 只作部分结构 gate。 + +promotion 只从 reproduction input/oracle 构建 fixture，不复制 assessment/excerpt/evidence 正文；要求 `privacy_review=approved`，再对全部字符串做 self-contained commit-safe 扫描。若 secret、绝对路径、remote/env 或 private marker 会被改写，则列出字段并拒绝，要求换成合成复现后重新批准，禁止静默写入失真的 fixture。 + +feedback 目录布局固定为 `{slug}-report.md / evidence.json / triage.json / public-issue-context.json? / github-issue.md? / regression-candidate.json?`，后三个问号项按 preview、上报和 fixture 交接需要生成；SKILL、shared-conventions 模板与 runtime copy 同步这一布局。 + +##### Interface 设计检查 + +- Module：反馈采集管线；由单文件改为 CLI orchestration + transcript normalization + privacy/projection + quality 模块。 +- Interface：collector CLI 的直接调用默认保持兼容；skill 编排显式选择 current。schema v2 明确 privacy、ordering、unknown、readiness 与 compatibility projection。 +- Seam：provider normalization 是真实变化点；Codex / Claude adapter 进入同一 record 接口。metadata-only reader 只从 path/top-level/session-meta 取 session/cwd/mtime，禁止 import/call `read_records`；cwd 只在正文时保持 unknown 并让用户选，用顶层 meta 正向、body-only 与输出子串负向测试锁定。 +- Depth / locality：provider 格式、脱敏策略和 readiness 规则分别封装，避免每个 caller 重复理解 transcript schema。 +- Dependency strategy：in-process，本地文件输入；不新增远程 adapter 或 LLM dependency。 +- Adapter：Codex / Claude 两个真实 source adapter；测试使用合成 transcript，不 mock 核心聚合逻辑。 +- Test surface：shipped capture/candidate CLI 与 repo promotion CLI 分别覆盖歧义、隐私、provider 漂移、config 和 fail-closed gate。 + +### 2.2 编排层 + +```mermaid +flowchart TD + A[用户显式调用 cs-feedback] --> B[定位当前 cwd 会话] + B -->|多候选| C[只让用户选择 session] + B -->|唯一候选| D[provider 规范化 records] + D --> E[检测 anchor 并聚合 FeedbackIncident] + E --> F[补 runtime / artifact / git 文件级上下文] + F --> G[写 local-private evidence.json] + G --> H[生成 triage 草稿并跑 Quality Gate] + H -->|缺关键字段| I[只问最高优先级缺口] + I --> H + H -->|triage ready| J[报告 + public allowlist preview] + J -->|用户确认| K[可选 GitHub issue] + H -->|regression ready| L[可选提升 regression fixture] +``` + +**现状**：默认扫描近三天文件，单条 record 独立匹配、取固定前后窗口并按 score 降序；agent 再从非结构化上下文手工写报告和 issue。 + +**变化**：skill 默认显式请求 current-session；collector 直接调用仍保留 v1 默认。先聚合事件包再排序。仓库 enrich 只读取模型/宿主 metadata、runtime manifest、相关 CodeStable artifact frontmatter/status 和 git 文件名状态。skill agent 基于 evidence 生成 triage，quality gate 决定追问、公开预览和 fixture candidate。 + +流程级约束： + +- session ambiguity 阶段只走 metadata-only adapter；不得调用 `read_records`，JSON 容器即使需要解析也不得 normalize、flatten、匹配、持久化、保留或返回 message 正文。用户选定后再采集。 +- transcript 顺序、role 和 tool pairing 必须保留；有 id、无 id 紧邻、无 id 歧义三种情况分别 exact/adjacency/unpaired，跨 user-turn incident 不因相同关键词合并。 +- 采集、报告、公开、上报、fixture promotion 五个阶段分别可重试且幂等；不得覆盖用户手工补充字段。 +- 推断值必须携带 source/confidence/evidence refs；无依据时保持 unknown。 +- public projection 永远从结构化 allowlist 生成；上传器按文件名硬拒 `evidence.json/triage.json/regression-candidate.json`，并按 `privacy=local-private` 二次拒绝。 +- quality score 只表达证据完备性，不代表问题严重度或 skill 一定有缺陷。 + +### 2.3 挂载点清单 + +1. `plugins/codestable/skills/cs-feedback/SKILL.md` 与 report-template：更新 incident/triage/quality、v1/v2 枚举分区和完整反馈目录布局。 +2. `collect_feedback_context.py` 公共 CLI 与 schema v2 JSON：提供 current-session 事件包、current/since-days 优先级和 v1 failure 映射。 +3. `feedback_to_fixture.py` 与 repo-local `eval-cs-skill/scripts/promote_feedback_fixture.py`：只用 candidate artifact 交接，后者拥有 config/validator/buildPrompt/scorer 与正式落盘。 +4. `cs-onboard/references/{execution,shared}-conventions.md` 模板及项目 runtime copy：增加显式反馈提示并同步 feedback 产物布局。 +5. tests、feedback decision fixtures、README / WORKFLOW / SKILL_CATALOG / system overview：锁定行为与对外投影。 + +### 2.4 推进策略 + +1. 行为等价拆分采集器与测试职责，保持现有 CLI/schema v1 测试全绿。 +2. 引入 metadata-only session 选择、current 绕过 `time_cutoff`、`trigger_cutoff`、normalized record、incident 聚合和 repo context，落 schema v2 evidence 与兼容投影。 +3. 引入 triage schema、字段来源和质量门，按缺口驱动 ask-user 与 public projection。 +4. 把 shipped converter 收紧为 candidate-only；在 repo-local eval skill 增 promotion 工具，读取 config、做 commit-safe/profile gate，迁移旧 skeleton 测试。 +5. 接入共享反馈提示，同步 SKILL/report-template 的 current 参数与两套枚举，并更新用户文档、runtime 模板/副本。 +6. 补跨 provider、隐私对抗、quality/promotion、skill decision fixtures，完成评测与全量 gate。 + +### 2.5 结构健康度与微重构 + +##### 评估 + +- 文件级 — `collect_feedback_context.py`：520 行，混合 transcript 解析、session 发现、匹配、脱敏、公共投影和 CLI 输出，新增 incident 会成为第六项职责。 +- 文件级 — `feedback_to_fixture.py`：81 行，收紧为 candidate-only；正式 promotion 新逻辑放 repo-local eval skill，避免 shipped skill 依赖维护者工具。 +- 文件级 — `tests/test_cs_feedback.py`：436 行，已混合采集、session 解析、隐私和 GitHub reporter；新增 quality/promotion 测试会跨过 500 行并进一步混杂。 +- 目录级 — `cs-feedback/scripts/` 当前 3 个同层文件；拆出 2-3 个职责模块后仍低于摊平阈值，命名可按 `feedback_*` 聚类。 +- 目录级 — `tests/` 已是仓库约定的扁平测试根；按反馈子职责拆测试文件，不另造嵌套结构。 +- compound 未发现目录组织或命名约定。 + +##### 结论：微重构（拆文件） + +##### 方案 + +- 搬什么：把 provider/session 规范化、隐私投影、数据类型从 collector CLI 中搬出；把现有 reporter 测试从采集测试中搬出。 +- 搬到哪：`scripts/feedback_transcripts.py`、`scripts/feedback_privacy.py`、`scripts/feedback_models.py`；测试按 capture / reporting / fixture promotion 分文件。 +- 行为不变怎么验证：拆分提交点前后既有 9 个 `test_cs_feedback.py` 用例全绿；CLI 参数、schema v1 字段和值不变；`git diff` 只含移动、import 与测试重排。 +- 步骤序列：先移动纯函数与 dataclass，再移动 source discovery，最后收薄 CLI；每次移动后跑 targeted tests。 + +超出范围的观察：`report_feedback_issue.py` 的 GitHub/proxy 逻辑与证据采集独立，本 feature 不重构其网络策略。 + +## 3. 验收契约 + +### 3.1 关键场景 + +1. 用户在当前 cwd 唯一会话调用 `cs-feedback` -> skill 不传 since-days；collector current 分支只绕过 `time_cutoff` 并自动选择，不扫描其他 repo 会话。 +2. 当前 cwd 有多个候选 -> metadata-only 路径不 flatten/match/persist 正文，候选输出不含 message/tool 子串并只问用户选哪一个。 +3. agent 动作、tool failure、user correction 连续出现 -> 有 id 精确配对、无 id 紧邻配对；歧义 call/result 各自按 source order 进 timeline 且不猜配对，仍生成有序 incident。 +4. 同一 session 的两个 CS 问题落在不重叠 user-turn 窗口 -> 生成两个 incident，不因 skill 名相同误合并。 +5. 用户纠正明确说明“应该怎样” -> expected source=user；actual 指向此前 assistant/tool observation。 +6. 只有“有问题”但缺 expected -> 保存反馈并将 `expected_behavior` 列入 quality gap，只追问这一项。 +7. runtime manifest 和相关 artifact 存在 -> triage 包含版本、repo-relative artifact/status；缺失则记录 unknown/mismatch，不失败。 +8. transcript 含 secret、绝对路径、remote URL、环境变量、原始工具 JSON 和代码块 -> local evidence 先脱敏，public preview 进一步移除；reporter 拒绝 evidence、triage 和 candidate 三类 local-private 文件。 +9. assessment 无 evidence ref -> quality gate 不允许 `triage_ready=true`。 +10. triage ready 但缺 reproduction/oracle -> 可生成维护报告和 issue preview，不能 promotion 为正式 fixture。 +11. regression ready -> shipped converter 只产 candidate；repo promotion 缺 config、空 input/占位、敏感字符串或不兼容 scorer/harness/judge 时不落盘，有效输入才写正式 fixture；旧直写非零。 +12. 旧 schema v1 public context -> reporter 仍可读取，converter 只生成未就绪 candidate；events 的 v1 `failure_type` 与 incidents 的 v2 `incident_kind` 分区且精确 8 字段不变。 +13. CS skill 遇到规则/工具问题 -> 共享约定只提示显式调用 `cs-feedback`，不自动采集或上传。 +14. 用户未确认 public preview -> reporter 不调用 `gh issue create`；确认后仍只上传 allowlist body。 +15. 冻结边界内找不到 user anchor -> `trigger_cutoff=unknown` 且 triage 未就绪；找到 anchor 时其后的 assistant/tool 记录绝不进入 evidence。 +16. Codex JSONL 与 Claude JSON/JSONL 表达同一 tool failure + user correction -> 生成字段、顺序、配对语义同构的 incident。 + +反向核对：代码和 skill 文本不应出现后台 telemetry、自动上传、默认全历史扫描、把 assessment 写入 observations、未就绪 skeleton 直接进入正式 fixtures 的路径。 + +### 3.2 Acceptance Coverage Matrix + +| Scenario | Covered By Step | Evidence Type | Command / Action | Core? | +|---|---|---|---|---| +| 1-2 current session 与 ambiguity | S2 | unit + CLI test | targeted pytest | yes | +| 3-4 tool 配对、incident 聚合与分离 | S2 | positive/ambiguous unit | targeted pytest | yes | +| 5-6,9 observation / assessment 与缺口 | S3 | schema + decision fixture | pytest / eval fixture | yes | +| 7 runtime/artifact enrich | S2 | isolated repo test | targeted pytest | yes | +| 8 private/public 脱敏与上传拒绝 | S3-S5 | adversarial negative test | targeted pytest | yes | +| 10-11 candidate/repo promotion | S4 | config/privacy/validator positive+negative | targeted/ADR lint pytest | yes | +| 12 v1 compatibility | S1-S4 | regression test | targeted pytest | yes | +| 13 显式提示、无自动采集 | S5 | static contract test | targeted pytest | yes | +| 14 上传确认 | S5 | reporter negative test | targeted pytest | yes | +| 15 trigger cutoff 锚点与后续排除 | S2 | positive/negative unit | targeted pytest | yes | +| 16 跨 provider 同构 | S2 | paired synthetic transcripts | targeted pytest | yes | +| 1-16 完整 gate | S6 | command + independent review | full pytest/runtime/diff | yes | + +### 3.3 DoD Contract + +| ID | 要求 | 证据 | 阻塞级别 | +|---|---|---|---| +| DOD-DESIGN-001 | design/checklist 经独立 Task agent review | design-review report | blocking | +| DOD-IMPL-001 | 六个 steps 完成且 schema/compat 证据落盘 | checklist + implementation report | blocking | +| DOD-REVIEW-001 | 代码 review 无 unresolved blocking/important | review report | blocking | +| DOD-QA-001 | 隐私、聚合、quality、promotion 和全量测试通过 | QA report | blocking | +| DOD-ACCEPT-001 | 16 个场景、文档/runtime 投影和最终清洁度核对 | acceptance report | blocking | + +Required Artifacts: design-review、implementation report、code review、QA、acceptance、feedback routing fixtures、测试输出。 + +## 4. 与项目级架构文档的关系 + +本 feature 落实 ADR-003 的“生产失败 -> regression fixture”，把旧直写脚本改成 candidate artifact 边界，并由 repo-local eval skill 拥有正式 promotion；需更新 ADR applies-to/Consequences 与 lint 套件，不新增 ADR。共享提示同步 execution conventions 模板/runtime；README、WORKFLOW、catalog 和 system overview 只投影用户入口。 diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-dod-results.json b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-dod-results.json new file mode 100644 index 0000000..b6c26a4 --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-dod-results.json @@ -0,0 +1,66 @@ +{ + "gate_id": "dod-runner", + "stage": "acceptance", + "status": "passed", + "blocking": [], + "warnings": [ + "CMD-004: non-core command failed with exit 1" + ], + "evidence": [ + { + "command": "PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests/test_cs_feedback*.py tests/test_cs_skill_bootstrap.py tests/test_skill_entry_simplification.py", + "exit_code": 0, + "stdout": "........................................................................ [ 42%]\n........................................................................ [ 84%]\n........................... [100%]\n171 passed in 0.49s\n", + "stderr": "", + "id": "CMD-001", + "core": true, + "failure_handling": "fix-or-block" + }, + { + "command": "PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests", + "exit_code": 0, + "stdout": "........................................................................ [ 21%]\n........................................................................ [ 42%]\n........................................................................ [ 63%]\n........................................................................ [ 84%]\n..................................................... [100%]\n341 passed in 5.47s\n", + "stderr": "", + "id": "CMD-002", + "core": true, + "failure_handling": "fix-or-block" + }, + { + "command": "PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests/test_cs_skill_eval.py tests/test_cs_skill_convergence.py tests/test_cs_skill_release.py tests/test_cs_skill_bootstrap.py tests/test_cs_skill_selfref.py", + "exit_code": 0, + "stdout": "................................................................. [100%]\n65 passed in 0.96s\n", + "stderr": "", + "id": "CMD-003", + "core": true, + "failure_handling": "fix-or-block" + }, + { + "command": "python3 tools/check-plugin-package.py --root . --json", + "exit_code": 1, + "stdout": "{\n \"ok\": false,\n \"findings\": [\n {\n \"path\": \"cs-onboard\",\n \"message\": \"root cs* skill entry must be moved under plugins/codestable/skills\"\n }\n ]\n}\n", + "stderr": "", + "id": "CMD-004", + "core": false, + "failure_handling": "document-baseline" + }, + { + "command": "python3 plugins/codestable/skills/cs-onboard/tools/codestable-runtime-sync.py --root . --source-skill-dir plugins/codestable/skills/cs-onboard --check --json", + "exit_code": 0, + "stdout": "{\n \"status\": \"ok\",\n \"ok\": true,\n \"hint\": \"runtime assets ok\",\n \"manifest\": {\n \"schema_version\": 1,\n \"plugin\": \"codestable\",\n \"plugin_version\": \"1.0.2\",\n \"runtime_version\": \"1.0.2\",\n \"tool_runtime\": \"skill-global\",\n \"managed_paths\": [\n \".codestable/gates\",\n \".codestable/reference\",\n \".codestable/.gitignore\",\n \".codestable/runtime-manifest.json\"\n ],\n \"updated_by\": \"codestable-runtime-sync\"\n },\n \"installed_plugin_version\": \"1.0.2\",\n \"expected_plugin_version\": \"1.0.2\",\n \"capabilities\": {\n \"base\": {\n \"ok\": true,\n \"required_paths\": [\n \".codestable/attention.md\",\n \".codestable/reference/execution-conventions.md\",\n \".codestable/reference/shared-conventions.md\",\n \".codestable/reference/agent-conventions.md\",\n \".codestable/reference/tools.md\",\n \".codestable/runtime-manifest.json\",\n \"tools/validate-yaml.py\",\n \"tools/search-yaml.py\",\n \"tools/codestable-doctor.py\",\n \"tools/build-review-packet.py\"\n ],\n \"repo_paths\": [\n \".codestable/attention.md\",\n \".codestable/reference/execution-conventions.md\",\n \".codestable/reference/shared-conventions.md\",\n \".codestable/reference/agent-conventions.md\",\n \".codestable/reference/tools.md\",\n \".codestable/runtime-manifest.json\"\n ],\n \"skill_tool_paths\": [\n \"tools/validate-yaml.py\",\n \"tools/search-yaml.py\",\n \"tools/codestable-doctor.py\",\n \"tools/build-review-packet.py\"\n ],\n \"missing\": [],\n \"missing_repo\": [],\n \"missing_skill_tools\": []\n },\n \"goal-gates\": {\n \"ok\": true,\n \"required_paths\": [\n \".codestable/gates/roadmap-goal-gates.yaml\",\n \"tools/codestable-scope-gate.py\",\n \"tools/codestable-dod-contract-gate.py\",\n \"tools/codestable-dod-runner.py\",\n \"tools/codestable-evidence-pack.py\",\n \"tools/codestable-goal-consistency-gate.py\"\n ],\n \"repo_paths\": [\n \".codestable/gates/roadmap-goal-gates.yaml\"\n ],\n \"skill_tool_paths\": [\n \"tools/codestable-scope-gate.py\",\n \"tools/codestable-dod-contract-gate.py\",\n \"tools/codestable-dod-runner.py\",\n \"tools/codestable-evidence-pack.py\",\n \"tools/codestable-goal-consistency-gate.py\"\n ],\n \"missing\": [],\n \"missing_repo\": [],\n \"missing_skill_tools\": []\n },\n \"workflow-next\": {\n \"ok\": true,\n \"required_paths\": [\n \"tools/codestable-workflow-next.py\"\n ],\n \"repo_paths\": [],\n \"skill_tool_paths\": [\n \"tools/codestable-workflow-next.py\"\n ],\n \"missing\": [],\n \"missing_repo\": [],\n \"missing_skill_tools\": []\n }\n },\n \"missing\": [],\n \"tool_runtime\": \"skill-global\"\n}\n", + "stderr": "", + "id": "CMD-005", + "core": true, + "failure_handling": "fix-or-block" + }, + { + "command": "git diff --check", + "exit_code": 0, + "stdout": "", + "stderr": "", + "id": "CMD-006", + "core": true, + "failure_handling": "fix-or-block" + } + ], + "providers": {} +} diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack-results.json b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack-results.json new file mode 100644 index 0000000..79c9519 --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack-results.json @@ -0,0 +1,25 @@ +{ + "gate_id": "evidence-pack", + "stage": "acceptance", + "status": "passed", + "blocking": [], + "warnings": [], + "evidence": [ + { + "out": ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack.md", + "providers": { + "archguard": { + "status": "skipped", + "reason": "archguard collection disabled", + "warnings": [] + }, + "meta_cc": { + "status": "skipped", + "reason": "meta-cc collection disabled", + "warnings": [] + } + } + } + ], + "providers": {} +} diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack.md new file mode 100644 index 0000000..d184850 --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack.md @@ -0,0 +1,222 @@ +--- +doc_type: feature-evidence-pack +feature: 2026-07-10-cs-feedback-evidence-pipeline +status: generated +--- + +# 2026-07-10-cs-feedback-evidence-pipeline evidence pack + +## 1. Scope + +- Design: `.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design.md` +- Checklist: `.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-checklist.yaml` + +## 2. DoD Results + +```json +{ + "gate_id": "dod-runner", + "stage": "acceptance", + "status": "passed", + "blocking": [], + "warnings": [ + "CMD-004: non-core command failed with exit 1" + ], + "evidence": [ + { + "command": "PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests/test_cs_feedback*.py tests/test_cs_skill_bootstrap.py tests/test_skill_entry_simplification.py", + "exit_code": 0, + "stdout": "........................................................................ [ 42%]\n........................................................................ [ 84%]\n........................... [100%]\n171 passed in 0.49s\n", + "stderr": "", + "id": "CMD-001", + "core": true, + "failure_handling": "fix-or-block" + }, + { + "command": "PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests", + "exit_code": 0, + "stdout": "........................................................................ [ 21%]\n........................................................................ [ 42%]\n........................................................................ [ 63%]\n........................................................................ [ 84%]\n..................................................... [100%]\n341 passed in 5.47s\n", + "stderr": "", + "id": "CMD-002", + "core": true, + "failure_handling": "fix-or-block" + }, + { + "command": "PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests/test_cs_skill_eval.py tests/test_cs_skill_convergence.py tests/test_cs_skill_release.py tests/test_cs_skill_bootstrap.py tests/test_cs_skill_selfref.py", + "exit_code": 0, + "stdout": "................................................................. [100%]\n65 passed in 0.96s\n", + "stderr": "", + "id": "CMD-003", + "core": true, + "failure_handling": "fix-or-block" + }, + { + "command": "python3 tools/check-plugin-package.py --root . --json", + "exit_code": 1, + "stdout": "{\n \"ok\": false,\n \"findings\": [\n {\n \"path\": \"cs-onboard\",\n \"message\": \"root cs* skill entry must be moved under plugins/codestable/skills\"\n }\n ]\n}\n", + "stderr": "", + "id": "CMD-004", + "core": false, + "failure_handling": "document-baseline" + }, + { + "command": "python3 plugins/codestable/skills/cs-onboard/tools/codestable-runtime-sync.py --root . --source-skill-dir plugins/codestable/skills/cs-onboard --check --json", + "exit_code": 0, + "stdout": "{\n \"status\": \"ok\",\n \"ok\": true,\n \"hint\": \"runtime assets ok\",\n \"manifest\": {\n \"schema_version\": 1,\n \"plugin\": \"codestable\",\n \"plugin_version\": \"1.0.2\",\n \"runtime_version\": \"1.0.2\",\n \"tool_runtime\": \"skill-global\",\n \"managed_paths\": [\n \".codestable/gates\",\n \".codestable/reference\",\n \".codestable/.gitignore\",\n \".codestable/runtime-manifest.json\"\n ],\n \"updated_by\": \"codestable-runtime-sync\"\n },\n \"installed_plugin_version\": \"1.0.2\",\n \"expected_plugin_version\": \"1.0.2\",\n \"capabilities\": {\n \"base\": {\n \"ok\": true,\n \"required_paths\": [\n \".codestable/attention.md\",\n \".codestable/reference/execution-conventions.md\",\n \".codestable/reference/shared-conventions.md\",\n \".codestable/reference/agent-conventions.md\",\n \".codestable/reference/tools.md\",\n \".codestable/runtime-manifest.json\",\n \"tools/validate-yaml.py\",\n \"tools/search-yaml.py\",\n \"tools/codestable-doctor.py\",\n \"tools/build-review-packet.py\"\n ],\n \"repo_paths\": [\n \".codestable/attention.md\",\n \".codestable/reference/execution-conventions.md\",\n \".codestable/reference/shared-conventions.md\",\n \".codestable/reference/agent-conventions.md\",\n \".codestable/reference/tools.md\",\n \".codestable/runtime-manifest.json\"\n ],\n \"skill_tool_paths\": [\n \"tools/validate-yaml.py\",\n \"tools/search-yaml.py\",\n \"tools/codestable-doctor.py\",\n \"tools/build-review-packet.py\"\n ],\n \"missing\": [],\n \"missing_repo\": [],\n \"missing_skill_tools\": []\n },\n \"goal-gates\": {\n \"ok\": true,\n \"required_paths\": [\n \".codestable/gates/roadmap-goal-gates.yaml\",\n \"tools/codestable-scope-gate.py\",\n \"tools/codestable-dod-contract-gate.py\",\n \"tools/codestable-dod-runner.py\",\n \"tools/codestable-evidence-pack.py\",\n \"tools/codestable-goal-consistency-gate.py\"\n ],\n \"repo_paths\": [\n \".codestable/gates/roadmap-goal-gates.yaml\"\n ],\n \"skill_tool_paths\": [\n \"tools/codestable-scope-gate.py\",\n \"tools/codestable-dod-contract-gate.py\",\n \"tools/codestable-dod-runner.py\",\n \"tools/codestable-evidence-pack.py\",\n \"tools/codestable-goal-consistency-gate.py\"\n ],\n \"missing\": [],\n \"missing_repo\": [],\n \"missing_skill_tools\": []\n },\n \"workflow-next\": {\n \"ok\": true,\n \"required_paths\": [\n \"tools/codestable-workflow-next.py\"\n ],\n \"repo_paths\": [],\n \"skill_tool_paths\": [\n \"tools/codestable-workflow-next.py\"\n ],\n \"missing\": [],\n \"missing_repo\": [],\n \"missing_skill_tools\": []\n }\n },\n \"missing\": [],\n \"tool_runtime\": \"skill-global\"\n}\n", + "stderr": "", + "id": "CMD-005", + "core": true, + "failure_handling": "fix-or-block" + }, + { + "command": "git diff --check", + "exit_code": 0, + "stdout": "", + "stderr": "", + "id": "CMD-006", + "core": true, + "failure_handling": "fix-or-block" + } + ], + "providers": {} +} +``` + +## 3. Validation Commands + +Extracted from checklist `dod.commands`; see DoD Results for command status. + +## 4. Scope And Cleanliness + +Design bytes: 19176 +Checklist bytes: 5289 + +## 5. Residual Risks + +- CMD-004: non-core command failed with exit 1 +- cleanliness marker TODO in .claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py +- cleanliness marker TODO in tests/test_cs_feedback_fixture_promotion.py + +## 6. Provider Signals + +```json +{ + "archguard": { + "status": "skipped", + "reason": "archguard collection disabled", + "warnings": [] + }, + "meta_cc": { + "status": "skipped", + "reason": "meta-cc collection disabled", + "warnings": [] + } +} +``` + +## 7. Gate Results + +```json +{ + "gate_id": "scope-gate", + "stage": "acceptance", + "status": "passed", + "blocking": [], + "warnings": [ + "cleanliness marker TODO in .claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py", + "cleanliness marker TODO in tests/test_cs_feedback_fixture_promotion.py" + ], + "evidence": [ + { + "changed_files": [ + ".codestable/.gitignore", + ".codestable/attention.md", + ".codestable/reference/execution-conventions.md", + ".codestable/reference/shared-conventions.md", + ".codestable/reference/system-overview.md", + "README.en.md", + "README.md", + "SKILL_CATALOG.en.md", + "SKILL_CATALOG.md", + "WORKFLOW.en.md", + "WORKFLOW.md", + "docs/adr/003-cs-skill-evaluation-loop.md", + "plugins/codestable/skills/cs-feedback/SKILL.md", + "plugins/codestable/skills/cs-feedback/references/report-template.md", + "plugins/codestable/skills/cs-feedback/scripts/collect_feedback_context.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_to_fixture.py", + "plugins/codestable/skills/cs-feedback/scripts/report_feedback_issue.py", + "plugins/codestable/skills/cs-onboard/codestable.gitignore", + "plugins/codestable/skills/cs-onboard/references/execution-conventions.md", + "plugins/codestable/skills/cs-onboard/references/shared-conventions.md", + "plugins/codestable/skills/cs-onboard/references/system-overview.md", + "tests/test_cs_feedback.py", + "tests/test_cs_skill_bootstrap.py", + "tests/test_skill_contracts.py", + "tests/test_skill_entry_simplification.py", + ".claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/approval-report.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-acceptance.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-checklist.yaml", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design-review.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation-review-fixes.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-qa.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review-history.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-plan.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-protocol.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-state.yaml", + ".codestable/requirements/VISION.md", + ".codestable/requirements/feedback-evidence-pipeline.md", + "experiments/cs-routing-001/fixtures/routing/rt-c17.json", + "plugins/codestable/skills/cs-feedback/scripts/feedback_incidents.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_models.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_privacy.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_repo_context.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_transcripts.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_triage.py", + "tests/test_cs_feedback_candidate.py", + "tests/test_cs_feedback_evidence_pipeline.py", + "tests/test_cs_feedback_fixture_promotion.py", + "tests/test_cs_feedback_reporting.py" + ], + "ignored_machine_artifacts": [ + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-dod-results.json", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack-results.json", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-gate-results.json" + ], + "allowed_prefixes": [ + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline", + ".claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py", + ".codestable/attention.md", + ".codestable/reference/", + ".codestable/.gitignore", + ".codestable/requirements/", + "README.en.md", + "README.md", + "SKILL_CATALOG.en.md", + "SKILL_CATALOG.md", + "WORKFLOW.en.md", + "WORKFLOW.md", + "docs/adr/003-cs-skill-evaluation-loop.md", + "experiments/cs-routing-001/fixtures/routing/rt-c17.json", + "plugins/codestable/skills/cs-feedback/", + "plugins/codestable/skills/cs-onboard/codestable.gitignore", + "plugins/codestable/skills/cs-onboard/references/", + "tests/test_cs_feedback.py", + "tests/test_cs_feedback_candidate.py", + "tests/test_cs_feedback_evidence_pipeline.py", + "tests/test_cs_feedback_fixture_promotion.py", + "tests/test_cs_feedback_reporting.py", + "tests/test_cs_skill_bootstrap.py", + "tests/test_skill_contracts.py", + "tests/test_skill_entry_simplification.py" + ] + } + ], + "providers": {} +} +``` diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-gate-results.json b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-gate-results.json new file mode 100644 index 0000000..10b3242 --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-gate-results.json @@ -0,0 +1,102 @@ +{ + "gate_id": "scope-gate", + "stage": "acceptance", + "status": "passed", + "blocking": [], + "warnings": [ + "cleanliness marker TODO in .claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py", + "cleanliness marker TODO in tests/test_cs_feedback_fixture_promotion.py" + ], + "evidence": [ + { + "changed_files": [ + ".codestable/.gitignore", + ".codestable/attention.md", + ".codestable/reference/execution-conventions.md", + ".codestable/reference/shared-conventions.md", + ".codestable/reference/system-overview.md", + "README.en.md", + "README.md", + "SKILL_CATALOG.en.md", + "SKILL_CATALOG.md", + "WORKFLOW.en.md", + "WORKFLOW.md", + "docs/adr/003-cs-skill-evaluation-loop.md", + "plugins/codestable/skills/cs-feedback/SKILL.md", + "plugins/codestable/skills/cs-feedback/references/report-template.md", + "plugins/codestable/skills/cs-feedback/scripts/collect_feedback_context.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_to_fixture.py", + "plugins/codestable/skills/cs-feedback/scripts/report_feedback_issue.py", + "plugins/codestable/skills/cs-onboard/codestable.gitignore", + "plugins/codestable/skills/cs-onboard/references/execution-conventions.md", + "plugins/codestable/skills/cs-onboard/references/shared-conventions.md", + "plugins/codestable/skills/cs-onboard/references/system-overview.md", + "tests/test_cs_feedback.py", + "tests/test_cs_skill_bootstrap.py", + "tests/test_skill_contracts.py", + "tests/test_skill_entry_simplification.py", + ".claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/approval-report.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-acceptance.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-checklist.yaml", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design-review.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation-review-fixes.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-qa.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review-history.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-plan.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-protocol.md", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-state.yaml", + ".codestable/requirements/VISION.md", + ".codestable/requirements/feedback-evidence-pipeline.md", + "experiments/cs-routing-001/fixtures/routing/rt-c17.json", + "plugins/codestable/skills/cs-feedback/scripts/feedback_incidents.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_models.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_privacy.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_repo_context.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_transcripts.py", + "plugins/codestable/skills/cs-feedback/scripts/feedback_triage.py", + "tests/test_cs_feedback_candidate.py", + "tests/test_cs_feedback_evidence_pipeline.py", + "tests/test_cs_feedback_fixture_promotion.py", + "tests/test_cs_feedback_reporting.py" + ], + "ignored_machine_artifacts": [ + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-dod-results.json", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-evidence-pack-results.json", + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-gate-results.json" + ], + "allowed_prefixes": [ + ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline", + ".claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py", + ".codestable/attention.md", + ".codestable/reference/", + ".codestable/.gitignore", + ".codestable/requirements/", + "README.en.md", + "README.md", + "SKILL_CATALOG.en.md", + "SKILL_CATALOG.md", + "WORKFLOW.en.md", + "WORKFLOW.md", + "docs/adr/003-cs-skill-evaluation-loop.md", + "experiments/cs-routing-001/fixtures/routing/rt-c17.json", + "plugins/codestable/skills/cs-feedback/", + "plugins/codestable/skills/cs-onboard/codestable.gitignore", + "plugins/codestable/skills/cs-onboard/references/", + "tests/test_cs_feedback.py", + "tests/test_cs_feedback_candidate.py", + "tests/test_cs_feedback_evidence_pipeline.py", + "tests/test_cs_feedback_fixture_promotion.py", + "tests/test_cs_feedback_reporting.py", + "tests/test_cs_skill_bootstrap.py", + "tests/test_skill_contracts.py", + "tests/test_skill_entry_simplification.py" + ] + } + ], + "providers": {} +} diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation-review-fixes.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation-review-fixes.md new file mode 100644 index 0000000..c2c902a --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation-review-fixes.md @@ -0,0 +1,56 @@ +--- +doc_type: feature-implementation-history +feature: 2026-07-10-cs-feedback-evidence-pipeline +status: active +updated: 2026-07-11 +--- + +# CS Feedback Evidence Pipeline Review Fix Continuation + +本文件承接主 implementation 报告的 Round 14+ 修复证据，避免单个 Markdown 超过 300 行。 + +## Review Fix Round 14 + +- Reviewer：有效 Paseo Fable 5/high Round 14 复审完成；本地复现七类开放终止符泄漏和五类 + 中文正文过吞，确认 REV-14-01/02；carry nits/suggestions 未修改。 +- RED：两层隐私矩阵新增 `…/——/（/(/·/～/“`；两层保真矩阵要求路径消失且 `，随后`、 + `应该 先跑 design-review.md`、`详见 3.2 节` 等正文保留。聚焦命令真实得到 `4 failed`。 +- GREEN：枚举式 suffix 替换为绝对路径核心识别 + 确定性 span 扫描；核心 segment 排除句读， + 引号内路径整段处理，带分隔符空格目录精确延伸，未加引号的文件尾部最多检查两个 token， + 多词只允许深路径。reporter 继续复用核心 `PATH_PATTERN`。 +- VERIFY：聚焦 `4 passed`、privacy/reporting `62 passed`、targeted `171 passed`、full + `341 passed`、eval `65 passed`；`/goal`、长文本和关键中文保真 sanity 均通过；runtime/diff + 通过，package 仅既有根 `cs-onboard/` baseline。 +- Resolution：REV-14-01 的开放终止符不再靠闭集枚举；REV-14-02 的核心标点与跨正文追逐均有 + 旧实现下失败的双向回归。未改 promotion/carry nit。 + +## Review Fix Round 15 + +- Reviewer：有效 Paseo Fable 5/high Round 15 复审完成；本地复现相对引用/版本号/dotted 中文 + 过吞及浅路径双词文件名泄漏，确认 REV-15-01/02/03；carry nits 未修改。 +- RED：双向矩阵补第一/第二词位、浅/深路径和 `.codestable/tests/docs` 相对引用；聚焦命令 + `4 failed`。 +- GREEN：spaced continuation 收紧为一个空格组件后立即接斜杠；扩展名拒绝纯数字和 4+ 字 + 非 ASCII 句子；双词文件名移除 depth gate。首次 GREEN 发现 core 会从相对路径内部斜杠重启， + 再增加 ASCII path-token 起始边界；最终聚焦 `4 passed`。 +- VERIFY：privacy/reporting `62 passed`、targeted `171 passed`、full `341 passed`、eval + `65 passed`；runtime/diff 通过，清理 reviewer import cache 后 package 仅既有 baseline。 +- Resolution：REV-15-01/02/03 均有旧实现失败的词位矩阵；相对 artifact 文本完整保留，浅路径 + 文件名完整脱敏。未改引号、`/goal*` 或其他 carry nit。 + +## Review Fix Round 16 + +- Reviewer：有效 Paseo Fable 5/high Round 16 复审完成；本地复现中文正文与相对路径无空格 + 粘连的单跳/多跳过吞，确认 REV-16-01；carry nits 未修改。 +- RED：两层保真矩阵新增 `没生成.codestable/design.md`、`后写到了build/output.json` 与 + 两跳链式样例；同时增加 `Program Files`、纯 CJK 目录和 `合同.docx` 隐私对照。旧实现聚焦 + 命令真实得到 `2 failed`。 +- GREEN：spaced continuation 暴露首组件并用共享 CJK→ASCII/`.` 粘连谓词停止延伸；终端 + 文件名仅在扩展名有效且后续不是分隔符时接受，保留 `计划.docx` / `合同.文档`。聚焦四函数 + `4 passed`，三条原反例正文与相对 artifact 均完整保留。 +- VERIFY：privacy/reporting `62 passed`、targeted `171 passed`、full `341 passed`、eval + `65 passed`；runtime/diff 通过，清理本轮 import cache 后 package 仅既有根 `cs-onboard/` + baseline。 +- Resolution：REV-16-01 有旧实现下真实失败的单跳/多跳双向回归。混合脚本真实路径组件如 + `项目logs/` 会停止绝对路径延伸，只留下相对尾段而不暴露根路径，作为窄 residual 交 QA； + 未改 Windows 伪盘符、引号、`/goal*`、transcript 死条件或其他 carry nit。 diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation.md new file mode 100644 index 0000000..fbfc6ff --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-implementation.md @@ -0,0 +1,297 @@ +--- +doc_type: feature-implementation +feature: 2026-07-10-cs-feedback-evidence-pipeline +status: ready-for-review +created: 2026-07-10 +baseline_ref: 30fecaae5e747dbad1f0e599d592d7c04cc7f0c4 +--- + +# CS Feedback Evidence Pipeline 实现报告 + +## 结果 + +实现已完成并通过本地 implementation gates，当前等待固定配置的独立代码评审。checklist +Step 1-6 已 done。Step 6 的 `done` 表示 implementation 验证与 review 输入已就绪；独立 review +仍由紧随其后的 review stage 执行并单独落盘，避免 `cs-code-review` 要求 steps 全 done 与 Step 6 +文案包含 review 的生命周期循环。 + +## Step 证据 + +### Step 1：行为等价拆分 + +- TDD exception：纯职责搬迁不改变外部行为，使用拆分前后兼容测试代替 RED。 +- 基线：拆分前 `tests/test_cs_feedback.py` 为 `9 passed`。 +- VERIFY：拆分后原 9 个 capture/reporting 行为分文件仍全绿；collector 降为 186 行编排层。 +- 交付：`feedback_models.py`、`feedback_privacy.py`、`feedback_transcripts.py`、 + `feedback_incidents.py`、`feedback_repo_context.py`、`feedback_triage.py`；reporting/candidate/ + promotion tests 分责。 + +### Step 2：证据编排 + +- RED：扩展契约测试首次运行 `18 failed, 20 passed`；失败覆盖弱 cwd 误选、无 anchor 丢 + incident、provider/adjacency 歧义、user-turn 未分离、Claude tool block 未规范化与 repo + context 缺失。 +- GREEN：current 只接受 cwd 精确唯一；ambiguity 走 metadata-only；JSONL/JSON 使用单次 + snapshot；trigger cutoff、跨 provider 规范化和 file-level repo enrich 已实现。 +- VERIFY：snapshot EOF、单 transcript 单次 snapshot、Codex/Claude JSON/JSONL 同构、两个 + user-turn incident 分离及 v1 compatibility tests 全绿。 + +### Step 3：分诊与隐私投影 + +- RED：同一轮首次失败包含 triage 混入 evidence、质量门信任缺 ref 字段、public body + 缺二次拒绝和手工字段被覆盖风险。 +- GREEN：`triage.json` 独立持久化；Assessment 带 source/confidence/evidence refs； + `triage_ready` 与 `regression_ready` 机械重算；重复采集保留用户 reproduction/privacy 补充。 +- VERIFY：secret、绝对路径、remote、env、raw tool JSON、代码块负向用例通过；public event + 精确 8 字段，public incident 精确 7 字段；reporter 未确认时不调用 `gh`。 + +### Step 4：candidate 与 promotion + +- RED：旧实现会接受非 canonical triage、把 assessment 回填 oracle，并在缺 scorer、mock + judge、target/profile 错配和通用绝对路径时 fail open。 +- GREEN：shipped converter 只写同目录 candidate；repo-local promotion 校验 config、profile、 + target、input、privacy、真实 scorer/judge/harness、buildPrompt 与 commit-safe 字段。 +- VERIFY：有效 findings/routing 正向 promotion、缺 config 与 15 类负向 gate、candidate 到 + promotion 的 JSON-only 串联均通过；旧 `--failure --experiment` 非零且无 fixture 目录。 + +### Step 5：入口与文档 + +- TDD exception：协议、模板、ADR 和公开说明属于声明投影，使用 static contract、runtime + copy `cmp`、Markdown 行数和 runtime sync 代替行为 RED。 +- 更新：cs-feedback current 默认、incident/triage/quality、candidate-only、确认上传； + shared/execution/system-overview 模板与 runtime copy；ADR-003；中英文 README/WORKFLOW/ + catalog；`rt-c17` feedback 路由 fixture。 +- VERIFY：目标文档均少于 300 行；README.en 从 363 行收敛到 291 行；三组 template/runtime + copy 逐字一致。 + +### Step 6：验证闭环 + +- unit/contract/decision fixtures 已覆盖 provider 同构、隐私边界、readiness、candidate/promotion + fail-closed 与 feedback 路由。 +- fresh DoD runner：CMD-001 `91 passed`、CMD-002 `261 passed`、CMD-003 `65 passed`；runtime + sync 与 `git diff --check` 通过。 +- package gate 仅返回 design 已记录的根 `cs-onboard/` legacy baseline，无新增 finding。 +- 独立代码评审不伪装成 implementation 证据；由 review stage 使用固定 Fable 5 high 配置执行并 + 回填 review report。 + +## 验证结果 + +| Gate | 结果 | +|---|---| +| `pytest tests/test_cs_feedback*.py tests/test_cs_skill_bootstrap.py tests/test_skill_entry_simplification.py` | 148 passed | +| `pytest tests` | 318 passed | +| eval/convergence/release/bootstrap/selfref pytest | 65 passed | +| runtime sync `--check --json` | status=ok | +| `git diff --check` | passed | +| plugin package | 仅既有根 `cs-onboard/` legacy finding；无新增 finding | + +所有 pytest 均使用 `PYTHONDONTWRITEBYTECODE=1`。package 诊断过程中生成的一次本地 pyc 已清理， +复测后无 `__pycache__`。 + +## 兼容与范围 + +- collector 直接调用仍保留 `since_days=3` v1 默认；skill 默认 current 且不传 since-days。 +- `matched_events` 与 public v1 events 继续保留，failure_type 值域不扩张。 +- reporter 网络/代理策略未重构，只增加确认和隐私边界。 +- shipped skill 不 import repo-local eval 工具；正式 promotion 只存在维护仓库。 +- 没有后台 telemetry、自动上传、默认全历史扫描或自动修改目标 skill。 + +## Review Fix Round 1 + +- triage 的 incident id 漂移或定位失败不再覆盖已有用户补充；保留 reproduction/privacy，清空 + 当前选择并写 previous/pending id，quality 强制未就绪，待用户重新选择。 +- public redaction 覆盖 pretty-printed JSON quoted secret 与 `/repo` 这类单段绝对路径；reporter + 网络边界同步拒绝单段路径和 env name，同时保留 `/goal` 命令文本。 +- Assessment 缺 source/evidence refs 或 inferred confidence 时，`triage_ready` 与 + `regression_ready` 均 fail closed，不再出现 ready 与 missing_fields 同时成立。 +- converter 对 canonical nested object 做类型验证；promotion 增 safe fixture id、目标目录 + containment 与 `fixture_classes` compatibility gate。 +- 配套修正显式 stale mtime 与超过 9999 records 的 numeric timeline ordering。 +- VERIFY：feedback 专项 `62 passed`，targeted `98 passed`，全量 `268 passed`，eval 专门 gate + `65 passed`。 + +## Review Fix Round 2 + +- RED：round 2 聚焦命令得到 `5 failed, 8 passed`，分别复现 brace-in-value/超长 JSON 穿透、 + pending incident 无采纳出口、same-id 语义漂移、reporter raw-json 漏检与 quoted-key secret + promotion fail-open。 +- GREEN：public JSON 清理改为无长度上限并迭代到不动点；triage 新增 local-private incident + fingerprint，same-id/different-fingerprint 也进入 unresolved 状态；collector 新增 + `--accept-incident`，严格核对 pending/current fingerprint，保留 reproduction、归档旧 + assessment/privacy，并把 active privacy review 重置为 pending。 +- 契约加固：reporter 复用同一 raw-json detector；promotion secret matcher 覆盖 quoted JSON + key；converter AST 守护分别读取 `Import` alias 与 `ImportFrom.module`；SKILL/report template + 明确人工采纳协议。 +- VERIFY：聚焦 GREEN `14 passed`，feedback 专项 `66 passed`，targeted `102 passed`，全量 + `272 passed`，eval 专门 gate `65 passed`；runtime sync 与 diff check 通过，package 仅既有 + 根 `cs-onboard/` legacy baseline。 + +## QA Fix Round 1 + +- QA 在真实 provider smoke 后按 review focus 复现 `password=hunter2` 与 + `authorization=abcdefg`：public redaction 原样放行且 confirmed-upload scanner 无 reason， + 因违反场景 8 核心隐私契约记为 `QA-PRIV-001`，未降级成 residual-risk。 +- RED：新增 public/reporter 两条短 secret 负向测试，首次运行 `2 failed`。 +- GREEN：仅把两条链路共用的 `feedback_privacy.SECRET_PATTERN` 最小长度从 8 对齐到 6； + promotion 侧已为 6，无第二套修改。 +- VERIFY：聚焦 `2 passed`、feedback 专项 `68 passed`、targeted `104 passed`、全量 + `274 passed`、eval `65 passed`；runtime sync/diff check 通过，package 仅既有 baseline。 + +## Review Fix Round 4 + +- RED：Fable/local 实跑确认 `Authorization: Basic ` 完全穿透，Bearer 仅擦除 + scheme 而保留 token；新增 public/reporter Basic、Bearer、大小写、Proxy-Authorization、 + standalone 与 sanitized-output 二次扫描测试，首次 `2 failed`。 +- GREEN：新增 shared `AUTH_SCHEME_PATTERN`，在普通 key=value redaction 前整段替换 credential； + reporter 的 `secret` reason 同时复用 assignment/auth 两个 matcher。 +- VERIFY：聚焦 `2 passed`、curl/Basic/Bearer 对抗 smoke 无 credential，feedback `70 passed`、 + targeted `106 passed`、full `276 passed`、eval `65 passed`；其余 gate 通过且 package baseline + 未扩张。 + +## Review Fix Round 5 + +- RED：新增 ENV 风格 Authorization、任意 auth scheme header、curl `-u/--user` 与 promotion + commit-safe 负向用例；聚焦命令真实得到 `7 failed, 9 passed`。 +- GREEN：新增 Authorization header、standalone Basic/Bearer、user-option 三类 credential + matcher；public redaction 在空白折叠和 ENV/PATH/URL 占位前清理 credential，reporter 复用 + shared matcher 集合，repo-local promotion 按独立安装边界复制同构规则。 +- VERIFY：聚焦 `16 passed`，feedback `77 passed`、targeted `113 passed`、full `283 passed`、 + eval `65 passed`；scope/evidence/runtime/diff gates 通过，package 仅既有 legacy baseline。 +- Resolution：R5-001 的 matcher ordering、R5-002 的 scheme 白名单缺口、R5-003 的 curl + userinfo 和 R5-004 的 promotion 边界不对称均有旧实现下会失败的回归测试锁定。 + +## Review Fix Round 6 + +- RED：特殊字符/quoted secret、confirmed title 与空 incident identity 聚焦命令真实得到 + `6 failed, 16 passed`；其中 promotion 两个新增值形态会实际写入 fixture。 +- GREEN：shared/promotion matcher 完整识别 quoted value 与特殊字符 bare token；confirmed + title 复用 private-reason scanner；canonical converter 拒绝非字符串或空白 `incident_id`。 +- VERIFY：聚焦 `22 passed`，值形态 smoke `6 passed` 且 sanitized 不自锁；feedback + `85 passed`、targeted `121 passed`、full `291 passed`、eval `65 passed`。 +- Resolution：R6-B1 public/reporter/promotion 三层盲区、R6-I1 title 旁路、R6-I2 空 incident + candidate 均由旧实现下会失败的行为回归锁定；package 仍只有既有 legacy baseline。 + +## Review Fix Round 7 + +- RED：angle/backtick/full-width separator、未闭合 quote、shell escape 与 placeholder 自锁 + 聚焦命令真实得到 `9 failed, 21 passed`；失败 promotion case 会实际写入 fixture。 +- GREEN：secret value parser 改为 double/single/backtick quoted 分支与 escaped bare 分支， + exact exempt ``，并支持 `：/＝`；shared/promotion 两份规则保持同构。 +- VERIFY：聚焦 `30 passed`，feedback `95 passed`、targeted `131 passed`、full `301 passed`、 + eval `65 passed`；sanitized placeholders 经 reporter 二次扫描保持无 reason。 +- Resolution：R7-B1 的值字符集、自锁张力、本地化 separator 与未闭合 value 终止规则均由 + 旧实现下会失败的三层回归锁定；carry nits 未借机修改。 + +## Review Fix Round 8 + +- Reviewer：Fable agent 异常提前结束；implementation driver 随后错误使用 Codex fallback。 + 用户未授权该降级，因此 Round 8 不计入有效 review gate；其 findings 仅作本地工程输入。 +- RED：六项 Round 8 finding 聚焦命令真实得到 `16 failed, 26 passed`；其中 shell segment、 + unsafe task key 与 correlation bridge 均复现 reviewer 反例。补充 shell expansion 对抗再得 + `3 failed, 31 passed`；targeted 首轮另捕获代码 fence ordering 回归 `1 failed, 147 passed`。 +- GREEN：secret assignment 改为连续 bare/quoted/backtick/escape parser，支持 ANSI-C quote 与 + command expansion fail-closed，同时保留 exact placeholder 和 code-fence 边界；promotion + 扫描 dict key/value、env/private marker，并按 profile 白名单 task keys。 +- 状态与持久化：correlation 窗口按连通分量全量合并；collector 拒绝损坏 triage 与输出路径 + 碰撞，三份输出先 staging 再原子 replace；triage 以 `observation_ids` 验证 source/ref；repo + context 先解析 Git top-level。 +- VERIFY：聚焦 `42 passed`，补充 shell `34 passed`，feedback `110 passed`，targeted + `148 passed`，full `318 passed`，eval `65 passed`；runtime sync/diff check 通过，package + 仍仅既有根 `cs-onboard/` legacy baseline。 +- Resolution：R8-B1/B2/B3 与 R8-I1/I2/I3 均有当前旧实现下真实失败的行为回归；Round 8 nits + 未借机修改。 + +## Review Fix Round 9 + +- Reviewer：Fable 与 OCR 均返回 `503 No available accounts`；implementation driver 再次错误 + 使用 Codex fallback。用户未授权该降级，因此 Round 9 不计入有效 review gate；其 findings + 仅作本地工程输入。 +- Privacy RED：跨物理换行 quote/ANSI-C/expansion、collector 泄漏与未闭合 fence 聚焦得到 + `6 failed, 35 passed`；GREEN 后同组 `41 passed`，privacy/reporting/promotion 三文件 + `107 passed`。shared/promotion parser 现在平衡扫描 nested `$()`/`${}`、quote/escape/CRLF， + 未闭合结构 fail-closed 到 EOF；未闭合 fence 整段替换。 +- Integrity/schema RED：伪造 incident/fingerprint/observation refs 与 object/list/string-bool + candidate 得到 `2 failed, 3 passed`；GREEN 后 `5 passed`、candidate+promotion `66 passed`。 + converter 以 sibling `evidence.json` 只做 canonical integrity 校验，candidate 内容仍只来自 + triage；promotion 在 validator/buildPrompt 前验证精确值类型，非法 JSON 统一 rc=2/no-write。 +- Persistence/privacy lifecycle RED：fsync temp 泄漏、replace mixed generation 与 Git ignore + 得到 `4 failed, 1 passed`；GREEN 后 `5 passed`，并补 write/flush 注入回归。writer 创建后立即 + 登记 temp、为旧 generation 建 rollback snapshot；onboard template/runtime 默认忽略 report、 + evidence、triage、candidate，public preview 与 GitHub body 仍可跟踪。 +- VERIFY：feedback targeted `166 passed`、full `336 passed`、eval `65 passed`；runtime sync、 + scope gate、evidence pack、template/runtime cmp 与 `git diff --check` 通过。package gate 仍仅 + 既有根 `cs-onboard/` legacy baseline，无新增 finding；无 bytecode/cache artifact。 +- Resolution：R9-B1/B2/B3/B4 与 R9-I1/I2 均由旧实现下真实失败的行为回归锁定；carry nit + 未借机修改。 + +## Review Fix Round 10 + +- Reviewer：有效 Paseo Fable 5/high Round 10 复审完成；本地核验后保留 REV-10-01/02 两个 + important，驳回 promotion temp 的不可复现碰撞/逃逸论据，REV-10-03/04 nits 未借机修改。 +- RED：新增环境 metadata 来源资格与无 user anchor public eligibility 两条行为断言，聚焦命令 + 真实得到 `2 failed`；分别复现 tool payload 的 model/version 污染和未就绪 public event。 +- GREEN：environment context 只接受 session/meta/turn-context 或无正文顶层 metadata，并支持 + Codex `cli_version`；collector 只在唯一 primary incident 且 triage-ready 时构建 public + events/incidents，local-private evidence/triage 仍保留。 +- VERIFY：聚焦 `2 passed`、feedback `131 passed`、targeted `167 passed`、full `337 passed`、 + eval `65 passed`；runtime sync 与 `git diff --check` 通过，package 仍仅既有根 `cs-onboard/` + legacy baseline。 +- Resolution：REV-10-01/02 均有旧实现下真实失败的回归测试锁定；旧 public redaction fixtures + 补为 tool failure + 明确 user correction，未放宽 public eligibility gate。 + +## Review Fix Round 11 + +- Reviewer：有效 Paseo Fable 5/high Round 11 复审完成；本地用 Windows、POSIX 和 + `Application Support` 三个反例确认 REV-11-01，REV-11-02/03 nits 未借机修改。 +- RED：在 public redaction 与 reporter 两层新增含空格绝对路径端到端断言，聚焦命令真实得到 + `2 failed`；旧实现分别残留 `plan.docx`、`contracts` 和 `Support`，且 reporter 放行脱敏结果。 +- GREEN：共享 `PATH_PATTERN` 只扩展两类可判定空格尾部——仍含目录分隔符或带扩展名文件； + reporter 继续直接复用同一 matcher，无复制规则。新增用例 `2 passed`。 +- VERIFY：evidence-pipeline + reporting `60 passed`；targeted `169 passed`、full `339 passed`、 + eval `65 passed`；runtime sync、scope gate、evidence pack 与 `git diff --check` 通过，package + 仍仅既有根 `cs-onboard/` baseline。 +- Resolution：REV-11-01 有旧实现下真实失败的两层回归锁定；保留 `/goal` 例外，未改 promotion + commit-safe 或两个非阻塞 nit。 + +## Review Fix Round 12 + +- Reviewer：有效 Paseo Fable 5/high Round 12 复审完成；本地复现引号包裹的 POSIX/Windows + 路径和句末句点三例，确认 REV-12-01；三个 nit 与一条 suggestion 未借机修改。 +- RED：扩展两层空格路径参数矩阵，覆盖单双引号、backtick、ASCII/CJK 句末标点及标点后闭合 + 引号；聚焦命令真实得到 `2 failed`，旧实现残留 `merger notes.txt` / `plan.docx` / `report.docx`。 +- GREEN：新增共享 `PATH_FILE_BOUNDARY_PATTERN`；直接闭合符可终止，句末标点仅在后续为空白、 + 闭合符或行尾时终止。reporter 继续复用 `PATH_PATTERN`，无第二份规则；新增矩阵 `2 passed`。 +- VERIFY：evidence-pipeline + reporting `60 passed`；targeted `169 passed`、full `339 passed`、 + eval `65 passed`；runtime sync、scope gate、evidence pack 与 `git diff --check` 通过，package + 仍仅既有根 `cs-onboard/` baseline。 +- Resolution：REV-12-01 的真实 transcript 定界符上下文由旧实现下失败的两层回归锁定；未改 + promotion commit-safe，也未把不可判定的无扩展名尾部目录纳入本轮范围。 + +## Review Fix Round 13 + +- Reviewer：有效 Paseo Fable 5/high Round 13 复审完成；本地复现 4 个中文连写标点反例， + 并确认长扩展名与 Unicode 扩展名残留；carry nits/suggestions 未修改。 +- RED：两层参数矩阵新增 `，。；）` 后直接连写正文、中文弯引号、`.presentation` 与 `.文档`； + 聚焦命令真实得到 `2 failed`，脱敏产物均被 reporter 放行。 +- GREEN：CJK 标点/全角闭合符本身改为终止边界；扩展名改用无固定长度的 Unicode + `word/+/-` 集合，删除 `{1,10}` 阈值。reporter 继续复用 shared matcher；矩阵 `2 passed`。 +- VERIFY：privacy/reporting `60 passed`、targeted `169 passed`、full `339 passed`、eval + `65 passed`；runtime、scope、evidence、diff 通过，package 仅既有根 `cs-onboard/` baseline。 +- Resolution：REV-13-01/02 均有旧实现下真实失败的两层回归；未扩大到无扩展名尾部目录或 + promotion/carry nit。 + +## Review Fix Continuation + +Round 14 起的修复证据见 `cs-feedback-evidence-pipeline-implementation-review-fixes.md`。 + +## 清洁度 + +- 无调试输出、临时 TODO/FIXME/XXX、注释掉代码、cache artifact 或方案外业务代码修改。 +- `print` 仅用于 shipped CLI 的结构化结果/错误；TODO/TBD 仅出现在 placeholder 拒绝规则与 + 对应负向测试。 +- `.codestable/attention.md` 与 feature 目录为本 goal 既有资产，未回滚用户改动。 + +## 下一步 + +按 goal protocol 只使用 Paseo Fable 5/high 只读审查本轮 diff；Fable 因额度/provider 异常 +无法完成时必须 handoff，不得换模型。 diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-qa.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-qa.md new file mode 100644 index 0000000..137be9a --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-qa.md @@ -0,0 +1,110 @@ +--- +doc_type: feature-qa +feature: 2026-07-10-cs-feedback-evidence-pipeline +status: passed +tested: 2026-07-11 +round: 2 +--- + +# cs-feedback-evidence-pipeline QA 报告 + +## 1. Scope And Inputs + +- Design/checklist：approved；6 steps done，21 checks 留给 acceptance。 +- Review：Round 17 `passed`，`reviewer: subagent`，无 unresolved blocking/important。 +- Evidence/gates：DoD、scope、evidence pack 均 passed；package 仅既有根 `cs-onboard/` baseline。 +- Diff basis：baseline `30fecaae5e747dbad1f0e599d592d7c04cc7f0c4` 加完整 tracked/untracked + 工作区；scope gate 证明全部可归因本 feature。 +- Feature type：functional。核心证据门覆盖 current/session CLI、snapshot/incident/triage 持久化、 + public privacy、candidate/promotion、v1/v2 兼容与 reporter 上传确认边界。 +- 真实 smoke 只读本机 transcript snapshot，输出落随机 `/tmp` 目录；未改 session、未联网、 + 未上传 GitHub。 + +## 2. Verification Matrix + +| ID | 来源 | 核心性 | 场景 / 风险 | 证据类型 | 结果 | +|---|---|---|---|---|---| +| QA-001 | S1-2 | core-functional | current 唯一/多候选、metadata-only、time cutoff | unit + 真实 CLI | pass | +| QA-002 | S3-4 | core-functional | provider/adjacency/unpaired、窗口合并与问题分离 | unit/integration | pass | +| QA-003 | S5-6,9 | core-functional | Observation/Assessment、source/ref/confidence、缺口单问 | unit | pass | +| QA-004 | S7 | supporting | runtime/artifact/git 文件级 context 与 unknown | integration | pass | +| QA-005 | S8 | core-functional | secret/path/remote/env/raw JSON/code 隔离与私有文件拒绝 | adversarial + CLI | pass | +| QA-006 | S10 | core-functional | triage-ready 可报告、未就绪不可 promotion | integration + CLI | pass | +| QA-007 | S11 | core-functional | candidate-only、config/profile/privacy/judge/harness fail-closed | integration | pass | +| QA-008 | S12 | core-functional | v1 events 8 字段/6 值域与 v2 incident 分区 | unit | pass | +| QA-009 | S13-14 | core-functional | 显式触发、未确认零网络、确认后 body/title 二次扫描 | contract + CLI | pass | +| QA-010 | S15 | core-functional | trigger cutoff、无 anchor、anchor 后排除 | unit | pass | +| QA-011 | S16 | core-functional | Codex/Claude JSON/JSONL 同构与真实 provider schema | unit + 真实 CLI | pass | +| QA-012 | review/gates | supporting | pending/accept/fingerprint、原子回滚、全量 gate | integration + command | pass | + +## 3. Command Results + +- 会话/incident 聚焦组 → `10 passed`；quality/privacy 聚焦组 → `15 passed`。 +- candidate/promotion 组 → `66 passed`；reporter/协议组 → `64 passed`。 +- targeted → `171 passed`；全量 → `341 passed`；eval/ADR lint → `65 passed`。 +- runtime sync → `status=ok`；三组 template/runtime `cmp` → exit 0；`git diff --check` → exit 0。 +- package → 仅既有根 `cs-onboard/` finding；无新增 cache/package finding。 +- 真实 `--session current --cwd` → exit 0，5 个 Codex/Claude metadata-only candidates；候选 + 精确只有 `cwd/mtime/path/provider/score/session`，无 message/tool/content 字段；triage 未就绪， + public events/incidents 为空。 +- 显式真实 Codex/Claude snapshot → 均 exit 0，生成 local-private evidence/triage 与有效 + `trigger_cutoff`；不完整 assessment 保持 fail-closed，public 字符串无隐私命中。 +- 真实 triage → candidate exit 0，只写同目录 local-private candidate、无 fixture；缺 config + promotion exit 2，目标目录未创建。 +- reporter 未确认 → exit 0/manual，reason 为 confirmation required；三类 local-private 文件即使 + 带确认也均在网络前 exit 1 拒绝。 + +## 4. Scenario Results + +- [x] QA-001：真实 current ambiguity 与 metadata-only 正文隔离均成立。 +- [x] QA-002：三态配对、歧义不猜、连通窗口 source order 与跨 user-turn 分离成立。 +- [x] QA-003：用户 expected、actual observation ref、unknown 单问及非法 source/ref fail-closed。 +- [x] QA-004：隔离 repo 的 runtime/artifact/git context 与 metadata 来源资格通过。 +- [x] QA-005：短/长/quoted/multiline/shell secret、JSON、路径和代码双层负向矩阵通过。 +- [x] QA-006：未就绪真实 candidate 保留缺口且不含 fixture；public 只在唯一 ready primary 生成。 +- [x] QA-007：routing/findings 正向 promotion 与全部 no-write 负向 gate 通过。 +- [x] QA-008：v1 精确字段和值域冻结，v2 `incident_kind` 不污染 events。 +- [x] QA-009：共享提示不自动采集；真实未确认 CLI 不触网，confirmed 调用边界由 subprocess + 断言锁定。 +- [x] QA-010：无 user anchor 未就绪；有 anchor 时 post-anchor assistant/tool 不进入 evidence。 +- [x] QA-011：三种 provider 容器同构测试与真实 Codex/Claude schema smoke 通过。 +- [x] QA-012：用户字段保留、same-id fingerprint 重选、三文件 rollback 与完整 gates 通过。 + +## 5. Findings + +### failed + +none。 + +### blocked + +none。 + +### residual-risk + +- CJK 直接粘 `.`+合法扩展名的句末/换行形状、假名/谚文粘连会落向隐私侧并损失正文； + 与真实文件名机械同形，不泄露绝对根路径，需继续依赖人工 preview 评估保真。 +- 混合脚本真实路径组件、CJK 相对路径重启、伪 Windows 多字母盘符与未闭合引号存在窄幅 + 过脱敏/尾段残留；本轮探针未发现可公开绝对根路径。 +- ENV_NAME 对普通全大写词过脱敏；方向 fail-closed。 +- 未真实创建 GitHub issue：真实上传需要用户逐次确认，本 QA 只执行未确认零网络 CLI、私有 + 文件真实拒绝，以及 confirmed `gh auth/issue create` 参数级集成测试。 +- recall_judge `[soft]` 与 k=1 variance、candidate 语义真实性留给 acceptance 记录。 + +## 6. Cleanliness + +- Debug output：pass。 +- Temporary TODO/FIXME/XXX：pass；scope 的两条 TODO warning 仅来自 placeholder 拒绝规则及其 + 负向测试字面量。 +- Commented-out code：pass。 +- Unused imports / dead code from this feature：pass。 +- Out-of-scope files：pass。 +- Cache/generated artifacts：pass；review/QA import 后无 `__pycache__`。 +- Markdown 300 行上限：pass；最长 design/implementation 各 297 行。 + +## 7. Verdict + +- Status：`passed`。16 个功能场景均有运行证据，核心隐私/聚合/quality/promotion/compatibility + 路径无 failed/blocked item。 +- Next：进入 `cs-feat` acceptance 阶段，逐项核验 21 checks、required artifacts、ADR/runtime + 投影和 residual-risk 记录。 diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review-history.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review-history.md new file mode 100644 index 0000000..e35d364 --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review-history.md @@ -0,0 +1,112 @@ +--- +doc_type: feature-review-history +feature: 2026-07-10-cs-feedback-evidence-pipeline +updated: 2026-07-11 +--- + +# CS Feedback Evidence Pipeline Review History + +本文件保留已完成轮次的压缩 resolution ledger；当前结论以 +`cs-feedback-evidence-pipeline-review.md` 为准。 + +## Round 1 + +- Verdict: changes-requested。 +- 解决：重采集保留用户 triage、多行 JSON 与单段路径脱敏、assessment readiness、fixture id + containment、malformed triage、regression class gate，以及测试的 stale mtime/长序号问题。 + +## Round 2 + +- Verdict: changes-requested。 +- 解决：nested/超长 JSON fail-closed、incident fingerprint、pending `--accept-incident` 出口、 + `ImportFrom.module` AST 守护、promotion quoted-key secret。 + +## Round 3 + +- Verdict: passed；reviewer `subagent+ocr`。 +- QA focus：真实 provider、problem-not-last-turn、reporter 误报、短 secret 与大历史性能。 + +## Round 4 + +- Verdict: changes-requested。 +- 解决：`Authorization: Basic/Bearer` credential 穿透；public 与 reporter 共用 auth matcher。 + +## Round 5 + +- Verdict: changes-requested。 +- 解决：ENV 替换顺序、任意 Authorization scheme header、curl `-u/--user` userinfo，及 + repo-local promotion credential matcher 对称性。 +- Round 5 DoD：feedback 70、targeted 106、full 276、eval 65 passed。 + +## Round 6 + +- Verdict: changes-requested。 +- 解决：特殊字符与 quoted secret 值、confirmed issue title 隐私扫描、canonical converter + 空 incident identity 拒绝；修复后 targeted 121、full 291、eval 65 passed。 + +## Round 7 + +- Verdict: changes-requested。 +- 解决：angle/backtick/full-width separator、未闭合 quote、shell escaped bare value，及 + exact sanitized placeholder exemption；修复后 targeted 131、full 301、eval 65 passed。 + +## Round 8 + +- Fable 异常提前结束后，implementation driver 错误使用 Codex fallback；用户未授权该降级， + 因此本轮不计入有效 review gate，findings 仅作本地工程输入。 +- 解决：连续 shell segment/expansion、promotion key/schema、correlation 连通分量、collector + 路径与 staging、triage source/ref 校验、Git root 恢复；修复后 targeted 148、full 318、 + eval 65 passed。 + +## Round 9 + +- Fable 与 OCR 均返回 503；implementation driver 再次错误使用 Codex fallback，用户未授权， + 因此本轮不计入有效 review gate。 +- 其本地 findings 已用于 review-fix：multiline privacy、canonical evidence binding、promotion + schema、atomic persistence 与 private gitignore；修复后 targeted 166、full 336、eval 65 passed。 + +## Round 10 + +- Verdict: changes-requested；有效 reviewer 为 Paseo Fable 5 / high。 +- 解决：环境上下文只接受 metadata 来源；无 user anchor / 非唯一 primary incident / triage + 未就绪时不生成 public events/incidents。修复后 targeted 167、full 337、eval 65 passed。 + +## Round 11 + +- Verdict: changes-requested；有效 reviewer 为 Paseo Fable 5 / high。 +- 解决：共享路径 matcher 覆盖仍含目录分隔符或带扩展名的空格尾部，reporter 继续复用同一规则； + 修复后 targeted 169、full 339、eval 65 passed。 + +## Round 12 + +- Verdict: changes-requested；有效 reviewer 为 Paseo Fable 5 / high。 +- 解决：含空格文件名支持单双引号、backtick、ASCII/CJK 句末标点及标点后闭合符；修复后 + targeted 169、full 339、eval 65 passed。Round 13 证明中文标点后连写正文仍需继续收敛。 + +## Round 13 + +- Verdict: changes-requested；有效 reviewer 为 Paseo Fable 5 / high。 +- 解决：CJK 标点自身作为终止边界，扩展名支持长值与 Unicode；targeted 169、full 339、 + eval 65 passed。Round 14 证明枚举终止符与无界 suffix 仍需改为结构化扫描。 + +## Round 14 + +- Verdict: changes-requested；有效 reviewer 为 Paseo Fable 5 / high。 +- 解决：改为绝对路径 core + 有界 span scanner，新增泄漏/保真双向矩阵；targeted 171、 + full 341、eval 65 passed。Round 15 继续定位 continuation 与词位谓词缺口。 + +## Round 15 + +- Verdict: changes-requested；有效 reviewer 为 Paseo Fable 5 / high。 +- 解决：spaced continuation 收紧为单步、spaced filename 拒绝纯数字/长 CJK 句子扩展名、 + 浅路径双词文件名完整脱敏，并阻止相对路径内部斜杠重启；targeted 171、full 341、 + eval 65 passed。Round 16 继续定位中文无空格粘连边界。 + +## Round 16 + +- Verdict: changes-requested；有效 reviewer 为 Paseo Fable 5 / high。 +- 解决：continuation 与终端文件名共享 CJK→ASCII/`.` 粘连谓词，补单跳/多跳保真矩阵及 + `Program Files`、纯 CJK 目录、Unicode 文件名隐私对照；targeted 171、full 341、 + eval 65 passed。Round 17 判定该可判定类别已闭合。 + +各轮完整审查输出亦保留在实现报告、goal ledger 与对应独立 reviewer transcript 中。 diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review.md new file mode 100644 index 0000000..b6c7dbb --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review.md @@ -0,0 +1,90 @@ +--- +doc_type: feature-review +feature: 2026-07-10-cs-feedback-evidence-pipeline +status: passed +reviewer: subagent +reviewed: 2026-07-11 +round: 17 +--- + +# cs-feedback-evidence-pipeline 代码审查报告 + +## 1. Scope And Inputs + +- Design/checklist：approved；6 implementation steps done，21 checks 等 acceptance。 +- Gate evidence：171 targeted、341 full、65 eval；scope/evidence/runtime/diff passed；package 仅既有 + `cs-onboard/` baseline。 +- Diff：baseline `30fecaae5e747dbad1f0e599d592d7c04cc7f0c4` 加完整工作区；无越界文件。 +- Implementation：主报告、review-fixes continuation、当前代码与测试。 + +### Independent Review + +- 环节 A：Paseo agent `2187c439-2b3a-4a32-bac9-c4fd0bf43d74`，Claude Fable 5 / high / + plan，completed；独立复跑 targeted 171 passed。 +- 环节 B：OCR `skipped-by-user-constraint`。 +- Merge：REV-16-01 类级闭合成立；新增反例、旧实现中性化探针和隐私对照均已核验。 +- Gate effect：无 blocking/important，允许进入 QA。 + +## 2. Diff Summary + +- 新增：反馈 normalization/incident/triage/privacy/repo-context 模块、promotion 工具、测试与 fixture。 +- 修改：collector/converter/reporter、skill/runtime/reference、ADR 与公开文档。 +- 风险热点：公开路径脱敏的隐私/保真张力，以及 candidate/promotion 的 fail-closed 边界。 + +## 3. Adversarial Pass + +- 假设：中文无空格粘连仍可被 continuation 吞掉，或新谓词使真实路径泄漏。 +- 反例：在原三例之外攻击不同词位、单跳/多跳、纯 CJK/ASCII、`Program Files`、Unicode + 文件名，并复核 trigger cutoff、配对、triage、原子写、promotion 与 reporter 门禁。 +- 结果：可判定的粘连类已闭合；只剩与真实终端文件名机械同形的不可判定保真 residual。 + +## 4. Findings + +### blocking + +none。 + +### important + +none。 + +### nit + +- [ ] **REV-17-01** `[paseo-fable]` CJK 直接粘 `.`+合法扩展名且位于句末时，与 + `合同.docx` 机械同形，当前落向隐私侧并可能吞掉正文；方向为过脱敏、无泄漏。 +- [ ] **REV-17-02** `[paseo-fable]` 粘连谓词只覆盖 CJK 统一表意文字，假名/谚文粘连仍可能 + 被 continuation 吞掉；本中文项目频率低。 +- [ ] **carry** Windows 伪盘符、CJK 相对目录内部斜杠重启、transcript 死条件、未闭合引号 + 远吞、converter 非原子单文件写等既有 nits 保持记录。 + +### suggestion + +- 若后续继续优化保真，可独立评估中文虚词/动词停用表；不应在本 feature 继续扩张扫描规则。 + +### learning + +- 隐私形状与正文形状机械同形时，应落向隐私侧并显式记录不可判定边界，而不是继续堆启发式。 + +### praise + +- 单一共享谓词同时约束 continuation 与终端文件名，双向矩阵锁定泄漏和过吞；public eligibility、 + reporter 单源、原子写、triage 状态机与 promotion fail-closed 均无回归。 + +## 5. Test And QA Focus + +- 用真实中文反馈语料评估句末 `没写.gitignore` 与换行折叠形状的过吞频率。 +- 中文 correction → public incident → confirmed issue 的真实 provider 全链路。 +- 真实 Codex/Claude 大历史 current 定位、stale mtime、多候选交互。 +- Windows 伪盘符、未闭合引号与 CJK 相对路径重启的实际出现率。 + +## 6. Residual Risk + +- CJK 粘 `.`+扩展名的句末/换行终端形状落向隐私侧，可能损失正文，但不会公开私有路径。 +- 已接受的无扩展名不可判界尾部目录与混合脚本真实路径组件继续依赖人工 preview。 +- ENV_NAME 过脱敏、candidate 语义真实性、CMD-004/TODO baseline 保持记录。 + +## 7. Verdict + +- Spec 合规：`passed`；design 16 个场景与关键兼容/隐私/fail-closed 契约均有证据。 +- 代码质量：`passed`；无 unresolved blocking/important。 +- Next：进入 `cs-feat` QA，重点执行本报告第 5 节和 design 场景矩阵。 diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-plan.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-plan.md new file mode 100644 index 0000000..6cd93e7 --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-plan.md @@ -0,0 +1,68 @@ +# CS Feedback Evidence Pipeline Goal Plan + +## 1. Inputs + +- Feature: `2026-07-10-cs-feedback-evidence-pipeline` +- Design: `.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design.md` +- Checklist: `.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-checklist.yaml` +- Design review: `.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design-review.md` +- Baseline ref: `30fecaae5e747dbad1f0e599d592d7c04cc7f0c4` +- User approval: 用户于 `2026-07-10T19:07:38+08:00` 明确回复“确认”。 +- Existing worktree changes: `.codestable/attention.md` 与本 feature 目录属于本 goal，driver 不得回滚；其他新增变化视为用户资产，先核验来源。 + +## 2. Objective + +把显式 `cs-feedback` 调用升级为本地证据管线：默认安全定位当前会话，生成可追溯 incident、evidence、triage 与 public preview；shipped skill 只产 local-private regression candidate，repo-local eval skill 负责正式 fail-closed promotion。 + +## 3. Execution Steps + +按 checklist 六步顺序推进，每完成一步立即更新 checklist status 与 `goal-state.yaml.ledger`： + +1. 行为等价拆分 collector、privacy/models 和 reporter tests。 +2. 完成 metadata-only current session、双 cutoff、normalized record、tool pairing、incident 聚合与 repo context。 +3. 完成 triage、字段来源、quality gate 和 public allowlist projection。 +4. 完成 shipped candidate-only converter 与 repo-local promotion 工具的 artifact handoff。 +5. 更新 cs-feedback 协议、report template、shared/execution runtime 模板与公开文档。 +6. 补齐 provider/隐私/profile/兼容 decision fixtures，运行全量 gate。 + +## 4. TDD Policy + +- 所有行为代码 step 默认使用 RED → GREEN → VERIFY micro-loop；每个 step 的 evidence 记录失败测试、最小实现和通过命令。 +- 纯移动的微重构先锁定既有 9 个 feedback tests，再移动、再验证；不得在同一 diff 混入行为变化。 +- 文档投影或纯声明无法合理先写失败测试时，记录 `TDD exception`，并提供 static contract test、diff review 或 schema validation 作为替代证据。 +- 缺 RED/GREEN/VERIFY 且无 `TDD exception` 的 step 不得进入 review。 + +## 5. Review Agent Constraint + +- 所有 design/code/re-review gate 必须使用 Paseo `provider=claude`、`model=claude-fable-5`、 + `thinkingOptionId=high`，只读等价 mode。 +- Fable 因额度或 provider 异常无法完成时必须 handoff；用户没有授权任何 reviewer 模型降级。 +- code review 结论必须分别覆盖 spec 合规与代码质量；blocking/important 全部解决后才能 passed。 + +## 6. Validation Commands + +```bash +PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests/test_cs_feedback*.py tests/test_cs_skill_bootstrap.py tests/test_skill_entry_simplification.py +PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests +PYTHONDONTWRITEBYTECODE=1 python3 -m pytest -q tests/test_cs_skill_eval.py tests/test_cs_skill_convergence.py tests/test_cs_skill_release.py tests/test_cs_skill_bootstrap.py tests/test_cs_skill_selfref.py +python3 tools/check-plugin-package.py --root . --json +python3 plugins/codestable/skills/cs-onboard/tools/codestable-runtime-sync.py --root . --source-skill-dir plugins/codestable/skills/cs-onboard --check --json +git diff --check +``` + +`check-plugin-package.py` 当前允许记录 ignored 根 `cs-onboard/` legacy 目录的既有非 core finding，但 findings 数量/类型不得新增。其余 core 命令失败必须修复或 handoff。 + +## 7. Core Acceptance Path + +- 逐项证明 design 的 16 个场景，尤其 current session ambiguity 不读正文、Codex/Claude 同构 incident、v1 public 8 字段/6 值域、三类 local-private 上传拒绝。 +- shipped runtime 不得 import repo-local eval skill；candidate→promotion 只通过 JSON artifact，跨单元连接只允许存在于 repo-local tests。 +- promotion 对缺 eval/config、空/占位/敏感 input、不兼容 scorer/harness/judge 全部非零且不落盘。 +- runtime template 与 repo-local copy 同步，`codestable-runtime-sync.py --check --json` 为 `status=ok`。 + +## 8. DoD And Handoff + +- Implementation：六个 steps done，TDD evidence 与 gate outputs 落盘。 +- Review：Fable 5 high reviewer 无 unresolved blocking/important。 +- QA：隐私、聚合、quality、promotion、兼容与全量测试通过。 +- Acceptance：16 个场景与 required artifacts 从仓库事实核验通过。 +- 需要改变 approved design/公开契约、同一失败三轮未过、Fable 5/high reviewer 不可用、外部环境阻止核心判断或用户要求暂停时，立即 handoff，不扩大范围。 diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-protocol.md b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-protocol.md new file mode 100644 index 0000000..429f0b9 --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-protocol.md @@ -0,0 +1,65 @@ +# CS Feedback Evidence Pipeline Goal Protocol + +## 1. Goal Mode + +本 goal 已通过独立 design review 并由用户确认。driver 必须连续执行 implementation → code review → QA → acceptance；普通阶段的用户 checkpoint 改为写报告、state 和证据，只有 handoff 条件命中时停止。 + +每轮开始先读取： + +- `goal-state.yaml` +- `goal-plan.md` +- `cs-feedback-evidence-pipeline-design.md` +- `cs-feedback-evidence-pipeline-checklist.yaml` +- `.codestable/attention.md` + +以仓库事实修正 stale state；保留用户已有改动，不做 destructive git 操作。 + +## 2. Implementation Loop + +1. 把 `goal-state.yaml` 更新为 `stage: implementation/status: running`。 +2. 按 checklist 顺序执行；行为 step 使用 RED → GREEN → VERIFY，例外写 `TDD exception` 和替代证据。 +3. 每个 step 完成立即把 checklist status 改为 `done`，并向 ledger 追加 `{step, status, evidence, commit_scope}`；续跑以 ledger + `git log` 为准，不重复已完成 step。 +4. 运行 goal-plan 的 implementation gates，保存 command output、diff summary、隐私负向证据与 DoD 结果。 +5. 全部通过后写 `stage: review/status: ready`。 + +## 3. Code Review Loop + +1. 运行 `cs-code-review`，必须启动 Paseo reviewer：`provider=claude`、`model=claude-fable-5`、`thinkingOptionId=high`，使用 provider 的 plan/read-only 等价 mode。 +2. review 必须分别给出 spec 合规与代码质量结论，并写 `cs-feedback-evidence-pipeline-review.md`。 +3. 有 blocking/important 时写 `stage: review/status: fixing`，回 implementation 做最小修复，再写 `review/ready` 并用同一模型约束重新独立审查。 +4. Fable 因额度或 provider 异常无法完成时必须 handoff；用户没有授权任何 reviewer 模型降级。 +5. passed 后写 `stage: qa/status: ready`。 + +## 4. QA Loop + +1. 运行 `cs-feat` QA，覆盖 design 的 16 个场景、全部命令、runtime/package baseline 和 cleanliness。 +2. QA failed/blocked 时写 `stage: qa/status: fixing`，回 implementation 修复；修完必须重新跑 code review 与 QA。 +3. QA passed 后写 `stage: acceptance/status: ready`。 + +## 5. Acceptance Loop + +1. 运行 `cs-feat` acceptance，从代码、tests、runtime copy、文档和报告核验 21 checks。 +2. 只更新 checklist checks 为 `passed/failed`，不改写 checks 内容。 +3. 所有 checks、review、QA 和 required artifacts 通过后，先写 `stage: complete/status: passed`，再输出 `CS_FEATURE_GOAL_COMPLETE`。 + +## 6. Handoff + +以下任一情况必须先写 `stage: handoff/status: blocked`、`handoff_reason`、`handoff_next`，再输出标记： + +- 需要改变 approved design、feature 范围、公开契约或 ADR 方向。 +- Fable 5 high 独立 reviewer pending/failed/unavailable。 +- 同一失败项三轮修复仍不通过。 +- 外部凭证或环境缺失导致核心行为无法判断。 +- 用户要求暂停、改方向或终止。 + +```text +CS_FEATURE_GOAL_HANDOFF +Reason: <具体阻塞> +Next: <建议动作> +``` + +## 7. Literal Goal Command + +```text +/goal "执行 CodeStable feature 目录 .codestable/features/2026-07-10-cs-feedback-evidence-pipeline 下的 goal 执行包。先读取 goal-protocol.md、goal-state.yaml、goal-plan.md、cs-feedback-evidence-pipeline-design.md、cs-feedback-evidence-pipeline-checklist.yaml；这是已由用户确认 design 后的 goal 模式。按 goal-protocol.md 连续执行 cs-feat implementation、cs-code-review、cs-feat QA、cs-feat acceptance；implementation 的代码行为 step 默认用 TDD micro-loop，必须留下 RED/GREEN/VERIFY evidence，不能 TDD 时写 TDD exception 和替代证据；review blocking 时做 review-fix并重跑 review；QA failed / blocked 时做 qa-fix 并重跑 review 和 QA。所有 review gate 固定使用 Paseo Claude Fable 5、high thinking，不可用时 handoff，不得降级。只有当 CS_FEATURE_GOAL_COMPLETE 出现在 transcript 中，且 review passed、QA passed、acceptance passed、没有 CS_FEATURE_GOAL_HANDOFF，本 goal 才算完成。" +``` diff --git a/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-state.yaml b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-state.yaml new file mode 100644 index 0000000..09acc7a --- /dev/null +++ b/.codestable/features/2026-07-10-cs-feedback-evidence-pipeline/goal-state.yaml @@ -0,0 +1,102 @@ +feature: "2026-07-10-cs-feedback-evidence-pipeline" +status: passed +baseline_ref: "30fecaae5e747dbad1f0e599d592d7c04cc7f0c4" +stage: complete +design: ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-design.md" +checklist: ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-checklist.yaml" +review: ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-review.md" +qa: ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-qa.md" +acceptance: ".codestable/features/2026-07-10-cs-feedback-evidence-pipeline/cs-feedback-evidence-pipeline-acceptance.md" +ledger: + - step: 1 + status: done + evidence: "TDD exception: 拆分前 9 feedback tests passed；拆分后兼容行为分文件持续通过，collector 收薄为 186 行" + commit_scope: "feedback models/privacy/transcripts/incidents/repo-context/triage modules and test split" + - step: 2 + status: done + evidence: "RED 18 failed/20 passed；GREEN 覆盖 metadata-only current、snapshot/cutoff、pairing、user-turn incidents、provider isomorphism、repo context" + commit_scope: "collector orchestration and transcript/incident/context modules" + - step: 3 + status: done + evidence: "quality ref 与隐私 RED 后 GREEN；triage 独立、readiness 重算、allowlist 8+7 字段、未确认 reporter 零网络调用" + commit_scope: "feedback_triage.py, feedback_privacy.py, reporter and privacy tests" + - step: 4 + status: done + evidence: "candidate/promotion fail-closed tests 通过；JSON-only 正向 promotion；旧 experiment 直写非零无目录" + commit_scope: "feedback_to_fixture.py, repo-local promotion tool and promotion tests" + - step: 5 + status: done + evidence: "TDD exception: static contracts + 91 targeted tests；runtime sync ok；template/runtime cmp；公开文档均 <=300 行" + commit_scope: "cs-feedback docs, onboard references/runtime copies, ADR-003, public docs and rt-c17" + - step: 6 + status: done + evidence: "implementation gate fresh passed：91 targeted、261 full、65 eval tests；runtime sync/diff check passed；仅既有非核心 package baseline；独立 review 由紧随其后的 review stage 回填" + commit_scope: "tests, gate artifacts, implementation report and review inputs" + - step: "review-fix-1" + status: done + evidence: "修复 7 个 round-1 important：triage mismatch 保留数据并阻断选择、multiline JSON/单段路径隐私、assessment readiness、safe fixture id/containment/regression class、malformed triage；62 feedback、98 targeted、268 full、65 eval tests passed" + commit_scope: "feedback privacy/triage/incidents/converter/reporter, promotion tool and focused regression tests" + - step: "review-fix-2" + status: done + evidence: "RED 5 failed/8 passed；修复 nested/超长 JSON、incident fingerprint 与显式 pending 采纳、ImportFrom 守护、quoted-key promotion secret；66 feedback、102 targeted、272 full、65 eval tests passed" + commit_scope: "feedback privacy/triage/collector/reporter protocol, promotion guard and focused regression tests" + - step: "qa-fix-1" + status: done + evidence: "QA 真实复现 6-7 字符 password/authorization 穿透；RED 2 failed，GREEN 2 passed；shared public/reporter secret 阈值对齐为 6；68 feedback、104 targeted、274 full、65 eval tests passed" + commit_scope: "feedback_privacy shared matcher and public/reporter short-secret regression tests" + - step: "review-fix-4" + status: done + evidence: "Round-4 复现 Authorization Basic 完全穿透与 Bearer 部分残留；RED 2 failed，GREEN 2 passed；新增 shared auth-scheme pattern 与双侧 Basic/Bearer/Proxy/sanitized 回归；70 feedback、106 targeted、276 full、65 eval tests passed" + commit_scope: "feedback_privacy auth-scheme redaction, reporter shared scan and focused tests" + - step: "review-fix-5" + status: done + evidence: "Round-5 credential ecosystem RED 7 failed/9 passed，GREEN 16 passed；修复 ENV ordering、任意 Authorization scheme、curl userinfo 与 promotion guard；77 feedback、113 targeted、283 full、65 eval tests passed" + commit_scope: "feedback_privacy credential ordering/matchers, reporter shared scanner, promotion guard and focused regression tests" + - step: "review-fix-6" + status: done + evidence: "Round-6 RED 6 failed/16 passed，GREEN 22 passed；修复特殊字符/quoted secret 三层穿透、confirmed title 旁路与空 incident converter；85 feedback、121 targeted、291 full、65 eval tests passed" + commit_scope: "feedback privacy/promotion value matcher, reporter title scan, canonical converter identity validation and regression tests" + - step: "review-fix-7" + status: done + evidence: "Round-7 RED 9 failed/21 passed，GREEN 30 passed；修复 angle/backtick/full-width/unbalanced/escaped value，并显式 exempt sanitized placeholder；95 feedback、131 targeted、301 full、65 eval tests passed" + commit_scope: "feedback privacy/promotion value parser and three-layer adversarial regression matrix" + - step: "review-fix-8" + status: done + evidence: "Round-8 RED 16 failed/26 passed，补充 shell RED 3 failed/31 passed；修复 segment parser、promotion key/schema、correlation bridge、atomic persistence、source/ref validation、repo root；110 feedback、148 targeted、318 full、65 eval tests passed" + commit_scope: "feedback privacy/reporter/promotion, incident merge, collector persistence, triage quality, repo context and focused regressions" + - step: "review-fix-9" + status: done + evidence: "Round-9 privacy RED 6 failed/35 passed、integrity/schema RED 2 failed/3 passed、persistence/ignore RED 4 failed/1 passed；修复 multiline parser/fence、evidence binding、strict promotion schema、generation rollback 与 private gitignore；166 targeted、336 full、65 eval tests passed" + commit_scope: "feedback privacy/converter/collector, reporter/promotion, onboard gitignore/runtime copy and focused regressions" + - step: "review-fix-10" + status: done + evidence: "Round-10 RED 2 failed；修复 metadata 来源污染与无 anchor public projection，GREEN 2 passed；131 feedback、167 targeted、337 full、65 eval tests passed" + commit_scope: "feedback repo-context/public eligibility and focused regression tests" + - step: "review-fix-11" + status: done + evidence: "Round-11 含空格路径 RED 2 failed，GREEN 2 passed；60 privacy/reporting、169 targeted、339 full、65 eval tests passed；runtime/scope/evidence/diff 通过，package 仅既有 baseline" + commit_scope: "shared public path matcher and evidence/reporting regression tests" + - step: "review-fix-12" + status: done + evidence: "Round-12 路径定界符 RED 2 failed，GREEN 2 passed；60 privacy/reporting、169 targeted、339 full、65 eval tests passed；runtime/scope/evidence/diff 通过，package 仅既有 baseline" + commit_scope: "shared path file-boundary matcher and quoted/punctuated path regression matrix" + - step: "review-fix-13" + status: done + evidence: "Round-13 CJK 连写/扩展名 RED 2 failed，GREEN 2 passed；60 privacy/reporting、169 targeted、339 full、65 eval tests passed；runtime/scope/evidence/diff 通过，package 仅既有 baseline" + commit_scope: "shared CJK path boundary, Unicode extension matcher and bilingual regression matrix" + - step: "review-fix-14" + status: done + evidence: "Round-14 泄漏/保真 RED 4 failed，GREEN 4 passed；62 privacy/reporting、171 targeted、341 full、65 eval tests passed；runtime/diff 通过，package 仅既有 baseline" + commit_scope: "bounded absolute-path span scanner and bidirectional privacy/fidelity regression matrix" + - step: "review-fix-15" + status: done + evidence: "Round-15 词位/相对引用 RED 4 failed；首次 GREEN 暴露 relative slash 重匹配，第二次 GREEN 4 passed；62 privacy/reporting、171 targeted、341 full、65 eval tests passed；package 仅既有 baseline" + commit_scope: "single-step path continuation, extension predicate, relative-path start boundary and word-position matrix" + - step: "review-fix-16" + status: done + evidence: "Round-16 中文粘连 RED 2 failed，GREEN 4 focused passed；62 privacy/reporting、171 targeted、341 full、65 eval tests passed；runtime/diff 通过，package 仅既有 baseline" + commit_scope: "CJK path-glue boundary, single/multi-hop fidelity matrix and pure-path privacy controls" + - step: "acceptance-1" + status: done + evidence: "Round-17 Fable review passed；Round-2 QA 16 scenarios passed；21 checks passed；acceptance DoD 171 targeted、341 full、65 eval，scope/evidence/runtime/diff passed" + commit_scope: "acceptance report, checklist closure, owner-approved requirement backfill and VISION index" diff --git a/.codestable/reference/execution-conventions.md b/.codestable/reference/execution-conventions.md index cfaf8b2..0fe9d52 100644 --- a/.codestable/reference/execution-conventions.md +++ b/.codestable/reference/execution-conventions.md @@ -21,6 +21,11 @@ `cs-note` 是唯一例外：`.codestable/` 存在但 `attention.md` 缺失时，它可以创建最小分节骨架 后写入。 +## CodeStable 自身反馈 + +遇到 CodeStable 规则不清、阶段跑偏或工具失败时，可以提示用户显式调用 `cs-feedback`。 +提示本身不得读取历史、后台采集、自动上传、自动修改目标 skill，也不得替用户确认 public preview。 + ## Skill 间同轮转交 公开 skill 选择另一个主入口后，按已安装 skill 名称加载目标协议，并在当前 run 继续。skill 是独立安装单元；不得靠读取 sibling skill 文件模拟转交。 diff --git a/.codestable/reference/shared-conventions.md b/.codestable/reference/shared-conventions.md index f340303..c851bf8 100644 --- a/.codestable/reference/shared-conventions.md +++ b/.codestable/reference/shared-conventions.md @@ -61,8 +61,11 @@ onboard 完成后骨架（`cs-onboard` 负责搭建）： ├── feedback/ CodeStable skill 使用反馈和上报证据 │ └── YYYY-MM-DD-{slug}/ │ ├── {slug}-report.md -│ ├── evidence.json -│ └── github-issue.md +│ ├── evidence.json local-private observations +│ ├── triage.json local-private assessments + quality +│ ├── public-issue-context.json allowlist preview，可选 +│ ├── github-issue.md 用户确认后可上传，可选 +│ └── regression-candidate.json local-private eval 交接，可选 ├── compound/ 沉淀类文档统一目录（cs-keep 产出） │ └── YYYY-MM-DD-{slug}.md │ 纯 markdown，无 frontmatter，grep 检索 @@ -77,7 +80,7 @@ onboard 完成后骨架（`cs-onboard` 负责搭建）： - 需求文档：`requirements/{slug}.md`（能力愿景，不带日期前缀，扁平不分组）；中心索引 `requirements/VISION.md` - roadmap：`roadmap/{slug}/`（不带日期前缀，平铺不嵌套） - feature / issue / refactor 目录：带日期前缀 `YYYY-MM-DD-{slug}` -- feedback 目录：带日期前缀 `YYYY-MM-DD-{slug}`，保存 feedback report、脱敏 evidence 和 GitHub issue body +- feedback 目录：带日期前缀 `YYYY-MM-DD-{slug}`，保存 report、local-private evidence/triage/candidate 与用户确认的 public preview - 沉淀类：`compound/YYYY-MM-DD-{slug}.md`，日期用**归档当天**，纯 markdown 无 frontmatter（cs-keep 产出） - 领域术语：`requirements/CONTEXT.md`（单 context）或 `requirements/{ctx}/CONTEXT.md`（多 context）；cs-domain lazy 创建 - 架构决策：`requirements/adrs/NNN-{slug}.md`（系统级）或 `requirements/{ctx}/adrs/NNN-{slug}.md`（子 context）；3 位编号，cs-domain 产出 @@ -105,7 +108,7 @@ onboard 完成后骨架（`cs-onboard` 负责搭建）： **归档类（compound）**：由 `cs-keep` 统一产出，写到 `.codestable/compound/YYYY-MM-DD-{slug}.md`。纯 markdown，**无 frontmatter**。三段足够：背景 / 结论 / 证据。检索靠 grep。 -**反馈类（feedback）**：由 `cs-feedback` 统一产出，写到 `.codestable/feedback/YYYY-MM-DD-{slug}/`。`{slug}-report.md` 用 `doc_type: codestable-feedback`；`evidence.json` 只放脱敏后的本机历史片段和上下文窗口；`github-issue.md` 是可公开上报前让用户确认的 issue body。 +**反馈类（feedback）**：由 `cs-feedback` 显式调用后产出，写到 `.codestable/feedback/YYYY-MM-DD-{slug}/`。`evidence.json` 保存脱敏 observation/incident，`triage.json` 保存 assessment 与 readiness，二者及 candidate 均为 local-private；`github-issue.md` 只能从 public allowlist 渲染并在上传前让用户确认。 **外部读者文档**（`cs-docs` tutorial / api mode）：frontmatter 由对应模式定义。无特殊说明：`draft` = 待 review，`current` = 当前有效，`outdated` = 代码已变更待同步。 diff --git a/.codestable/reference/system-overview.md b/.codestable/reference/system-overview.md index 918aefb..5f3faff 100644 --- a/.codestable/reference/system-overview.md +++ b/.codestable/reference/system-overview.md @@ -31,7 +31,7 @@ CodeStable 把常见开发活动各配一套流程，产物放进统一的 `.cod - `cs-req` — 起草或刷新 `.codestable/requirements/` 下的需求文档。 - `cs-domain` — 维护 CONTEXT.md 术语、ADR 决策和单/多 context 拓扑。 - `cs-audit` — 主动扫描 bug、安全、性能、可维护性和架构偏离。 -- `cs-feedback` — 收集 CodeStable skill 使用问题，自动采集本机 Codex/Claude 历史并准备 GitHub issue。 +- `cs-feedback` — 显式调用后把当前会话整理为 local-private incident/triage；公开预览经用户确认后才可上报。 - `cs-docs` — 写给外部读者的开发者指南、用户指南或 API 参考。 - `cs-docs-neat` — 阶段/里程碑收尾时整理 `.codestable/`、README/docs、`CLAUDE.md` / `AGENTS.md` 和 agent 记忆。 diff --git a/.codestable/requirements/VISION.md b/.codestable/requirements/VISION.md new file mode 100644 index 0000000..cd1dcd5 --- /dev/null +++ b/.codestable/requirements/VISION.md @@ -0,0 +1,14 @@ +# CodeStable 能力清单 + +## Current + +- `[current]` [反馈证据整理](feedback-evidence-pipeline.md) — 把一次 CodeStable 使用问题整理成可安全分享、可分诊、可复现的反馈证据。 +- `[current]` [插件安装与升级](plugin-market-distribution.md) — 让 CodeStable 像插件一样被安装和升级。 + +## Draft + +none。 + +## Outdated + +none。 diff --git a/.codestable/requirements/feedback-evidence-pipeline.md b/.codestable/requirements/feedback-evidence-pipeline.md new file mode 100644 index 0000000..1392c37 --- /dev/null +++ b/.codestable/requirements/feedback-evidence-pipeline.md @@ -0,0 +1,41 @@ +--- +doc_type: requirement +slug: feedback-evidence-pipeline +pitch: 把一次 CodeStable 使用问题整理成可安全分享、可分诊、可复现的反馈证据 +status: current +last_reviewed: 2026-07-11 +implemented_by: + - 2026-07-10-cs-feedback-evidence-pipeline +tags: [codestable, feedback, evaluation, privacy] +--- + +# 把 CodeStable 使用问题整理成反馈证据 + +## 用户故事 + +- 作为遇到 CS skill 跑偏、跳过 gate 或工具失败的开发者，我希望显式发起反馈后自动保留当前问题现场，而不是自己翻会话记录拼线索。 +- 作为担心本机路径、凭证和业务内容泄露的用户，我希望先看到经过整理的公开预览并逐次确认，再决定是否分享。 +- 作为维护 CodeStable 的人，我希望收到带时间线、预期行为、实际行为和证据缺口的反馈，而不是从零散描述中重新猜发生了什么。 +- 作为维护回归样本的人，我希望只有可重放、结果明确且通过隐私检查的反馈才能进入正式评测，避免低信号样本污染优化。 + +## 为什么需要 + +一次 CS skill 使用问题通常散落在 agent 动作、工具结果、用户纠正和仓库状态里。只留一句“跑偏了”很难定位，只复制整段会话又可能泄露私密内容。维护者需要足够完整的证据，用户也需要清楚知道哪些内容会离开本机。 + +## 怎么解决 + +用户主动发起反馈时，CodeStable 围绕当前问题整理时间线、使用环境和用户纠正，区分客观观察与分析判断，并指出还缺哪些关键信息。证据不足时仍可留在本机继续补充；需要公开时先生成安全预览；信息足够时再交给维护者制作回归样本。 + +## 边界 + +- 只在用户显式发起后工作，不做后台遥测或自动采集。 +- 默认聚焦当前相关会话，不批量扫描并公开全部历史。 +- 不自动上传；任何公开分享都必须先由用户确认。 +- 不把推测当成事实，也不自动认定问题一定来自 skill。 +- 不自动修改被反馈的 skill 或替用户决定修复方案。 +- 不完整反馈可以保存和分诊，但不保证能成为正式回归样本。 +- 只有可重放、结果明确且通过隐私检查的反馈才能进入正式评测。 + +## 实现记录 + +- 2026-07-11：`2026-07-10-cs-feedback-evidence-pipeline` 完成当前会话证据整理、结构化分诊、安全公开预览与回归样本交接。 diff --git a/.codestable/runtime-manifest.json b/.codestable/runtime-manifest.json index 106bb6f..a72b81e 100644 --- a/.codestable/runtime-manifest.json +++ b/.codestable/runtime-manifest.json @@ -1,8 +1,8 @@ { "schema_version": 1, "plugin": "codestable", - "plugin_version": "1.0.2", - "runtime_version": "1.0.2", + "plugin_version": "1.0.3", + "runtime_version": "1.0.3", "tool_runtime": "skill-global", "managed_paths": [ ".codestable/gates", diff --git a/CHANGELOG.md b/CHANGELOG.md index cb45e48..ac3eff9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 1.0.3 + +- Added the `cs-feedback` evidence pipeline for collecting local Codex and Claude session context, triaging incidents, and producing privacy-safe public issue previews. +- Added candidate-only fixture conversion in the shipped plugin and fail-closed regression fixture promotion in the repository evaluation tooling. +- Hardened current-session discovery, trigger cutoffs, provider-aware tool pairing, public redaction, and upload confirmation gates. + ## 1.0.2 - Hardened runtime upgrades: versionless or mismatched manifests now require synchronization, while `/cs-onboard --mode refresh-runtime` refreshes package-owned assets without overwriting dirty managed paths. diff --git a/README.en.md b/README.en.md index 5d6475a..71bd02c 100644 --- a/README.en.md +++ b/README.en.md @@ -189,7 +189,7 @@ CodeStable models real coding work as a set of **entities** and **flows**. | Refactor | `cs-refactor` | Behavior-preserving refactor workflow | | Review | `cs-code-review` | Cross-cutting read-only implementation review gate | | Audit | `cs-audit` | Scan for bugs, security, performance, maintainability, and architecture drift | -| Feedback | `cs-feedback` | Capture CodeStable skill usage problems, collect local history, and prepare a GitHub issue | +| Feedback | `cs-feedback` | Explicitly capture current-session incidents/triage; upload only after preview confirmation | | Knowledge | `cs-keep` / `cs-note` | Capture durable knowledge or short startup-critical notes | | External docs | `cs-docs` | Developer guides, user guides, and API references | | Docs hygiene | `cs-docs-neat` | Sync `.codestable/`, README/docs, agent entries, and memory | @@ -233,7 +233,7 @@ How to read it: - `cs-feat`, `cs-issue`, and `cs-refactor` resume from repository facts. `cs-issue` and `cs-refactor` stop at review, blocking, or user-confirmation checkpoints; `cs-feat` stops only at the design gate, then runs impl, review, QA, and accept long-range via a visible goal driver. - `cs-epic` prepares planning and goal packages, then dispatches a visible goal driver; v1 still writes `.codestable/roadmap/`. - `cs-code-review` is the cross-cutting gate; `cs-docs-neat` handles hygiene; `cs-docs` writes outward docs. -- `cs-feedback` captures failures and detours while using CodeStable skills, collects local Codex/Claude history, and prepares an issue. +- `cs-feedback` explicitly captures a local-private current-session evidence package; public issue upload remains separately confirmed. - Old stage skills are long-term compatibility entries for historical users. See [WORKFLOW.en.md](./WORKFLOW.en.md) for the compact diagram. @@ -242,91 +242,19 @@ See [WORKFLOW.en.md](./WORKFLOW.en.md) for the compact diagram. ## Runtime structure -After `/cs-onboard`, a `.codestable/` directory appears at your project root as the aggregate root for requirements, roadmap, goals, features, issues, refactors, audits, compound, gates, and reference. Python tool scripts run from the installed `cs-onboard` skill package instead of being copied into each repo. +After `/cs-onboard`, project artifacts aggregate under `.codestable/`. Python tool scripts run from the installed `cs-onboard` skill package instead of being copied into each repo. ```text -your-project/ -├── .codestable/ -│ ├── attention.md # required preflight for CodeStable skills -│ ├── requirements/ # requirements + domain model -│ │ ├── VISION.md # capability index -│ │ ├── {slug}.md # one capability per flat file -│ │ ├── CONTEXT.md # domain glossary -│ │ ├── CONTEXT-MAP.md # multi-context topology, when needed -│ │ ├── adrs/ # architecture decisions -│ │ │ └── NNN-{slug}.md # Nygard four sections + status machine -│ │ └── {ctx}/ # bounded-context subdir, when needed -│ │ ├── CONTEXT.md -│ │ ├── adrs/ -│ │ └── {capability}.md -│ │ -│ ├── roadmap/ # roadmaps ("how we plan to walk next") -│ │ └── {slug}/ -│ │ ├── {slug}-roadmap.md # main doc: background / breakdown / sequencing -│ │ ├── {slug}-items.yaml # machine-readable sub-feature list -│ │ ├── {slug}-roadmap-review.md # planning review before human approval -│ │ └── drafts/ # optional drafts / research -│ │ -│ ├── goals/ # goal-driven workflow aggregate root -│ │ └── {slug}/ -│ │ ├── {slug}-start-report.md -│ │ ├── {slug}-state.yaml -│ │ ├── {slug}-iteration-*.md -│ │ └── {slug}-functional-acceptance.md -│ │ -│ ├── features/ # feature flow aggregate root -│ │ └── YYYY-MM-DD-{slug}/ # one directory per feature -│ │ ├── {slug}-brainstorm.md # optional cs-brainstorm output -│ │ ├── {slug}-design.md # design -│ │ ├── {slug}-checklist.yaml # implementation checklist -│ │ ├── {slug}-design-review.md # pre-human design review -│ │ ├── {slug}-review.md # post-implementation code review -│ │ ├── {slug}-qa.md # QA gate after code review -│ │ └── {slug}-acceptance.md # acceptance report -│ │ -│ ├── issues/ # issue flow aggregate root -│ │ └── YYYY-MM-DD-{slug}/ -│ │ ├── {slug}-report.md -│ │ ├── {slug}-analysis.md # only when root cause is non-obvious -│ │ └── {slug}-fix-note.md -│ │ -│ ├── refactors/ # refactor flow aggregate root -│ │ └── YYYY-MM-DD-{slug}/ -│ │ ├── {slug}-scan.md -│ │ ├── {slug}-refactor-design.md -│ │ ├── {slug}-checklist.yaml -│ │ └── {slug}-apply-notes.md -│ │ -│ ├── audits/ # audit findings and scan outputs -│ ├── brainstorms/ # standalone brainstorm outputs -│ ├── compound/ # unified knowledge sink -│ │ └── YYYY-MM-DD-{slug}.md -│ │ # plain markdown, no frontmatter, grep to search -│ │ -│ ├── gates/ # workflow gate config released by onboard -│ └── reference/ # shared references released by onboard -│ ├── shared-conventions.md # cross-skill conventions / paths / metadata -│ ├── system-overview.md # system overview + scenario routing -│ └── ... -│ -└── AGENTS.md # project root, not under .codestable/ +.codestable/ +├── attention.md +├── requirements/ roadmap/ goals/ +├── features/ issues/ refactors/ +├── audits/ brainstorms/ feedback/ compound/ +├── gates/ +└── reference/ ``` -**Key points:** - -- All artifacts aggregate under `.codestable/`, so "how did we handle that feature / bug last time" is three seconds away. -- `requirements/` is the **long-lived archive** (capability vision + domain glossary CONTEXT.md + decisions adrs/); `roadmap/` is the **planning layer** (what's next), deliberately separated. -- `features/` `issues/` `refactors/` use `YYYY-MM-DD-{slug}/` to bundle all related specs in one directory, no crossing. -- `compound/` is the **single** knowledge sink directory: plain markdown, no frontmatter, searched via `grep -r`. -- `.codestable/reference/` is copied in by `cs-onboard` from `plugins/codestable/skills/cs-onboard/references/`; to change shared conventions, edit those skill-package templates so new projects pick them up at onboard time. - -### Hard constraint - -> A skill is an independent install unit. At runtime, **each skill can only see files inside its own package**. References like `B-skill/references/xxx.md` written in skill A's SKILL.md are **simply unreachable** at runtime. -> -> Cross-skill shared references must go through the "working project" layer: `cs-onboard` copies them from the skill package to the project's `.codestable/reference/`, and other skills read them via the project-relative path. - -To change shared conventions, edit the templates under `plugins/codestable/skills/cs-onboard/references/`; new projects pick them up at onboard time. See [WORKFLOW.en.md](./WORKFLOW.en.md) for the full directory model and cross-skill reference constraints. +`requirements/` is the long-lived archive, `roadmap/` is planning, dated work-item directories bundle one workflow, and `compound/` is the single knowledge sink. A skill is an independent install unit: cross-skill references must be released by `cs-onboard` into project-local `.codestable/reference/`, never read from a sibling skill package. See [WORKFLOW.en.md](./WORKFLOW.en.md) for the directory contract. --- diff --git a/README.md b/README.md index 3c583ed..5010c3e 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ CodeStable 顺着软件编码的真实流程来设计，把开发活动建模成 | 重构流程 | `cs-refactor` | 行为等价重构，含标准模式和 fastforward mode | | 横切审查 | `cs-code-review` | 实现完成后、commit 前的只读代码审查 gate | | 审计 | `cs-audit` | 主动扫描 bug、安全、性能、可维护性和架构偏离 | -| 反馈 | `cs-feedback` | 收集 CodeStable skill 使用问题，自动采集本机历史并准备 GitHub issue | +| 反馈 | `cs-feedback` | 显式采集当前会话为 local-private incident/triage；确认 preview 后才可上报 | | 知识沉淀 | `cs-keep` / `cs-note` | 沉淀 compound 知识或短项目注意事项 | | 对外文档 | `cs-docs` | 写开发者指南、用户指南、API 参考 | | 文档收尾 | `cs-docs-neat` | 同步 `.codestable/`、README/docs、agent 入口和记忆 | @@ -215,7 +215,7 @@ CodeStable 顺着软件编码的真实流程来设计，把开发活动建模成 CodeStable 是分层、事件驱动的：`cs` 先判入口模式，行动请求同轮直转，咨询请求只给建议；`cs-feat` / `cs-issue` / `cs-refactor` 按仓库事实恢复阶段并经过 `cs-code-review`，其中 issue / refactor 在 review、blocking 或用户确认 checkpoint 停下；`cs-epic` 编排 planning、批量子 design 和 goal driver；旧阶段技能只保留为兼容入口。 -`cs-onboard` 在项目根生成 `.codestable/`，集中保存 requirements、roadmap、goals、features、issues、refactors、audits、compound、gates 与共享 reference。Python 工具脚本从已安装的 `cs-onboard` skill 包运行，不再复制到每个 repo。 +`cs-onboard` 在项目根生成 `.codestable/`，集中保存 requirements、roadmap、goals、features、issues、refactors、audits、feedback、compound、gates 与共享 reference。Python 工具脚本从已安装的 `cs-onboard` skill 包运行，不再复制到每个 repo。 - `requirements/` 保存长期能力、术语和 ADR；`roadmap/` 保存待执行规划。 - feature / issue / refactor 各自按工作项聚合产物；`compound/` 是统一知识沉淀目录。 diff --git a/SKILL_CATALOG.en.md b/SKILL_CATALOG.en.md index 44d02ae..3d1230e 100644 --- a/SKILL_CATALOG.en.md +++ b/SKILL_CATALOG.en.md @@ -18,7 +18,7 @@ Main entries accept optional stage / mode flags (for example `/cs-feat --stage q | Refactor flow | `cs-refactor` | Behavior-preserving refactor entry with standard and fastforward modes | | Cross-cutting review | `cs-code-review` | Read-only implementation review gate | | Audit | `cs-audit` | Scan for bugs, security, performance, maintainability, and architecture drift | -| Feedback | `cs-feedback` | Capture CodeStable skill usage problems, collect local history, and prepare a GitHub issue | +| Feedback | `cs-feedback` | Explicitly capture current-session incidents/triage; upload only after preview confirmation | | Knowledge | `cs-keep` | Capture lessons, tricks, decisions, and research in `.codestable/compound/` | | Knowledge | `cs-note` | Append short startup-critical notes to `.codestable/attention.md` | | External docs | `cs-docs` | Write or update developer guides, user guides, and API references | diff --git a/SKILL_CATALOG.md b/SKILL_CATALOG.md index 7e8023a..0b115e4 100644 --- a/SKILL_CATALOG.md +++ b/SKILL_CATALOG.md @@ -18,7 +18,7 @@ | 重构流程 | `cs-refactor` | 行为等价重构入口：标准模式或 fastforward mode | | 横切审查 | `cs-code-review` | 实现完成后的只读代码审查 gate | | 审计 | `cs-audit` | 主动扫描 bug、安全、性能、可维护性和架构偏离 | -| 反馈 | `cs-feedback` | 收集 CodeStable skill 使用问题，自动采集本机历史并准备 GitHub issue | +| 反馈 | `cs-feedback` | 显式采集当前会话为 local-private incident/triage；确认 preview 后才可上报 | | 知识沉淀 | `cs-keep` | 把坑点、技巧、决策、调研沉淀到 `.codestable/compound/` | | 知识沉淀 | `cs-note` | 把一两行启动必读项目注意事项追加到 `.codestable/attention.md` | | 对外文档 | `cs-docs` | 写或更新开发者指南、用户指南、API 参考 | diff --git a/VERSION b/VERSION index 6d7de6e..21e8796 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.2 +1.0.3 diff --git a/WORKFLOW.en.md b/WORKFLOW.en.md index 611980d..b1f286d 100644 --- a/WORKFLOW.en.md +++ b/WORKFLOW.en.md @@ -28,7 +28,7 @@ The vertical layout is layering, not strict time order. Long-lived records are r The event entries are `cs-feat` for new capability, `cs-issue` for bugs, `cs-refactor` for behavior-preserving cleanup, and `cs-docs` for outward documentation. `cs-code-review` remains the cross-cutting implementation review gate. -The knowledge and feedback loop remains cross-cutting: any workflow can use `cs-keep`; `cs-feedback` captures skill failures and rule gaps for issue reporting; milestone cleanup uses `cs-docs-neat` to sync `.codestable/`, README/docs, agent entry files, and memory. +The knowledge and feedback loop remains cross-cutting: `cs-keep` compounds knowledge; explicit `cs-feedback` calls produce local-private incidents/triage and require separate preview confirmation before upload; `cs-docs-neat` handles milestone hygiene. Old stage skills remain long-term compatibility entries: @@ -52,6 +52,7 @@ After `/cs-onboard`, the project root contains `.codestable/`: ├── refactors/ ├── audits/ ├── brainstorms/ +├── feedback/ ├── compound/ ├── tools/ └── reference/ diff --git a/WORKFLOW.md b/WORKFLOW.md index 9f123fd..ec87192 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -28,7 +28,7 @@ cs 第 3 层是事件入口：新需求走 `cs-feat`，bug 走 `cs-issue`，腐化走 `cs-refactor`，对外文档走 `cs-docs`。`cs-code-review` 是横切代码审查 gate，feature / issue / refactor 链路都经它产 `{slug}-review.md`。 -横切层是知识与反馈飞轮：任何流程都可以把值得复用的经验经 `cs-keep` 沉淀到 compound；`cs-feedback` 收集 skill 使用失败和规则缺口，准备上报 issue；`cs-docs-neat` 在里程碑收尾时同步 `.codestable/`、README/docs、`CLAUDE.md` / `AGENTS.md` 和 agent 记忆。 +横切层是知识与反馈飞轮：`cs-keep` 沉淀 compound；`cs-feedback` 仅在显式调用后生成 local-private incident/triage，public preview 经确认后才可上报；`cs-docs-neat` 在里程碑收尾时同步文档与记忆。 旧阶段技能仍是长期兼容入口，但不再作为主路径展示： @@ -52,6 +52,7 @@ cs ├── refactors/ ├── audits/ ├── brainstorms/ +├── feedback/ ├── compound/ ├── tools/ └── reference/ diff --git a/docs/adr/003-cs-skill-evaluation-loop.md b/docs/adr/003-cs-skill-evaluation-loop.md index 95dc08a..82da3c7 100644 --- a/docs/adr/003-cs-skill-evaluation-loop.md +++ b/docs/adr/003-cs-skill-evaluation-loop.md @@ -7,6 +7,7 @@ applies-to: - ".claude/skills/eval-cs-skill/" - "experiments/" - "plugins/codestable/skills/cs-feedback/scripts/feedback_to_fixture.py" + - ".claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py" enforcement: test stage: [author, eval, optimize, release] lint: "python3 -m pytest tests/test_cs_skill_eval.py tests/test_cs_skill_convergence.py tests/test_cs_skill_release.py tests/test_cs_skill_bootstrap.py tests/test_cs_skill_selfref.py" @@ -31,12 +32,13 @@ CodeStable 原有 `tests/test_skill_*` 只验证 skill **写得对不对**（路 5. **release 两步走**：`knowledge-extractor` 产草稿 → `adapt_extracted_skill.py` 翻译成 CS 合规结构（禁止 extractor 直写 plugins/），再 `regression.py` + `bump_version.py`。 6. **自治默认轻量 cron**（`enqueue_experiment.py`），BAIME `loop-backlog` 为可选宿主。 7. **自指**：`experiments/eval-cs-skill-001/` 用同一 runner/scorer 评 `eval-cs-skill` 自身。 +8. **反馈交接边界**：shipped `cs-feedback` 只把 local-private `triage.json` 转成同目录 candidate；正式 fixture 由 repo-local promotion 工具读取 experiment config，校验 profile/input/privacy/scorer/harness/judge 后 fail-closed 落盘。两单元只通过 JSON artifact 连接，运行时互不 import。 ## Consequences - skill 效果可跨 model/harness 量化，改进有硬 verdict 而非直觉。 - 新 skill 接入只需加 `experiments/` 数据（自举）；加 harness 只需加一个 adapter。 -- 生产失败经 `cs-feedback/feedback_to_fixture.py` 转 regression fixture，闭环回评测。 +- 生产失败先经 `cs-feedback/feedback_to_fixture.py` 形成未入库 candidate；只有 readiness、隐私与目标 experiment gates 全过，repo-local promotion 才写 regression fixture。 - eval-cs-skill 自身可被同一闭环评测优化（自指）。 - 真实多模型运行需 API/CLI 鉴权并产生成本，受 `--dry-run` + `budget_usd` 护栏约束。 - **评测效度是头等风险**（首轮真实 campaign 教训）：必须复现 skill 的设计运行环境（`inject_context` 补 onboard 上下文）、用语义 oracle（`recall_judge`）判散文 answer、fixture 内嵌被操作的 subject matter；否则测到的是「skill 在残缺环境下的反应」而非真实能力。核查须分模型看 + 手工读原始输出 + 认 k=1 variance。细则见 `references/eval/protocol.md` 效度三铁律。 diff --git a/experiments/cs-routing-001/fixtures/routing/rt-c17.json b/experiments/cs-routing-001/fixtures/routing/rt-c17.json new file mode 100644 index 0000000..5eda12a --- /dev/null +++ b/experiments/cs-routing-001/fixtures/routing/rt-c17.json @@ -0,0 +1,16 @@ +{ + "id": "rt-c17", + "answerType": "routing-decision", + "task": { + "kind": "routing", + "state": { + ".codestable/attention.md": "present", + "installed_skills": ["cs-feedback", "cs-feat"] + }, + "utterance": "刚才 cs-feat 跳过了独立 review gate。请收集这次 CodeStable 使用反馈，不要修业务代码。" + }, + "expect": { + "result_type": "RoutedTo", + "target": "cs-feedback" + } +} diff --git a/plugins/codestable/.claude-plugin/plugin.json b/plugins/codestable/.claude-plugin/plugin.json index de2da03..1f69772 100644 --- a/plugins/codestable/.claude-plugin/plugin.json +++ b/plugins/codestable/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codestable", - "version": "1.0.2", + "version": "1.0.3", "description": "CodeStable AI coding workflow skills.", "author": { "name": "CodeStable" diff --git a/plugins/codestable/.codex-plugin/plugin.json b/plugins/codestable/.codex-plugin/plugin.json index e7d1e4b..6aae916 100644 --- a/plugins/codestable/.codex-plugin/plugin.json +++ b/plugins/codestable/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codestable", - "version": "1.0.2", + "version": "1.0.3", "description": "CodeStable AI coding workflow skills.", "skills": "./skills/" } diff --git a/plugins/codestable/skills/cs-feedback/SKILL.md b/plugins/codestable/skills/cs-feedback/SKILL.md index b15b19d..1aba256 100644 --- a/plugins/codestable/skills/cs-feedback/SKILL.md +++ b/plugins/codestable/skills/cs-feedback/SKILL.md @@ -1,7 +1,7 @@ --- name: cs-feedback -description: CodeStable 使用反馈闭环。触发：用户反馈 cs skill 跑偏、工具失败、规则没讲清、agent 被用户纠正，自动采集本机 Codex/Claude 历史并生成可上报 issue。 -argument-hint: "[--since-days N] [--session current|] [--github] " +description: CodeStable 使用反馈闭环。触发：用户反馈 cs skill 跑偏、工具失败、规则没讲清、agent 被用户纠正；显式调用后采集本机证据并准备可确认的公开 issue。 +argument-hint: "[--since-days N | --session current|] [--accept-incident ] [--github] " --- # cs-feedback @@ -10,129 +10,148 @@ argument-hint: "[--since-days N] [--session current|] [--github] ` | 只扫某个 Codex/Claude session id 或 jsonl 路径 | -| `--session current` | 尝试按 cwd、mtime 和 transcript metadata 推断当前会话；命中多个候选时必须问用户选择 | -| `--github` | 生成 GitHub issue public preview；用户确认后才用 `gh issue create` 上报 | +| `--session current` | metadata-only 定位当前 cwd 的唯一会话；弱匹配或多候选必须让用户选择 | +| `--session ` | 只采集用户选定的 Codex / Claude 会话 | +| `--since-days N` | 用户显式要求时跨会话扫描最近 N 天；不再传 `--session current` | +| `--accept-incident ` | 只在 triage 已记录同一 pending incident 时显式采纳；不得由 agent 自行推断 | +| `--github` | 生成 public preview；不是上传授权 | -其余文本作为用户原始反馈，必须原样写入报告。 +collector 直接调用仍保留 v1 的 `session=None + since_days=3` 默认。若调用方同时传 current +和 since-days，current 只绕过 `time_cutoff`，输出 `since_days_ignored=true`；最后 user +record 的 `trigger_cutoff` 始终生效。 ---- - -## 文件放哪儿 +## 文件布局 ```text -.codestable/feedback/{YYYY-MM-DD}-{slug}/ +.codestable/feedback/YYYY-MM-DD-{slug}/ ├── {slug}-report.md -├── evidence.json # local-private，不上传 -├── public-issue-context.json # 可公开证据摘要，可选 -└── github-issue.md +├── evidence.json # local-private observations +├── triage.json # local-private assessments + quality +├── public-issue-context.json # public allowlist projection，可选 +├── github-issue.md # 用户确认的公开正文，可选 +└── regression-candidate.json # local-private eval 交接，可选 ``` -如果当前仓库还没接入 `.codestable/`，先提示 `cs-onboard`；但仍可把 evidence/report 写到 `/tmp/codestable-feedback-{slug}/`，方便用户手动带走。 - ---- +未 onboard 时先提示 `cs-onboard`；仍需保留现场时可写 +`/tmp/codestable-feedback-{slug}/`，但继续按 local-private 处理。 ## 工作流 -### 1. 收集线索 +### 1. 定位并冻结会话 -先把用户原话分类，不急着定责： - -- 工具调用失败：file read、apply patch、MCP、Paseo、git、GitHub、网络、权限、超时。 -- 规则不清：skill 没讲清阶段、参数、fallback、handoff、goal driver、review gate。 -- 流程绕路：agent 做了不必要步骤，被用户纠正后才回到正确路径。 -- 分发安装：Codex / Claude marketplace、版本缓存、branch/ref、插件安装。 - -### 2. 自动采集 - -运行： +默认运行： ```bash -python3 {skill_dir}/scripts/collect_feedback_context.py --since-days 3 --feedback "{用户原话}" --output {feedback-dir}/evidence.json --public-output {feedback-dir}/public-issue-context.json +python3 {skill_dir}/scripts/collect_feedback_context.py --session current --cwd "$(pwd)" --feedback "{用户原话}" --output {feedback-dir}/evidence.json --triage-output {feedback-dir}/triage.json --public-output {feedback-dir}/public-issue-context.json ``` -如果用户给了 session id/path，加 `--session`。如果用户要“当前会话”，传 -`--session current --cwd "$(pwd)"`；这是 best-effort，不是可靠 oracle。脚本会扫： +显式 `--since-days N` 时去掉 current/cwd。`ambiguity.candidates` 非空时，metadata-only +路径不得 flatten、匹配或持久化 message/tool 正文；只让用户选择 session，再用 +`--session ` 重跑。 -- Codex：`~/.codex/sessions/**/*.jsonl` -- Claude：`~/.claude/projects/**/*.jsonl` 和 `~/.claude/sessions/*.json` +采集器对选定输入只读一次 snapshot：JSONL 记录完整 record EOF，单 JSON 使用不可变 byte +snapshot；`trigger_cutoff` 后的 assistant/tool record 不进入 evidence。没有 user anchor 时仍可 +保存 incident，但 triage 保持未就绪。 -脚本输出两层证据：`evidence.json` 是 local-private，只留本机；`public-issue-context.json` 是 allowlist 摘要，可用于生成 `github-issue.md`。不要把完整 transcript 或 `evidence.json` 原文贴进 GitHub issue。 -当 `evidence.json` 里 `ambiguity.candidates` 非空时，先让用户选 session，不能假装已经定位。 +### 2. 检查 incident 与 triage -### 3. 归纳报告 +`evidence.json` schema v2 以有序 `incidents` 为 canonical 单元，保留 role、tool pairing、 +用户纠正、observation ids、runtime/artifact/git 文件级上下文；旧 `matched_events` 继续作为 v1 +兼容投影。provider id 精确配对标 `provider`，唯一无 id 紧邻配对标 `adjacency`，其他均为 +`unpaired`。 -读取 `references/report-template.md`，写 `{slug}-report.md`。报告必须包含： +`triage.json` 把 Observation 与 Assessment 分开。判断字段必须带 `source` 和 +`evidence_refs`；`source=inferred` 还必须有 `confidence`。未知就写 `unknown`，不猜根因。 -- 用户原始反馈。 -- 自动采集范围：provider、时间窗口、命中条数；session 只写短标签或 hash，不写本机路径。 -- 失败点清单：每条有现象、上下文、疑似根因、涉及 skill/reference/script。 -- 用户纠正信号：用户说了什么，agent 之前做了什么绕路。 -- 可执行建议：改哪个 skill / reference / script / test，或标为需要更多样本。 -- 隐私说明：`evidence.json` 只保存在本机，best-effort 脱敏后仍按私有文件处理。 +canonical `incident_kind`：`wrong-route / skipped-gate / missing-artifact / tool-failure / +goal-driver / unnecessary-detour / install-version / privacy-reporting / unclear-rule / unknown`。 +v1 public `events[].failure_type` 只保留原 6 值域，不写 v2 枚举。 -没有自动命中时也要写报告：明确“自动采集未命中”，并列出已尝试的历史位置。 +### 3. Ask User（缺口驱动） -### 4. Ask User（缺口驱动） +不要固定三问。按 `quality.next_questions` 每次只问最高优先级缺口： -报告落盘后只问当前阻塞质量或隐私确认的那个问题；不要固定三问。按优先级： +1. session / primary incident 仍歧义：只让用户选择。若 triage 已有 + `pending_incident_id`，展示 previous/pending 值；用户明确采纳后，用原采集参数加 + `--accept-incident ` 重跑。采纳必须匹配当前 primary 的 fingerprint，保留 + reproduction，归档旧 assessment/privacy，并把 active privacy review 重置为 `pending`。 +2. target skill、expected、actual 或 observation ref 缺失：只补当前第一项。 +3. triage 已就绪但 regression 缺 input/oracle：只在用户要做回归样本时追问。 +4. 用户要求 GitHub：只问 public preview 是否可以公开。 -1. `ambiguity.candidates` 非空：只问用户选哪个 session。 -2. `github-issue.md` 还没生成：先生成 public preview，不问泛泛问题。 -3. preview 缺 expected behavior：问“你期望 agent 当时怎么做？” -4. preview 缺相关 skill/reference：问“最相关的是哪个 skill 或流程入口？” -5. 用户要求上报 GitHub：只问“这个 GitHub issue preview 可以公开上报吗？” +`triage_ready` 要求唯一 incident、trigger cutoff、target skill、expected/actual 和 observation +refs；`regression_ready` 还要求兼容 profile、最小可重放 input 与 oracle。quality 表示证据完备性， +不表示问题严重度，也不证明一定是 skill 缺陷。重复采集不得覆盖用户手工补充字段；位置编号 +相同但 fingerprint 不同也必须重新选择。 -用户已在原始反馈里说清的，不重复问。 +### 4. 写本地报告与公开投影 -### 5. GitHub 上报 +按 `references/report-template.md` 写 `{slug}-report.md`。报告引用 observation ids 与 triage +quality，不贴完整 transcript。`evidence.json`、`triage.json`、`regression-candidate.json` 永远 +local-private。 +local evidence 只做 best-effort 脱敏，不能因此视为可公开文件。 -先生成 `{slug}/github-issue.md` public preview，列出将公开包含 / 不会公开包含的内容。即使用户传 `--github`，也必须先让用户确认 preview；未确认前不调用 `gh issue create`。 +public preview 只能从结构化 allowlist 构建： -允许公开的内容只来自 allowlist：provider、session_label、timestamp_bucket、failure_type、match_type、tool_name、CodeStable skill/reference/script 相对路径、sanitized_excerpt、expected_behavior、actual_behavior、proposed_fix。 +- v1 event 精确 8 字段：provider、session_label、timestamp_bucket、failure_type、match_type、 + tool_name、skill_or_reference、sanitized_excerpt。 +- v2 incident 精确 7 字段：incident_kind、target_skill、stage_hint、expected_behavior、 + actual_behavior、impact、proposed_fix。 -禁止公开：完整 transcript、`evidence.json` 原文、本机绝对路径、私有 repo 名/remote URL、环境变量、token/secret/API key、大段业务代码或用户长对话、MCP/tool 原始 JSON 参数。 +禁止公开：完整 transcript、本机绝对路径、repo/remote、环境变量、secret、原始 tool JSON、 +代码块或业务代码。不要从人写报告反向抓字段生成 preview。 -用户确认可上报后： +### 5. 可选 regression candidate + +shipped skill 只产同目录 local-private candidate： ```bash -python3 {skill_dir}/scripts/report_feedback_issue.py --repo liuzhengdongfortest/CodeStable --title "{title}" --body-file {feedback-dir}/github-issue.md +python3 {skill_dir}/scripts/feedback_to_fixture.py --triage {feedback-dir}/triage.json ``` -脚本检测 `gh`： +兼容 `--evidence ` 也只生成未就绪 candidate。旧 `--failure --experiment` +/ shipped `--experiment` 直写正式 fixture 必须非零。正式 promotion +只由 CodeStable 维护仓库的 repo-local `eval-cs-skill` 工具消费 JSON artifact；普通用户仓库 +没有该工具时保留 candidate，不形成运行时跨 skill 依赖。 -- `gh` 可用且已登录：直接 `gh issue create`，回填 issue URL 到报告。 -- `gh` 不可用或未登录：不失败，把命令和 issue body 路径交给用户手动执行。 +### 6. GitHub 上报 ---- +即使用户传 `--github`，也必须先让用户确认 preview；未确认不调用 GitHub。确认后只上传 +`github-issue.md`： + +```bash +python3 {skill_dir}/scripts/report_feedback_issue.py --repo liuzhengdongfortest/CodeStable --title "{title}" --body-file {feedback-dir}/github-issue.md --confirm-public-preview +``` + +reporter 按文件名硬拒 evidence、triage、candidate，按 `privacy=local-private` 二次拒绝，并在 +网络边界再次扫描 public body。`gh` 缺失、未登录或网络失败时只返回 manual fallback；若需要 +访问 GitHub，按宿主规则检测本机代理后重试。 ## 退出条件 -- [ ] `{slug}-report.md` 已落盘，含用户反馈、自动采集范围、失败点和建议。 -- [ ] `evidence.json` 已落盘并标记 local-private，或报告说明自动采集为什么不可用。 -- [ ] 已生成 `github-issue.md` public preview，且不含禁止公开内容。 -- [ ] 用户要求上报时，已创建 GitHub issue 或给出可手动执行的 `gh issue create` 命令。 -- [ ] 没有把 token、密钥、完整无关 transcript 写进报告。 - ---- +- [ ] report、evidence、triage 已落盘，或报告说明采集不可用。 +- [ ] primary incident 与 `quality.triage_ready` 状态明确；未知字段未编造。 +- [ ] public preview（如生成）只含 allowlist，用户未确认时没有网络上报。 +- [ ] candidate（如生成）仍在 feedback 目录，未就绪输入未进入正式 fixtures。 +- [ ] 没有后台采集、自动上传、自动修改目标 skill 或默认全历史扫描。 ## 相关文档 -- `references/report-template.md` — feedback report 和 GitHub issue 模板。 -- `scripts/collect_feedback_context.py` — 本机 Codex / Claude 历史采集。 -- `scripts/report_feedback_issue.py` — GitHub issue 创建 / fallback。 +- `references/report-template.md` — 本地报告、triage 补充和 GitHub body 模板。 +- `scripts/collect_feedback_context.py` — evidence / triage / public projection 编排。 +- `scripts/feedback_to_fixture.py` — candidate-only 转换。 +- `scripts/report_feedback_issue.py` — 确认后的 GitHub 创建 / fallback。 diff --git a/plugins/codestable/skills/cs-feedback/references/report-template.md b/plugins/codestable/skills/cs-feedback/references/report-template.md index 1a8c449..2f748c3 100644 --- a/plugins/codestable/skills/cs-feedback/references/report-template.md +++ b/plugins/codestable/skills/cs-feedback/references/report-template.md @@ -21,78 +21,121 @@ github_issue: "" ## 自动采集范围 -- since_days: {N} -- session_filter: {session_filter_or_none} +- mode: {current | selected-session | since-days} +- session_filter: {current_or_selected_or_none} +- since_days_ignored: {true_or_false} - local_private_evidence: `evidence.json` -- public_preview: `github-issue.md` -- matched_events: {count} +- local_private_triage: `triage.json` +- public_preview: `public-issue-context.json` +- incidents: {count} +- primary_incident: {incident_id_or_unknown} -## 失败点清单 +## 反馈事件包 -| # | 类型 | 相关 skill | 现象 | 证据 | +| Incident | Kind | Target / Stage | Cutoff | Observation refs | |---|---|---|---|---| -| 1 | tool-failure / unclear-rule / agent-detour / goal-driver / install-distribution / privacy-reporting | `cs-feat` | {一句话} | {provider/session_label/timestamp_bucket} | +| `incident-01` | {incident_kind} | `{skill}` / `{stage}` | `{record_or_unknown}` | `obs-0001`, `obs-0002` | -## 关键上下文 +## 客观观察 -{按失败点摘要 evidence 里的 context，不贴完整 transcript。} +| Ref | Role / Type | 事实摘要 | +|---|---|---| +| `obs-0001` | assistant / message | {只摘要脱敏事实，不写根因} | +| `obs-0002` | tool / tool_result | {工具结果摘要} | + +## 分析判断 + +| 字段 | 值 | Source | Confidence | Evidence refs | +|---|---|---|---|---| +| expected_behavior | {value_or_unknown} | {user_or_unknown} | - | {refs} | +| actual_behavior | {value_or_unknown} | {transcript_or_unknown} | - | {refs} | +| impact | {value_or_unknown} | {inferred_or_unknown} | {confidence_or_dash} | {refs} | +| proposed_fix | {value_or_unknown} | {source_or_unknown} | {confidence_or_dash} | {refs} | + +`cause_status` 默认 `unclassified`。Observation 不写疑似根因；Assessment 无依据时保持 +`unknown`，`source=inferred` 必须同时有 confidence 和 evidence refs。 + +## 质量门 + +- triage_ready: {true_or_false} +- regression_ready: {true_or_false} +- incident_fingerprint: {sha256_or_unknown} +- previous_incident: {id_or_none} +- pending_incident: {id_or_none} +- missing_fields: {list} +- next_questions: {最多当前最高优先级一项} +- reasons: {list} + +`pending_incident` 非空时，先让用户核对 previous/pending。只有用户明确采纳，才用同一组采集 +参数追加 `--accept-incident {pending-id}`；采纳后重新检查 assessment 与 public preview。 + +## 本机环境 + +- provider / model / host: {values_or_unknown} +- runtime: {version_and_status_or_unknown} +- related_artifacts: {仅 repo-relative path + status} +- git_status: {仅 status + repo-relative filename，不贴 diff 或文件内容} ## 隐私说明 -- `evidence.json` 是本机私有证据，`public_upload_allowed=false`。 -- GitHub issue 只使用 `github-issue.md` public preview，不上传 `evidence.json` 原文。 -- 虽然 evidence 已 best-effort 脱敏，仍可能含业务上下文，应按私有文件处理。 +- `evidence.json`、`triage.json`、`regression-candidate.json` 是 local-private。 +- evidence 已 best-effort 脱敏，仍可能含业务上下文，不得上传。 +- GitHub 只使用用户确认后的 `github-issue.md`，且该正文只从 public allowlist 渲染。 +- 不公开完整 transcript、绝对路径、remote/env、secret、原始工具参数或代码块。 -## 用户纠正信号 +## Regression 交接 -{用户指出 agent 绕路、做错阶段、没有按 skill 行为执行的原话和前后动作。} - -## 疑似根因 - -{规则缺口 / 脚本缺口 / 分发缓存 / agent 执行偏差 / 需要更多样本。} - -## 建议修改 - -- {改哪个 skill / reference / script / test} +- candidate: {path_or_not_requested} +- promotion_blockers: {list} +- 正式 fixture: {repo_local_promotion_result_or_not_ready} ## 上报状态 +- Public preview confirmed: {yes_or_no} - GitHub issue: {url_or_pending} - Manual fallback: {command_or_none} ``` ## `github-issue.md` +该正文只从 `public-issue-context.json` 的 allowlist 字段渲染；不要复制用户原话、报告路径或 +local-private 文件正文。 + ```markdown ## Summary -{一句话说明 CodeStable skill 使用问题。} +{sanitized one-line incident summary} -## User Feedback +## Incident -{用户原始反馈} - -## Evidence - -- Report: `{report_path}` -- Local private evidence: kept on the user's machine, not uploaded -- Matched events: {count} -- Public evidence fields: provider, session_label, timestamp_bucket, failure_type, match_type, tool_name, skill_or_reference, sanitized_excerpt - -## Suspected Area - -- Skill/reference/script/test: {paths_or_unknown} -- Failure type: {tool-failure|unclear-rule|agent-detour|goal-driver|install-distribution|privacy-reporting|unknown} - -## Context - -{只放 public allowlist 摘要；不贴完整 transcript、本机绝对路径、私有 repo 名、remote URL、环境变量、token、MCP/tool 原始 JSON 参数或大段业务代码。} +- Kind: {incident_kind} +- Target skill: {target_skill} +- Stage hint: {stage_hint} ## Expected Behavior -{用户期望或报告推断出的正确行为。} +{public expected_behavior or unknown} + +## Actual Behavior + +{public actual_behavior or unknown} + +## Impact + +{public impact or unknown} ## Proposed Fix -{建议补规则、脚本或测试。} +{public proposed_fix or unknown} + +## Evidence + +- Matched public events: {count} +- Public evidence fields: provider, session_label, timestamp_bucket, failure_type, match_type, + tool_name, skill_or_reference, sanitized_excerpt, incident_kind, target_skill, stage_hint, + expected_behavior, actual_behavior, impact, proposed_fix +- Local private evidence remains on the user's machine and is not uploaded. ``` + +生成后再次确认：正文不含完整 transcript、本机绝对路径、repo/remote、环境变量、secret、 +原始 MCP/tool JSON 参数、代码块或大段业务代码。 diff --git a/plugins/codestable/skills/cs-feedback/scripts/collect_feedback_context.py b/plugins/codestable/skills/cs-feedback/scripts/collect_feedback_context.py index f7a88db..4a4855c 100644 --- a/plugins/codestable/skills/cs-feedback/scripts/collect_feedback_context.py +++ b/plugins/codestable/skills/cs-feedback/scripts/collect_feedback_context.py @@ -1,453 +1,142 @@ #!/usr/bin/env python3 -"""Collect local Codex/Claude history snippets for CodeStable feedback.""" +"""Collect local Codex/Claude history for a CodeStable feedback evidence package.""" from __future__ import annotations import argparse -import hashlib import json -import re -import time -from dataclasses import asdict, dataclass +import os +import sys +import tempfile +from dataclasses import asdict from pathlib import Path from typing import Any +sys.dont_write_bytecode = True +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) -CS_PATTERN = re.compile(r"(?:\b(?:cs-[a-z0-9-]+|codestable)\b|\.codestable\b|/goal\b)", re.IGNORECASE) -FAILURE_PATTERN = re.compile( - r"(failed|failure|error|exception|traceback|timeout|timed out|permission|denied|not found|" - r"no such file|tool call|apply_patch|file read|read failed|mcp|paseo|gh issue|git clone|early EOF)", - re.IGNORECASE, +from feedback_incidents import ( # noqa: E402,F401 + build_incident_payload, + collect_file, + failure_type_for, + feedback_tokens, + incident_kind_for, + is_relevant_event, + match_types_for, + public_incident, + public_summary_for, + records_through_trigger, + score_text, + skill_reference_from, + timestamp_bucket, + tool_name_from, ) -USER_CORRECTION_PATTERN = re.compile( - r"(不对|不是|应该|你没有|你刚才|绕|错|确认后|没有用|没用|wrong|should have|" - r"you didn't|not what|instead)", - re.IGNORECASE, +from feedback_models import ( # noqa: E402,F401 + Event, + PUBLIC_EVENT_FIELDS, + PUBLIC_INCIDENT_FIELDS, + V1_FAILURE_MAP, ) -GOAL_PATTERN = re.compile(r"/goal|CS_FEATURE_GOAL_|CS_ROADMAP_GOAL_|goal driver|handoff", re.IGNORECASE) -INSTALL_PATTERN = re.compile(r"(plugin|marketplace|install|update|cache|version|codex|claude)", re.IGNORECASE) -PATH_PATTERN = re.compile(r"(?:~[/\\][^\s`'\"<>]+|/(?:[^\s`'\"<>/]+/)+[^\s`'\"<>]+|[A-Za-z]:\\[^\s`'\"<>]+)") -URL_PATTERN = re.compile(r"(?:https?|ssh|git)://[^\s`'\"<>]+") -EMAIL_PATTERN = re.compile(r"[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}") -REMOTE_PATTERN = re.compile(r"(?:[\w.+-]+@[\w.-]+:[^\s`'\"<>]+)") -SECRET_PATTERN = re.compile( - r"(?i)(api[_-]?key|token|secret|password|authorization|bearer)\s*[:=]\s*['\"]?([A-Za-z0-9._~+/=-]{8,})" +from feedback_repo_context import session_label # noqa: E402 +from feedback_transcripts import ( # noqa: E402,F401 + discover_files, + normalize_records, + provider_from_path, + read_transcript_snapshot, + session_id_from, +) +from feedback_triage import ( # noqa: E402 + accept_pending_incident, + build_triage, + merge_existing_triage, ) -FEEDBACK_TOKEN_STOPWORDS = { - "agent", - "call", - "current", - "error", - "failed", - "failure", - "file", - "read", - "rule", - "session", - "should", - "tool", - "unclear", -} -@dataclass -class Event: - provider: str - session: str - path: str - timestamp: str - kind: str - score: int - reasons: list[str] - match_types: list[str] - public_summary: dict[str, str] - text: str - context: list[str] +def _load_existing_triage(path: Path) -> dict[str, Any] | None: + if not path.exists(): + return None + if not path.is_file(): + raise ValueError(f"existing triage is not a file: {path}") + try: + loaded = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"cannot read existing triage: {exc}") from exc + if not isinstance(loaded, dict): + raise ValueError("existing triage must be a JSON object") + if loaded.get("schema_version") != 2 or loaded.get("privacy") != "local-private": + raise ValueError("existing triage must be schema v2 local-private") + for key in ("target", "assessment", "reproduction", "privacy_review"): + if not isinstance(loaded.get(key), dict): + raise ValueError(f"existing triage {key} must be a JSON object") + return loaded -@dataclass(frozen=True) -class Candidate: - path: str - provider: str - session: str - cwd: str - mtime: float - score: int - - -def redact(text: str, limit: int = 1200) -> str: - text = SECRET_PATTERN.sub(lambda match: f"{match.group(1)}=", text) - text = re.sub(r"sk-[A-Za-z0-9]{20,}", "sk-", text) - text = re.sub(r"gh[pousr]_[A-Za-z0-9_]{20,}", "gh_", text) - text = text.replace("\x00", "") - if len(text) > limit: - return text[:limit] + "..." - return text - - -def public_redact(text: str, limit: int = 300) -> str: - text = redact(text, limit=limit * 2) - text = REMOTE_PATTERN.sub("", text) - text = URL_PATTERN.sub("", text) - text = PATH_PATTERN.sub("", text) - text = EMAIL_PATTERN.sub("", text) - text = re.sub(r"```.*?```", "", text, flags=re.DOTALL) - text = re.sub(r"\s+", " ", text).strip() - if len(text) > limit: - return text[:limit] + "..." - return text - - -def flatten(value: Any) -> str: - if value is None: - return "" - if isinstance(value, str): - return value - if isinstance(value, list): - return "\n".join(flatten(item) for item in value) - if isinstance(value, dict): - parts: list[str] = [] - for key in ("message", "text", "output", "content", "arguments", "name", "type", "role"): - if key in value: - parts.append(flatten(value[key])) - if parts: - return "\n".join(part for part in parts if part) - return json.dumps(value, ensure_ascii=False, sort_keys=True) - return str(value) - - -def event_text(record: dict[str, Any]) -> str: - payload = record.get("payload", record) - return flatten(payload) - - -def event_kind(record: dict[str, Any]) -> str: - payload = record.get("payload") - if isinstance(payload, dict): - for key in ("type", "name", "role"): - if payload.get(key): - return str(payload[key]) - return str(record.get("type", "unknown")) - - -def score_text(text: str, feedback: str) -> tuple[int, list[str]]: - score = 0 - reasons: list[str] = [] - if CS_PATTERN.search(text): - score += 2 - reasons.append("codestable") - if FAILURE_PATTERN.search(text): - score += 3 - reasons.append("failure") - if USER_CORRECTION_PATTERN.search(text): - score += 3 - reasons.append("user-correction") - for token in feedback_tokens(feedback): - if token.lower() in text.lower(): - score += 1 - if "feedback-token" not in reasons: - reasons.append("feedback-token") - return score, reasons - - -def feedback_tokens(feedback: str) -> list[str]: - tokens: list[str] = [] - for token in re.findall(r"[A-Za-z0-9_-]{4,}", feedback): - normalized = token.lower() - if normalized in FEEDBACK_TOKEN_STOPWORDS: - continue - if normalized.startswith("cs-") or "-" in normalized or len(normalized) >= 6: - tokens.append(token) - return tokens - - -def match_types_for(text: str) -> list[str]: - match_types: list[str] = [] - if FAILURE_PATTERN.search(text): - match_types.append("tool-failure") - if GOAL_PATTERN.search(text): - match_types.append("goal-driver") - if USER_CORRECTION_PATTERN.search(text): - match_types.append("user-correction") - if CS_PATTERN.search(text): - match_types.append("skill-reference") - if INSTALL_PATTERN.search(text): - match_types.append("install-distribution") - return match_types - - -def is_relevant_event(match_types: list[str], reasons: list[str]) -> bool: - if not match_types: - return False - if any(match_type in match_types for match_type in ("skill-reference", "user-correction", "goal-driver", "install-distribution")): - return True - return "tool-failure" in match_types and "feedback-token" in reasons - - -def failure_type_for(match_types: list[str], text: str) -> str: - if "goal-driver" in match_types: - return "goal-driver" - if "tool-failure" in match_types: - return "tool-failure" - if "install-distribution" in match_types: - return "install-distribution" - if "user-correction" in match_types: - if re.search(r"(规则|没讲清|unclear|should have|应该|没有用|没用)", text, re.IGNORECASE): - return "unclear-rule" - return "agent-detour" - return "unknown" - - -def skill_reference_from(text: str) -> str: - match = re.search(r"\b(cs-[a-z0-9-]+)(?:/(references/[^\s`'\"<>]+\.md|scripts/[^\s`'\"<>]+\.py))?", text, re.IGNORECASE) - if not match: - return "unknown" - skill = match.group(1) - rel = match.group(2) - return f"{skill}/{rel}" if rel else skill - - -def tool_name_from(record: dict[str, Any], text: str) -> str: - payload = record.get("payload") - if isinstance(payload, dict): - name = payload.get("name") or payload.get("tool_name") or payload.get("tool") - if name: - return public_redact(str(name), limit=80) - for candidate in ("apply_patch", "read_file", "git", "gh", "paseo", "mcp"): - if candidate in text.lower(): - return candidate - return "unknown" - - -def session_label(session: str) -> str: - digest = hashlib.sha256(session.encode("utf-8")).hexdigest()[:10] - return f"session-{digest}" - - -def timestamp_bucket(timestamp: str) -> str: - if not timestamp: - return "unknown" - day = timestamp[:10] if len(timestamp) >= 10 else timestamp - hour_match = re.search(r"T(\d{2})", timestamp) - if not hour_match: - return day - hour = int(hour_match.group(1)) - if hour < 6: - part = "night" - elif hour < 12: - part = "morning" - elif hour < 18: - part = "afternoon" - else: - part = "evening" - return f"{day} {part}" - - -def public_summary_for(record: dict[str, Any], provider: str, session: str, timestamp: str, text: str, match_types: list[str]) -> dict[str, str]: - return { - "provider": provider, - "session_label": session_label(session), - "timestamp_bucket": timestamp_bucket(timestamp), - "failure_type": failure_type_for(match_types, text), - "match_type": ",".join(match_types), - "tool_name": tool_name_from(record, text), - "skill_or_reference": skill_reference_from(text), - "sanitized_excerpt": public_redact(text), - } - - -def normalize_json_records(value: Any) -> list[dict[str, Any]]: - if isinstance(value, list): - return [item if isinstance(item, dict) else {"payload": item} for item in value] - if not isinstance(value, dict): - return [{"payload": value}] - - collection_keys = ("messages", "events", "entries", "items", "transcript") - records: list[dict[str, Any]] = [] - meta = {key: item for key, item in value.items() if key not in collection_keys} - if meta: - records.append(meta) - for key in collection_keys: - items = value.get(key) - if not isinstance(items, list): - continue - for item in items: - records.append(item if isinstance(item, dict) else {"payload": item}) - return records or [value] - - -def read_records(path: Path) -> list[dict[str, Any]]: - if path.suffix == ".json": - try: - value = json.loads(path.read_text(encoding="utf-8", errors="ignore")) - except json.JSONDecodeError: - return [] - return normalize_json_records(value) - - records: list[dict[str, Any]] = [] - with path.open(encoding="utf-8", errors="ignore") as handle: - for line in handle: - line = line.strip() - if not line: - continue - try: - value = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(value, dict): - records.append(value) - return records - - -def session_id_from(path: Path, records: list[dict[str, Any]]) -> str: - for record in records: - payload = record.get("payload") - if isinstance(payload, dict): - session_id = payload.get("session_id") or payload.get("sessionId") or payload.get("id") - if session_id: - return str(session_id) - session_id = record.get("session_id") or record.get("sessionId") or record.get("sessionid") or record.get("id") - if session_id: - return str(session_id) - return path.stem - - -def cwd_from(records: list[dict[str, Any]]) -> str: - for record in records: - payload = record.get("payload") - if isinstance(payload, dict) and payload.get("cwd"): - return str(payload["cwd"]) - if record.get("cwd"): - return str(record["cwd"]) - return "" - - -def provider_from_path(path: Path) -> str: - text = str(path) - if ".codex" in text: - return "codex" - if ".claude" in text: - return "claude" - return "unknown" - - -def collect_file(path: Path, feedback: str, max_events: int, context_window: int) -> list[Event]: - records = read_records(path) - if not records: - return [] - provider = provider_from_path(path) - session = session_id_from(path, records) - texts = [redact(event_text(record), limit=800) for record in records] - events: list[Event] = [] - for index, record in enumerate(records): - text = texts[index] - score, reasons = score_text(text, feedback) - match_types = match_types_for(text) - if not is_relevant_event(match_types, reasons): - continue - start = max(0, index - context_window) - end = min(len(texts), index + context_window + 1) - timestamp = str(record.get("timestamp") or record.get("created_at") or "") - summary = public_summary_for(record, provider, session, timestamp, text, match_types) - events.append( - Event( - provider=provider, - session=session, - path=str(path), - timestamp=timestamp, - kind=event_kind(record), - score=score, - reasons=reasons, - match_types=match_types, - public_summary=summary, - text=text, - context=[texts[pos] for pos in range(start, end)], +def _write_text_files_atomically(files: list[tuple[Path, str]]) -> None: + staged: list[tuple[Path, Path]] = [] + backups: dict[Path, Path | None] = {} + replaced: list[Path] = [] + try: + for target, text in files: + target.parent.mkdir(parents=True, exist_ok=True) + handle = tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=target.parent, + prefix=f".{target.name}.", + suffix=".tmp", + delete=False, ) - ) - events.sort(key=lambda event: event.score, reverse=True) - return events[:max_events] - - -def candidate_for(path: Path, cwd: str | None) -> Candidate: - records = read_records(path) - session = session_id_from(path, records) - transcript_cwd = cwd_from(records) - score = 0 - if cwd and transcript_cwd == cwd: - score += 5 - elif cwd and transcript_cwd and (cwd.startswith(transcript_cwd) or transcript_cwd.startswith(cwd)): - score += 2 - score += int(path.stat().st_mtime // 60) - return Candidate( - path=str(path), - provider=provider_from_path(path), - session=session, - cwd=transcript_cwd, - mtime=path.stat().st_mtime, - score=score, - ) - - -def resolve_current_session(files: list[Path], cwd: str | None) -> tuple[list[Path], list[Candidate]]: - candidates = [candidate_for(path, cwd) for path in files if path.suffix in {".jsonl", ".json"}] - candidates.sort(key=lambda candidate: candidate.score, reverse=True) - if not candidates: - return [], [] - if cwd: - exact = [candidate for candidate in candidates if candidate.cwd == cwd] - if len(exact) == 1: - return [Path(exact[0].path)], [] - if len(exact) > 1: - return [], exact[:5] - containing = [ - candidate - for candidate in candidates - if candidate.cwd and (cwd.startswith(candidate.cwd) or candidate.cwd.startswith(cwd)) - ] - if len(containing) == 1: - return [Path(containing[0].path)], [] - if len(containing) > 1: - return [], containing[:5] - return [], candidates[:5] - - -def discover_files( - home: Path, - since_days: int, - session_filter: str | None, - cwd: str | None, -) -> tuple[list[Path], list[Candidate]]: - roots = [ - home / ".codex/sessions", - home / ".claude/projects", - home / ".claude/sessions", - ] - if session_filter and session_filter != "current": - candidate = Path(session_filter).expanduser() - if candidate.is_file(): - return [candidate], [] - cutoff = time.time() - since_days * 86400 - files: list[Path] = [] - for root in roots: - if not root.exists(): - continue - for path in root.rglob("*"): - if not path.is_file() or path.suffix not in {".jsonl", ".json"}: + temporary = Path(handle.name) + staged.append((temporary, target)) + with handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + for _temporary, target in staged: + if not target.exists(): + backups[target] = None continue - if path.stat().st_mtime < cutoff: - continue - if session_filter and session_filter != "current": - if session_filter in path.name or session_filter in str(path): - files.append(path) - continue - records = read_records(path) - if session_filter not in session_id_from(path, records): - continue - files.append(path) - files = sorted(files) - if session_filter == "current": - return resolve_current_session(files, cwd) - return files, [] + handle = tempfile.NamedTemporaryFile( + mode="wb", + dir=target.parent, + prefix=f".{target.name}.rollback.", + suffix=".tmp", + delete=False, + ) + backup = Path(handle.name) + backups[target] = backup + with handle: + handle.write(target.read_bytes()) + handle.flush() + os.fsync(handle.fileno()) + try: + for temporary, target in staged: + temporary.replace(target) + replaced.append(target) + except OSError as exc: + rollback_errors: list[str] = [] + for target in reversed(replaced): + backup = backups[target] + try: + if backup is None: + target.unlink(missing_ok=True) + else: + os.replace(backup, target) + except OSError as rollback_exc: + rollback_errors.append(f"{target}: {rollback_exc}") + if rollback_errors: + raise OSError( + "feedback output rollback failed: " + "; ".join(rollback_errors) + ) from exc + raise + finally: + for temporary, _target in staged: + temporary.unlink(missing_ok=True) + for backup in backups.values(): + if backup is not None: + backup.unlink(missing_ok=True) def main_with_args_for_test(argv: list[str] | None = None) -> int: @@ -456,59 +145,156 @@ def main_with_args_for_test(argv: list[str] | None = None) -> int: parser.add_argument("--since-days", type=int, default=3) parser.add_argument("--session", default=None, help="Session id substring or transcript path") parser.add_argument("--output", required=True) + parser.add_argument("--triage-output", default=None, help="Write local-private triage JSON") parser.add_argument("--public-output", default=None, help="Write public allowlist context JSON") parser.add_argument("--history-root", default=None, help="Override home directory for tests") parser.add_argument("--cwd", default=None, help="Current working directory, used by --session current") parser.add_argument("--max-events-per-file", type=int, default=5) parser.add_argument("--context-window", type=int, default=2) + parser.add_argument( + "--accept-incident", + default=None, + help="Explicitly accept the current pending primary incident", + ) args = parser.parse_args(argv) + output = Path(args.output).expanduser() + triage_output = ( + Path(args.triage_output).expanduser() + if args.triage_output + else output.with_name("triage.json") + ) + public_output = ( + Path(args.public_output).expanduser() + if args.public_output + else output.with_name("public-issue-context.json") + ) + try: + resolved_outputs = [path.resolve() for path in (output, triage_output, public_output)] + except OSError as exc: + print(f"feedback output blocked: {exc}", file=sys.stderr) + return 2 + if len(set(resolved_outputs)) != len(resolved_outputs): + print("feedback output blocked: output paths must be distinct", file=sys.stderr) + return 2 + try: + existing_triage = _load_existing_triage(triage_output) + except ValueError as exc: + print(f"feedback output blocked: {exc}", file=sys.stderr) + return 2 + home = Path(args.history_root).expanduser() if args.history_root else Path.home() cwd = str(Path(args.cwd).expanduser()) if args.cwd else None files, ambiguity = discover_files(home, args.since_days, args.session, cwd) + + records_by_path: dict[Path, list[dict[str, Any]]] = {} + captures_by_path: dict[Path, dict[str, Any]] = {} + for path in files: + records_by_path[path], captures_by_path[path] = read_transcript_snapshot(path) + events: list[Event] = [] for path in files: - events.extend(collect_file(path, args.feedback, args.max_events_per_file, args.context_window)) + events.extend( + collect_file( + path, + args.feedback, + args.max_events_per_file, + args.context_window, + records_through_trigger(path, records_by_path[path]), + ) + ) events.sort(key=lambda event: (event.score, event.timestamp), reverse=True) + incidents, primary_incident = build_incident_payload( + files, + args.feedback, + cwd, + records_by_path, + captures_by_path, + ) + + generated_triage = build_triage(incidents, primary_incident) + try: + triage = ( + accept_pending_incident( + generated_triage, existing_triage, args.accept_incident + ) + if args.accept_incident + else merge_existing_triage(generated_triage, existing_triage) + ) + except ValueError as exc: + print(f"incident acceptance blocked: {exc}", file=sys.stderr) + return 2 + quality = triage.get("quality") + public_projection_ready = ( + primary_incident is not None + and triage.get("incident_id") == primary_incident.get("id") + and isinstance(quality, dict) + and quality.get("triage_ready") is True + ) + public_incidents: list[dict[str, str]] = [] + if public_projection_ready: + for incident in incidents: + incident_triage = ( + triage + if incident.get("id") == triage.get("incident_id") + else build_triage([incident], incident) + ) + public_incidents.append(public_incident(incident, incident_triage)) public_issue_context = { "privacy": "public-preview", "source": "derived-from-local-private-evidence", - "allowed_fields": [ - "provider", - "session_label", - "timestamp_bucket", - "failure_type", - "match_type", - "tool_name", - "skill_or_reference", - "sanitized_excerpt", - "expected_behavior", - "actual_behavior", - "proposed_fix", - ], - "events": [event.public_summary for event in events[:8]], + "allowed_fields": list( + dict.fromkeys(PUBLIC_EVENT_FIELDS + PUBLIC_INCIDENT_FIELDS) + ), + "events": ( + [event.public_summary for event in events[:8]] + if public_projection_ready + else [] + ), + "incidents": public_incidents, } payload = { + "schema_version": 2, "feedback": args.feedback, "privacy": "local-private", "public_upload_allowed": False, "redaction": "best-effort", "since_days": args.since_days, + "since_days_ignored": args.session == "current", "session_filter": args.session, "history_root": str(home), "cwd": cwd, "searched_files": [str(path) for path in files], "ambiguity": {"candidates": [asdict(candidate) for candidate in ambiguity]}, + "captures": [ + { + "provider": provider_from_path(path), + "session_label": session_label( + session_id_from(path, records_by_path[path]) + ), + **captures_by_path[path], + } + for path in files + ], "matched_events": [asdict(event) for event in events], + "incidents": incidents, "public_issue_context": public_issue_context, } - output = Path(args.output) - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - public_output = Path(args.public_output).expanduser() if args.public_output else output.with_name("public-issue-context.json") - public_output.parent.mkdir(parents=True, exist_ok=True) - public_output.write_text(json.dumps(public_issue_context, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + try: + _write_text_files_atomically( + [ + (triage_output, json.dumps(triage, ensure_ascii=False, indent=2) + "\n"), + (output, json.dumps(payload, ensure_ascii=False, indent=2) + "\n"), + ( + public_output, + json.dumps(public_issue_context, ensure_ascii=False, indent=2) + "\n", + ), + ] + ) + except OSError as exc: + print(f"feedback output blocked: {exc}", file=sys.stderr) + return 2 return 0 diff --git a/plugins/codestable/skills/cs-feedback/scripts/feedback_incidents.py b/plugins/codestable/skills/cs-feedback/scripts/feedback_incidents.py new file mode 100644 index 0000000..57ea785 --- /dev/null +++ b/plugins/codestable/skills/cs-feedback/scripts/feedback_incidents.py @@ -0,0 +1,507 @@ +from __future__ import annotations + +import re +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from feedback_models import ( + Event, + FeedbackIncident, + NormalizedRecord, + PUBLIC_INCIDENT_FIELDS, + V1_FAILURE_MAP, +) +from feedback_privacy import public_redact, redact +from feedback_repo_context import build_repo_context, environment_context, session_label +from feedback_transcripts import ( + event_kind, + event_text, + normalize_records, + provider_from_path, + read_transcript_snapshot, + session_id_from, +) + + +CS_PATTERN = re.compile( + r"(?:\b(?:cs-[a-z0-9-]+|codestable)\b|\.codestable\b|/goal\b)", re.IGNORECASE +) +FAILURE_PATTERN = re.compile( + r"(failed|failure|error|exception|traceback|timeout|timed out|permission|denied|not found|" + r"no such file|tool call|apply_patch|file read|read failed|mcp|paseo|gh issue|git clone|early EOF)", + re.IGNORECASE, +) +USER_CORRECTION_PATTERN = re.compile( + r"(不对|不是|应该|应当|需要|必须|你没有|你刚才|绕|错|确认后|没有用|没用|wrong|should have|" + r"you didn't|not what|instead)", + re.IGNORECASE, +) +GOAL_PATTERN = re.compile( + r"/goal|CS_FEATURE_GOAL_|CS_ROADMAP_GOAL_|goal driver|handoff", re.IGNORECASE +) +INSTALL_PATTERN = re.compile( + r"(plugin|marketplace|install|update|cache|version|codex|claude)", re.IGNORECASE +) +FEEDBACK_TOKEN_STOPWORDS = { + "agent", + "call", + "current", + "error", + "failed", + "failure", + "file", + "read", + "rule", + "session", + "should", + "tool", + "unclear", +} + + +def score_text(text: str, feedback: str) -> tuple[int, list[str]]: + score = 0 + reasons: list[str] = [] + if CS_PATTERN.search(text): + score += 2 + reasons.append("codestable") + if FAILURE_PATTERN.search(text): + score += 3 + reasons.append("failure") + if USER_CORRECTION_PATTERN.search(text): + score += 3 + reasons.append("user-correction") + for token in feedback_tokens(feedback): + if token.lower() in text.lower(): + score += 1 + if "feedback-token" not in reasons: + reasons.append("feedback-token") + return score, reasons + + +def feedback_tokens(feedback: str) -> list[str]: + tokens: list[str] = [] + for token in re.findall(r"[A-Za-z0-9_-]{4,}", feedback): + normalized = token.lower() + if normalized in FEEDBACK_TOKEN_STOPWORDS: + continue + if normalized.startswith("cs-") or "-" in normalized or len(normalized) >= 6: + tokens.append(token) + return tokens + + +def match_types_for(text: str) -> list[str]: + match_types: list[str] = [] + if FAILURE_PATTERN.search(text): + match_types.append("tool-failure") + if GOAL_PATTERN.search(text): + match_types.append("goal-driver") + if USER_CORRECTION_PATTERN.search(text): + match_types.append("user-correction") + if CS_PATTERN.search(text): + match_types.append("skill-reference") + if INSTALL_PATTERN.search(text): + match_types.append("install-distribution") + return match_types + + +def is_relevant_event(match_types: list[str], reasons: list[str]) -> bool: + if not match_types: + return False + if any( + match_type in match_types + for match_type in ( + "skill-reference", + "user-correction", + "goal-driver", + "install-distribution", + ) + ): + return True + return "tool-failure" in match_types and "feedback-token" in reasons + + +def incident_kind_for(match_types: list[str], text: str) -> str: + if re.search(r"(privacy|隐私|上传|public preview|reporting)", text, re.IGNORECASE): + return "privacy-reporting" + if re.search(r"(missing artifact|缺少.{0,8}(?:产物|文件)|未生成)", text, re.IGNORECASE): + return "missing-artifact" + if re.search(r"(绕路|多余|unnecessary|detour)", text, re.IGNORECASE): + return "unnecessary-detour" + if "user-correction" in match_types and re.search( + r"(gate|review|确认后|跳过|skipped)", text, re.IGNORECASE + ): + return "skipped-gate" + if "goal-driver" in match_types: + return "goal-driver" + if "tool-failure" in match_types: + return "tool-failure" + if "install-distribution" in match_types: + return "install-version" + if "user-correction" in match_types: + if re.search( + r"(规则|没讲清|unclear|should have|应该|没有用|没用)", text, re.IGNORECASE + ): + return "unclear-rule" + return "wrong-route" + return "unknown" + + +def failure_type_for(match_types: list[str], text: str) -> str: + return V1_FAILURE_MAP.get(incident_kind_for(match_types, text), "unknown") + + +def skill_reference_from(text: str) -> str: + match = re.search( + r"\b(cs-[a-z0-9-]+)(?:/(references/[^\s`'\"<>]+\.md|scripts/[^\s`'\"<>]+\.py))?", + text, + re.IGNORECASE, + ) + if not match: + return "unknown" + skill = match.group(1) + rel = match.group(2) + return f"{skill}/{rel}" if rel else skill + + +def tool_name_from(record: dict[str, Any], text: str) -> str: + payload = record.get("payload") + if isinstance(payload, dict): + name = payload.get("name") or payload.get("tool_name") or payload.get("tool") + if name: + return public_redact(str(name), limit=80) + for candidate in ("apply_patch", "read_file", "git", "gh", "paseo", "mcp"): + if candidate in text.lower(): + return candidate + return "unknown" + + +def timestamp_bucket(timestamp: str) -> str: + if not timestamp: + return "unknown" + day = timestamp[:10] if len(timestamp) >= 10 else timestamp + hour_match = re.search(r"T(\d{2})", timestamp) + if not hour_match: + return day + hour = int(hour_match.group(1)) + if hour < 6: + part = "night" + elif hour < 12: + part = "morning" + elif hour < 18: + part = "afternoon" + else: + part = "evening" + return f"{day} {part}" + + +def public_summary_for( + record: dict[str, Any], + provider: str, + session: str, + timestamp: str, + text: str, + match_types: list[str], +) -> dict[str, str]: + return { + "provider": provider, + "session_label": session_label(session), + "timestamp_bucket": timestamp_bucket(timestamp), + "failure_type": failure_type_for(match_types, text), + "match_type": ",".join(match_types), + "tool_name": tool_name_from(record, text), + "skill_or_reference": skill_reference_from(text), + "sanitized_excerpt": public_redact(text), + } + + +def collect_file( + path: Path, + feedback: str, + max_events: int, + context_window: int, + records: list[dict[str, Any]] | None = None, +) -> list[Event]: + if records is None: + records, _capture = read_transcript_snapshot(path) + if not records: + return [] + provider = provider_from_path(path) + session = session_id_from(path, records) + texts = [redact(event_text(record), limit=800) for record in records] + events: list[Event] = [] + for index, record in enumerate(records): + text = texts[index] + score, reasons = score_text(text, feedback) + match_types = match_types_for(text) + if not is_relevant_event(match_types, reasons): + continue + start = max(0, index - context_window) + end = min(len(texts), index + context_window + 1) + timestamp = str(record.get("timestamp") or record.get("created_at") or "") + summary = public_summary_for(record, provider, session, timestamp, text, match_types) + events.append( + Event( + provider=provider, + session=session, + path=str(path), + timestamp=timestamp, + kind=event_kind(record), + score=score, + reasons=reasons, + match_types=match_types, + public_summary=summary, + text=text, + context=[texts[pos] for pos in range(start, end)], + ) + ) + events.sort(key=lambda event: event.score, reverse=True) + return events[:max_events] + + +def _trigger_cutoff(records: list[NormalizedRecord]) -> int | None: + for index in range(len(records) - 1, -1, -1): + if records[index].role == "user": + return index + return None + + +def public_incident( + incident: dict[str, object], triage: dict[str, object] +) -> dict[str, str]: + assessment = ( + triage.get("assessment", {}) if isinstance(triage.get("assessment"), dict) else {} + ) + + def value(name: str) -> str: + item = assessment.get(name, {}) + if isinstance(item, dict): + return public_redact(str(item.get("value") or "unknown")) + return "unknown" + + projected = { + "incident_kind": public_redact(str(incident.get("incident_kind") or "unknown")), + "target_skill": public_redact(str(incident.get("target_skill") or "unknown")), + "stage_hint": public_redact(str(incident.get("stage_hint") or "unknown")), + "expected_behavior": value("expected_behavior"), + "actual_behavior": value("actual_behavior"), + "impact": value("impact"), + "proposed_fix": value("proposed_fix"), + } + return {field: projected[field] for field in PUBLIC_INCIDENT_FIELDS} + + +def _incident_windows( + records: list[NormalizedRecord], cutoff: int | None +) -> list[list[NormalizedRecord]]: + eligible = records[: cutoff + 1] if cutoff is not None else records + if cutoff is None: + return [eligible] if eligible else [] + windows: list[list[NormalizedRecord]] = [] + start = 0 + for index, record in enumerate(eligible): + if record.role == "user": + windows.append(eligible[start : index + 1]) + start = index + 1 + return windows + + +def _merge_correlated_windows( + windows: list[list[NormalizedRecord]], +) -> list[list[NormalizedRecord]]: + merged: list[list[NormalizedRecord]] = [] + for window in windows: + correlations = {record.correlation_id for record in window if record.correlation_id} + merge_at = [ + index + for index, existing in enumerate(merged) + if correlations + & {record.correlation_id for record in existing if record.correlation_id} + ] + if not merge_at: + merged.append(window) + else: + insert_at = merge_at[0] + combined = [*window] + for index in reversed(merge_at): + combined.extend(merged.pop(index)) + unique = { + (record.provider, record.session, record.id): record for record in combined + } + merged.insert( + insert_at, + sorted( + unique.values(), + key=lambda record: int(record.id.rsplit("-", 1)[-1]), + ), + ) + return merged + + +INCIDENT_KIND_PRIORITY = [ + "privacy-reporting", + "skipped-gate", + "missing-artifact", + "wrong-route", + "unnecessary-detour", + "goal-driver", + "tool-failure", + "install-version", + "unclear-rule", +] + + +def _incident_kind(records: list[NormalizedRecord]) -> str: + kinds = { + incident_kind_for(match_types_for(record.text), record.text) for record in records + } + return next((kind for kind in INCIDENT_KIND_PRIORITY if kind in kinds), "unknown") + + +def _incident_from_window( + window: list[NormalizedRecord], + feedback: str, + incident_number: int, + observation_start: int, + environment: dict[str, object], + repo_context: dict[str, object], +) -> tuple[dict[str, object], int] | None: + meaningful = [record for record in window if record.record_type != "session_meta"] + relevant = [] + for record in meaningful: + score, reasons = score_text(record.text, feedback) + match_types = match_types_for(record.text) + if is_relevant_event(match_types, reasons): + relevant.append(record) + if not relevant: + return None + + correction_records = [ + record + for record in meaningful + if record.role == "user" and USER_CORRECTION_PATTERN.search(record.text) + ] + target_skill = "unknown" + for record in [*reversed(correction_records), *relevant]: + target_skill = skill_reference_from(record.text) + if target_skill != "unknown": + target_skill = target_skill.split("/", 1)[0] + break + stage_hint = "unknown" + for record in [*reversed(correction_records), *relevant]: + stage_match = re.search( + r"\b(design-review|design|review|qa|acceptance|implementation|goal)\b", + record.text, + re.IGNORECASE, + ) + if stage_match: + stage_hint = stage_match.group(1).lower() + break + + observations: list[dict[str, object]] = [] + for offset, record in enumerate(meaningful): + observations.append( + { + "id": f"obs-{observation_start + offset:04d}", + "record_id": record.id, + "source_index": record.source_index, + "role": record.role, + "record_type": record.record_type, + "text": record.text, + } + ) + obs_by_record = { + str(observation["record_id"]): str(observation["id"]) + for observation in observations + } + timeline = [ + { + "record_id": record.id, + "role": record.role, + "record_type": record.record_type, + "tool_name": record.tool_name, + "correlation_id": record.correlation_id, + "correlation_source": record.correlation_source, + "observation_id": obs_by_record.get(record.id, ""), + } + for record in meaningful + ] + correction_ids = {record.id for record in correction_records} + user_correction = next( + ( + observation + for observation in reversed(observations) + if observation["record_id"] in correction_ids + ), + {}, + ) + user_records = [record for record in meaningful if record.role == "user"] + incident = FeedbackIncident( + id=f"incident-{incident_number:02d}", + target_skill=target_skill, + stage_hint=stage_hint, + incident_kind=_incident_kind(relevant), + observations=observations, + timeline=timeline, + environment_context=environment, + repo_context=repo_context, + user_correction=user_correction, + capture_cutoff=user_records[-1].id if user_records else "unknown", + ) + return asdict(incident), observation_start + len(observations) + + +def build_incident_payload( + paths: list[Path], + feedback: str, + cwd: str | None, + records_by_path: dict[Path, list[dict[str, Any]]] | None = None, + captures_by_path: dict[Path, dict[str, Any]] | None = None, +) -> tuple[list[dict[str, object]], dict[str, object] | None]: + incidents: list[dict[str, object]] = [] + primary_candidates: list[dict[str, object]] = [] + observation_number = 1 + repo_context = build_repo_context(cwd) + for path in paths: + if records_by_path is not None and path in records_by_path: + raw_records = records_by_path[path] + capture = (captures_by_path or {}).get(path, {}) + else: + raw_records, capture = read_transcript_snapshot(path) + records = normalize_records(path, raw_records) + cutoff = _trigger_cutoff(records) + trigger_id = records[cutoff].id if cutoff is not None else None + windows = _merge_correlated_windows(_incident_windows(records, cutoff)) + environment = environment_context(path, raw_records, capture) + for window in windows: + built = _incident_from_window( + window, + feedback, + len(incidents) + 1, + observation_number, + environment, + repo_context, + ) + if built is None: + continue + incident, observation_number = built + incidents.append(incident) + if trigger_id and any( + item.get("record_id") == trigger_id + for item in incident.get("timeline", []) + if isinstance(item, dict) + ): + primary_candidates.append(incident) + primary = primary_candidates[0] if len(primary_candidates) == 1 else None + return incidents, primary + + +def records_through_trigger( + path: Path, records: list[dict[str, Any]] +) -> list[dict[str, Any]]: + normalized = normalize_records(path, records) + cutoff = _trigger_cutoff(normalized) + if cutoff is None: + return records + return records[: normalized[cutoff].source_index + 1] diff --git a/plugins/codestable/skills/cs-feedback/scripts/feedback_models.py b/plugins/codestable/skills/cs-feedback/scripts/feedback_models.py new file mode 100644 index 0000000..e63d249 --- /dev/null +++ b/plugins/codestable/skills/cs-feedback/scripts/feedback_models.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +V1_FAILURE_MAP = { + "wrong-route": "agent-detour", + "skipped-gate": "agent-detour", + "missing-artifact": "agent-detour", + "unnecessary-detour": "agent-detour", + "tool-failure": "tool-failure", + "goal-driver": "goal-driver", + "install-version": "install-distribution", + "unclear-rule": "unclear-rule", + "privacy-reporting": "unknown", + "unknown": "unknown", +} + +PUBLIC_EVENT_FIELDS = [ + "provider", + "session_label", + "timestamp_bucket", + "failure_type", + "match_type", + "tool_name", + "skill_or_reference", + "sanitized_excerpt", +] + +PUBLIC_INCIDENT_FIELDS = [ + "incident_kind", + "target_skill", + "stage_hint", + "expected_behavior", + "actual_behavior", + "impact", + "proposed_fix", +] + + +@dataclass +class Event: + provider: str + session: str + path: str + timestamp: str + kind: str + score: int + reasons: list[str] + match_types: list[str] + public_summary: dict[str, str] + text: str + context: list[str] + + +@dataclass(frozen=True) +class Candidate: + path: str + provider: str + session: str + cwd: str + mtime: float + score: int + + +@dataclass(frozen=True) +class SessionMeta: + session: str + cwd: str + + +@dataclass +class NormalizedRecord: + id: str + provider: str + session: str + timestamp: str + role: str + record_type: str + tool_name: str + correlation_id: str + correlation_source: str + text: str + source_index: int + + +@dataclass +class FeedbackIncident: + id: str + target_skill: str + stage_hint: str + incident_kind: str + observations: list[dict[str, object]] + timeline: list[dict[str, object]] + environment_context: dict[str, object] + repo_context: dict[str, object] + user_correction: dict[str, object] + capture_cutoff: str diff --git a/plugins/codestable/skills/cs-feedback/scripts/feedback_privacy.py b/plugins/codestable/skills/cs-feedback/scripts/feedback_privacy.py new file mode 100644 index 0000000..d52d88c --- /dev/null +++ b/plugins/codestable/skills/cs-feedback/scripts/feedback_privacy.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +import re + + +PATH_SEGMENT_PATTERN = ( + r"[^\s`'\"<>/\\,;:!?()\[\]{}，。；：！？、（）【】《》〈〉「」『』" + r"〔〕〖〗〘〙〚〛“”‘’…—·～]+" +) +PATH_PATTERN = re.compile( + rf"(?]+") +EMAIL_PATTERN = re.compile(r"[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}") +REMOTE_PATTERN = re.compile(r"(?:[\w.+-]+@[\w.-]+:[^\s`'\"<>]+)") +SECRET_KEY_PATTERN = re.compile( + r"""(?imx)['"]?(?Papi[_-]?key|token|secret|password|authorization|bearer)['"]? + \s*[:=：＝]\s*""" +) +AUTHORIZATION_HEADER_PATTERN = re.compile( + r"(?im)(?"), + (USER_CREDENTIAL_PATTERN, ""), + (AUTH_SCHEME_PATTERN, ""), +) +ENV_PATTERN = re.compile(r"\b[A-Z][A-Z0-9_]{2,}\s*=\s*[^\s`'\"<>]+") +ENV_NAME_PATTERN = re.compile(r"\b[A-Z][A-Z0-9_]{2,}\b") +INLINE_JSON_PATTERN = re.compile( + r"(?:\{[^{}]*\}|\[[^\[\]]*\])", re.DOTALL +) +JSON_DELIMITER_PATTERN = re.compile(r"[{}\[\]]") + + +def _quoted_segment_end(text: str, start: int) -> tuple[int, int]: + quote = text[start] + index = start + 1 + logical_length = 0 + while index < len(text): + char = text[index] + if char == "\\" and index + 1 < len(text): + if text[index + 1] == "\r" and index + 2 < len(text) and text[index + 2] == "\n": + index += 3 + elif text[index + 1] in "\r\n": + index += 2 + else: + logical_length += 1 + index += 2 + continue + if char == quote: + return index + 1, logical_length + if char not in "\r\n": + logical_length += 1 + index += 1 + return len(text), logical_length + + +def _shell_expansion_end(text: str, start: int) -> int: + stack = [")" if text.startswith("$(", start) else "}"] + index = start + 2 + while index < len(text) and stack: + char = text[index] + if char == "\\" and index + 1 < len(text): + if text[index + 1] == "\r" and index + 2 < len(text) and text[index + 2] == "\n": + index += 3 + else: + index += 2 + continue + if char in "'\"`": + index, _length = _quoted_segment_end(text, index) + continue + if text.startswith("$(", index): + stack.append(")") + index += 2 + continue + if text.startswith("${", index): + stack.append("}") + index += 2 + continue + opener = "(" if stack[-1] == ")" else "{" + if char == opener: + stack.append(stack[-1]) + elif char == stack[-1]: + stack.pop() + index += 1 + return index + + +def _secret_value_end(text: str, start: int) -> int | None: + placeholder_end = start + len("") + if text.startswith("", start) and ( + placeholder_end == len(text) or text[placeholder_end].isspace() + ): + return None + + index = start + logical_length = 0 + has_expansion = False + starts_quoted = ( + index < len(text) and text[index] in "'\"`" + ) or text.startswith(("$'", '$"'), index) + while index < len(text) and not text[index].isspace(): + char = text[index] + if text.startswith(("$(", "${"), index): + has_expansion = True + index = _shell_expansion_end(text, index) + continue + if char == "$" and index + 1 < len(text) and text[index + 1] in "'\"": + index += 1 + continue + if char in "'\"`": + index, segment_length = _quoted_segment_end(text, index) + logical_length += segment_length + continue + if char == "\\" and index + 1 < len(text): + if text[index + 1] == "\r" and index + 2 < len(text) and text[index + 2] == "\n": + index += 3 + elif text[index + 1] == "\n": + index += 2 + else: + logical_length += 1 + index += 2 + continue + logical_length += 1 + index += 1 + + minimum = 4 if starts_quoted else 6 + return index if has_expansion or logical_length >= minimum else None + + +def secret_assignment_spans(text: str) -> list[tuple[int, int, str]]: + spans: list[tuple[int, int, str]] = [] + cursor = 0 + while match := SECRET_KEY_PATTERN.search(text, cursor): + end = _secret_value_end(text, match.end()) + if end is None: + cursor = match.end() + continue + spans.append((match.start(), end, match.group("key"))) + cursor = end + return spans + + +def contains_secret_assignment(text: str) -> bool: + return bool(secret_assignment_spans(text)) + + +def redact_secret_assignments(text: str) -> str: + spans = secret_assignment_spans(text) + if not spans: + return text + pieces: list[str] = [] + cursor = 0 + for start, end, key in spans: + pieces.extend((text[cursor:start], f"{key}=")) + cursor = end + pieces.append(text[cursor:]) + return "".join(pieces) + + +def redact_credentials(text: str) -> str: + for pattern, replacement in CREDENTIAL_REDACTIONS: + text = pattern.sub(replacement, text) + return text + + +def redact(text: str, limit: int = 1200) -> str: + text = redact_credentials(text) + text = redact_secret_assignments(text) + text = re.sub(r"sk-[A-Za-z0-9]{20,}", "sk-", text) + text = re.sub(r"gh[pousr]_[A-Za-z0-9_]{20,}", "gh_", text) + text = text.replace("\x00", "") + if len(text) > limit: + return text[:limit] + "..." + return text + + +def redact_inline_json(text: str) -> str: + while True: + redacted, count = INLINE_JSON_PATTERN.subn("", text) + text = redacted + if count == 0: + break + if JSON_DELIMITER_PATTERN.search(text): + return "" + return text + + +def contains_inline_json(text: str) -> bool: + return bool(INLINE_JSON_PATTERN.search(text) or JSON_DELIMITER_PATTERN.search(text)) + + +def _filename_extension(value: str) -> str | None: + base, separator, extension = value.rpartition(".") + if not base or not separator or not PATH_EXTENSION_PATTERN.fullmatch(extension): + return None + if extension.isnumeric(): + return None + if not extension.isascii() and len(extension) > 3: + return None + return extension + + +def _has_cjk_path_glue(value: str, *, terminal_filename: bool = False) -> bool: + extension = _filename_extension(value) if terminal_filename else None + for transition in CJK_PATH_GLUE_PATTERN.finditer(value): + if extension is not None and value[transition.end()] == ".": + continue + return True + return False + + +def _absolute_path_end(text: str, match: re.Match[str]) -> int: + end = match.end() + while end > match.start() and text[end - 1] == ".": + end -= 1 + + opener = text[match.start() - 1] if match.start() else "" + closer = PATH_QUOTE_PAIRS.get(opener) + if closer: + close = text.find(closer, end) + if close >= 0: + return close + + while continuation := PATH_SPACED_CONTINUATION_PATTERN.match(text, end): + if _has_cjk_path_glue(continuation.group(1)): + break + end = continuation.end() + while end > match.start() and text[end - 1] == ".": + end -= 1 + + cursor = end + for _ in range(2): + word = PATH_SPACED_WORD_PATTERN.match(text, cursor) + if not word: + break + token_end = word.end(1) + while token_end > word.start(1) and text[token_end - 1] == ".": + token_end -= 1 + token = text[word.start(1):token_end] + if ( + _filename_extension(token) is not None + and not _has_cjk_path_glue(token, terminal_filename=True) + and (token_end == len(text) or text[token_end] not in "/\\") + ): + return token_end + cursor = word.end() + return end + + +def redact_absolute_paths(text: str) -> str: + pieces: list[str] = [] + cursor = 0 + for match in PATH_PATTERN.finditer(text): + if match.start() < cursor: + continue + pieces.extend((text[cursor:match.start()], "")) + cursor = _absolute_path_end(text, match) + pieces.append(text[cursor:]) + return "".join(pieces) + + +def public_redact(text: str, limit: int = 300) -> str: + text = re.sub(r"```.*?(?:```|\Z)", "", text, flags=re.DOTALL) + text = redact_credentials(text) + text = re.sub(r"\s+", " ", text).strip() + text = redact_inline_json(text) + text = ENV_PATTERN.sub("", text) + text = redact(text, limit=limit * 4) + text = REMOTE_PATTERN.sub("", text) + text = URL_PATTERN.sub("", text) + text = redact_absolute_paths(text) + text = EMAIL_PATTERN.sub("", text) + text = ENV_NAME_PATTERN.sub("", text) + if len(text) > limit: + return text[:limit] + "..." + return text diff --git a/plugins/codestable/skills/cs-feedback/scripts/feedback_repo_context.py b/plugins/codestable/skills/cs-feedback/scripts/feedback_repo_context.py new file mode 100644 index 0000000..65367af --- /dev/null +++ b/plugins/codestable/skills/cs-feedback/scripts/feedback_repo_context.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path +from typing import Any + +try: + import yaml # type: ignore +except ImportError: # Repo context degrades to unknown without optional PyYAML. + yaml = None + +from feedback_privacy import public_redact +from feedback_transcripts import METADATA_TYPES, provider_from_path, session_id_from + + +ENVIRONMENT_METADATA_TYPES = METADATA_TYPES | {"turn_context"} +BODY_FIELDS = {"message", "content", "text", "output", "arguments", "input"} + + +def session_label(session: str) -> str: + digest = hashlib.sha256(session.encode("utf-8")).hexdigest()[:10] + return f"session-{digest}" + + +def _artifact_metadata(path: Path, root: Path) -> dict[str, str] | None: + if yaml is None: + return None + try: + if path.suffix in {".yaml", ".yml"}: + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) + elif path.suffix == ".md": + prefix = path.read_text(encoding="utf-8", errors="ignore")[:8192] + if not prefix.startswith("---\n") or "\n---\n" not in prefix[4:]: + return None + frontmatter = prefix[4:].split("\n---\n", 1)[0] + loaded = yaml.safe_load(frontmatter) + else: + return None + except (OSError, UnicodeError, yaml.YAMLError): + return None + if not isinstance(loaded, dict): + return None + keys = ("doc_type", "status", "stage", "feature", "issue", "goal", "refactor") + metadata = { + key: str(loaded[key]) + for key in keys + if loaded.get(key) is not None and not isinstance(loaded.get(key), (dict, list)) + } + if not metadata or not {"doc_type", "status", "stage"}.intersection(metadata): + return None + return {"path": path.relative_to(root).as_posix(), **metadata} + + +def _repo_root(cwd: Path) -> Path: + try: + completed = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=cwd, + text=True, + capture_output=True, + timeout=5, + check=False, + ) + if completed.returncode == 0 and completed.stdout.strip(): + root = Path(completed.stdout.strip()).resolve() + if root.is_dir(): + return root + except (OSError, subprocess.TimeoutExpired): + pass + return cwd.resolve() + + +def build_repo_context(cwd: str | None) -> dict[str, object]: + empty = { + "runtime": {"status": "unknown"}, + "artifacts": [], + "git_status": [], + } + if not cwd: + return empty + requested_root = Path(cwd) + if not requested_root.is_dir(): + return empty + root = _repo_root(requested_root) + + runtime: dict[str, object] = {"status": "unknown"} + manifest = root / ".codestable/runtime-manifest.json" + if manifest.is_file(): + try: + data = json.loads(manifest.read_text(encoding="utf-8")) + runtime_version = str(data.get("runtime_version") or "unknown") + plugin_version = str(data.get("plugin_version") or "unknown") + runtime = { + "status": "present" if runtime_version == plugin_version else "mismatch", + "runtime_version": runtime_version, + "plugin_version": plugin_version, + } + except (OSError, json.JSONDecodeError): + runtime = {"status": "invalid"} + + artifacts: list[dict[str, str]] = [] + codestable = root / ".codestable" + if codestable.is_dir(): + for path in sorted(codestable.rglob("*")): + if not path.is_file() or path == manifest: + continue + metadata = _artifact_metadata(path, root) + if metadata: + artifacts.append(metadata) + if len(artifacts) >= 30: + break + + git_status: list[dict[str, str]] = [] + try: + completed = subprocess.run( + ["git", "status", "--short", "--untracked-files=all"], + cwd=root, + text=True, + capture_output=True, + timeout=5, + check=False, + ) + if completed.returncode == 0: + for line in completed.stdout.splitlines()[:100]: + if len(line) >= 4: + git_status.append({"status": line[:2], "path": line[3:]}) + except (OSError, subprocess.TimeoutExpired): + pass + return {"runtime": runtime, "artifacts": artifacts, "git_status": git_status} + + +def environment_context( + path: Path, + records: list[dict[str, Any]], + capture: dict[str, Any], +) -> dict[str, object]: + model = "unknown" + host_version = "unknown" + for record in records: + payload = record.get("payload") if isinstance(record.get("payload"), dict) else record + record_type = str(record.get("type") or payload.get("type") or "") + top_level_metadata = any( + key in record for key in ("session_id", "sessionId", "sessionid", "cwd") + ) and not BODY_FIELDS.intersection(record) + if record_type not in ENVIRONMENT_METADATA_TYPES and not top_level_metadata: + continue + if model == "unknown" and payload.get("model"): + model = public_redact(str(payload["model"]), limit=120) + if host_version == "unknown" and ( + payload.get("version") + or payload.get("client_version") + or payload.get("cli_version") + ): + host_version = public_redact( + str( + payload.get("version") + or payload.get("client_version") + or payload.get("cli_version") + ), + limit=120, + ) + if model != "unknown" and host_version != "unknown": + break + return { + "provider": provider_from_path(path), + "session": session_label(session_id_from(path, records)), + "model": model, + "host_version": host_version, + "capture": capture, + } diff --git a/plugins/codestable/skills/cs-feedback/scripts/feedback_to_fixture.py b/plugins/codestable/skills/cs-feedback/scripts/feedback_to_fixture.py index 095ed5d..794c22f 100644 --- a/plugins/codestable/skills/cs-feedback/scripts/feedback_to_fixture.py +++ b/plugins/codestable/skills/cs-feedback/scripts/feedback_to_fixture.py @@ -1,10 +1,9 @@ #!/usr/bin/env python3 -"""把 cs-feedback 采集到的失败案例转成 cs-skill 的 regression fixture + hypothesis 骨架。 +"""Convert cs-feedback triage into a local regression candidate artifact. -闭环起点：生产失败 → 评测 fixture。cs-feedback 自有脚本（不跨 skill 依赖 cs-skill 内部）。 -输入：--failure 文本 或 --evidence public-issue-context.json（collect_feedback_context 产出）。 -输出：experiments//fixtures/regression/reg-.json + 打印建议 hypothesis 行。 -生成的是**骨架**：diff/answer 需人工补全为可复现样本后再冻结。 +The shipped cs-feedback skill does not write official experiment fixtures. +It only creates `regression-candidate.json` next to `triage.json`; the +repo-local eval skill owns promotion into an experiment directory. """ from __future__ import annotations @@ -16,6 +15,16 @@ import sys from pathlib import Path sys.dont_write_bytecode = True +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from feedback_triage import ( # noqa: E402 + incident_fingerprint, + profile_for, + recompute_quality, + task_kind_for, +) def _slug(text: str, n: int = 6) -> str: @@ -23,57 +32,223 @@ def _slug(text: str, n: int = 6) -> str: return "-".join(words[:n]) or "case" -def _fixture(kind: str, summary: str, spec: str, diff: str) -> dict: - return { - "id": f"reg-{_slug(summary)}", - "answerType": "findings-recall", - "answer": [summary], - "task": {"kind": kind, "spec": spec, "diff": diff or "TODO: 补全可复现代码"}, +def _field(data: dict, *path: str, default=None): + current = data + for key in path: + if not isinstance(current, dict): + return default + current = current.get(key) + return current if current is not None else default + + +def _load_canonical_triage(path: Path) -> dict: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("triage must be a JSON object") + if path.name != "triage.json": + raise ValueError("canonical input must be named triage.json") + if data.get("schema_version") != 2: + raise ValueError("triage schema_version must be 2") + if data.get("privacy") != "local-private": + raise ValueError("triage privacy must be local-private") + incident_id = data.get("incident_id") + if not isinstance(incident_id, str) or not incident_id.strip(): + raise ValueError("triage incident_id must be selected") + for key in ("target", "assessment", "reproduction", "privacy_review"): + if not isinstance(data.get(key), dict): + raise ValueError(f"triage {key} must be a JSON object") + _validate_evidence_binding(path.with_name("evidence.json"), data) + return data + + +def _validate_evidence_binding(path: Path, triage: dict) -> None: + evidence = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(evidence, dict): + raise ValueError("evidence must be a JSON object") + if evidence.get("schema_version") != 2 or evidence.get("privacy") != "local-private": + raise ValueError("evidence must be canonical local-private schema v2") + incidents = evidence.get("incidents") + if not isinstance(incidents, list): + raise ValueError("evidence incidents must be a JSON array") + selected = [ + incident + for incident in incidents + if isinstance(incident, dict) and incident.get("id") == triage["incident_id"] + ] + if len(selected) != 1: + raise ValueError("triage incident_id must select exactly one evidence incident") + incident = selected[0] + observations = incident.get("observations") + if not isinstance(observations, list) or not all( + isinstance(observation, dict) for observation in observations + ): + raise ValueError("selected evidence incident observations must be objects") + canonical_ids = [observation.get("id") for observation in observations] + if not all(isinstance(item, str) and item.strip() for item in canonical_ids): + raise ValueError("selected evidence observations must have string ids") + if len(set(canonical_ids)) != len(canonical_ids): + raise ValueError("selected evidence observation ids must be unique") + triage_ids = triage.get("observation_ids") + if triage_ids != canonical_ids: + raise ValueError("triage observation_ids do not match selected evidence incident") + fingerprint = triage.get("incident_fingerprint") + if not isinstance(fingerprint, str) or fingerprint != incident_fingerprint(incident): + raise ValueError("triage incident_fingerprint does not match evidence") + if triage.get("trigger_cutoff") != incident.get("capture_cutoff"): + raise ValueError("triage trigger_cutoff does not match evidence") + + referenced_ids: list[str] = [] + assessment = triage["assessment"] + for field_name in ( + "expected_behavior", + "actual_behavior", + "impact", + "proposed_fix", + ): + field = assessment.get(field_name) + if not isinstance(field, dict): + continue + refs = field.get("evidence_refs", []) + if not isinstance(refs, list) or not all(isinstance(ref, str) for ref in refs): + raise ValueError(f"triage assessment.{field_name}.evidence_refs must be strings") + referenced_ids.extend(refs) + reproduction_refs = triage["reproduction"].get("evidence_refs", []) + if not isinstance(reproduction_refs, list) or not all( + isinstance(ref, str) for ref in reproduction_refs + ): + raise ValueError("triage reproduction.evidence_refs must be strings") + referenced_ids.extend(reproduction_refs) + if set(referenced_ids) - set(canonical_ids): + raise ValueError("triage evidence_refs must belong to selected evidence incident") + + +def _candidate_from_triage(path: Path) -> dict: + data = _load_canonical_triage(path) + target_skill = _field(data, "target", "skill", default="unknown") + incident_kind = data.get("incident_kind", "unknown") + reproduction = data["reproduction"] + profile = reproduction.get("eval_profile") or "unknown" + expected_profile = profile_for(str(incident_kind)) + task_kind = task_kind_for(str(target_skill), str(profile)) + input_data = reproduction.get("input") if isinstance(reproduction.get("input"), dict) else {} + oracle = reproduction.get("oracle") if isinstance(reproduction.get("oracle"), dict) else {} + quality = recompute_quality(data) + privacy_review = data["privacy_review"] + + candidate = { + "id": ( + f"reg-{_slug(str(target_skill))}-{_slug(str(incident_kind), 3)}-" + f"{_slug(str(data.get('incident_id') or 'unselected'), 2)}" + ), + "privacy": "local-private", "_source": "cs-feedback", - "_status": "skeleton", # 需人工补全 diff/answer 后方可用于正式实验 + "_status": "candidate", + "_profile": profile, + "incident_id": data.get("incident_id", ""), + "target_skill": target_skill, + "incident_kind": incident_kind, + "quality": quality, + "privacy_review": privacy_review, + } + if profile == "routing-decision": + candidate.update( + { + "answerType": "routing-decision", + "expect": oracle.get("expect", {}), + "task": { + "kind": "routing", + **{ + key: input_data[key] + for key in ("state", "intent", "utterance") + if key in input_data + }, + }, + } + ) + elif profile == "findings-recall": + candidate.update( + { + "answerType": "findings-recall", + "answer": oracle.get("coverage_points") + if isinstance(oracle.get("coverage_points"), list) + else [], + "task": { + "kind": task_kind, + **{ + key: input_data[key] + for key in ("spec", "diff", "context", "audience") + if key in input_data + }, + }, + } + ) + else: + candidate.update({"answerType": "unknown", "task": {"kind": "unknown"}}) + missing = list(quality.get("missing_fields") or []) + if not quality.get("regression_ready"): + missing.append("quality.regression_ready") + if profile != expected_profile: + missing.append("reproduction.eval_profile") + if privacy_review.get("status") != "approved": + missing.append("privacy_review.status") + candidate["promotion_blockers"] = sorted(set(missing)) + return candidate + + +def _candidate_from_v1_public_context(path: Path) -> dict: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("v1 public context must be a JSON object") + events = data.get("events") or data.get("candidates") or [] + if not isinstance(events, list): + raise ValueError("v1 public context events must be a JSON array") + event = events[0] if events and isinstance(events[0], dict) else {} + summary = event.get("actual_behavior") or event.get("sanitized_excerpt") or event.get("failure_type") or "unknown failure" + return { + "id": f"reg-{_slug(str(summary))}", + "privacy": "local-private", + "_source": "cs-feedback", + "_status": "candidate", + "answerType": "findings-recall", + "answer": [str(summary)[:120]], + "task": {"kind": event.get("kind", "review")}, + "quality": {"triage_ready": False, "regression_ready": False, "missing_fields": ["triage.json", "reproduction.input", "reproduction.oracle"]}, + "privacy_review": {"status": "pending"}, + "promotion_blockers": ["triage.json", "quality.regression_ready", "privacy_review.status"], } -def from_evidence(path: Path) -> list[dict]: - data = json.loads(path.read_text(encoding="utf-8")) - events = data.get("events") or data.get("candidates") or [] - out = [] - for ev in events: - summary = ev.get("actual_behavior") or ev.get("sanitized_excerpt") or ev.get("failure_type") or "unknown failure" - spec = ev.get("expected_behavior") or "" - out.append(_fixture(ev.get("kind", "review"), summary[:120], spec[:200], "")) - return out - - def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser(description="feedback 失败 → regression fixture 骨架") - p.add_argument("--experiment", required=True, help="目标实验目录，如 experiments/cs-code-review-001") - p.add_argument("--failure", help="单条失败描述") - p.add_argument("--evidence", help="public-issue-context.json 路径") - p.add_argument("--kind", default="review", choices=["review", "fix", "audit"]) - p.add_argument("--spec", default="") - p.add_argument("--diff", default="") - args = p.parse_args(argv) + parser = argparse.ArgumentParser(description="feedback triage -> local regression candidate") + parser.add_argument("--triage", help="canonical triage.json") + parser.add_argument("--evidence", help="compat public-issue-context.json; produces not-ready candidate") + parser.add_argument("--experiment", help=argparse.SUPPRESS) + parser.add_argument("--failure", help=argparse.SUPPRESS) + parser.add_argument("--kind", default="review", help=argparse.SUPPRESS) + parser.add_argument("--spec", default="", help=argparse.SUPPRESS) + parser.add_argument("--diff", default="", help=argparse.SUPPRESS) + args = parser.parse_args(argv) - exp = Path(args.experiment).resolve() - reg_dir = exp / "fixtures" / "regression" - reg_dir.mkdir(parents=True, exist_ok=True) - - fixtures: list[dict] = [] - if args.evidence: - fixtures += from_evidence(Path(args.evidence)) - if args.failure: - fixtures.append(_fixture(args.kind, args.failure[:120], args.spec, args.diff)) - if not fixtures: - print("需 --failure 或 --evidence", file=sys.stderr) + if args.experiment or args.failure: + print("legacy direct fixture writing is disabled; run --triage and promote with eval-cs-skill", file=sys.stderr) + return 2 + if bool(args.triage) == bool(args.evidence): + print("需且只能提供 --triage 或 --evidence", file=sys.stderr) return 2 - for fx in fixtures: - out = reg_dir / f"{fx['id']}.json" - out.write_text(json.dumps(fx, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - print(f"[cs-feedback] 写 regression fixture 骨架 → {out}") - print(f" 建议 hypothesis: H-regression-{fx['id']}: 该 skill 应识别「{fx['answer'][0]}」（recall=1.0）") - print("提示：补全 diff/answer 为可复现样本后，把 hypotheses.md 冻结并 git commit 再评测。") + source = Path(args.triage or args.evidence).expanduser() + try: + candidate = ( + _candidate_from_triage(source) + if args.triage + else _candidate_from_v1_public_context(source) + ) + except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + print(f"candidate blocked: {exc}", file=sys.stderr) + return 2 + output = source.parent / "regression-candidate.json" + output.write_text(json.dumps(candidate, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(f"[cs-feedback] wrote local regression candidate -> {output}") return 0 diff --git a/plugins/codestable/skills/cs-feedback/scripts/feedback_transcripts.py b/plugins/codestable/skills/cs-feedback/scripts/feedback_transcripts.py new file mode 100644 index 0000000..00212e6 --- /dev/null +++ b/plugins/codestable/skills/cs-feedback/scripts/feedback_transcripts.py @@ -0,0 +1,547 @@ +from __future__ import annotations + +import json +import time +from collections import defaultdict +from pathlib import Path +from typing import Any + +from feedback_models import Candidate, NormalizedRecord, SessionMeta +from feedback_privacy import public_redact, redact + + +METADATA_TYPES = {"session_meta", "metadata", "system_meta"} +TOOL_CALL_TYPES = {"function_call", "tool_call", "tool_use"} +TOOL_RESULT_TYPES = {"function_call_output", "tool_result", "tool_output"} + + +def flatten(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, list): + return "\n".join(flatten(item) for item in value) + if isinstance(value, dict): + parts: list[str] = [] + for key in ( + "message", + "text", + "output", + "content", + "arguments", + "input", + "name", + "type", + "role", + ): + if key in value: + parts.append(flatten(value[key])) + if parts: + return "\n".join(part for part in parts if part) + return json.dumps(value, ensure_ascii=False, sort_keys=True) + return str(value) + + +def event_text(record: dict[str, Any]) -> str: + payload = record.get("payload", record) + return flatten(payload) + + +def event_kind(record: dict[str, Any]) -> str: + payload = record.get("payload") + if isinstance(payload, dict): + for key in ("type", "name", "role"): + if payload.get(key): + return str(payload[key]) + message = record.get("message") + if isinstance(message, dict): + for key in ("type", "name", "role"): + if message.get(key): + return str(message[key]) + return str(record.get("type", record.get("role", "unknown"))) + + +def normalize_json_records(value: Any) -> list[dict[str, Any]]: + if isinstance(value, list): + return [item if isinstance(item, dict) else {"payload": item} for item in value] + if not isinstance(value, dict): + return [{"payload": value}] + + collection_keys = ("messages", "events", "entries", "items", "transcript") + records: list[dict[str, Any]] = [] + meta = {key: item for key, item in value.items() if key not in collection_keys} + if meta: + records.append(meta) + for key in collection_keys: + items = value.get(key) + if not isinstance(items, list): + continue + for item in items: + records.append(item if isinstance(item, dict) else {"payload": item}) + return records or [value] + + +def read_transcript_snapshot(path: Path) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Read one immutable byte snapshot and report the last complete record boundary.""" + raw = path.read_bytes() + if path.suffix == ".json": + try: + value = json.loads(raw.decode("utf-8", errors="ignore")) + except json.JSONDecodeError: + return [], {"format": "json", "byte_length": len(raw), "complete_record_eof": 0} + return normalize_json_records(value), { + "format": "json", + "byte_length": len(raw), + "complete_record_eof": len(raw), + } + + records: list[dict[str, Any]] = [] + offset = 0 + complete_record_eof = 0 + for raw_line in raw.splitlines(keepends=True): + offset += len(raw_line) + line = raw_line.strip() + if not line: + continue + try: + value = json.loads(line.decode("utf-8", errors="ignore")) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + records.append(value) + complete_record_eof = offset + return records, { + "format": "jsonl", + "byte_length": len(raw), + "complete_record_eof": complete_record_eof, + } + + +def read_records(path: Path) -> list[dict[str, Any]]: + records, _capture = read_transcript_snapshot(path) + return records + + +def _session_id_from_record(record: dict[str, Any]) -> str: + payload = record.get("payload") + if isinstance(payload, dict): + session_id = ( + payload.get("session_id") + or payload.get("sessionId") + or payload.get("sessionid") + or payload.get("id") + ) + if session_id: + return str(session_id) + session_id = ( + record.get("session_id") + or record.get("sessionId") + or record.get("sessionid") + or record.get("id") + ) + return str(session_id) if session_id else "" + + +def _cwd_from_record(record: dict[str, Any]) -> str: + payload = record.get("payload") + payload_type = payload.get("type") if isinstance(payload, dict) else None + record_type = record.get("type") + if ( + isinstance(payload, dict) + and payload.get("cwd") + and (record_type == "session_meta" or payload_type == "session_meta") + ): + return str(payload["cwd"]) + body_keys = {"message", "content", "text", "output", "arguments", "input"} + if record.get("cwd") and not body_keys.intersection(record): + return str(record["cwd"]) + return "" + + +def read_session_metadata(path: Path) -> SessionMeta: + """Read only top-level or session-meta fields; never normalize message bodies.""" + if path.suffix == ".json": + try: + value = json.loads(path.read_bytes().decode("utf-8", errors="ignore")) + except json.JSONDecodeError: + return SessionMeta(path.stem, "") + if isinstance(value, dict): + session = str( + value.get("session_id") + or value.get("sessionId") + or value.get("id") + or path.stem + ) + cwd = str(value.get("cwd") or "") + return SessionMeta(session, cwd) + return SessionMeta(path.stem, "") + + session = "" + cwd = "" + with path.open(encoding="utf-8", errors="ignore") as handle: + for line in handle: + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(record, dict): + continue + if not session: + session = _session_id_from_record(record) + if not cwd: + cwd = _cwd_from_record(record) + if session and cwd: + break + return SessionMeta(session or path.stem, cwd) + + +def session_id_from(path: Path, records: list[dict[str, Any]]) -> str: + for record in records: + session_id = _session_id_from_record(record) + if session_id: + return session_id + return path.stem + + +def cwd_from(records: list[dict[str, Any]]) -> str: + for record in records: + cwd = _cwd_from_record(record) + if cwd: + return cwd + return "" + + +def provider_from_path(path: Path) -> str: + text = str(path) + if ".codex" in text: + return "codex" + if ".claude" in text: + return "claude" + return "unknown" + + +def candidate_for(path: Path, cwd: str | None) -> Candidate: + meta = read_session_metadata(path) + score = 0 + if cwd and meta.cwd == cwd: + score += 5 + elif cwd and meta.cwd and (cwd.startswith(meta.cwd) or meta.cwd.startswith(cwd)): + score += 2 + score += int(path.stat().st_mtime // 60) + return Candidate( + path=str(path), + provider=provider_from_path(path), + session=meta.session, + cwd=meta.cwd, + mtime=path.stat().st_mtime, + score=score, + ) + + +def resolve_current_session( + files: list[Path], cwd: str | None +) -> tuple[list[Path], list[Candidate]]: + candidates = [ + candidate_for(path, cwd) for path in files if path.suffix in {".jsonl", ".json"} + ] + candidates.sort(key=lambda candidate: candidate.score, reverse=True) + if not candidates: + return [], [] + if cwd: + exact = [candidate for candidate in candidates if candidate.cwd == cwd] + if len(exact) == 1: + return [Path(exact[0].path)], [] + if len(exact) > 1: + return [], exact[:5] + containing = [ + candidate + for candidate in candidates + if candidate.cwd and (cwd.startswith(candidate.cwd) or candidate.cwd.startswith(cwd)) + ] + if containing: + return [], containing[:5] + return [], candidates[:5] + + +def _history_files(home: Path) -> list[Path]: + roots = [ + home / ".codex/sessions", + home / ".claude/projects", + home / ".claude/sessions", + ] + files: list[Path] = [] + for root in roots: + if not root.exists(): + continue + for path in root.rglob("*"): + if path.is_file() and path.suffix in {".jsonl", ".json"}: + files.append(path) + return sorted(files) + + +def discover_files( + home: Path, + since_days: int, + session_filter: str | None, + cwd: str | None, +) -> tuple[list[Path], list[Candidate]]: + if session_filter and session_filter != "current": + candidate = Path(session_filter).expanduser() + if candidate.is_file(): + return [candidate], [] + + all_files = _history_files(home) + if session_filter == "current": + return resolve_current_session(all_files, cwd) + + cutoff = time.time() - since_days * 86400 + files: list[Path] = [] + for path in all_files: + if path.stat().st_mtime < cutoff: + continue + if session_filter and session_filter != "current": + if session_filter in path.name or session_filter in str(path): + files.append(path) + continue + if session_filter not in read_session_metadata(path).session: + continue + files.append(path) + return files, [] + + +def _payload(record: dict[str, Any]) -> dict[str, Any]: + payload = record.get("payload") + return payload if isinstance(payload, dict) else record + + +def _metadata_record(record: dict[str, Any]) -> bool: + payload = _payload(record) + kind = str(payload.get("type") or record.get("type") or "") + if kind in METADATA_TYPES: + return True + body_keys = {"message", "content", "text", "output", "arguments", "input"} + has_meta = any( + key in record for key in ("session_id", "sessionId", "sessionid", "cwd") + ) + return has_meta and not body_keys.intersection(record) + + +def _message_container(record: dict[str, Any]) -> dict[str, Any] | None: + message = record.get("message") + if isinstance(message, dict): + return message + if "content" in record and (record.get("role") or record.get("type") in {"user", "assistant"}): + return record + return None + + +def _expanded_records(record: dict[str, Any]) -> list[dict[str, Any]]: + if _metadata_record(record): + return [ + { + "role": "system", + "record_type": "session_meta", + "tool_name": "unknown", + "call_id": "", + "text": "", + } + ] + + container = _message_container(record) + if container is not None: + role = str(container.get("role") or record.get("role") or record.get("type") or "unknown") + content = container.get("content") + blocks = content if isinstance(content, list) else [content] + expanded: list[dict[str, Any]] = [] + for block in blocks: + if block is None: + continue + if isinstance(block, dict): + kind = str(block.get("type") or "") + if kind in TOOL_CALL_TYPES: + name = str(block.get("name") or block.get("tool_name") or "unknown") + expanded.append( + { + "role": "assistant", + "record_type": "tool_call", + "tool_name": name, + "call_id": str(block.get("id") or block.get("call_id") or ""), + "text": "\n".join( + part + for part in (name, flatten(block.get("input") or block.get("arguments"))) + if part + ), + } + ) + continue + if kind in TOOL_RESULT_TYPES: + expanded.append( + { + "role": "tool", + "record_type": "tool_result", + "tool_name": str(block.get("name") or block.get("tool_name") or "unknown"), + "call_id": str( + block.get("tool_use_id") + or block.get("tool_call_id") + or block.get("call_id") + or "" + ), + "text": flatten(block.get("content") or block.get("output")), + } + ) + continue + text = flatten(block.get("text") if kind == "text" else block) + else: + text = flatten(block) + if text: + expanded.append( + { + "role": role if role in {"user", "assistant", "system"} else "unknown", + "record_type": "message", + "tool_name": "unknown", + "call_id": "", + "text": text, + } + ) + if expanded: + return expanded + + payload = _payload(record) + kind = str(payload.get("type") or record.get("type") or "unknown") + role = str(payload.get("role") or record.get("role") or "") + if kind in TOOL_CALL_TYPES: + name = str(payload.get("name") or payload.get("tool_name") or payload.get("tool") or "unknown") + return [ + { + "role": "assistant", + "record_type": "tool_call", + "tool_name": name, + "call_id": str( + payload.get("call_id") + or payload.get("tool_call_id") + or payload.get("id") + or "" + ), + "text": event_text(record), + } + ] + if kind in TOOL_RESULT_TYPES: + return [ + { + "role": "tool", + "record_type": "tool_result", + "tool_name": str(payload.get("name") or payload.get("tool_name") or "unknown"), + "call_id": str( + payload.get("call_id") + or payload.get("tool_call_id") + or payload.get("tool_use_id") + or "" + ), + "text": event_text(record), + } + ] + if not role: + if "user" in kind: + role = "user" + elif "assistant" in kind: + role = "assistant" + return [ + { + "role": role if role in {"user", "assistant", "tool", "system"} else "unknown", + "record_type": "message" if "message" in kind or role else kind, + "tool_name": "unknown", + "call_id": "", + "text": event_text(record), + } + ] + + +def _fallback_tool_name(text: str) -> str: + for candidate in ("apply_patch", "read_file", "git", "gh", "paseo", "mcp"): + if candidate in text.lower(): + return candidate + return "unknown" + + +def normalize_records(path: Path, records: list[dict[str, Any]]) -> list[NormalizedRecord]: + provider = provider_from_path(path) + session = session_id_from(path, records) + entries: list[dict[str, Any]] = [] + for source_index, record in enumerate(records): + timestamp = str(record.get("timestamp") or record.get("created_at") or "") + for expanded in _expanded_records(record): + text = redact(str(expanded["text"]), limit=800) + tool_name = str(expanded["tool_name"]) + if tool_name == "unknown": + tool_name = _fallback_tool_name(text) + entries.append( + { + **expanded, + "timestamp": timestamp, + "text": text, + "tool_name": public_redact(tool_name, limit=80), + "source_index": source_index, + "correlation_id": "", + "correlation_source": "unpaired", + } + ) + + calls_by_id: dict[str, list[int]] = defaultdict(list) + results_by_id: dict[str, list[int]] = defaultdict(list) + for index, entry in enumerate(entries): + call_id = str(entry["call_id"]) + if not call_id: + continue + if entry["record_type"] == "tool_call": + calls_by_id[call_id].append(index) + elif entry["record_type"] == "tool_result": + results_by_id[call_id].append(index) + for call_id in set(calls_by_id) & set(results_by_id): + calls = calls_by_id[call_id] + results = results_by_id[call_id] + if len(calls) == 1 and len(results) == 1: + for index in (calls[0], results[0]): + entries[index]["correlation_id"] = call_id + entries[index]["correlation_source"] = "provider" + + pending_calls: list[int] = [] + previous_significant: int | None = None + for index, entry in enumerate(entries): + if entry["record_type"] in METADATA_TYPES: + continue + record_type = entry["record_type"] + call_id = str(entry["call_id"]) + if entry["correlation_source"] == "provider" or call_id: + pending_calls = [] + elif record_type == "tool_call": + pending_calls.append(index) + elif record_type == "tool_result": + if len(pending_calls) == 1 and previous_significant == pending_calls[0]: + call_index = pending_calls[0] + correlation_id = f"adjacent-record-{call_index:04d}" + entries[call_index]["correlation_id"] = correlation_id + entries[call_index]["correlation_source"] = "adjacency" + entry["correlation_id"] = correlation_id + entry["correlation_source"] = "adjacency" + pending_calls = [] + else: + pending_calls = [] + previous_significant = index + + return [ + NormalizedRecord( + id=f"record-{index:04d}", + provider=provider, + session=session, + timestamp=str(entry["timestamp"]), + role=str(entry["role"]), + record_type=str(entry["record_type"]), + tool_name=str(entry["tool_name"]), + correlation_id=str(entry["correlation_id"]), + correlation_source=str(entry["correlation_source"]), + text=str(entry["text"]), + source_index=int(entry["source_index"]), + ) + for index, entry in enumerate(entries) + ] diff --git a/plugins/codestable/skills/cs-feedback/scripts/feedback_triage.py b/plugins/codestable/skills/cs-feedback/scripts/feedback_triage.py new file mode 100644 index 0000000..81a0f68 --- /dev/null +++ b/plugins/codestable/skills/cs-feedback/scripts/feedback_triage.py @@ -0,0 +1,576 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import re +from typing import Any + + +ROUTING_INCIDENT_KINDS = { + "wrong-route", + "skipped-gate", + "missing-artifact", + "goal-driver", + "unnecessary-detour", +} +FINDINGS_INCIDENT_KINDS = { + "tool-failure", + "install-version", + "privacy-reporting", + "unclear-rule", +} +ASSESSMENT_SOURCES = {"user", "transcript", "inferred"} +TASK_KIND_BY_TARGET = { + "cs-code-review": "review", + "cs-issue": "fix", + "cs-audit": "audit", + "cs-feat": "design", + "cs-refactor": "design", + "cs-epic": "design", + "cs-req": "design", + "cs-domain": "design", + "cs-docs": "docs", + "cs-docs-neat": "docs", +} +EXPECTED_PATTERN = re.compile( + r"(应该|应当|本应|必须|需要|要先|要再|正确.{0,8}(?:是|为)|should|expected|must|instead)", + re.IGNORECASE, +) + + +def field( + value: str, + source: str, + refs: list[str], + confidence: str | None = None, +) -> dict[str, object]: + out: dict[str, object] = { + "value": value, + "source": source, + "evidence_refs": refs, + } + if source == "inferred": + out["confidence"] = confidence or "medium" + return out + + +def profile_for(incident_kind: str) -> str: + if incident_kind in ROUTING_INCIDENT_KINDS: + return "routing-decision" + if incident_kind in FINDINGS_INCIDENT_KINDS: + return "findings-recall" + return "unknown" + + +def task_kind_for(target_skill: str, profile: str) -> str: + if profile == "routing-decision": + return "routing" + if profile == "findings-recall": + return TASK_KIND_BY_TARGET.get(target_skill, "unknown") + return "unknown" + + +def empty_triage() -> dict[str, Any]: + triage = { + "schema_version": 2, + "privacy": "local-private", + "incident_id": "", + "incident_fingerprint": "", + "observation_ids": [], + "trigger_cutoff": "unknown", + "target": { + "skill": "unknown", + "stage_hint": "unknown", + "suspected_area": "unknown", + }, + "incident_kind": "unknown", + "assessment": { + "expected_behavior": field("unknown", "unknown", []), + "actual_behavior": field("unknown", "unknown", []), + "impact": field("unknown", "unknown", []), + "proposed_fix": field("unknown", "unknown", []), + "cause_status": "unclassified", + }, + "reproduction": { + "eval_profile": "unknown", + "task_kind": "unknown", + "input": None, + "oracle": None, + "evidence_refs": [], + }, + "environment_context": {"provider": "unknown", "session": "unknown"}, + "repo_context": { + "runtime": {"status": "unknown"}, + "artifacts": [], + "git_status": [], + }, + "quality": {}, + "privacy_review": {"status": "pending"}, + } + triage["quality"] = recompute_quality(triage) + return triage + + +def _meaningful(value: Any) -> bool: + if value is None: + return False + if isinstance(value, str): + return bool(value.strip()) and value.strip().lower() not in {"unknown", "todo", "tbd"} + if isinstance(value, (list, dict)): + return bool(value) + return True + + +def _assessment_from_incident(incident: dict[str, Any]) -> dict[str, Any]: + observations = [ + observation + for observation in incident.get("observations", []) + if isinstance(observation, dict) + ] + correction = incident.get("user_correction") + if not isinstance(correction, dict): + correction = {} + correction_text = str(correction.get("text") or "") + expected = field("unknown", "unknown", []) + if correction_text and EXPECTED_PATTERN.search(correction_text): + expected = field(correction_text, "user", [str(correction.get("id"))]) + + correction_index = correction.get("source_index") + actual_candidates = [ + observation + for observation in observations + if observation.get("role") in {"assistant", "tool"} + and ( + not isinstance(correction_index, int) + or not isinstance(observation.get("source_index"), int) + or int(observation["source_index"]) < correction_index + ) + ] + actual = field("unknown", "unknown", []) + if actual_candidates: + observation = actual_candidates[-1] + actual = field( + str(observation.get("text") or "unknown"), + "transcript", + [str(observation.get("id"))], + ) + + expected_refs = list(expected.get("evidence_refs") or []) + actual_refs = list(actual.get("evidence_refs") or []) + impact = field("unknown", "unknown", []) + if expected_refs and actual_refs: + impact = field( + "反馈 gate 或工具行为偏离预期，需要维护者分诊", + "inferred", + [*actual_refs, *expected_refs], + "medium", + ) + return { + "expected_behavior": expected, + "actual_behavior": actual, + "impact": impact, + "proposed_fix": field("unknown", "unknown", []), + "cause_status": "unclassified", + } + + +def incident_fingerprint(incident: dict[str, Any]) -> str: + environment = incident.get("environment_context") + environment = environment if isinstance(environment, dict) else {} + observations = [ + { + "record_id": str(observation.get("record_id") or ""), + "role": str(observation.get("role") or "unknown"), + "record_type": str(observation.get("record_type") or "unknown"), + "text": str(observation.get("text") or ""), + } + for observation in incident.get("observations", []) + if isinstance(observation, dict) + ] + identity = { + "provider": str(environment.get("provider") or "unknown"), + "session": str(environment.get("session") or "unknown"), + "capture_cutoff": str(incident.get("capture_cutoff") or "unknown"), + "target_skill": str(incident.get("target_skill") or "unknown"), + "stage_hint": str(incident.get("stage_hint") or "unknown"), + "incident_kind": str(incident.get("incident_kind") or "unknown"), + "observations": observations, + } + encoded = json.dumps( + identity, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def build_triage( + incidents: list[dict[str, Any]], primary_incident: dict[str, Any] | None +) -> dict[str, Any]: + context_incident = primary_incident + if context_incident is None and len(incidents) == 1: + context_incident = incidents[0] + if context_incident is None: + return empty_triage() + + incident_kind = str(context_incident.get("incident_kind") or "unknown") + target_skill = str(context_incident.get("target_skill") or "unknown") + profile = profile_for(incident_kind) + trigger_cutoff = ( + str(context_incident.get("capture_cutoff") or "unknown") + if primary_incident is not None + else "unknown" + ) + triage = { + "schema_version": 2, + "privacy": "local-private", + "incident_id": str(context_incident.get("id") or "") + if primary_incident is not None + else "", + "incident_fingerprint": incident_fingerprint(context_incident) + if primary_incident is not None + else "", + "observation_ids": [ + str(observation.get("id")) + for observation in context_incident.get("observations", []) + if isinstance(observation, dict) and _meaningful(observation.get("id")) + ], + "trigger_cutoff": trigger_cutoff, + "target": { + "skill": target_skill, + "stage_hint": str(context_incident.get("stage_hint") or "unknown"), + "suspected_area": "unknown", + }, + "incident_kind": incident_kind, + "assessment": _assessment_from_incident(context_incident), + "reproduction": { + "eval_profile": profile, + "task_kind": task_kind_for(target_skill, profile), + "input": None, + "oracle": None, + "evidence_refs": [], + }, + "environment_context": copy.deepcopy( + context_incident.get("environment_context") + or {"provider": "unknown", "session": "unknown"} + ), + "repo_context": copy.deepcopy( + context_incident.get("repo_context") + or {"runtime": {"status": "unknown"}, "artifacts": [], "git_status": []} + ), + "quality": {}, + "privacy_review": {"status": "pending"}, + } + triage["quality"] = recompute_quality(triage) + return triage + + +def _field_gaps( + triage: dict[str, Any], name: str, *, required: bool = True +) -> list[str]: + assessment = triage.get("assessment") + value = assessment.get(name) if isinstance(assessment, dict) else None + prefix = f"assessment.{name}" + if not isinstance(value, dict): + return [prefix] if required else [] + gaps: list[str] = [] + has_value = _meaningful(value.get("value")) + active = required or has_value + if required and not has_value: + gaps.append(prefix) + source = value.get("source") + if active and source not in ASSESSMENT_SOURCES: + gaps.append(f"{prefix}.source") + refs = value.get("evidence_refs") + observation_ids = triage.get("observation_ids") + allowed_refs = ( + {ref for ref in observation_ids if isinstance(ref, str) and _meaningful(ref)} + if isinstance(observation_ids, list) + else set() + ) + valid_refs = ( + isinstance(refs, list) + and bool(refs) + and all(isinstance(ref, str) and _meaningful(ref) for ref in refs) + and set(refs).issubset(allowed_refs) + ) + if active and not valid_refs: + gaps.append(f"{prefix}.evidence_refs") + if source == "inferred" and not ( + isinstance(value.get("confidence"), str) + and _meaningful(value.get("confidence")) + ): + gaps.append(f"{prefix}.confidence") + return gaps + + +def _reproduction_gaps(triage: dict[str, Any]) -> tuple[list[str], list[str]]: + reproduction = triage.get("reproduction") + if not isinstance(reproduction, dict): + return ["reproduction"], [] + target = triage.get("target") if isinstance(triage.get("target"), dict) else {} + target_skill = str(target.get("skill") or "unknown") + incident_kind = str(triage.get("incident_kind") or "unknown") + profile = str(reproduction.get("eval_profile") or "unknown") + task_kind = str(reproduction.get("task_kind") or "unknown") + input_data = reproduction.get("input") + oracle = reproduction.get("oracle") + gaps: list[str] = [] + reasons: list[str] = [] + + if profile not in {"routing-decision", "findings-recall"}: + gaps.extend( + ["reproduction.eval_profile", "reproduction.input", "reproduction.oracle"] + ) + return gaps, reasons + if profile == "routing-decision": + if incident_kind not in ROUTING_INCIDENT_KINDS: + gaps.append("reproduction.eval_profile") + reasons.append("profile_incident_kind_mismatch") + if task_kind != "routing": + gaps.append("reproduction.task_kind") + if not isinstance(input_data, dict) or not any( + _meaningful(input_data.get(key)) for key in ("state", "intent", "utterance") + ): + gaps.append("reproduction.input") + expect = oracle.get("expect") if isinstance(oracle, dict) else None + if not isinstance(expect, dict) or not _meaningful(expect.get("result_type")): + gaps.append("reproduction.oracle.expect.result_type") + else: + if incident_kind not in FINDINGS_INCIDENT_KINDS: + gaps.append("reproduction.eval_profile") + reasons.append("profile_incident_kind_mismatch") + derived_kind = TASK_KIND_BY_TARGET.get(target_skill) + if not derived_kind: + gaps.append("reproduction.task_kind") + reasons.append("unsupported_target") + elif task_kind != derived_kind: + gaps.append("reproduction.task_kind") + if not isinstance(input_data, dict): + gaps.append("reproduction.input") + else: + if task_kind in {"review", "audit"} and not _meaningful(input_data.get("diff")): + gaps.append("reproduction.input.diff") + if task_kind == "fix" and ( + not _meaningful(input_data.get("spec")) + or not _meaningful(input_data.get("diff")) + ): + gaps.append("reproduction.input.spec_or_diff") + if task_kind == "design" and not _meaningful(input_data.get("spec")): + gaps.append("reproduction.input.spec") + if task_kind == "docs" and ( + not _meaningful(input_data.get("spec")) + or not _meaningful(input_data.get("diff")) + ): + gaps.append("reproduction.input.spec_or_diff") + coverage = oracle.get("coverage_points") if isinstance(oracle, dict) else None + if not isinstance(coverage, list) or not coverage or not all( + _meaningful(item) for item in coverage + ): + gaps.append("reproduction.oracle.coverage_points") + return gaps, reasons + + +def recompute_quality(triage: dict[str, Any]) -> dict[str, Any]: + missing: list[str] = [] + incident_id = str(triage.get("incident_id") or "") + if not incident_id: + missing.append("incident") + if str(triage.get("trigger_cutoff") or "unknown") == "unknown": + missing.append("trigger_cutoff") + target = triage.get("target") if isinstance(triage.get("target"), dict) else {} + if not _meaningful(target.get("skill")): + missing.append("target.skill") + triage_gaps = [ + *_field_gaps(triage, "expected_behavior"), + *_field_gaps(triage, "actual_behavior"), + *_field_gaps(triage, "impact", required=False), + *_field_gaps(triage, "proposed_fix", required=False), + ] + missing.extend(triage_gaps) + identity_gaps = {"incident", "trigger_cutoff", "target.skill"} + triage_ready = not identity_gaps.intersection(missing) and not triage_gaps + + reproduction_gaps, reasons = _reproduction_gaps(triage) + missing.extend(reproduction_gaps) + missing = list(dict.fromkeys(missing)) + if reproduction_gaps: + reasons.insert(0, "regression_requires_replayable_input_and_oracle") + priority = [ + "incident", + "trigger_cutoff", + "target.skill", + *triage_gaps, + *reproduction_gaps, + ] + next_question = next((item for item in priority if item in missing), None) + return { + "triage_ready": triage_ready, + "regression_ready": triage_ready and not reproduction_gaps, + "missing_fields": missing, + "reasons": list(dict.fromkeys(reasons)), + "next_questions": [next_question] if next_question else [], + } + + +def _merge_prefer_existing(generated: Any, existing: Any) -> Any: + if isinstance(generated, dict) and isinstance(existing, dict): + return { + key: _merge_prefer_existing(generated.get(key), existing.get(key)) + if key in existing + else copy.deepcopy(value) + for key, value in generated.items() + } | { + key: copy.deepcopy(value) + for key, value in existing.items() + if key not in generated + } + return copy.deepcopy(existing) if _meaningful(existing) else copy.deepcopy(generated) + + +def _has_user_supplements(existing: dict[str, Any]) -> bool: + reproduction = existing.get("reproduction") + if isinstance(reproduction, dict) and any( + _meaningful(reproduction.get(key)) for key in ("input", "oracle", "evidence_refs") + ): + return True + privacy_review = existing.get("privacy_review") + if isinstance(privacy_review, dict): + status = privacy_review.get("status") + if _meaningful(status) and str(status) != "pending": + return True + assessment = existing.get("assessment") + if isinstance(assessment, dict): + return any( + isinstance(item, dict) and _meaningful(item.get("value")) + for item in assessment.values() + ) + return False + + +def _preserve_unresolved_triage( + generated: dict[str, Any], existing: dict[str, Any] +) -> dict[str, Any]: + preserved = copy.deepcopy(existing) + existing_id = str(existing.get("incident_id") or "") + generated_id = str(generated.get("incident_id") or "") + existing_fingerprint = str(existing.get("incident_fingerprint") or "") + generated_fingerprint = str(generated.get("incident_fingerprint") or "") + preserved["previous_incident_id"] = existing_id or str( + existing.get("previous_incident_id") or "" + ) + preserved["previous_incident_fingerprint"] = existing_fingerprint or str( + existing.get("previous_incident_fingerprint") or "" + ) + if generated_id: + preserved["pending_incident_id"] = generated_id + preserved["pending_incident_fingerprint"] = generated_fingerprint + else: + preserved["pending_incident_id"] = str( + existing.get("pending_incident_id") or "" + ) + preserved["pending_incident_fingerprint"] = str( + existing.get("pending_incident_fingerprint") or "" + ) + preserved["incident_id"] = "" + preserved["incident_fingerprint"] = "" + preserved["observation_ids"] = [] + preserved["trigger_cutoff"] = "unknown" + preserved["environment_context"] = generated.get("environment_context", {}) + preserved["repo_context"] = generated.get("repo_context", {}) + quality = recompute_quality(preserved) + if generated_id and existing_id == generated_id: + reason = "incident_identity_changed" + elif generated_id: + reason = "incident_id_changed" + else: + reason = "incident_resolution_failed" + quality["reasons"] = list(dict.fromkeys([reason, *quality["reasons"]])) + preserved["quality"] = quality + return preserved + + +def merge_existing_triage( + generated: dict[str, Any], existing: dict[str, Any] | None +) -> dict[str, Any]: + if not isinstance(existing, dict) or existing.get("privacy") != "local-private": + return generated + generated_id = str(generated.get("incident_id") or "") + existing_id = str(existing.get("incident_id") or "") + generated_fingerprint = str(generated.get("incident_fingerprint") or "") + existing_fingerprint = str(existing.get("incident_fingerprint") or "") + fingerprint_changed = bool( + generated_id + and generated_id == existing_id + and generated_fingerprint != existing_fingerprint + ) + if generated_id != existing_id or fingerprint_changed: + if ( + existing_id + or existing.get("pending_incident_id") + or _has_user_supplements(existing) + ): + return _preserve_unresolved_triage(generated, existing) + return generated + if not generated_id: + if existing.get("pending_incident_id") or _has_user_supplements(existing): + return _preserve_unresolved_triage(generated, existing) + return generated + + preserved = copy.deepcopy(existing) + preserved.pop("quality", None) + merged = _merge_prefer_existing(generated, preserved) + merged["schema_version"] = 2 + merged["privacy"] = "local-private" + merged["incident_fingerprint"] = generated_fingerprint + merged["observation_ids"] = copy.deepcopy(generated.get("observation_ids", [])) + merged["trigger_cutoff"] = generated.get("trigger_cutoff", "unknown") + merged["environment_context"] = generated.get("environment_context", {}) + merged["repo_context"] = generated.get("repo_context", {}) + merged.pop("previous_incident_id", None) + merged.pop("previous_incident_fingerprint", None) + merged.pop("pending_incident_id", None) + merged.pop("pending_incident_fingerprint", None) + merged["quality"] = recompute_quality(merged) + return merged + + +def accept_pending_incident( + generated: dict[str, Any], + existing: dict[str, Any] | None, + accepted_incident_id: str, +) -> dict[str, Any]: + accepted_id = accepted_incident_id.strip() + if not accepted_id: + raise ValueError("accepted incident id must not be empty") + if not isinstance(existing, dict) or existing.get("privacy") != "local-private": + raise ValueError("existing local-private triage is required") + pending_id = str(existing.get("pending_incident_id") or "") + generated_id = str(generated.get("incident_id") or "") + if pending_id != accepted_id: + raise ValueError( + f"accepted incident {accepted_id} does not match pending incident {pending_id or 'none'}" + ) + if generated_id != accepted_id: + raise ValueError( + f"accepted incident {accepted_id} is not the current primary incident" + ) + generated_fingerprint = str(generated.get("incident_fingerprint") or "") + if not generated_fingerprint: + raise ValueError("current primary incident has no fingerprint") + pending_fingerprint = str(existing.get("pending_incident_fingerprint") or "") + if pending_fingerprint and pending_fingerprint != generated_fingerprint: + raise ValueError("pending incident fingerprint no longer matches current primary") + + accepted = copy.deepcopy(generated) + reproduction = existing.get("reproduction") + if isinstance(reproduction, dict): + accepted["reproduction"] = copy.deepcopy(reproduction) + assessment = existing.get("assessment") + if isinstance(assessment, dict): + accepted["previous_assessment"] = copy.deepcopy(assessment) + privacy_review = existing.get("privacy_review") + if isinstance(privacy_review, dict): + accepted["previous_privacy_review"] = copy.deepcopy(privacy_review) + accepted["privacy_review"] = {"status": "pending"} + accepted["quality"] = recompute_quality(accepted) + return accepted diff --git a/plugins/codestable/skills/cs-feedback/scripts/report_feedback_issue.py b/plugins/codestable/skills/cs-feedback/scripts/report_feedback_issue.py index 1bc9709..ff07781 100644 --- a/plugins/codestable/skills/cs-feedback/scripts/report_feedback_issue.py +++ b/plugins/codestable/skills/cs-feedback/scripts/report_feedback_issue.py @@ -6,13 +6,32 @@ from __future__ import annotations import argparse import os import json +import re import shlex import shutil import socket import subprocess +import sys from urllib.parse import urlparse from pathlib import Path +sys.dont_write_bytecode = True +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from feedback_privacy import ( # noqa: E402 + CREDENTIAL_REDACTIONS, + EMAIL_PATTERN, + ENV_PATTERN, + ENV_NAME_PATTERN, + PATH_PATTERN, + REMOTE_PATTERN, + URL_PATTERN, + contains_secret_assignment, + contains_inline_json, +) + NETWORK_ERROR_PATTERN = ( "could not resolve host", @@ -27,6 +46,29 @@ NETWORK_ERROR_PATTERN = ( ) +def public_body_private_reasons(text: str) -> list[str]: + checks = { + "absolute-path": PATH_PATTERN, + "remote": REMOTE_PATTERN, + "url": URL_PATTERN, + "email": EMAIL_PATTERN, + "environment": ENV_PATTERN, + "environment-name": ENV_NAME_PATTERN, + } + reasons = [name for name, pattern in checks.items() if pattern.search(text)] + if contains_secret_assignment(text) or any( + pattern.search(text) for pattern, _replacement in CREDENTIAL_REDACTIONS + ): + reasons.append("secret") + if contains_inline_json(text): + reasons.append("raw-json") + if "```" in text: + reasons.append("code-block") + if re.search(r"(?:sk-[A-Za-z0-9]{20,}|gh[pousr]_[A-Za-z0-9_]{20,})", text): + reasons.append("secret-token") + return reasons + + def run(command: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: return subprocess.run(command, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) @@ -97,13 +139,18 @@ def main_with_args_for_test(argv: list[str] | None = None) -> int: parser.add_argument("--title", required=True) parser.add_argument("--body-file", required=True) parser.add_argument("--json-output", default=None) + parser.add_argument( + "--confirm-public-preview", + action="store_true", + help="Confirm the reviewed public preview may be sent to GitHub", + ) args = parser.parse_args(argv) body_file = Path(args.body_file).expanduser() if not body_file.is_file(): raise SystemExit(f"body file not found: {body_file}") - if body_file.name == "evidence.json": - raise SystemExit("refusing to upload local-private evidence.json; use github-issue.md public preview") + if body_file.name in {"evidence.json", "triage.json", "regression-candidate.json"}: + raise SystemExit(f"refusing to upload local-private {body_file.name}; use github-issue.md public preview") if body_file.suffix == ".json": try: payload = json.loads(body_file.read_text(encoding="utf-8")) @@ -111,6 +158,19 @@ def main_with_args_for_test(argv: list[str] | None = None) -> int: payload = {} if isinstance(payload, dict) and payload.get("privacy") == "local-private": raise SystemExit("refusing to upload local-private evidence; generate a public preview first") + if args.confirm_public_preview: + if body_file.name != "github-issue.md": + raise SystemExit("confirmed upload requires the reviewed github-issue.md public preview") + reasons = public_body_private_reasons(body_file.read_text(encoding="utf-8")) + if reasons: + raise SystemExit( + "public preview contains private content: " + ", ".join(reasons) + ) + title_reasons = public_body_private_reasons(args.title) + if title_reasons: + raise SystemExit( + "issue title contains private content: " + ", ".join(title_reasons) + ) gh = shutil.which("gh") command = [ @@ -126,7 +186,14 @@ def main_with_args_for_test(argv: list[str] | None = None) -> int: ] result_payload: dict[str, object] - if not gh: + if gh and not args.confirm_public_preview: + result_payload = { + "status": "manual", + "reason": "public preview confirmation required", + "command": shell_join(command), + "body_file": str(body_file), + } + elif not gh: result_payload = { "status": "manual", "reason": "gh not found", diff --git a/plugins/codestable/skills/cs-onboard/codestable.gitignore b/plugins/codestable/skills/cs-onboard/codestable.gitignore index 7c3a8d3..6e971db 100644 --- a/plugins/codestable/skills/cs-onboard/codestable.gitignore +++ b/plugins/codestable/skills/cs-onboard/codestable.gitignore @@ -1,2 +1,6 @@ **/__pycache__/ **/*.pyc +feedback/*/*-report.md +feedback/*/evidence.json +feedback/*/triage.json +feedback/*/regression-candidate.json diff --git a/plugins/codestable/skills/cs-onboard/references/execution-conventions.md b/plugins/codestable/skills/cs-onboard/references/execution-conventions.md index cfaf8b2..0fe9d52 100644 --- a/plugins/codestable/skills/cs-onboard/references/execution-conventions.md +++ b/plugins/codestable/skills/cs-onboard/references/execution-conventions.md @@ -21,6 +21,11 @@ `cs-note` 是唯一例外：`.codestable/` 存在但 `attention.md` 缺失时，它可以创建最小分节骨架 后写入。 +## CodeStable 自身反馈 + +遇到 CodeStable 规则不清、阶段跑偏或工具失败时，可以提示用户显式调用 `cs-feedback`。 +提示本身不得读取历史、后台采集、自动上传、自动修改目标 skill，也不得替用户确认 public preview。 + ## Skill 间同轮转交 公开 skill 选择另一个主入口后，按已安装 skill 名称加载目标协议，并在当前 run 继续。skill 是独立安装单元；不得靠读取 sibling skill 文件模拟转交。 diff --git a/plugins/codestable/skills/cs-onboard/references/shared-conventions.md b/plugins/codestable/skills/cs-onboard/references/shared-conventions.md index f340303..c851bf8 100644 --- a/plugins/codestable/skills/cs-onboard/references/shared-conventions.md +++ b/plugins/codestable/skills/cs-onboard/references/shared-conventions.md @@ -61,8 +61,11 @@ onboard 完成后骨架（`cs-onboard` 负责搭建）： ├── feedback/ CodeStable skill 使用反馈和上报证据 │ └── YYYY-MM-DD-{slug}/ │ ├── {slug}-report.md -│ ├── evidence.json -│ └── github-issue.md +│ ├── evidence.json local-private observations +│ ├── triage.json local-private assessments + quality +│ ├── public-issue-context.json allowlist preview，可选 +│ ├── github-issue.md 用户确认后可上传，可选 +│ └── regression-candidate.json local-private eval 交接，可选 ├── compound/ 沉淀类文档统一目录（cs-keep 产出） │ └── YYYY-MM-DD-{slug}.md │ 纯 markdown，无 frontmatter，grep 检索 @@ -77,7 +80,7 @@ onboard 完成后骨架（`cs-onboard` 负责搭建）： - 需求文档：`requirements/{slug}.md`（能力愿景，不带日期前缀，扁平不分组）；中心索引 `requirements/VISION.md` - roadmap：`roadmap/{slug}/`（不带日期前缀，平铺不嵌套） - feature / issue / refactor 目录：带日期前缀 `YYYY-MM-DD-{slug}` -- feedback 目录：带日期前缀 `YYYY-MM-DD-{slug}`，保存 feedback report、脱敏 evidence 和 GitHub issue body +- feedback 目录：带日期前缀 `YYYY-MM-DD-{slug}`，保存 report、local-private evidence/triage/candidate 与用户确认的 public preview - 沉淀类：`compound/YYYY-MM-DD-{slug}.md`，日期用**归档当天**，纯 markdown 无 frontmatter（cs-keep 产出） - 领域术语：`requirements/CONTEXT.md`（单 context）或 `requirements/{ctx}/CONTEXT.md`（多 context）；cs-domain lazy 创建 - 架构决策：`requirements/adrs/NNN-{slug}.md`（系统级）或 `requirements/{ctx}/adrs/NNN-{slug}.md`（子 context）；3 位编号，cs-domain 产出 @@ -105,7 +108,7 @@ onboard 完成后骨架（`cs-onboard` 负责搭建）： **归档类（compound）**：由 `cs-keep` 统一产出，写到 `.codestable/compound/YYYY-MM-DD-{slug}.md`。纯 markdown，**无 frontmatter**。三段足够：背景 / 结论 / 证据。检索靠 grep。 -**反馈类（feedback）**：由 `cs-feedback` 统一产出，写到 `.codestable/feedback/YYYY-MM-DD-{slug}/`。`{slug}-report.md` 用 `doc_type: codestable-feedback`；`evidence.json` 只放脱敏后的本机历史片段和上下文窗口；`github-issue.md` 是可公开上报前让用户确认的 issue body。 +**反馈类（feedback）**：由 `cs-feedback` 显式调用后产出，写到 `.codestable/feedback/YYYY-MM-DD-{slug}/`。`evidence.json` 保存脱敏 observation/incident，`triage.json` 保存 assessment 与 readiness，二者及 candidate 均为 local-private；`github-issue.md` 只能从 public allowlist 渲染并在上传前让用户确认。 **外部读者文档**（`cs-docs` tutorial / api mode）：frontmatter 由对应模式定义。无特殊说明：`draft` = 待 review，`current` = 当前有效，`outdated` = 代码已变更待同步。 diff --git a/plugins/codestable/skills/cs-onboard/references/system-overview.md b/plugins/codestable/skills/cs-onboard/references/system-overview.md index 918aefb..5f3faff 100644 --- a/plugins/codestable/skills/cs-onboard/references/system-overview.md +++ b/plugins/codestable/skills/cs-onboard/references/system-overview.md @@ -31,7 +31,7 @@ CodeStable 把常见开发活动各配一套流程，产物放进统一的 `.cod - `cs-req` — 起草或刷新 `.codestable/requirements/` 下的需求文档。 - `cs-domain` — 维护 CONTEXT.md 术语、ADR 决策和单/多 context 拓扑。 - `cs-audit` — 主动扫描 bug、安全、性能、可维护性和架构偏离。 -- `cs-feedback` — 收集 CodeStable skill 使用问题，自动采集本机 Codex/Claude 历史并准备 GitHub issue。 +- `cs-feedback` — 显式调用后把当前会话整理为 local-private incident/triage；公开预览经用户确认后才可上报。 - `cs-docs` — 写给外部读者的开发者指南、用户指南或 API 参考。 - `cs-docs-neat` — 阶段/里程碑收尾时整理 `.codestable/`、README/docs、`CLAUDE.md` / `AGENTS.md` 和 agent 记忆。 diff --git a/tests/test_cs_feedback.py b/tests/test_cs_feedback.py index dcbe63d..4b141c5 100644 --- a/tests/test_cs_feedback.py +++ b/tests/test_cs_feedback.py @@ -8,7 +8,6 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "plugins/codestable/skills/cs-feedback/scripts/collect_feedback_context.py" -REPORT_SCRIPT = ROOT / "plugins/codestable/skills/cs-feedback/scripts/report_feedback_issue.py" def load_collector(): @@ -23,18 +22,6 @@ def load_collector(): collector = load_collector() -def load_reporter(): - spec = importlib.util.spec_from_file_location("report_feedback_issue", REPORT_SCRIPT) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -reporter = load_reporter() - - def write_jsonl(path: Path, records: list[dict]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text("\n".join(json.dumps(record, ensure_ascii=False) for record in records) + "\n", encoding="utf-8") @@ -69,7 +56,7 @@ def test_collects_codestable_tool_failures_and_user_corrections(tmp_path: Path) { "timestamp": "2026-07-03T01:03:00Z", "type": "event_msg", - "payload": {"type": "user_message", "message": "不对，你没有按 cs-feat 的 goal driver 规则走。token=secret123456"}, + "payload": {"type": "user_message", "message": "不对，应该按 cs-feat 的 goal driver 规则走。token=secret123456"}, }, ], ) @@ -345,15 +332,24 @@ def test_public_summary_redacts_paths_and_remotes(tmp_path: Path) -> None: }, { "timestamp": "2026-07-03T04:01:00Z", - "type": "event_msg", + "type": "response_item", "payload": { - "message": ( + "type": "function_call_output", + "output": ( "cs-feat failed reading /Users/me/private/repo/.codestable/attention.md, " "~/work/private-client/secrets.md, and /opt/acme/customer_data.md from " "https://github.com/acme/private?token=abc and git@gitlab.company.com:secret/private-repo.git" ) }, }, + { + "timestamp": "2026-07-03T04:02:00Z", + "type": "event_msg", + "payload": { + "type": "user_message", + "message": "不对，应该只报告脱敏后的 cs-feat failure。", + }, + }, ], ) @@ -372,7 +368,10 @@ def test_public_summary_redacts_paths_and_remotes(tmp_path: Path) -> None: ) payload = json.loads(output.read_text(encoding="utf-8")) - excerpt = payload["public_issue_context"]["events"][0]["sanitized_excerpt"] + excerpt = " ".join( + event["sanitized_excerpt"] + for event in payload["public_issue_context"]["events"] + ) assert "/Users/me" not in excerpt assert "~/work" not in excerpt assert "/opt/acme" not in excerpt @@ -382,55 +381,3 @@ def test_public_summary_redacts_paths_and_remotes(tmp_path: Path) -> None: assert "token=abc" not in excerpt assert "" in excerpt assert "" in excerpt - - -def test_reporter_falls_back_when_gh_is_missing(tmp_path: Path, monkeypatch) -> None: - body = tmp_path / "github-issue.md" - body.write_text("## Summary\n\ncs-feedback issue\n", encoding="utf-8") - output = tmp_path / "result.json" - monkeypatch.setattr(reporter.shutil, "which", lambda name: None) - - exit_code = reporter.main_with_args_for_test( - [ - "--repo", - "owner/repo", - "--title", - "Feedback: cs skill failed", - "--body-file", - str(body), - "--json-output", - str(output), - ] - ) - - assert exit_code == 0 - payload = json.loads(output.read_text(encoding="utf-8")) - assert payload["status"] == "manual" - assert payload["reason"] == "gh not found" - assert "gh issue create" in payload["command"] - assert "'Feedback: cs skill failed'" in payload["command"] - - -def test_reporter_refuses_local_private_evidence(tmp_path: Path, monkeypatch) -> None: - evidence = tmp_path / "evidence.json" - evidence.write_text( - json.dumps({"privacy": "local-private", "public_upload_allowed": False}, ensure_ascii=False), - encoding="utf-8", - ) - monkeypatch.setattr(reporter.shutil, "which", lambda name: None) - - try: - reporter.main_with_args_for_test( - [ - "--repo", - "owner/repo", - "--title", - "Feedback: cs skill failed", - "--body-file", - str(evidence), - ] - ) - except SystemExit as exc: - assert "refusing to upload local-private" in str(exc) - else: - raise AssertionError("expected reporter to reject evidence.json") diff --git a/tests/test_cs_feedback_candidate.py b/tests/test_cs_feedback_candidate.py new file mode 100644 index 0000000..96a5738 --- /dev/null +++ b/tests/test_cs_feedback_candidate.py @@ -0,0 +1,322 @@ +from __future__ import annotations + +import ast +import importlib.util +import json +import sys +from copy import deepcopy +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +CONVERTER_SCRIPT = ROOT / "plugins/codestable/skills/cs-feedback/scripts/feedback_to_fixture.py" + + +def load_converter(): + spec = importlib.util.spec_from_file_location("feedback_to_fixture_candidate", CONVERTER_SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +converter = load_converter() +triage_module = sys.modules["feedback_triage"] + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") + + +def write_canonical_feedback_pair(path: Path, payload: dict) -> dict: + triage = deepcopy(payload) + triage.setdefault("trigger_cutoff", "record-0003") + observations = [ + { + "id": observation_id, + "record_id": f"record-{index:04d}", + "role": "assistant", + "record_type": "message", + "text": f"synthetic observation {observation_id}", + } + for index, observation_id in enumerate(triage.get("observation_ids", []), 1) + ] + incident = { + "id": triage["incident_id"], + "target_skill": triage.get("target", {}).get("skill", "unknown"), + "stage_hint": triage.get("target", {}).get("stage_hint", "unknown"), + "incident_kind": triage.get("incident_kind", "unknown"), + "observations": observations, + "capture_cutoff": triage["trigger_cutoff"], + "environment_context": {"provider": "codex", "session": "session-test"}, + } + triage["incident_fingerprint"] = triage_module.incident_fingerprint(incident) + write_json(path, triage) + write_json( + path.with_name("evidence.json"), + { + "schema_version": 2, + "privacy": "local-private", + "incidents": [incident], + }, + ) + return triage + + +def test_feedback_converter_writes_local_candidate_and_rejects_legacy_direct_write(tmp_path: Path) -> None: + feedback_dir = tmp_path / "feedback/case-1" + triage = feedback_dir / "triage.json" + write_canonical_feedback_pair( + triage, + { + "schema_version": 2, + "privacy": "local-private", + "incident_id": "incident-01", + "observation_ids": ["obs-1", "obs-2"], + "target": {"skill": "cs-code-review", "stage_hint": "review"}, + "incident_kind": "tool-failure", + "assessment": { + "expected_behavior": { + "value": "发现阻塞问题", + "source": "user", + "evidence_refs": ["obs-1"], + }, + "actual_behavior": { + "value": "漏报阻塞问题", + "source": "transcript", + "evidence_refs": ["obs-2"], + }, + }, + "reproduction": { + "eval_profile": "findings-recall", + "task_kind": "review", + "input": {"spec": "review spec", "diff": "+ unsafe_call()"}, + "oracle": {"coverage_points": ["unsafe_call must be reported"]}, + }, + "quality": {"triage_ready": True, "regression_ready": True}, + "privacy_review": {"status": "approved"}, + }, + ) + + assert converter.main(["--triage", str(triage)]) == 0 + candidate = feedback_dir / "regression-candidate.json" + data = json.loads(candidate.read_text(encoding="utf-8")) + assert data["_status"] == "candidate" + assert data["privacy"] == "local-private" + assert data["answerType"] == "findings-recall" + + legacy_exp = tmp_path / "experiment" + assert converter.main(["--experiment", str(legacy_exp), "--failure", "old path"]) != 0 + assert not (legacy_exp / "fixtures/regression").exists() + + +def test_v1_public_context_only_creates_a_not_ready_candidate(tmp_path: Path) -> None: + evidence = tmp_path / "feedback/public-issue-context.json" + write_json( + evidence, + { + "privacy": "public-preview", + "events": [ + { + "failure_type": "tool-failure", + "sanitized_excerpt": "synthetic tool failure", + } + ], + }, + ) + + assert converter.main(["--evidence", str(evidence)]) == 0 + candidate = json.loads( + (evidence.parent / "regression-candidate.json").read_text(encoding="utf-8") + ) + assert candidate["quality"]["regression_ready"] is False + assert "triage.json" in candidate["promotion_blockers"] + + +def test_feedback_converter_rejects_noncanonical_triage_without_writing(tmp_path: Path) -> None: + triage = tmp_path / "feedback/triage.json" + write_json( + triage, + { + "schema_version": 2, + "privacy": "public-preview", + "incident_id": "incident-01", + "target": {"skill": "cs-feat"}, + "incident_kind": "wrong-route", + }, + ) + + assert converter.main(["--triage", str(triage)]) != 0 + assert not (triage.parent / "regression-candidate.json").exists() + + +def test_feedback_converter_rejects_malformed_nested_triage_objects(tmp_path: Path) -> None: + triage = tmp_path / "feedback/triage.json" + base = { + "schema_version": 2, + "privacy": "local-private", + "incident_id": "incident-01", + "target": {}, + "assessment": {}, + "reproduction": {}, + "privacy_review": {}, + } + for key in ("target", "assessment", "reproduction", "privacy_review"): + payload = {**base, key: None} + write_json(triage, payload) + candidate = triage.parent / "regression-candidate.json" + candidate.unlink(missing_ok=True) + assert converter.main(["--triage", str(triage)]) != 0 + assert not candidate.exists() + + +def test_feedback_converter_rejects_unselected_incident_without_writing(tmp_path: Path) -> None: + triage = tmp_path / "feedback/triage.json" + write_json( + triage, + { + "schema_version": 2, + "privacy": "local-private", + "incident_id": " ", + "target": {}, + "assessment": {}, + "reproduction": {}, + "privacy_review": {}, + }, + ) + + assert converter.main(["--triage", str(triage)]) != 0 + assert not (triage.parent / "regression-candidate.json").exists() + + +def test_feedback_converter_rejects_triage_not_bound_to_canonical_evidence( + tmp_path: Path, +) -> None: + triage_path = tmp_path / "feedback/triage.json" + base = write_canonical_feedback_pair( + triage_path, + { + "schema_version": 2, + "privacy": "local-private", + "incident_id": "incident-01", + "observation_ids": ["obs-01", "obs-02"], + "target": {"skill": "cs-code-review", "stage_hint": "review"}, + "incident_kind": "tool-failure", + "assessment": { + "expected_behavior": { + "value": "report the finding", + "source": "user", + "evidence_refs": ["obs-01"], + }, + "actual_behavior": { + "value": "finding was missed", + "source": "transcript", + "evidence_refs": ["obs-02"], + }, + }, + "reproduction": { + "eval_profile": "findings-recall", + "task_kind": "review", + "input": {"spec": "synthetic spec", "diff": "+ unsafe_call()"}, + "oracle": {"coverage_points": ["report unsafe_call"]}, + "evidence_refs": ["obs-01", "obs-02"], + }, + "quality": {"triage_ready": True, "regression_ready": True}, + "privacy_review": {"status": "approved"}, + }, + ) + mutations = ( + {**base, "incident_id": "incident-02"}, + {**base, "incident_fingerprint": "sha256:forged"}, + { + **base, + "observation_ids": ["obs-does-not-exist"], + "assessment": { + **base["assessment"], + "expected_behavior": { + **base["assessment"]["expected_behavior"], + "evidence_refs": ["obs-does-not-exist"], + }, + "actual_behavior": { + **base["assessment"]["actual_behavior"], + "evidence_refs": ["obs-does-not-exist"], + }, + }, + "reproduction": { + **base["reproduction"], + "evidence_refs": ["obs-does-not-exist"], + }, + }, + ) + for mutation in mutations: + write_json(triage_path, mutation) + candidate = triage_path.with_name("regression-candidate.json") + candidate.unlink(missing_ok=True) + assert converter.main(["--triage", str(triage_path)]) == 2 + assert not candidate.exists() + + +def test_feedback_converter_recomputes_blockers_and_never_copies_assessment_fallback( + tmp_path: Path, +) -> None: + triage = tmp_path / "feedback/triage.json" + write_canonical_feedback_pair( + triage, + { + "schema_version": 2, + "privacy": "local-private", + "incident_id": "incident-01", + "observation_ids": ["obs-02"], + "target": {"skill": "cs-code-review", "stage_hint": "review"}, + "incident_kind": "tool-failure", + "assessment": { + "expected_behavior": { + "value": "PRIVATE ASSESSMENT MUST NOT BECOME ORACLE", + "source": "user", + "evidence_refs": [], + }, + "actual_behavior": { + "value": "PRIVATE ACTUAL MUST NOT BECOME FIXTURE", + "source": "transcript", + "evidence_refs": ["obs-02"], + }, + }, + "reproduction": { + "eval_profile": "findings-recall", + "task_kind": "review", + "input": {"diff": "+ synthetic_bug()"}, + "oracle": None, + "evidence_refs": [], + }, + "quality": {"triage_ready": True, "regression_ready": True}, + "privacy_review": {"status": "approved"}, + }, + ) + + assert converter.main(["--triage", str(triage)]) == 0 + candidate = json.loads( + (triage.parent / "regression-candidate.json").read_text(encoding="utf-8") + ) + candidate_text = json.dumps(candidate, ensure_ascii=False) + assert candidate["answer"] == [] + assert "reproduction.oracle.coverage_points" in candidate["promotion_blockers"] + assert "assessment.expected_behavior.evidence_refs" in candidate["promotion_blockers"] + assert "PRIVATE ASSESSMENT" not in candidate_text + assert "PRIVATE ACTUAL" not in candidate_text + + +def test_shipped_converter_has_no_runtime_import_of_repo_local_eval_tools() -> None: + source = CONVERTER_SCRIPT.read_text(encoding="utf-8") + tree = ast.parse(source) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + if node.module: + imported.add(node.module) + imported.update(alias.name for alias in node.names) + assert not {"config", "fixtures", "buildprompt", "promote_feedback_fixture"} & imported + assert ".claude/skills/eval-cs-skill" not in source diff --git a/tests/test_cs_feedback_evidence_pipeline.py b/tests/test_cs_feedback_evidence_pipeline.py new file mode 100644 index 0000000..748da1b --- /dev/null +++ b/tests/test_cs_feedback_evidence_pipeline.py @@ -0,0 +1,1525 @@ +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +COLLECTOR_SCRIPT = ROOT / "plugins/codestable/skills/cs-feedback/scripts/collect_feedback_context.py" + + +def load_module(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +collector = load_module(COLLECTOR_SCRIPT, "collect_feedback_context_v2") +transcripts = sys.modules["feedback_transcripts"] +triage_module = sys.modules.get("feedback_triage") +incidents_module = sys.modules["feedback_incidents"] +models_module = sys.modules["feedback_models"] +privacy_module = sys.modules["feedback_privacy"] +repo_context_module = sys.modules["feedback_repo_context"] + + +def write_jsonl(path: Path, records: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(json.dumps(record, ensure_ascii=False) for record in records) + "\n", encoding="utf-8") + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") + + +def test_current_session_metadata_only_ignores_stale_mtime_and_body_cwd(tmp_path: Path) -> None: + home = tmp_path / "home" + stale = home / ".codex/sessions/2026/01/01/stale.jsonl" + body_only = home / ".codex/sessions/2026/01/01/body-only.jsonl" + write_jsonl( + stale, + [ + {"type": "session_meta", "timestamp": "2026-01-01T00:00:00Z", "payload": {"session_id": "stale", "cwd": "/repo"}}, + {"type": "event_msg", "timestamp": "2026-01-01T00:01:00Z", "payload": {"message": "cs-feedback failed"}}, + ], + ) + write_jsonl( + body_only, + [ + {"type": "event_msg", "timestamp": "2026-01-01T00:00:00Z", "payload": {"message": "cwd=/repo secret-token-123456"}}, + ], + ) + old = 1_700_000_000 + os.utime(stale, (old, old)) + os.utime(body_only, (old, old)) + stale.chmod(0o600) + body_only.chmod(0o600) + + output = tmp_path / "evidence.json" + rc = collector.main_with_args_for_test( + [ + "--history-root", + str(home), + "--since-days", + "0", + "--session", + "current", + "--cwd", + "/repo", + "--feedback", + "cs-feedback failed", + "--output", + str(output), + ] + ) + + assert rc == 0 + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["searched_files"] == [str(stale)] + assert payload["since_days_ignored"] is True + assert "secret-token-123456" not in json.dumps(payload["ambiguity"], ensure_ascii=False) + + +def test_current_session_ambiguity_never_reads_or_returns_message_bodies( + tmp_path: Path, monkeypatch +) -> None: + home = tmp_path / "home" + for session, marker in (("a", "PRIVATE_TOOL_ALPHA"), ("b", "PRIVATE_TOOL_BETA")): + write_jsonl( + home / f".codex/sessions/2026/07/03/{session}.jsonl", + [ + { + "type": "session_meta", + "payload": {"session_id": session, "cwd": "/same/repo"}, + }, + {"type": "event_msg", "payload": {"message": marker}}, + ], + ) + + def forbidden(*_args, **_kwargs): + raise AssertionError("ambiguity resolution must not read or flatten transcript bodies") + + monkeypatch.setattr(transcripts, "read_records", forbidden) + monkeypatch.setattr(transcripts, "flatten", forbidden) + output = tmp_path / "evidence.json" + assert ( + collector.main_with_args_for_test( + [ + "--history-root", + str(home), + "--session", + "current", + "--cwd", + "/same/repo", + "--output", + str(output), + ] + ) + == 0 + ) + + payload = json.loads(output.read_text(encoding="utf-8")) + ambiguity_text = json.dumps(payload["ambiguity"], ensure_ascii=False) + assert payload["searched_files"] == [] + assert len(payload["ambiguity"]["candidates"]) == 2 + assert "PRIVATE_TOOL_ALPHA" not in ambiguity_text + assert "PRIVATE_TOOL_BETA" not in ambiguity_text + + +def test_current_session_weak_cwd_match_requires_user_selection(tmp_path: Path) -> None: + home = tmp_path / "home" + transcript = home / ".codex/sessions/2026/07/03/parent.jsonl" + write_jsonl( + transcript, + [ + { + "type": "session_meta", + "payload": {"session_id": "parent", "cwd": "/repo"}, + }, + {"type": "event_msg", "payload": {"message": "cs-feat failed"}}, + ], + ) + output = tmp_path / "evidence.json" + collector.main_with_args_for_test( + [ + "--history-root", + str(home), + "--session", + "current", + "--cwd", + "/repo/subdir", + "--output", + str(output), + ] + ) + + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["searched_files"] == [] + assert [item["session"] for item in payload["ambiguity"]["candidates"]] == ["parent"] + + +def test_jsonl_snapshot_freezes_only_complete_records_at_eof(tmp_path: Path) -> None: + transcript = tmp_path / ".codex/sessions/frozen.jsonl" + complete = ( + json.dumps({"type": "session_meta", "payload": {"session_id": "s1"}}) + "\n" + + json.dumps({"type": "event_msg", "payload": {"message": "first"}}) + + "\n" + ).encode() + transcript.parent.mkdir(parents=True) + transcript.write_bytes(complete + b'{"type":"event_msg"') + + records, capture = transcripts.read_transcript_snapshot(transcript) + frozen = json.dumps(records, sort_keys=True) + with transcript.open("ab") as handle: + handle.write(b',"payload":{"message":"late"}}\n') + + assert len(records) == 2 + assert json.dumps(records, sort_keys=True) == frozen + assert capture["complete_record_eof"] == len(complete) + assert capture["byte_length"] > capture["complete_record_eof"] + + +def test_collector_reads_each_selected_transcript_snapshot_once(tmp_path: Path, monkeypatch) -> None: + transcript = tmp_path / ".codex/sessions/once.jsonl" + write_jsonl( + transcript, + [ + {"type": "session_meta", "payload": {"session_id": "s1", "cwd": "/repo"}}, + {"type": "response_item", "payload": {"type": "function_call_output", "output": "cs-feat failed"}}, + {"type": "event_msg", "payload": {"type": "user_message", "message": "不对，应该 review。"}}, + ], + ) + real_read = collector.read_transcript_snapshot + calls: list[Path] = [] + + def counting_read(path: Path): + calls.append(path) + return real_read(path) + + monkeypatch.setattr(collector, "read_transcript_snapshot", counting_read) + collector.main_with_args_for_test( + ["--session", str(transcript), "--output", str(tmp_path / "evidence.json")] + ) + assert calls == [transcript] + + +def test_evidence_v2_incident_triage_and_public_projection_are_structured(tmp_path: Path) -> None: + home = tmp_path / "home" + transcript = home / ".codex/sessions/2026/07/03/incident.jsonl" + write_jsonl( + transcript, + [ + {"type": "session_meta", "timestamp": "2026-07-03T01:00:00Z", "payload": {"session_id": "s1", "cwd": "/repo"}}, + { + "type": "response_item", + "timestamp": "2026-07-03T01:01:00Z", + "payload": {"type": "function_call", "call_id": "call-1", "name": "read_file", "arguments": "cs-feat gate"}, + }, + { + "type": "response_item", + "timestamp": "2026-07-03T01:02:00Z", + "payload": {"type": "function_call_output", "call_id": "call-1", "output": "tool call failed: /Users/me/private token=secret123456"}, + }, + { + "type": "event_msg", + "timestamp": "2026-07-03T01:03:00Z", + "payload": {"type": "user_message", "message": "不对，应该执行 cs-feat design review gate。"}, + }, + {"type": "event_msg", "timestamp": "2026-07-03T01:04:00Z", "payload": {"message": "anchor 后的内容不应进入 incident"}}, + ], + ) + + output = tmp_path / "evidence.json" + collector.main_with_args_for_test( + [ + "--history-root", + str(home), + "--since-days", + "9999", + "--feedback", + "cs-feat tool failed should execute gate", + "--output", + str(output), + ] + ) + + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["schema_version"] == 2 + assert payload["incidents"][0]["target_skill"] == "cs-feat" + assert payload["incidents"][0]["capture_cutoff"] == "record-0003" + assert "anchor 后" not in json.dumps(payload["incidents"], ensure_ascii=False) + assert payload["incidents"][0]["timeline"][1]["correlation_source"] == "provider" + assert "triage" not in payload + triage = json.loads((tmp_path / "triage.json").read_text(encoding="utf-8")) + assert triage["quality"]["triage_ready"] is True + assert triage["quality"]["regression_ready"] is False + assert triage["assessment"]["expected_behavior"]["source"] == "user" + public_payload = payload["public_issue_context"] + assert set(public_payload["events"][0]) == { + "provider", + "session_label", + "timestamp_bucket", + "failure_type", + "match_type", + "tool_name", + "skill_or_reference", + "sanitized_excerpt", + } + public_text = json.dumps(public_payload, ensure_ascii=False) + assert "/Users/me" not in public_text + assert "secret123456" not in public_text + assert "incident_kind" in public_payload["incidents"][0] + assert set(public_payload["incidents"][0]) == { + "incident_kind", + "target_skill", + "stage_hint", + "expected_behavior", + "actual_behavior", + "impact", + "proposed_fix", + } + + +def test_missing_user_anchor_keeps_incident_but_blocks_triage(tmp_path: Path) -> None: + transcript = tmp_path / ".codex/sessions/no-anchor.jsonl" + write_jsonl( + transcript, + [ + {"type": "session_meta", "payload": {"session_id": "s1", "cwd": "/repo"}}, + { + "type": "response_item", + "payload": { + "type": "function_call", + "call_id": "c1", + "name": "read_file", + "arguments": "cs-feat gate", + }, + }, + { + "type": "response_item", + "payload": { + "type": "function_call_output", + "call_id": "c1", + "output": "tool call failed", + }, + }, + ], + ) + output = tmp_path / "evidence.json" + collector.main_with_args_for_test( + ["--session", str(transcript), "--feedback", "cs-feat failed", "--output", str(output)] + ) + + evidence = json.loads(output.read_text(encoding="utf-8")) + triage = json.loads((tmp_path / "triage.json").read_text(encoding="utf-8")) + public = json.loads((tmp_path / "public-issue-context.json").read_text(encoding="utf-8")) + assert evidence["incidents"][0]["capture_cutoff"] == "unknown" + assert triage["quality"]["triage_ready"] is False + assert "trigger_cutoff" in triage["quality"]["missing_fields"] + assert public["events"] == [] + assert public["incidents"] == [] + + +def test_tool_pairing_prefers_provider_ids_and_fails_closed_on_ambiguity(tmp_path: Path) -> None: + path = tmp_path / ".codex/sessions/pairing.jsonl" + records = [ + {"payload": {"type": "function_call", "call_id": "p1", "name": "one"}}, + {"payload": {"type": "function_call_output", "call_id": "p1", "output": "ok"}}, + {"payload": {"type": "function_call", "name": "two"}}, + {"type": "session_meta", "payload": {"session_id": "s1"}}, + {"payload": {"type": "function_call_output", "output": "ok"}}, + {"payload": {"type": "function_call", "name": "three"}}, + {"payload": {"type": "function_call", "name": "four"}}, + {"payload": {"type": "function_call_output", "output": "ambiguous"}}, + {"payload": {"type": "function_call", "name": "five"}}, + {"payload": {"type": "function_call_output", "call_id": "missing", "output": "no"}}, + ] + + normalized = collector.normalize_records(path, records) + assert [record.correlation_source for record in normalized[:2]] == ["provider", "provider"] + assert normalized[2].correlation_source == "adjacency" + assert normalized[4].correlation_source == "adjacency" + assert len({normalized[2].correlation_id, normalized[4].correlation_id}) == 1 + assert [record.correlation_source for record in normalized[5:8]] == [ + "unpaired", + "unpaired", + "unpaired", + ] + assert [record.correlation_source for record in normalized[8:10]] == ["unpaired", "unpaired"] + + +def test_non_overlapping_user_turns_create_separate_incidents(tmp_path: Path) -> None: + transcript = tmp_path / ".codex/sessions/two-incidents.jsonl" + write_jsonl( + transcript, + [ + {"type": "session_meta", "payload": {"session_id": "s1", "cwd": "/repo"}}, + { + "type": "response_item", + "payload": {"type": "function_call_output", "output": "cs-feat tool call failed"}, + }, + { + "type": "event_msg", + "payload": {"type": "user_message", "message": "不对，应该先运行 cs-feat design review。"}, + }, + { + "type": "response_item", + "payload": {"type": "function_call_output", "output": "cs-feat second tool call failed"}, + }, + { + "type": "event_msg", + "payload": {"type": "user_message", "message": "又错了，应该进入 cs-feat QA。"}, + }, + ], + ) + output = tmp_path / "evidence.json" + collector.main_with_args_for_test( + ["--session", str(transcript), "--feedback", "cs-feat failed", "--output", str(output)] + ) + + incidents = json.loads(output.read_text(encoding="utf-8"))["incidents"] + assert len(incidents) == 2 + first = json.dumps(incidents[0], ensure_ascii=False) + second = json.dumps(incidents[1], ensure_ascii=False) + assert "design review" in first and "进入 cs-feat QA" not in first + assert "进入 cs-feat QA" in second and "design review" not in second + + +def _semantic_timeline(module, path: Path) -> list[tuple[str, str, str, str]]: + records = module.read_records(path) + normalized = collector.normalize_records(path, records) + return [ + (record.role, record.record_type, record.tool_name, record.correlation_source) + for record in normalized + if record.record_type != "session_meta" + ] + + +def test_codex_and_claude_json_variants_normalize_tool_incidents_isomorphically(tmp_path: Path) -> None: + codex = tmp_path / ".codex/sessions/codex.jsonl" + codex_records = [ + {"type": "session_meta", "payload": {"session_id": "cx", "cwd": "/repo"}}, + { + "type": "response_item", + "payload": {"type": "function_call", "call_id": "c1", "name": "read_file", "arguments": "cs-feat review"}, + }, + { + "type": "response_item", + "payload": {"type": "function_call_output", "call_id": "c1", "output": "tool call failed"}, + }, + {"type": "event_msg", "payload": {"type": "user_message", "message": "不对，应该 review。"}}, + ] + write_jsonl(codex, codex_records) + + claude_messages = [ + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "c1", "name": "read_file", "input": {"intent": "cs-feat review"}} + ], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "c1", "content": "tool call failed"}], + }, + {"role": "user", "content": "不对，应该 review。"}, + ] + claude_json = tmp_path / ".claude/sessions/claude.json" + write_json(claude_json, {"session_id": "cl", "cwd": "/repo", "messages": claude_messages}) + claude_jsonl = tmp_path / ".claude/sessions/claude.jsonl" + write_jsonl( + claude_jsonl, + [ + {"type": "session_meta", "session_id": "cl", "cwd": "/repo"}, + *claude_messages, + ], + ) + + expected = _semantic_timeline(transcripts, codex) + assert _semantic_timeline(transcripts, claude_json) == expected + assert _semantic_timeline(transcripts, claude_jsonl) == expected + + +def test_repo_context_records_runtime_artifacts_and_file_level_git_status(tmp_path: Path) -> None: + repo = tmp_path / "repo" + child = repo / "nested/child" + (repo / ".codestable/features/demo").mkdir(parents=True) + child.mkdir(parents=True) + write_json( + repo / ".codestable/runtime-manifest.json", + {"schema_version": 1, "runtime_version": "9.9.9", "plugin_version": "9.9.9"}, + ) + (repo / ".codestable/features/demo/demo-design.md").write_text( + "---\ndoc_type: feature-design\nstatus: approved\n---\n# Demo\n", + encoding="utf-8", + ) + (repo / ".codestable/features/demo/goal-state.yaml").write_text( + "stage: implementation\nstatus: running\n", + encoding="utf-8", + ) + (repo / "changed.txt").write_text("private business content", encoding="utf-8") + subprocess.run(["git", "init", "-q"], cwd=repo, check=True) + + transcript = tmp_path / ".codex/sessions/repo-context.jsonl" + write_jsonl( + transcript, + [ + {"type": "session_meta", "payload": {"session_id": "s1", "cwd": str(child)}}, + {"type": "response_item", "payload": {"type": "function_call_output", "output": "cs-feat failed"}}, + {"type": "event_msg", "payload": {"type": "user_message", "message": "不对，应该 review。"}}, + ], + ) + output = tmp_path / "evidence.json" + collector.main_with_args_for_test( + ["--session", str(transcript), "--cwd", str(child), "--output", str(output)] + ) + + context = json.loads(output.read_text(encoding="utf-8"))["incidents"][0]["repo_context"] + assert context["runtime"]["runtime_version"] == "9.9.9" + artifact_paths = {item["path"] for item in context["artifacts"]} + assert ".codestable/features/demo/demo-design.md" in artifact_paths + assert ".codestable/features/demo/goal-state.yaml" in artifact_paths + assert any(item["path"] == "changed.txt" for item in context["git_status"]) + assert "private business content" not in json.dumps(context, ensure_ascii=False) + triage = json.loads((tmp_path / "triage.json").read_text(encoding="utf-8")) + assert triage["repo_context"]["runtime"]["runtime_version"] == "9.9.9" + + +def test_environment_context_uses_metadata_records_not_tool_payload_fields() -> None: + path = Path("/tmp/.codex/sessions/environment.jsonl") + tool_record = { + "type": "response_item", + "payload": { + "type": "function_call_output", + "model": "business-model-name", + "version": "internal-service-v42", + "output": "tool failed", + }, + } + + contaminated = repo_context_module.environment_context( + path, + [ + {"type": "session_meta", "payload": {"session_id": "s1"}}, + tool_record, + ], + {"format": "jsonl"}, + ) + assert contaminated["model"] == "unknown" + assert contaminated["host_version"] == "unknown" + + metadata = repo_context_module.environment_context( + path, + [ + { + "type": "session_meta", + "payload": {"session_id": "s1", "cli_version": "codex-cli-1"}, + }, + {"type": "turn_context", "model": "review-model"}, + tool_record, + ], + {"format": "jsonl"}, + ) + assert metadata["model"] == "review-model" + assert metadata["host_version"] == "codex-cli-1" + + +def test_collector_rejects_invalid_existing_triage_without_overwriting(tmp_path: Path) -> None: + invalid_payloads = ( + "{broken", + "[]\n", + '{"schema_version":2,"privacy":"public-preview"}\n', + ) + for index, invalid in enumerate(invalid_payloads): + case = tmp_path / f"invalid-{index}" + transcript = case / ".codex/sessions/feedback.jsonl" + write_jsonl( + transcript, + [ + {"type": "session_meta", "payload": {"session_id": "s1", "cwd": "/repo"}}, + {"type": "response_item", "payload": {"type": "function_call_output", "output": "cs-feat failed"}}, + {"type": "event_msg", "payload": {"type": "user_message", "message": "不对，应该 review。"}}, + ], + ) + output = case / "evidence.json" + triage_path = case / "triage.json" + triage_path.write_text(invalid, encoding="utf-8") + + assert collector.main_with_args_for_test( + ["--session", str(transcript), "--output", str(output)] + ) == 2 + assert triage_path.read_text(encoding="utf-8") == invalid + assert not output.exists() + assert not (case / "public-issue-context.json").exists() + + +def test_collector_rejects_colliding_output_paths_without_writing(tmp_path: Path) -> None: + transcript = tmp_path / ".codex/sessions/feedback.jsonl" + write_jsonl( + transcript, + [ + {"type": "session_meta", "payload": {"session_id": "s1", "cwd": "/repo"}}, + {"type": "response_item", "payload": {"type": "function_call_output", "output": "cs-feat failed"}}, + {"type": "event_msg", "payload": {"type": "user_message", "message": "不对，应该 review。"}}, + ], + ) + collision = tmp_path / "same.json" + + assert collector.main_with_args_for_test( + [ + "--session", + str(transcript), + "--output", + str(collision), + "--triage-output", + str(collision), + "--public-output", + str(collision), + ] + ) == 2 + assert not collision.exists() + + +def test_collector_atomically_replaces_each_output(tmp_path: Path, monkeypatch) -> None: + transcript = tmp_path / ".codex/sessions/feedback.jsonl" + write_jsonl( + transcript, + [ + {"type": "session_meta", "payload": {"session_id": "s1", "cwd": "/repo"}}, + {"type": "response_item", "payload": {"type": "function_call_output", "output": "cs-feat failed"}}, + {"type": "event_msg", "payload": {"type": "user_message", "message": "不对，应该 review。"}}, + ], + ) + replaced: list[tuple[Path, Path]] = [] + real_replace = Path.replace + + def track_replace(source: Path, target: Path) -> Path: + replaced.append((source, Path(target))) + return real_replace(source, target) + + monkeypatch.setattr(Path, "replace", track_replace) + output = tmp_path / "evidence.json" + + assert collector.main_with_args_for_test( + ["--session", str(transcript), "--output", str(output)] + ) == 0 + assert {target.name for _source, target in replaced} == { + "evidence.json", + "triage.json", + "public-issue-context.json", + } + assert all(source.parent == target.parent and source != target for source, target in replaced) + assert not list(tmp_path.glob(".*.tmp")) + + +def test_atomic_writer_cleans_private_temps_when_staging_fails( + tmp_path: Path, + monkeypatch, +) -> None: + targets = [tmp_path / name for name in ("triage.json", "evidence.json", "public.json")] + for target in targets: + target.write_text(f"old-{target.name}", encoding="utf-8") + + def fail_fsync(_fd: int) -> None: + raise OSError("injected fsync failure") + + monkeypatch.setattr(collector.os, "fsync", fail_fsync) + with pytest.raises(OSError, match="injected fsync failure"): + collector._write_text_files_atomically( + [(target, f"new-{target.name}") for target in targets] + ) + + for target in targets: + assert target.read_text(encoding="utf-8") == f"old-{target.name}" + assert not list(tmp_path.glob(".*.tmp")) + + +@pytest.mark.parametrize("operation", ["write", "flush"]) +def test_atomic_writer_tracks_temp_before_write_or_flush_failure( + tmp_path: Path, + monkeypatch, + operation: str, +) -> None: + target = tmp_path / "triage.json" + target.write_text("old-triage", encoding="utf-8") + real_named_temporary_file = collector.tempfile.NamedTemporaryFile + + class FailingHandle: + def __init__(self, handle) -> None: + self.handle = handle + + @property + def name(self) -> str: + return self.handle.name + + def __enter__(self): + self.handle.__enter__() + return self + + def __exit__(self, *args): + return self.handle.__exit__(*args) + + def write(self, text: str) -> int: + if operation == "write": + raise OSError("injected write failure") + return self.handle.write(text) + + def flush(self) -> None: + if operation == "flush": + raise OSError("injected flush failure") + self.handle.flush() + + def fileno(self) -> int: + return self.handle.fileno() + + def failing_named_temporary_file(*args, **kwargs): + return FailingHandle(real_named_temporary_file(*args, **kwargs)) + + monkeypatch.setattr( + collector.tempfile, + "NamedTemporaryFile", + failing_named_temporary_file, + ) + with pytest.raises(OSError, match=f"injected {operation} failure"): + collector._write_text_files_atomically([(target, "new-triage")]) + + assert target.read_text(encoding="utf-8") == "old-triage" + assert not list(tmp_path.glob(".*.tmp")) + + +@pytest.mark.parametrize("failure_at", [1, 2, 3]) +def test_atomic_writer_rolls_back_all_outputs_when_replace_fails( + tmp_path: Path, + monkeypatch, + failure_at: int, +) -> None: + targets = [tmp_path / name for name in ("triage.json", "evidence.json", "public.json")] + for target in targets: + target.write_text(f"old-{target.name}", encoding="utf-8") + real_replace = Path.replace + calls = 0 + + def fail_replace(source: Path, target: Path) -> Path: + nonlocal calls + calls += 1 + if calls == failure_at: + raise OSError(f"injected replace failure {failure_at}") + return real_replace(source, target) + + monkeypatch.setattr(Path, "replace", fail_replace) + with pytest.raises(OSError, match="injected replace failure"): + collector._write_text_files_atomically( + [(target, f"new-{target.name}") for target in targets] + ) + + for target in targets: + assert target.read_text(encoding="utf-8") == f"old-{target.name}" + assert not list(tmp_path.glob(".*.tmp")) + + +def test_onboard_gitignore_excludes_only_feedback_private_artifacts(tmp_path: Path) -> None: + repo = tmp_path / "repo" + runtime_dir = repo / ".codestable" + feedback_dir = runtime_dir / "feedback/2026-07-11-private-case" + feedback_dir.mkdir(parents=True) + source_gitignore = ROOT / "plugins/codestable/skills/cs-onboard/codestable.gitignore" + runtime_gitignore = ROOT / ".codestable/.gitignore" + assert source_gitignore.read_text(encoding="utf-8") == runtime_gitignore.read_text( + encoding="utf-8" + ) + (runtime_dir / ".gitignore").write_text( + source_gitignore.read_text(encoding="utf-8"), + encoding="utf-8", + ) + subprocess.run(["git", "init", "-q"], cwd=repo, check=True) + + private_names = ( + "private-case-report.md", + "evidence.json", + "triage.json", + "regression-candidate.json", + ) + public_names = ("public-issue-context.json", "github-issue.md") + for name in private_names + public_names: + (feedback_dir / name).write_text("synthetic\n", encoding="utf-8") + + for name in private_names: + result = subprocess.run( + ["git", "check-ignore", "--quiet", str(feedback_dir / name)], + cwd=repo, + check=False, + ) + assert result.returncode == 0, name + for name in public_names: + result = subprocess.run( + ["git", "check-ignore", "--quiet", str(feedback_dir / name)], + cwd=repo, + check=False, + ) + assert result.returncode == 1, name + + +def test_quality_gate_rejects_assessment_without_observation_reference() -> None: + assert triage_module is not None + triage = { + "incident_id": "incident-01", + "observation_ids": ["obs-02"], + "trigger_cutoff": "record-0003", + "target": {"skill": "cs-feat"}, + "incident_kind": "wrong-route", + "assessment": { + "expected_behavior": {"value": "route to design", "source": "user", "evidence_refs": []}, + "actual_behavior": {"value": "route to QA", "source": "transcript", "evidence_refs": ["obs-02"]}, + }, + "reproduction": { + "eval_profile": "routing-decision", + "task_kind": "routing", + "input": {"utterance": "continue"}, + "oracle": {"expect": {"result_type": "RoutedTo"}}, + }, + } + + quality = triage_module.recompute_quality(triage) + assert quality["triage_ready"] is False + assert quality["regression_ready"] is False + assert "assessment.expected_behavior.evidence_refs" in quality["missing_fields"] + + +def test_quality_gate_keeps_unknown_expected_and_prioritizes_that_question(tmp_path: Path) -> None: + transcript = tmp_path / ".codex/sessions/missing-expected.jsonl" + write_jsonl( + transcript, + [ + {"type": "session_meta", "payload": {"session_id": "s1", "cwd": "/repo"}}, + {"type": "response_item", "payload": {"type": "function_call_output", "output": "cs-feat tool call failed"}}, + {"type": "event_msg", "payload": {"type": "user_message", "message": "有问题。"}}, + ], + ) + output = tmp_path / "evidence.json" + collector.main_with_args_for_test(["--session", str(transcript), "--output", str(output)]) + + triage = json.loads((tmp_path / "triage.json").read_text(encoding="utf-8")) + assert triage["assessment"]["expected_behavior"] == { + "value": "unknown", + "source": "unknown", + "evidence_refs": [], + } + assert triage["quality"]["triage_ready"] is False + assert "assessment.expected_behavior" in triage["quality"]["missing_fields"] + assert triage["quality"]["next_questions"] == ["assessment.expected_behavior"] + + +def test_public_projection_removes_tool_json_code_paths_env_remote_and_secrets(tmp_path: Path) -> None: + transcript = tmp_path / ".codex/sessions/privacy.jsonl" + write_jsonl( + transcript, + [ + {"type": "session_meta", "payload": {"session_id": "s1", "cwd": "/repo"}}, + { + "type": "response_item", + "payload": { + "type": "function_call", + "name": "apply_patch", + "arguments": '{"path":"/Users/me/private.py","env":"API_TOKEN=topsecret123","remote":"https://github.com/acme/private"}', + }, + }, + { + "type": "response_item", + "payload": { + "type": "function_call_output", + "output": ( + 'cs-feat failed {\n "token": "multilinesecret123456",' + '\n "path": "/repo"\n} token=secret123456 ' + "```python\nprivate_code()\n```" + ), + }, + }, + { + "type": "event_msg", + "payload": {"type": "user_message", "message": "不对，应该先 review，不要读取 /opt/private。"}, + }, + ], + ) + output = tmp_path / "evidence.json" + collector.main_with_args_for_test(["--session", str(transcript), "--output", str(output)]) + + evidence_text = output.read_text(encoding="utf-8") + public_path = tmp_path / "public-issue-context.json" + public_text = public_path.read_text(encoding="utf-8") + public_payload = json.loads(public_text) + + def strings(value) -> list[str]: + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [item for child in value for item in strings(child)] + if isinstance(value, dict): + return [item for child in value.values() for item in strings(child)] + return [] + + assert "secret123456" not in evidence_text + for forbidden in ( + "/Users/me", + "/opt/private", + "topsecret123", + "multilinesecret123456", + "API_TOKEN", + "github.com/acme/private", + "/repo", + '"token"', + '"path"', + "private_code()", + "```", + ): + assert forbidden not in public_text + for value in strings(public_payload): + assert "{" not in value and "}" not in value + assert "API_TOKEN" not in value + + +def test_public_redaction_removes_absolute_paths_with_spaces() -> None: + cases = ( + (r"error at C:\Users\bob\secret plan.docx", ("plan.docx",)), + ( + "/Users/alice/customer contracts/acme-pricing-2026.md", + ("contracts", "acme-pricing-2026.md"), + ), + ( + "/Users/alice/Library/Application Support/CodeStable/session.json", + ("Support", "CodeStable", "session.json"), + ), + ( + 'cat "/Users/alice/acme merger notes.txt" done', + ("merger", "notes.txt"), + ), + ( + "cat '/Users/alice/acme merger notes.txt' done", + ("merger", "notes.txt"), + ), + ( + "cat `/Users/alice/acme merger notes.txt` done", + ("merger", "notes.txt"), + ), + ( + 'open "C:\\Users\\bob\\secret plan.docx" now', + ("plan.docx",), + ), + ("see /Users/bob/my report.docx.", ("report.docx",)), + ("see /Users/bob/my report.docx!", ("report.docx",)), + ("see /Users/bob/my report.docx?", ("report.docx",)), + ("see /Users/bob/my report.docx。", ("report.docx",)), + ("see /Users/bob/my report.docx！", ("report.docx",)), + ("see /Users/bob/my report.docx？", ("report.docx",)), + ( + 'see "/Users/bob/my report.docx." next', + ("report.docx",), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx，其中第二节有问题", + ("合并", "计划.docx"), + ), + ( + "路径是 /Users/alice/acme 合并 计划.docx。下一步继续", + ("合并", "计划.docx"), + ), + ( + "（详见 /Users/alice/merger plan.docx）之后再说", + ("merger", "plan.docx"), + ), + ( + "见 /Users/alice/plan file.docx；继续", + ("plan", "file.docx"), + ), + ( + "见“/Users/alice/acme 合并 计划.docx”，继续", + ("合并", "计划.docx"), + ), + ( + "见 /Users/alice/plan long.presentation，继续", + ("long.presentation",), + ), + ( + "见 /Users/alice/客户 合同.文档，继续", + ("合同.文档",), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx…其中有问题", + ("合并", "计划.docx"), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx——其中有问题", + ("合并", "计划.docx"), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx（内含预算）", + ("合并", "计划.docx"), + ), + ( + "see /Users/alice/acme merger plan.docx(v2)", + ("merger", "plan.docx"), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx·补充", + ("合并", "计划.docx"), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx～补充", + ("合并", "计划.docx"), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx“重点”", + ("合并", "计划.docx"), + ), + ( + "/tmp/x my report.docx 之后继续", + ("my", "report.docx"), + ), + ( + r"open C:\Program Files\CodeStable\session.json", + ("Files", "CodeStable", "session.json"), + ), + ( + "/Users/alice/项目 文档/计划.docx", + ("文档", "计划.docx"), + ), + ("/tmp/x 合同.docx", ("合同.docx",)), + ) + for raw, forbidden_parts in cases: + public = privacy_module.public_redact(raw) + assert "" in public + assert all(part not in public for part in forbidden_parts) + + +def test_public_redaction_preserves_text_after_absolute_paths() -> None: + cases = ( + ( + "报错发生在 /tmp/build，随后 agent 跳过了 review gate", + "，随后 agent 跳过了 review gate", + ), + ( + "/Users/alice/acme，其中第二节有问题", + "，其中第二节有问题", + ), + ( + "cs-feat 在 /tmp/repo 之后 应该 先跑 design-review.md 再实现", + "之后 应该 先跑 design-review.md 再实现", + ), + ( + "/tmp/a 运行 失败.详情如下：日志已附", + "运行 失败.详情如下：日志已附", + ), + ( + "/tmp/build 失败 详见 3.2 节", + "失败 详见 3.2 节", + ), + ( + "agent 在 /tmp/repo 没有读 .codestable/attention.md 就开工", + "没有读 .codestable/attention.md 就开工", + ), + ( + "/tmp/build 失败 详见 docs/report.md", + "失败 详见 docs/report.md", + ), + ( + "/tmp/app v1.2.3 crashed", + "v1.2.3 crashed", + ), + ( + "/tmp/build 失败.详情如下 请看日志", + "失败.详情如下 请看日志", + ), + ( + "/Users/a/b/c 详见 3.2 节", + "详见 3.2 节", + ), + ( + "cs-feat 在 /tmp/repo 没生成.codestable/design.md", + "没生成.codestable/design.md", + ), + ( + "报错在 /tmp/repo 后写到了build/output.json", + "后写到了build/output.json", + ), + ( + "/tmp/x 输出在logs/a.txt 又读了docs/b.md 然后停了", + "输出在logs/a.txt 又读了docs/b.md 然后停了", + ), + ) + for raw, expected_text in cases: + public = privacy_module.public_redact(raw, limit=4000) + assert "" in public + assert expected_text in public + + +def test_public_redaction_removes_nested_braced_and_unbounded_tool_json() -> None: + nested = ( + 'cs-feat failed {"patch":"def f() { return {} }",' + '"note":"private business logic"}' + ) + oversized = '{"note":"private prefix","payload":"' + ("x" * 2500) + '"}' + + for raw, forbidden in ( + (nested, "private business logic"), + (oversized, "private prefix"), + ): + public = privacy_module.public_redact(raw, limit=4000) + assert public == "cs-feat failed " or public == "" + assert forbidden not in public + assert "{" not in public and "}" not in public + + +def test_public_redaction_removes_six_and_seven_character_explicit_secrets() -> None: + for raw in ("password=hunter2", "authorization=abcdefg", '"token":"secret"'): + public = privacy_module.public_redact(raw) + assert raw not in public + assert "" in public + + +def test_public_redaction_removes_special_and_quoted_secret_values() -> None: + cases = ( + ("password=p@ssw0rd!123", ("p@ssw0rd!123",)), + ('password: "correct horse battery staple"', ("correct", "horse", "battery", "staple")), + ('export password="s3cr3tv@lue"', ("s3cr3tv@lue", "@lue")), + ('token = "abc def ghi jkl"', ("abc", "def", "ghi", "jkl")), + ("password=abcghi", ("abcghi",)), + ("password=`s3cr3tv@l!`", ("s3cr3tv@l!",)), + ("password：hunter22!", ("hunter22!",)), + ("token＝abcghi", ("abcghi",)), + ('password="s3cr3tv@lue', ("s3cr3tv@lue",)), + ("token='unterminated secret value", ("unterminated", "secret", "value")), + (r"password=secret\ pass", (r"secret\ pass", " pass")), + ("password：`abcghi`", ("abcghi",)), + ) + for raw, forbidden_parts in cases: + public = privacy_module.public_redact(raw) + assert "" in public + assert all(part not in public for part in forbidden_parts) + + +def test_public_redaction_removes_shell_segmented_secret_values() -> None: + cases = ( + ("password=abc'def'ghi", ("abc", "def", "ghi")), + ('password=abc"def"ghi', ("abc", "def", "ghi")), + ("password=abc`def`ghi", ("abc", "def", "ghi")), + ("password=abc\\\ndef", ("abc", "def")), + ("password=$'abcd'", ("abcd",)), + ("password=$(printf secretvalue)", ("secretvalue",)), + ) + for raw, forbidden_parts in cases: + public = privacy_module.public_redact(raw) + assert "" in public + assert all(part not in public for part in forbidden_parts) + + +def test_multiline_shell_secret_values_are_removed_from_local_and_public_text() -> None: + cases = ( + ('password="abcd\nsecretvalue"', ("abcd", "secretvalue")), + ("password='ab\r\ncdefgh'", ("ab", "cdefgh")), + ("password=$'a\nbcdefgh'", ("bcdefgh",)), + ("password=$(\nprintf secretvalue\n)", ("printf", "secretvalue")), + ("password=${TOKEN:-\nsecretvalue\n}", ("secretvalue",)), + ) + for raw, forbidden_parts in cases: + local = privacy_module.redact(raw, limit=4000) + public = privacy_module.public_redact(raw, limit=4000) + assert "" in local + assert "" in public + assert all(part not in local for part in forbidden_parts) + assert all(part not in public for part in forbidden_parts) + + +def test_collector_removes_multiline_secret_from_private_and_public_artifacts( + tmp_path: Path, +) -> None: + transcript = tmp_path / ".codex/sessions/multiline-secret.jsonl" + write_jsonl( + transcript, + [ + {"type": "session_meta", "payload": {"session_id": "s1", "cwd": "/repo"}}, + { + "type": "response_item", + "payload": { + "type": "function_call_output", + "output": 'cs-feat failed password="abcd\nsecretvalue"', + }, + }, + { + "type": "event_msg", + "payload": {"type": "user_message", "message": "不对，应该先 review。"}, + }, + ], + ) + evidence = tmp_path / "evidence.json" + + assert ( + collector.main_with_args_for_test( + ["--session", str(transcript), "--output", str(evidence)] + ) + == 0 + ) + assert "secretvalue" not in evidence.read_text(encoding="utf-8") + assert "secretvalue" not in ( + tmp_path / "public-issue-context.json" + ).read_text(encoding="utf-8") + + +def test_public_redaction_fail_closes_unterminated_code_fence() -> None: + public = privacy_module.public_redact( + "cs-feat failed ```python\nprivate_code('customer logic')", + limit=4000, + ) + + assert public == "cs-feat failed " + assert "private_code" not in public + assert "customer logic" not in public + assert "```" not in public + + +def test_public_secret_placeholders_do_not_self_lock_reporter() -> None: + for sanitized in ( + "password=", + "password：", + "", + "", + "", + "", + "", + ): + assert privacy_module.public_redact(sanitized) == sanitized + + +def test_public_redaction_removes_http_auth_scheme_credentials() -> None: + cases = ( + ("Authorization: Basic dXNlcjpwYXNz", "dXNlcjpwYXNz"), + ("authorization: bearer abc123def", "abc123def"), + ("Proxy-Authorization=Basic cHJveHk6cGFzcw==", "cHJveHk6cGFzcw=="), + ("Bearer standalone123", "standalone123"), + ) + for raw, credential in cases: + public = privacy_module.public_redact(raw) + assert credential not in public + assert "" in public + + +def test_public_redaction_removes_auth_headers_before_env_and_curl_userinfo() -> None: + headers = ( + ("AUTHORIZATION=Basic dXNlcjpwYXNz", "dXNlcjpwYXNz"), + ("HTTP_AUTHORIZATION=Basic aHR0cDpwYXNz", "aHR0cDpwYXNz"), + ("Authorization: token ghp_short12", "ghp_short12"), + ( + 'Authorization: Digest username="u", response="6629fae49393a05397450978507c4ef1"', + "6629fae49393a05397450978507c4ef1", + ), + ) + for raw, credential in headers: + public = privacy_module.public_redact(raw) + assert credential not in public + assert "" in public + + userinfo = privacy_module.public_redact("curl -u deploy:password123 endpoint") + assert "deploy:password123" not in userinfo + assert "" in userinfo + + +def test_rerun_preserves_user_supplied_triage_fields(tmp_path: Path) -> None: + transcript = tmp_path / ".codex/sessions/idempotent.jsonl" + records = [ + {"type": "session_meta", "payload": {"session_id": "s1", "cwd": "/repo"}}, + {"type": "response_item", "payload": {"type": "function_call_output", "output": "cs-feat failed"}}, + {"type": "event_msg", "payload": {"type": "user_message", "message": "不对，应该 review。"}}, + ] + write_jsonl(transcript, records) + output = tmp_path / "evidence.json" + args = ["--session", str(transcript), "--output", str(output)] + collector.main_with_args_for_test(args) + triage_path = tmp_path / "triage.json" + triage = json.loads(triage_path.read_text(encoding="utf-8")) + triage["reproduction"] = { + "eval_profile": "routing-decision", + "task_kind": "routing", + "input": {"utterance": "synthetic user request"}, + "oracle": {"expect": {"result_type": "RoutedTo"}}, + "evidence_refs": ["obs-01"], + } + triage["privacy_review"] = {"status": "approved"} + write_json(triage_path, triage) + + collector.main_with_args_for_test(args) + rerun = json.loads(triage_path.read_text(encoding="utf-8")) + assert rerun["reproduction"] == triage["reproduction"] + assert rerun["privacy_review"] == {"status": "approved"} + + records.extend( + [ + {"type": "response_item", "payload": {"type": "function_call_output", "output": "cs-feedback failed again"}}, + {"type": "event_msg", "payload": {"type": "user_message", "message": "不对，应该保留补充字段。"}}, + ] + ) + write_jsonl(transcript, records) + collector.main_with_args_for_test(args) + shifted = json.loads(triage_path.read_text(encoding="utf-8")) + assert shifted["reproduction"] == triage["reproduction"] + assert shifted["privacy_review"] == {"status": "approved"} + assert shifted["incident_id"] == "" + assert shifted["previous_incident_id"] == "incident-01" + assert shifted["pending_incident_id"] == "incident-02" + assert shifted["previous_incident_fingerprint"] + assert shifted["pending_incident_fingerprint"] + assert shifted["previous_incident_fingerprint"] != shifted["pending_incident_fingerprint"] + assert shifted["quality"]["triage_ready"] is False + assert "incident_id_changed" in shifted["quality"]["reasons"] + + unresolved_again = triage_module.merge_existing_triage( + triage_module.empty_triage(), shifted + ) + assert unresolved_again["previous_incident_id"] == "incident-01" + assert unresolved_again["pending_incident_id"] == "incident-02" + assert ( + unresolved_again["pending_incident_fingerprint"] + == shifted["pending_incident_fingerprint"] + ) + + before_invalid_accept = triage_path.read_text(encoding="utf-8") + assert collector.main_with_args_for_test([*args, "--accept-incident", "incident-01"]) == 2 + assert triage_path.read_text(encoding="utf-8") == before_invalid_accept + + assert collector.main_with_args_for_test([*args, "--accept-incident", "incident-02"]) == 0 + accepted = json.loads(triage_path.read_text(encoding="utf-8")) + assert accepted["incident_id"] == "incident-02" + assert accepted["incident_fingerprint"] == shifted["pending_incident_fingerprint"] + assert accepted["reproduction"] == triage["reproduction"] + assert accepted["privacy_review"] == {"status": "pending"} + assert accepted["previous_privacy_review"] == {"status": "approved"} + assert accepted["previous_assessment"] == shifted["assessment"] + assert "previous_incident_id" not in accepted + assert "pending_incident_id" not in accepted + assert accepted["quality"]["triage_ready"] is True + + unresolved = triage_module.merge_existing_triage( + triage_module.empty_triage(), triage + ) + assert unresolved["reproduction"] == triage["reproduction"] + assert unresolved["incident_id"] == "" + assert "incident_resolution_failed" in unresolved["quality"]["reasons"] + + +def test_same_position_incident_id_with_different_fingerprint_requires_reselection() -> None: + def incident(session: str, cutoff: str, actual: str) -> dict: + observations = [ + { + "id": "obs-0001", + "record_id": "record-0002", + "source_index": 2, + "role": "assistant", + "record_type": "message", + "text": actual, + }, + { + "id": "obs-0002", + "record_id": cutoff, + "source_index": 3, + "role": "user", + "record_type": "message", + "text": "不对，应该先 review。", + }, + ] + return { + "id": "incident-01", + "target_skill": "cs-feat", + "stage_hint": "review", + "incident_kind": "skipped-gate", + "observations": observations, + "user_correction": observations[-1], + "capture_cutoff": cutoff, + "environment_context": {"provider": "codex", "session": session}, + "repo_context": {}, + } + + old_incident = incident("session-old", "record-0003", "skipped the old review") + new_incident = incident("session-new", "record-0042", "skipped a different review") + existing = triage_module.build_triage([old_incident], old_incident) + existing["reproduction"]["input"] = {"utterance": "keep this input"} + existing["privacy_review"] = {"status": "approved"} + generated = triage_module.build_triage([new_incident], new_incident) + + assert existing["incident_id"] == generated["incident_id"] == "incident-01" + assert existing["incident_fingerprint"] != generated["incident_fingerprint"] + merged = triage_module.merge_existing_triage(generated, existing) + assert merged["incident_id"] == "" + assert merged["previous_incident_id"] == "incident-01" + assert merged["pending_incident_id"] == "incident-01" + assert merged["reproduction"]["input"] == {"utterance": "keep this input"} + assert merged["privacy_review"] == {"status": "approved"} + assert "incident_identity_changed" in merged["quality"]["reasons"] + + stale_incident = incident( + "session-new", "record-0042", "the primary changed after selection" + ) + stale_generated = triage_module.build_triage([stale_incident], stale_incident) + try: + triage_module.accept_pending_incident( + stale_generated, merged, "incident-01" + ) + except ValueError as exc: + assert "fingerprint" in str(exc) + else: + raise AssertionError("stale pending incident acceptance must fail closed") + + +def test_quality_gate_blocks_missing_assessment_source_or_inferred_confidence() -> None: + triage = { + "incident_id": "incident-01", + "observation_ids": ["obs-01", "obs-02"], + "trigger_cutoff": "record-0003", + "target": {"skill": "cs-feat"}, + "incident_kind": "wrong-route", + "assessment": { + "expected_behavior": { + "value": "route to design", + "source": "inferred", + "evidence_refs": ["obs-01"], + }, + "actual_behavior": { + "value": "route to QA", + "source": "", + "evidence_refs": ["obs-02"], + }, + }, + "reproduction": { + "eval_profile": "routing-decision", + "task_kind": "routing", + "input": {"utterance": "continue"}, + "oracle": {"expect": {"result_type": "RoutedTo"}}, + }, + } + + quality = triage_module.recompute_quality(triage) + assert quality["triage_ready"] is False + assert quality["regression_ready"] is False + assert "assessment.expected_behavior.confidence" in quality["missing_fields"] + assert "assessment.actual_behavior.source" in quality["missing_fields"] + assert quality["next_questions"] == ["assessment.expected_behavior.confidence"] + + +def test_quality_gate_rejects_unknown_source_values_and_dangling_observation_refs() -> None: + triage = { + "incident_id": "incident-01", + "observation_ids": ["obs-01", "obs-02"], + "trigger_cutoff": "record-0003", + "target": {"skill": "cs-feat"}, + "incident_kind": "wrong-route", + "assessment": { + "expected_behavior": { + "value": "route to design", + "source": "fabricated", + "evidence_refs": ["obs-01"], + }, + "actual_behavior": { + "value": "route to QA", + "source": "transcript", + "evidence_refs": ["obs-does-not-exist"], + }, + }, + "reproduction": { + "eval_profile": "routing-decision", + "task_kind": "routing", + "input": {"utterance": "continue"}, + "oracle": {"expect": {"result_type": "RoutedTo"}}, + }, + } + + quality = triage_module.recompute_quality(triage) + assert quality["triage_ready"] is False + assert quality["regression_ready"] is False + assert "assessment.expected_behavior.source" in quality["missing_fields"] + assert "assessment.actual_behavior.evidence_refs" in quality["missing_fields"] + + +def test_correlated_windows_keep_numeric_record_order_after_9999() -> None: + def record(record_id: str): + return models_module.NormalizedRecord( + id=record_id, + provider="codex", + session="s1", + timestamp="", + role="tool", + record_type="tool_result", + tool_name="mcp", + correlation_id="call-1", + correlation_source="provider", + text="failed", + source_index=int(record_id.rsplit("-", 1)[-1]), + ) + + merged = incidents_module._merge_correlated_windows( + [[record("record-9999")], [record("record-10000")]] + ) + assert [item.id for item in merged[0]] == ["record-9999", "record-10000"] + + +def test_correlation_bridge_merges_every_connected_window_in_source_order() -> None: + def record(record_id: str, correlation_id: str): + return models_module.NormalizedRecord( + id=record_id, + provider="codex", + session="s1", + timestamp="", + role="tool", + record_type="tool_result", + tool_name="mcp", + correlation_id=correlation_id, + correlation_source="provider", + text="failed", + source_index=int(record_id.rsplit("-", 1)[-1]), + ) + + merged = incidents_module._merge_correlated_windows( + [ + [record("record-0001", "x")], + [record("record-0002", "y")], + [record("record-0003", "x"), record("record-0004", "y")], + ] + ) + + assert len(merged) == 1 + assert [item.id for item in merged[0]] == [ + "record-0001", + "record-0002", + "record-0003", + "record-0004", + ] + + +def test_v1_failure_mapping_and_public_event_fields_remain_frozen() -> None: + assert collector.V1_FAILURE_MAP == { + "wrong-route": "agent-detour", + "skipped-gate": "agent-detour", + "missing-artifact": "agent-detour", + "unnecessary-detour": "agent-detour", + "tool-failure": "tool-failure", + "goal-driver": "goal-driver", + "install-version": "install-distribution", + "unclear-rule": "unclear-rule", + "privacy-reporting": "unknown", + "unknown": "unknown", + } + assert collector.PUBLIC_EVENT_FIELDS == [ + "provider", + "session_label", + "timestamp_bucket", + "failure_type", + "match_type", + "tool_name", + "skill_or_reference", + "sanitized_excerpt", + ] diff --git a/tests/test_cs_feedback_fixture_promotion.py b/tests/test_cs_feedback_fixture_promotion.py new file mode 100644 index 0000000..21eafb7 --- /dev/null +++ b/tests/test_cs_feedback_fixture_promotion.py @@ -0,0 +1,401 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from copy import deepcopy +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +PROMOTE_SCRIPT = ROOT / ".claude/skills/eval-cs-skill/scripts/promote_feedback_fixture.py" +CONVERTER_SCRIPT = ROOT / "plugins/codestable/skills/cs-feedback/scripts/feedback_to_fixture.py" + + +def load_module(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +promote = load_module(PROMOTE_SCRIPT, "promote_feedback_fixture_contract") +converter = load_module(CONVERTER_SCRIPT, "feedback_to_fixture_contract") +triage_module = sys.modules["feedback_triage"] + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def write_canonical_feedback_pair(path: Path, payload: dict) -> None: + triage = deepcopy(payload) + observations = [ + { + "id": observation_id, + "record_id": f"record-{index:04d}", + "role": "assistant", + "record_type": "message", + "text": f"synthetic observation {observation_id}", + } + for index, observation_id in enumerate(triage.get("observation_ids", []), 1) + ] + incident = { + "id": triage["incident_id"], + "target_skill": triage.get("target", {}).get("skill", "unknown"), + "stage_hint": triage.get("target", {}).get("stage_hint", "unknown"), + "incident_kind": triage.get("incident_kind", "unknown"), + "observations": observations, + "capture_cutoff": triage["trigger_cutoff"], + "environment_context": {"provider": "codex", "session": "session-test"}, + } + triage["incident_fingerprint"] = triage_module.incident_fingerprint(incident) + write_json(path, triage) + write_json( + path.with_name("evidence.json"), + { + "schema_version": 2, + "privacy": "local-private", + "incidents": [incident], + }, + ) + + +def compatible_config(**overrides) -> dict: + config = { + "name": "feedback-promotion-test", + "skill_under_test": "cs-code-review", + "variants": ["baseline"], + "model_list": ["model-under-test"], + "k": 1, + "harnesses": ["api"], + "scorers": ["recall_judge"], + "fixture_classes": ["regression"], + "judge_model": "independent-judge", + } + config.update(overrides) + return config + + +def findings_candidate(**overrides) -> dict: + candidate = { + "id": "reg-cs-code-review-tool-failure", + "privacy": "local-private", + "_source": "cs-feedback", + "_status": "candidate", + "_profile": "findings-recall", + "incident_id": "incident-01", + "target_skill": "cs-code-review", + "incident_kind": "tool-failure", + "answerType": "findings-recall", + "answer": ["unsafe_call must be reported"], + "task": { + "kind": "review", + "spec": "synthetic review contract", + "diff": "+ unsafe_call()", + }, + "quality": {"triage_ready": True, "regression_ready": True, "missing_fields": []}, + "privacy_review": {"status": "approved"}, + "promotion_blockers": [], + } + candidate.update(overrides) + return candidate + + +def run_promotion(tmp_path: Path, candidate: dict, config: dict) -> tuple[int, Path]: + candidate_path = tmp_path / "feedback/regression-candidate.json" + experiment = tmp_path / "experiment" + write_json(candidate_path, candidate) + write_json(experiment / "config.json", config) + rc = promote.main(["--candidate", str(candidate_path), "--experiment", str(experiment)]) + target = experiment / "fixtures/regression" / f"{candidate['id']}.json" + return rc, target + + +def test_valid_findings_candidate_promotes_only_commit_safe_fixture_fields(tmp_path: Path) -> None: + rc, target = run_promotion(tmp_path, findings_candidate(), compatible_config()) + + assert rc == 0 + fixture = json.loads(target.read_text(encoding="utf-8")) + assert fixture == { + "id": "reg-cs-code-review-tool-failure", + "incident_id": "incident-01", + "answerType": "findings-recall", + "answer": ["unsafe_call must be reported"], + "task": { + "kind": "review", + "spec": "synthetic review contract", + "diff": "+ unsafe_call()", + }, + } + + +def test_triage_candidate_to_repo_promotion_uses_only_json_artifact_handoff(tmp_path: Path) -> None: + triage = tmp_path / "feedback/triage.json" + write_canonical_feedback_pair( + triage, + { + "schema_version": 2, + "privacy": "local-private", + "incident_id": "incident-01", + "observation_ids": ["obs-02", "obs-03"], + "trigger_cutoff": "record-0003", + "target": {"skill": "cs-code-review", "stage_hint": "review"}, + "incident_kind": "tool-failure", + "assessment": { + "expected_behavior": { + "value": "report the synthetic unsafe call", + "source": "user", + "evidence_refs": ["obs-03"], + }, + "actual_behavior": { + "value": "the synthetic unsafe call was missed", + "source": "transcript", + "evidence_refs": ["obs-02"], + }, + }, + "reproduction": { + "eval_profile": "findings-recall", + "task_kind": "review", + "input": { + "spec": "synthetic review contract", + "diff": "+ unsafe_call()", + }, + "oracle": {"coverage_points": ["unsafe_call must be reported"]}, + "evidence_refs": ["obs-02", "obs-03"], + }, + "quality": {"triage_ready": False, "regression_ready": False}, + "privacy_review": {"status": "approved"}, + }, + ) + assert converter.main(["--triage", str(triage)]) == 0 + candidate = triage.parent / "regression-candidate.json" + experiment = tmp_path / "experiment" + write_json(experiment / "config.json", compatible_config()) + + assert ( + promote.main(["--candidate", str(candidate), "--experiment", str(experiment)]) + == 0 + ) + promoted = list((experiment / "fixtures/regression").glob("*.json")) + assert len(promoted) == 1 + fixture_text = promoted[0].read_text(encoding="utf-8") + assert "unsafe_call must be reported" in fixture_text + assert "the synthetic unsafe call was missed" not in fixture_text + + +def test_valid_routing_candidate_promotes_with_routing_scorer(tmp_path: Path) -> None: + candidate = findings_candidate( + id="reg-cs-feat-wrong-route", + _profile="routing-decision", + target_skill="cs-feat", + incident_kind="wrong-route", + answerType="routing-decision", + expect={"result_type": "RoutedTo", "target": "design-review"}, + task={"kind": "routing", "utterance": "continue the approved feature"}, + ) + candidate.pop("answer") + config = compatible_config( + skill_under_test="cs-feat", + scorers=["routing_decision"], + judge_model=None, + ) + + rc, target = run_promotion(tmp_path, candidate, config) + assert rc == 0 + fixture = json.loads(target.read_text(encoding="utf-8")) + assert fixture["answerType"] == "routing-decision" + assert fixture["expect"]["result_type"] == "RoutedTo" + + +def test_promotion_without_experiment_config_never_creates_fixture_directory(tmp_path: Path) -> None: + candidate_path = tmp_path / "feedback/regression-candidate.json" + experiment = tmp_path / "experiment" + write_json(candidate_path, findings_candidate()) + experiment.mkdir() + + assert ( + promote.main( + ["--candidate", str(candidate_path), "--experiment", str(experiment)] + ) + != 0 + ) + assert not (experiment / "fixtures").exists() + + +@pytest.mark.parametrize( + ("candidate_patch", "config_patch"), + [ + ({}, {"scorers": ["planted_defect"]}), + ({}, {"judge_model": None}), + ({}, {"judge_model": "mock-judge"}), + ({}, {"judge_model": "model-under-test"}), + ({"target_skill": "cs-audit"}, {}), + ({"_profile": "routing-decision"}, {}), + ({"promotion_blockers": ["reproduction.input"]}, {}), + ({"privacy_review": {"status": "pending"}}, {}), + ({"id": "../escape"}, {}), + ({}, {"fixture_classes": ["planted-defect"]}), + ], +) +def test_findings_promotion_gates_fail_closed_without_writing( + tmp_path: Path, candidate_patch: dict, config_patch: dict +) -> None: + candidate = findings_candidate(**deepcopy(candidate_patch)) + config = compatible_config(**deepcopy(config_patch)) + + rc, target = run_promotion(tmp_path, candidate, config) + assert rc != 0 + assert not target.exists() + + +@pytest.mark.parametrize( + "unsafe_value", + [ + "/home/alice/private-spec.md", + "/repo", + "API_TOKEN=topsecret123", + "https://github.com/acme/private", + "token=secret123456", + '{"token":"secret123456"}', + "password=p@ssw0rd!123", + 'password: "correct horse battery staple"', + 'export password="s3cr3tv@lue"', + 'token = "abc def ghi jkl"', + "password=abcghi", + "password=`s3cr3tv@l!`", + "password：hunter22!", + "token＝abcghi", + 'password="s3cr3tv@lue', + "token='unterminated secret value", + r"password=secret\ pass", + "password：`abcghi`", + "password=abc'def'ghi", + 'password=abc"def"ghi', + "password=abc`def`ghi", + "password=abc\\\ndef", + "password=$'abcd'", + "password=$(printf secretvalue)", + 'password="abcd\nsecretvalue"', + "password='ab\r\ncdefgh'", + "password=$'a\nbcdefgh'", + "password=$(\nprintf secretvalue\n)", + "password=${TOKEN:-\nsecretvalue\n}", + "Authorization: Basic dXNlcjpwYXNz", + "Bearer standalone123", + "Proxy-Authorization=Basic cHJveHk6cGFzcw==", + "Authorization: token ghp_short12", + "curl -u deploy:password123 endpoint", + "TODO replace this placeholder", + "unknown", + " ", + ], +) +def test_commit_safe_scan_rejects_each_private_or_placeholder_field( + tmp_path: Path, unsafe_value: str +) -> None: + candidate = findings_candidate() + candidate["task"] = {**candidate["task"], "spec": unsafe_value} + + rc, target = run_promotion(tmp_path, candidate, compatible_config()) + assert rc != 0 + assert not target.exists() + + +@pytest.mark.parametrize( + ("key", "value", "reason"), + [ + ("password=hunter22", "safe", "secret"), + ("API_TOKEN", "safe", "environment-name"), + ("marker", "local-private", "private-marker"), + ], +) +def test_commit_safe_scan_checks_dict_keys_and_private_markers( + key: str, value: str, reason: str +) -> None: + problems = promote._commit_safe_issues({"task": {key: value}}) + assert any(reason in problem for problem in problems) + + +def test_promotion_rejects_task_keys_outside_profile_schema_without_writing( + tmp_path: Path, +) -> None: + candidate = findings_candidate() + candidate["task"] = {**candidate["task"], "unexpected": "safe"} + + rc, target = run_promotion(tmp_path, candidate, compatible_config()) + assert rc != 0 + assert not target.exists() + + +def test_promotion_rejects_invalid_candidate_value_types_without_writing( + tmp_path: Path, +) -> None: + routing = findings_candidate( + id="reg-cs-feat-wrong-route", + _profile="routing-decision", + target_skill="cs-feat", + incident_kind="wrong-route", + answerType="routing-decision", + expect={"result_type": "RoutedTo", "target": "design-review"}, + task={"kind": "routing", "state": {"stage": "review"}}, + ) + routing.pop("answer") + routing_config = compatible_config( + skill_under_test="cs-feat", + scorers=["routing_decision"], + judge_model=None, + ) + cases = ( + ( + {**routing, "expect": {"result_type": {"nested": "safe"}}}, + routing_config, + ), + ({**routing, "task": {"kind": "routing", "state": ["review"]}}, routing_config), + ( + findings_candidate( + task={"kind": "review", "spec": ["not", "text"], "diff": "+ safe"} + ), + compatible_config(), + ), + ( + findings_candidate( + quality={ + "triage_ready": "true", + "regression_ready": True, + "missing_fields": [], + } + ), + compatible_config(), + ), + (findings_candidate(incident_id=["incident-01"]), compatible_config()), + ) + for index, (candidate, config) in enumerate(cases): + rc, target = run_promotion(tmp_path / str(index), candidate, config) + assert rc == 2 + assert not target.exists() + + +@pytest.mark.parametrize("kind", ["design", "docs"]) +def test_design_and_docs_candidates_require_non_mock_harness(tmp_path: Path, kind: str) -> None: + target_skill = "cs-feat" if kind == "design" else "cs-docs" + candidate = findings_candidate( + id=f"reg-{target_skill}-unclear-rule", + target_skill=target_skill, + incident_kind="unclear-rule", + task={ + "kind": kind, + "spec": "synthetic requirement", + "diff": "+ synthetic material" if kind == "docs" else "", + }, + ) + config = compatible_config(skill_under_test=target_skill, harnesses=["mock"]) + + rc, target = run_promotion(tmp_path, candidate, config) + assert rc != 0 + assert not target.exists() diff --git a/tests/test_cs_feedback_reporting.py b/tests/test_cs_feedback_reporting.py new file mode 100644 index 0000000..c7b0036 --- /dev/null +++ b/tests/test_cs_feedback_reporting.py @@ -0,0 +1,500 @@ +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +REPORT_SCRIPT = ROOT / "plugins/codestable/skills/cs-feedback/scripts/report_feedback_issue.py" + + +def load_reporter(): + spec = importlib.util.spec_from_file_location("report_feedback_issue_contract", REPORT_SCRIPT) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +reporter = load_reporter() +privacy = sys.modules["feedback_privacy"] + + +def test_public_body_scanner_rejects_single_segment_path_and_env_name() -> None: + reasons = reporter.public_body_private_reasons( + "failed in /repo while reading API_TOKEN" + ) + assert "absolute-path" in reasons + assert "environment-name" in reasons + assert "absolute-path" not in reporter.public_body_private_reasons( + "run /goal for this feature" + ) + + +def test_public_body_scanner_handles_absolute_paths_with_spaces_end_to_end() -> None: + cases = ( + (r"error at C:\Users\bob\secret plan.docx", ("plan.docx",)), + ( + "/Users/alice/customer contracts/acme-pricing-2026.md", + ("contracts", "acme-pricing-2026.md"), + ), + ( + "/Users/alice/Library/Application Support/CodeStable/session.json", + ("Support", "CodeStable", "session.json"), + ), + ( + 'cat "/Users/alice/acme merger notes.txt" done', + ("merger", "notes.txt"), + ), + ( + "cat '/Users/alice/acme merger notes.txt' done", + ("merger", "notes.txt"), + ), + ( + "cat `/Users/alice/acme merger notes.txt` done", + ("merger", "notes.txt"), + ), + ( + 'open "C:\\Users\\bob\\secret plan.docx" now', + ("plan.docx",), + ), + ("see /Users/bob/my report.docx.", ("report.docx",)), + ("see /Users/bob/my report.docx!", ("report.docx",)), + ("see /Users/bob/my report.docx?", ("report.docx",)), + ("see /Users/bob/my report.docx。", ("report.docx",)), + ("see /Users/bob/my report.docx！", ("report.docx",)), + ("see /Users/bob/my report.docx？", ("report.docx",)), + ( + 'see "/Users/bob/my report.docx." next', + ("report.docx",), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx，其中第二节有问题", + ("合并", "计划.docx"), + ), + ( + "路径是 /Users/alice/acme 合并 计划.docx。下一步继续", + ("合并", "计划.docx"), + ), + ( + "（详见 /Users/alice/merger plan.docx）之后再说", + ("merger", "plan.docx"), + ), + ( + "见 /Users/alice/plan file.docx；继续", + ("plan", "file.docx"), + ), + ( + "见“/Users/alice/acme 合并 计划.docx”，继续", + ("合并", "计划.docx"), + ), + ( + "见 /Users/alice/plan long.presentation，继续", + ("long.presentation",), + ), + ( + "见 /Users/alice/客户 合同.文档，继续", + ("合同.文档",), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx…其中有问题", + ("合并", "计划.docx"), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx——其中有问题", + ("合并", "计划.docx"), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx（内含预算）", + ("合并", "计划.docx"), + ), + ( + "see /Users/alice/acme merger plan.docx(v2)", + ("merger", "plan.docx"), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx·补充", + ("合并", "计划.docx"), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx～补充", + ("合并", "计划.docx"), + ), + ( + "请看 /Users/alice/acme 合并 计划.docx“重点”", + ("合并", "计划.docx"), + ), + ( + "/tmp/x my report.docx 之后继续", + ("my", "report.docx"), + ), + ( + r"open C:\Program Files\CodeStable\session.json", + ("Files", "CodeStable", "session.json"), + ), + ( + "/Users/alice/项目 文档/计划.docx", + ("文档", "计划.docx"), + ), + ("/tmp/x 合同.docx", ("合同.docx",)), + ) + for raw, forbidden_parts in cases: + assert "absolute-path" in reporter.public_body_private_reasons(raw) + sanitized = privacy.public_redact(raw) + assert all(part not in sanitized for part in forbidden_parts) + assert reporter.public_body_private_reasons(sanitized) == [] + + +def test_public_body_scanner_preserves_text_after_absolute_paths() -> None: + cases = ( + ( + "报错发生在 /tmp/build，随后 agent 跳过了 review gate", + "，随后 agent 跳过了 review gate", + ), + ( + "/Users/alice/acme，其中第二节有问题", + "，其中第二节有问题", + ), + ( + "cs-feat 在 /tmp/repo 之后 应该 先跑 design-review.md 再实现", + "之后 应该 先跑 design-review.md 再实现", + ), + ( + "/tmp/a 运行 失败.详情如下：日志已附", + "运行 失败.详情如下：日志已附", + ), + ( + "/tmp/build 失败 详见 3.2 节", + "失败 详见 3.2 节", + ), + ( + "agent 在 /tmp/repo 没有读 .codestable/attention.md 就开工", + "没有读 .codestable/attention.md 就开工", + ), + ( + "/tmp/build 失败 详见 docs/report.md", + "失败 详见 docs/report.md", + ), + ( + "/tmp/app v1.2.3 crashed", + "v1.2.3 crashed", + ), + ( + "/tmp/build 失败.详情如下 请看日志", + "失败.详情如下 请看日志", + ), + ( + "/Users/a/b/c 详见 3.2 节", + "详见 3.2 节", + ), + ( + "cs-feat 在 /tmp/repo 没生成.codestable/design.md", + "没生成.codestable/design.md", + ), + ( + "报错在 /tmp/repo 后写到了build/output.json", + "后写到了build/output.json", + ), + ( + "/tmp/x 输出在logs/a.txt 又读了docs/b.md 然后停了", + "输出在logs/a.txt 又读了docs/b.md 然后停了", + ), + ) + for raw, expected_text in cases: + assert "absolute-path" in reporter.public_body_private_reasons(raw) + sanitized = privacy.public_redact(raw, limit=4000) + assert expected_text in sanitized + assert reporter.public_body_private_reasons(sanitized) == [] + + +def test_public_body_scanner_rejects_nested_and_unbounded_tool_json() -> None: + nested = '{"patch":"def f() { return {} }","note":"private logic"}' + oversized = '{"note":"private prefix","payload":"' + ("x" * 2500) + '"}' + + assert "raw-json" in reporter.public_body_private_reasons(nested) + assert "raw-json" in reporter.public_body_private_reasons(oversized) + + +def test_public_body_scanner_rejects_six_and_seven_character_explicit_secrets() -> None: + for raw in ("password=hunter2", "authorization=abcdefg", '"token":"secret"'): + assert "secret" in reporter.public_body_private_reasons(raw) + + +def test_public_body_scanner_rejects_special_and_quoted_secret_values() -> None: + cases = ( + ("password=p@ssw0rd!123", ("p@ssw0rd!123",)), + ('password: "correct horse battery staple"', ("correct", "horse", "battery", "staple")), + ('export password="s3cr3tv@lue"', ("s3cr3tv@lue", "@lue")), + ('token = "abc def ghi jkl"', ("abc", "def", "ghi", "jkl")), + ("password=abcghi", ("abcghi",)), + ("password=`s3cr3tv@l!`", ("s3cr3tv@l!",)), + ("password：hunter22!", ("hunter22!",)), + ("token＝abcghi", ("abcghi",)), + ('password="s3cr3tv@lue', ("s3cr3tv@lue",)), + ("token='unterminated secret value", ("unterminated", "secret", "value")), + (r"password=secret\ pass", (r"secret\ pass", " pass")), + ("password：`abcghi`", ("abcghi",)), + ) + for raw, forbidden_parts in cases: + assert "secret" in reporter.public_body_private_reasons(raw) + sanitized = privacy.public_redact(raw) + assert all(part not in sanitized for part in forbidden_parts) + assert reporter.public_body_private_reasons(sanitized) == [] + + +def test_public_body_scanner_rejects_shell_segmented_secret_values() -> None: + cases = ( + ("password=abc'def'ghi", ("abc", "def", "ghi")), + ('password=abc"def"ghi', ("abc", "def", "ghi")), + ("password=abc`def`ghi", ("abc", "def", "ghi")), + ("password=abc\\\ndef", ("abc", "def")), + ("password=$'abcd'", ("abcd",)), + ("password=$(printf secretvalue)", ("secretvalue",)), + ) + for raw, forbidden_parts in cases: + assert "secret" in reporter.public_body_private_reasons(raw) + sanitized = privacy.public_redact(raw) + assert all(part not in sanitized for part in forbidden_parts) + assert reporter.public_body_private_reasons(sanitized) == [] + + +def test_public_body_scanner_rejects_multiline_shell_secret_values() -> None: + cases = ( + ('password="abcd\nsecretvalue"', ("abcd", "secretvalue")), + ("password='ab\r\ncdefgh'", ("ab", "cdefgh")), + ("password=$'a\nbcdefgh'", ("bcdefgh",)), + ("password=$(\nprintf secretvalue\n)", ("printf", "secretvalue")), + ("password=${TOKEN:-\nsecretvalue\n}", ("secretvalue",)), + ) + for raw, forbidden_parts in cases: + assert "secret" in reporter.public_body_private_reasons(raw) + sanitized = privacy.public_redact(raw, limit=4000) + assert all(part not in sanitized for part in forbidden_parts) + assert reporter.public_body_private_reasons(sanitized) == [] + + +def test_public_body_scanner_accepts_sanitized_privacy_placeholders() -> None: + for sanitized in ( + "password=", + "password：", + "", + "", + "", + "", + "", + ): + assert reporter.public_body_private_reasons(sanitized) == [] + + +def test_public_body_scanner_rejects_http_auth_schemes_and_accepts_sanitized_text() -> None: + cases = ( + ("Authorization: Basic dXNlcjpwYXNz", "dXNlcjpwYXNz"), + ("authorization: bearer abc123def", "abc123def"), + ("Proxy-Authorization=Basic cHJveHk6cGFzcw==", "cHJveHk6cGFzcw=="), + ("Bearer standalone123", "standalone123"), + ) + for raw, credential in cases: + assert "secret" in reporter.public_body_private_reasons(raw) + sanitized = privacy.public_redact(raw) + assert credential not in sanitized + assert reporter.public_body_private_reasons(sanitized) == [] + + +def test_public_body_scanner_rejects_auth_headers_env_order_and_curl_userinfo() -> None: + cases = ( + ("AUTHORIZATION=Basic dXNlcjpwYXNz", "dXNlcjpwYXNz"), + ("HTTP_AUTHORIZATION=Basic aHR0cDpwYXNz", "aHR0cDpwYXNz"), + ("Authorization: token ghp_short12", "ghp_short12"), + ( + 'Authorization: Digest username="u", response="6629fae49393a05397450978507c4ef1"', + "6629fae49393a05397450978507c4ef1", + ), + ("curl -u deploy:password123 endpoint", "deploy:password123"), + ) + for raw, credential in cases: + assert "secret" in reporter.public_body_private_reasons(raw) + sanitized = privacy.public_redact(raw) + assert credential not in sanitized + assert reporter.public_body_private_reasons(sanitized) == [] + + +def test_reporter_falls_back_when_gh_is_missing(tmp_path: Path, monkeypatch) -> None: + body = tmp_path / "github-issue.md" + body.write_text("## Summary\n\ncs-feedback issue\n", encoding="utf-8") + output = tmp_path / "result.json" + monkeypatch.setattr(reporter.shutil, "which", lambda name: None) + + exit_code = reporter.main_with_args_for_test( + [ + "--repo", + "owner/repo", + "--title", + "Feedback: cs skill failed", + "--body-file", + str(body), + "--json-output", + str(output), + ] + ) + + assert exit_code == 0 + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["status"] == "manual" + assert payload["reason"] == "gh not found" + assert "gh issue create" in payload["command"] + assert "'Feedback: cs skill failed'" in payload["command"] + + +def test_reporter_refuses_local_private_evidence(tmp_path: Path, monkeypatch) -> None: + evidence = tmp_path / "evidence.json" + evidence.write_text( + json.dumps({"privacy": "local-private", "public_upload_allowed": False}), + encoding="utf-8", + ) + monkeypatch.setattr(reporter.shutil, "which", lambda name: None) + + try: + reporter.main_with_args_for_test( + [ + "--repo", + "owner/repo", + "--title", + "Feedback: cs skill failed", + "--body-file", + str(evidence), + ] + ) + except SystemExit as exc: + assert "refusing to upload local-private" in str(exc) + else: + raise AssertionError("expected reporter to reject evidence.json") + + +def test_reporter_requires_explicit_public_preview_confirmation(tmp_path: Path, monkeypatch) -> None: + body = tmp_path / "github-issue.md" + body.write_text("# Public issue\n\nSanitized summary.\n", encoding="utf-8") + calls: list[list[str]] = [] + monkeypatch.setattr(reporter.shutil, "which", lambda _name: "/usr/local/bin/gh") + + def fake_run(command): + calls.append(command) + raise AssertionError("gh must not run before explicit public-preview confirmation") + + monkeypatch.setattr(reporter, "run_with_proxy_retry", fake_run) + assert ( + reporter.main_with_args_for_test( + ["--repo", "owner/repo", "--title", "feedback", "--body-file", str(body)] + ) + == 0 + ) + assert calls == [] + + +def test_reporter_confirmed_preview_calls_only_auth_and_issue_create(tmp_path: Path, monkeypatch) -> None: + body = tmp_path / "github-issue.md" + body.write_text("# Public issue\n\nSanitized summary.\n", encoding="utf-8") + calls: list[list[str]] = [] + monkeypatch.setattr(reporter.shutil, "which", lambda _name: "/usr/local/bin/gh") + + def fake_run(command): + calls.append(command) + stdout = "https://github.com/owner/repo/issues/1\n" if "create" in command else "" + return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr=""), None + + monkeypatch.setattr(reporter, "run_with_proxy_retry", fake_run) + assert ( + reporter.main_with_args_for_test( + [ + "--repo", + "owner/repo", + "--title", + "feedback", + "--body-file", + str(body), + "--confirm-public-preview", + ] + ) + == 0 + ) + assert len(calls) == 2 + assert calls[0][-2:] == ["auth", "status"] + assert calls[1][1:3] == ["issue", "create"] + + +def test_reporter_rejects_private_content_even_after_confirmation(tmp_path: Path, monkeypatch) -> None: + body = tmp_path / "github-issue.md" + body.write_text("Private path: /Users/me/client/repo/secret.md\n", encoding="utf-8") + calls: list[list[str]] = [] + monkeypatch.setattr(reporter.shutil, "which", lambda _name: "/usr/local/bin/gh") + + def fake_run(command): + calls.append(command) + return subprocess.CompletedProcess(command, 0, stdout="", stderr=""), None + + monkeypatch.setattr(reporter, "run_with_proxy_retry", fake_run) + try: + reporter.main_with_args_for_test( + [ + "--repo", + "owner/repo", + "--title", + "feedback", + "--body-file", + str(body), + "--confirm-public-preview", + ] + ) + except SystemExit as exc: + assert "public preview contains private content" in str(exc) + else: + raise AssertionError("expected private public-preview body to be rejected") + assert calls == [] + + +def test_reporter_rejects_private_title_even_after_confirmation(tmp_path: Path, monkeypatch) -> None: + body = tmp_path / "github-issue.md" + body.write_text("# Public issue\n\nSanitized summary.\n", encoding="utf-8") + calls: list[list[str]] = [] + monkeypatch.setattr(reporter.shutil, "which", lambda _name: "/usr/local/bin/gh") + + def fake_run(command): + calls.append(command) + return subprocess.CompletedProcess(command, 0, stdout="", stderr=""), None + + monkeypatch.setattr(reporter, "run_with_proxy_retry", fake_run) + try: + reporter.main_with_args_for_test( + [ + "--repo", + "owner/repo", + "--title", + "fails in /Users/me/client-x", + "--body-file", + str(body), + "--confirm-public-preview", + ] + ) + except SystemExit as exc: + assert "issue title contains private content" in str(exc) + else: + raise AssertionError("expected private issue title to be rejected") + assert calls == [] + + +def test_reporter_refuses_triage_and_regression_candidate_files(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(reporter.shutil, "which", lambda name: None) + for name in ("triage.json", "regression-candidate.json"): + private = tmp_path / name + private.write_text(json.dumps({"privacy": "local-private"}), encoding="utf-8") + try: + reporter.main_with_args_for_test( + ["--repo", "owner/repo", "--title", "x", "--body-file", str(private)] + ) + except SystemExit as exc: + assert "refusing to upload local-private" in str(exc) + else: + raise AssertionError(f"expected reporter to reject {name}") diff --git a/tests/test_cs_skill_bootstrap.py b/tests/test_cs_skill_bootstrap.py index 20d468f..4116d72 100644 --- a/tests/test_cs_skill_bootstrap.py +++ b/tests/test_cs_skill_bootstrap.py @@ -98,17 +98,24 @@ def test_buildprompt_dispatch(): assert "SKILL BODY" in review # 被测 skill 快照注入 -# ---- cs-feedback → regression fixture ---- +# ---- cs-feedback → local candidate ---- -def test_feedback_to_fixture_skeleton(tmp_path): +def test_feedback_to_fixture_candidate_only(tmp_path): exp = tmp_path / "experiments/cs-code-review-007" exp.mkdir(parents=True) rc = f2f.main(["--experiment", str(exp), "--failure", "agent 漏报了 SQL 注入", "--kind", "review"]) - assert rc == 0 - regs = list((exp / "fixtures/regression").glob("*.json")) - assert regs, "应生成 regression fixture" - data = json.loads(regs[0].read_text(encoding="utf-8")) + assert rc != 0 + assert not (exp / "fixtures/regression").exists() + + evidence = tmp_path / "feedback/public-issue-context.json" + evidence.parent.mkdir() + evidence.write_text( + json.dumps({"privacy": "public-preview", "events": [{"sanitized_excerpt": "agent 漏报 SQL 注入"}]}), + encoding="utf-8", + ) + assert f2f.main(["--evidence", str(evidence)]) == 0 + data = json.loads((evidence.parent / "regression-candidate.json").read_text(encoding="utf-8")) assert data["_source"] == "cs-feedback" - assert data["_status"] == "skeleton" - # 骨架 schema 合规（除 diff 需人工补全外） + assert data["_status"] == "candidate" + assert data["quality"]["regression_ready"] is False assert fx_mod.validate_fixture_dict(data) == [] diff --git a/tests/test_skill_contracts.py b/tests/test_skill_contracts.py index 59b4c60..a467985 100644 --- a/tests/test_skill_contracts.py +++ b/tests/test_skill_contracts.py @@ -172,7 +172,7 @@ def test_goal_routing_fixtures_use_current_state_schema() -> None: def test_cs_router_fixtures_cover_modes_conflicts_and_recovery() -> None: fixtures = _routing_fixture_states("cs-routing-001") - assert set(fixtures) == {f"rt-c{i:02d}" for i in range(1, 17)} + assert set(fixtures) == {f"rt-c{i:02d}" for i in range(1, 18)} assert fixtures["rt-c01"]["expect"]["result_type"] == "RoutedTo" assert fixtures["rt-c01"]["expect"]["target"] == "cs-issue" @@ -196,6 +196,10 @@ def test_cs_router_fixtures_cover_modes_conflicts_and_recovery() -> None: assert fixtures["rt-c15"]["expect"]["result_type"] == "HumanCheckpoint" assert fixtures["rt-c16"]["expect"]["result_type"] == "Completed" assert "issue workflow" in fixtures["rt-c16"]["expect"]["target_any"] + assert fixtures["rt-c17"]["expect"] == { + "result_type": "RoutedTo", + "target": "cs-feedback", + } # result type 的禁止分支由精确 outcome 断言完成，不能误用只检查 target 的字段。 for fixture_id in ("rt-c02", "rt-c03", "rt-c04", "rt-c13", "rt-c15"): diff --git a/tests/test_skill_entry_simplification.py b/tests/test_skill_entry_simplification.py index 029d296..e0cbd5c 100644 --- a/tests/test_skill_entry_simplification.py +++ b/tests/test_skill_entry_simplification.py @@ -206,7 +206,7 @@ MAIN_ENTRY_ARGUMENT_HINTS = { "cs-refactor": "[--stage scan|design|apply] [--mode standard|fastforward] ", "cs-docs": "[--mode tutorial|api] ", "cs-epic": "[--stage planning|review|goal-package] ", - "cs-feedback": "[--since-days N] [--session current|] [--github] ", + "cs-feedback": "[--since-days N | --session current|] [--accept-incident ] [--github] ", "cs-code-review": "[--range ] [scope]", "cs-docs-neat": "[scope]", } @@ -658,14 +658,21 @@ def test_feedback_skill_is_registered_and_uses_progressive_disclosure() -> None: assert "CodeStable 使用反馈闭环" in feedback assert "--session current" in feedback + assert "--accept-incident" in feedback assert "best-effort" in feedback assert "ambiguity.candidates" in feedback assert "Ask User（缺口驱动）" in feedback assert "不要固定三问" in feedback assert "即使用户传 `--github`，也必须先让用户确认 preview" in feedback assert "public-issue-context.json" in feedback + assert "--triage-output" in feedback + assert "triage_ready" in feedback and "regression_ready" in feedback + assert "regression-candidate.json" in feedback + assert "--confirm-public-preview" in feedback + assert "旧 `--failure --experiment`" in feedback assert "禁止公开" in feedback assert "local_private_evidence" in template + assert "local_private_triage" in template assert "Public evidence fields" in template assert "完整 transcript" in template assert collector.is_file()