mirror of
https://github.com/boshu2/agentops.git
synced 2026-09-14 15:08:13 +08:00
ci(reconcile): wire factory-claim-ledger validator as advisory job (Wave 1C — soc-lmww1)
Adds `factory-claim-ledger-strict (advisory)` job to .github/workflows/validate.yml:
- runs `bash scripts/check-factory-claim-ledger.sh` via continue-on-error: true
- emits {run_id, pr_number, verdict, surfaces_touched, timestamp} JSON observation
- publishes observation as workflow artifact (not committed to repo)
- merged_anyway / ledger_updated backfill is Wave 1E concern, documented in-job
Also adds `tests/integration/test-factory-claim-ledger-ci-job.sh` to lock in the
job shape (T1-T6 contract assertions) for Wave 1E to invert when promoting to
blocking.
Validator stays at exit 0 on this branch.
This commit is contained in:
Boden Fuller
@@ -447,6 +447,122 @@ jobs:
|
||||
chmod +x scripts/eval-agentops.sh
|
||||
./scripts/eval-agentops.sh --fast
|
||||
|
||||
factory-claim-ledger-strict:
|
||||
# Wave 1C of epic soc-e4ulx (Reconciliation Engine).
|
||||
# Bead: soc-lmww1. Plan: .agents/plans/2026-05-07-reconciliation-engine-full-arc.md
|
||||
# Advisory-only: failures here MUST NOT fail the PR check. Promotion to
|
||||
# blocking is gated on Wave 1E structured sampling, not wall-clock time.
|
||||
# NOTE: `merged_anyway` and `ledger_updated` (referenced in the plan's
|
||||
# observation schema) cannot be set during PR CI — they are backfilled by
|
||||
# Wave 1E aggregation or by the operator post-merge. We emit only the
|
||||
# fields the runner can know: run_id, pr_number, verdict, surfaces_touched,
|
||||
# timestamp. Aggregation into .agents/reconcile/observation-log.jsonl is
|
||||
# explicitly out of scope for this wave; observations are published as a
|
||||
# workflow artifact instead of committed back to the repo.
|
||||
needs: [changes]
|
||||
if: needs.changes.outputs.docs == 'true' || needs.changes.outputs.contracts == 'true' || needs.changes.outputs.ci == 'true' || needs.changes.outputs.markdown == 'true'
|
||||
name: factory-claim-ledger-strict (advisory)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install jq (for observation construction)
|
||||
run: |
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
sudo apt-get update -y
|
||||
sudo apt-get install -y jq
|
||||
fi
|
||||
|
||||
- name: Run factory claim-ledger validator
|
||||
id: validator
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set +e
|
||||
bash scripts/check-factory-claim-ledger.sh
|
||||
rc=$?
|
||||
set -e
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
verdict=pass
|
||||
else
|
||||
verdict=fail
|
||||
fi
|
||||
echo "verdict=$verdict" >> "$GITHUB_OUTPUT"
|
||||
echo "rc=$rc" >> "$GITHUB_OUTPUT"
|
||||
# Do not propagate non-zero — continue-on-error keeps this advisory.
|
||||
exit 0
|
||||
|
||||
- name: Emit structured observation
|
||||
id: observation
|
||||
continue-on-error: true
|
||||
env:
|
||||
VERDICT: ${{ steps.validator.outputs.verdict }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$RUNNER_TEMP/factory-claim-ledger"
|
||||
out="$RUNNER_TEMP/factory-claim-ledger/observation.jsonl"
|
||||
|
||||
# Compute touched surfaces relative to the validator's source-set.
|
||||
# On pull_request: diff base..head. On push: diff HEAD~1..HEAD as a
|
||||
# reasonable approximation. Filter against the source-set patterns
|
||||
# collected by scripts/check-factory-claim-ledger.sh.
|
||||
if [ -n "${BASE_SHA:-}" ]; then
|
||||
range="${BASE_SHA}..${HEAD_SHA}"
|
||||
else
|
||||
range="HEAD~1..HEAD"
|
||||
fi
|
||||
|
||||
changed_files="$(git diff --name-only "$range" 2>/dev/null || true)"
|
||||
|
||||
# Source-set patterns mirror collect_repo_sources() in the validator.
|
||||
surfaces="$(printf '%s\n' "$changed_files" | awk '
|
||||
/^README\.md$/ ||
|
||||
/^PRODUCT\.md$/ ||
|
||||
/^GOALS\.md$/ ||
|
||||
/^docs\/index\.md$/ ||
|
||||
/^docs\/agentops-brief\.md$/ ||
|
||||
/^docs\/assurance-profile\.md$/ ||
|
||||
/^docs\/software-factory\.md$/ ||
|
||||
/^docs\/trust-factory\.md$/ ||
|
||||
/^docs\/wiki-for-agents\.md$/ ||
|
||||
/^docs\/comparisons\/.*\.md$/ ||
|
||||
/^docs\/contracts\/factory-.*\.md$/
|
||||
')"
|
||||
|
||||
# Build a JSON array of surfaces (jq handles empty input safely).
|
||||
surfaces_json="$(printf '%s\n' "$surfaces" | jq -R . | jq -s 'map(select(length > 0))')"
|
||||
|
||||
jq -n \
|
||||
--arg run_id "${GITHUB_RUN_ID:-}" \
|
||||
--arg pr_number "${PR_NUMBER:-}" \
|
||||
--arg verdict "${VERDICT:-unknown}" \
|
||||
--argjson surfaces_touched "$surfaces_json" \
|
||||
--arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
'{
|
||||
"run_id": $run_id,
|
||||
"pr_number": (if $pr_number == "" then null else $pr_number end),
|
||||
"verdict": $verdict,
|
||||
"surfaces_touched": $surfaces_touched,
|
||||
"timestamp": $timestamp
|
||||
}' > "$out"
|
||||
|
||||
echo "Observation:"
|
||||
cat "$out"
|
||||
echo "path=$out" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload claim-ledger observation artifact
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: factory-claim-ledger-observation
|
||||
path: ${{ runner.temp }}/factory-claim-ledger/
|
||||
retention-days: 30
|
||||
if-no-files-found: warn
|
||||
|
||||
eval-skill-delta:
|
||||
needs: [changes]
|
||||
if: needs.changes.outputs.eval == 'true' || needs.changes.outputs.skills == 'true' || needs.changes.outputs.ci == 'true'
|
||||
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bash
|
||||
# test-factory-claim-ledger-ci-job.sh — assert Wave 1C wiring of the
|
||||
# factory-claim-ledger validator into validate.yml as an advisory job.
|
||||
#
|
||||
# Bead: soc-lmww1 (Wave 1C of epic soc-e4ulx).
|
||||
# Contract: .agents/specs/contract-soc-lmww1.md
|
||||
#
|
||||
# Asserts:
|
||||
# T1. validate.yml is valid YAML.
|
||||
# T2. jobs.factory-claim-ledger-strict exists.
|
||||
# T3. Some step in that job has continue-on-error: true.
|
||||
# T4. Some step runs `bash scripts/check-factory-claim-ledger.sh`.
|
||||
# T5. Some step emits a JSON line containing the required observation keys
|
||||
# (run_id, pr_number, verdict, surfaces_touched, timestamp).
|
||||
# T6. Some step uses actions/upload-artifact to publish the observation.
|
||||
#
|
||||
# Usage: bash tests/integration/test-factory-claim-ledger-ci-job.sh
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
WORKFLOW="$REPO_ROOT/.github/workflows/validate.yml"
|
||||
JOB="factory-claim-ledger-strict"
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source "$REPO_ROOT/tests/lib/colors.sh"
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
ok() {
|
||||
pass "$1"
|
||||
PASS=$((PASS + 1))
|
||||
}
|
||||
|
||||
ko() {
|
||||
fail "$1"
|
||||
FAIL=$((FAIL + 1))
|
||||
}
|
||||
|
||||
if ! python3 -c 'import yaml' >/dev/null 2>&1; then
|
||||
red "python3 PyYAML is required for this test (pip install pyyaml)" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ ! -f "$WORKFLOW" ]]; then
|
||||
ko "validate.yml exists at $WORKFLOW"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mapfile -t LINES < <(python3 - "$WORKFLOW" "$JOB" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
workflow_path, job_name = sys.argv[1], sys.argv[2]
|
||||
REQUIRED_OBSERVATION_KEYS = ["run_id", "pr_number", "verdict", "surfaces_touched", "timestamp"]
|
||||
|
||||
try:
|
||||
with open(workflow_path, "r", encoding="utf-8") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
print("T1\t1\tvalidate.yml parses as YAML")
|
||||
except Exception as exc: # noqa: BLE001 - test surface
|
||||
print(f"T1\t0\tvalidate.yml parses as YAML ({exc!s})")
|
||||
sys.exit(0)
|
||||
|
||||
jobs = (data or {}).get("jobs") or {}
|
||||
job = jobs.get(job_name)
|
||||
print(f"T2\t{1 if isinstance(job, dict) else 0}\tjobs.{job_name} is defined")
|
||||
|
||||
steps = job.get("steps") if isinstance(job, dict) else []
|
||||
if not isinstance(steps, list):
|
||||
steps = []
|
||||
|
||||
run_steps = [s for s in steps if isinstance(s, dict) and isinstance(s.get("run"), str)]
|
||||
uses_steps = [s for s in steps if isinstance(s, dict) and isinstance(s.get("uses"), str)]
|
||||
|
||||
t3 = any(
|
||||
s.get("continue-on-error") in (True, "true")
|
||||
for s in steps
|
||||
if isinstance(s, dict)
|
||||
)
|
||||
print(f"T3\t{1 if t3 else 0}\ta step has continue-on-error: true")
|
||||
|
||||
validator_re = re.compile(r"\bbash\s+scripts/check-factory-claim-ledger\.sh\b")
|
||||
t4 = any(validator_re.search(s["run"]) for s in run_steps)
|
||||
print(f"T4\t{1 if t4 else 0}\ta step runs bash scripts/check-factory-claim-ledger.sh")
|
||||
|
||||
t5 = False
|
||||
for s in run_steps:
|
||||
body = s["run"]
|
||||
if all(
|
||||
re.search(rf'["\']{re.escape(k)}["\']', body) for k in REQUIRED_OBSERVATION_KEYS
|
||||
):
|
||||
t5 = True
|
||||
break
|
||||
print(f"T5\t{1 if t5 else 0}\ta step constructs the observation JSON")
|
||||
|
||||
t6 = any(s["uses"].startswith("actions/upload-artifact") for s in uses_steps)
|
||||
print(f"T6\t{1 if t6 else 0}\ta step uses actions/upload-artifact")
|
||||
PY
|
||||
)
|
||||
|
||||
for line in "${LINES[@]}"; do
|
||||
[[ -z "$line" ]] && continue
|
||||
IFS=$'\t' read -r tag flag msg <<<"$line"
|
||||
if [[ "$flag" == "1" ]]; then
|
||||
ok "$tag $msg"
|
||||
else
|
||||
ko "$tag $msg"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
if [[ "$FAIL" -gt 0 ]]; then
|
||||
red "FAILED — $FAIL/$((PASS + FAIL)) checks failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
green "PASSED — $PASS/$PASS checks passed"
|
||||
exit 0
|
||||
Reference in New Issue
Block a user