diff --git a/cli/cmd/ao/eval_scenario_moat.go b/cli/cmd/ao/eval_scenario_moat.go new file mode 100644 index 000000000..9a3b73c71 --- /dev/null +++ b/cli/cmd/ao/eval_scenario_moat.go @@ -0,0 +1,73 @@ +// practices: [llm-eval-harness] +package main + +import ( + "errors" + "fmt" + + "github.com/spf13/cobra" + + aoeval "github.com/boshu2/agentops/cli/internal/eval" +) + +var ( + evalScenarioMoatScorecards []string + evalScenarioMoatOutput string +) + +var evalScenarioMoatCmd = &cobra.Command{ + Use: "scenario-moat", + Short: "Aggregate moat-eligible scenario A/B scorecards into a publication verdict", + Long: `Render a moat positive/null/inconclusive verdict over one or more +ScenarioDeltaScorecard JSON artifacts from ao eval scenario-ab. + +The claim surface fail-closes on any scorecard with moat_eligible=false — a +fact-recall/plumbing scorecard can pass its own gate but must NEVER be aggregated +into a moat verdict (age-6ys/age-sb0). See docs/evals/applied-ood-claim-rule.md.`, + Args: cobra.NoArgs, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, _ []string) error { + if len(evalScenarioMoatScorecards) == 0 { + return fmt.Errorf("at least one --scorecard path is required") + } + cards := make([]aoeval.ScenarioDeltaScorecard, 0, len(evalScenarioMoatScorecards)) + for _, path := range evalScenarioMoatScorecards { + card, err := aoeval.LoadScenarioDeltaScorecard(path) + if err != nil { + return err + } + cards = append(cards, card) + } + result, err := aoeval.AggregateMoatClaim(cards) + if err != nil { + var ineligible aoeval.ErrMoatIneligibleScorecard + if errors.As(err, &ineligible) { + fmt.Fprintf(cmd.ErrOrStderr(), "REJECTED: %s\n", err) + } + return err + } + if err := aoeval.WriteMoatClaimResult(evalScenarioMoatOutput, result); err != nil { + return err + } + if GetOutput() == "json" { + return writeEvalJSON(cmd, result) + } + fmt.Fprintf(cmd.OutOrStdout(), + "scenario-moat: verdict=%s scenarios=%d mean_delta=%.4f\n %s\n", + result.Verdict, result.ScenarioCount, result.MeanAggregateDelta, result.Reason, + ) + if evalScenarioMoatOutput != "" { + fmt.Fprintf(cmd.OutOrStdout(), "Moat claim result: %s\n", evalScenarioMoatOutput) + } + if result.Verdict == aoeval.MoatVerdictInconclusive { + return fmt.Errorf("moat claim inconclusive — cannot publish positive or honest null") + } + return nil + }, +} + +func init() { + evalScenarioMoatCmd.Flags().StringArrayVar(&evalScenarioMoatScorecards, "scorecard", nil, "Path to a ScenarioDeltaScorecard JSON (repeatable)") + evalScenarioMoatCmd.Flags().StringVar(&evalScenarioMoatOutput, "output", "", "Write the MoatClaimResult JSON to this path") + evalCmd.AddCommand(evalScenarioMoatCmd) +} diff --git a/cli/cmd/ao/eval_scenario_moat_test.go b/cli/cmd/ao/eval_scenario_moat_test.go new file mode 100644 index 000000000..3434b4327 --- /dev/null +++ b/cli/cmd/ao/eval_scenario_moat_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "bytes" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func withMoatCmdReset(t *testing.T) { + t.Helper() + t.Cleanup(func() { + evalScenarioMoatScorecards = nil + evalScenarioMoatOutput = "" + }) +} + +func runScenarioMoatCmd(t *testing.T, scorecards []string, outPath string) (string, error) { + t.Helper() + cmd := evalScenarioMoatCmd + var buf bytes.Buffer + cmd.SetOut(&buf) + cmd.SetErr(&buf) + evalScenarioMoatScorecards = scorecards + evalScenarioMoatOutput = outPath + err := cmd.RunE(cmd, nil) + return buf.String(), err +} + +func TestEvalScenarioMoatRejectsPlumbingScorecard(t *testing.T) { + withMoatCmdReset(t) + root := repoRootForEvalCmd(t) + plumbing := filepath.Join(root, "evals/scenarios/fixtures/scenario-ab-fact-recall-plumbing.scorecard.json") + _, err := runScenarioMoatCmd(t, []string{plumbing}, "") + if err == nil { + t.Fatal("expected error when aggregating moat_eligible=false scorecard") + } + if !strings.Contains(err.Error(), "moat_eligible=false") { + t.Fatalf("error = %q, want moat_eligible=false rejection", err.Error()) + } +} + +func TestEvalScenarioMoatPositiveFromFixture(t *testing.T) { + withMoatCmdReset(t) + root := repoRootForEvalCmd(t) + valid := filepath.Join(root, "evals/scenarios/fixtures/scenario-ab-valid-redacted.scorecard.json") + out := filepath.Join(t.TempDir(), "moat-claim.json") + _, err := runScenarioMoatCmd(t, []string{valid}, out) + if err != nil { + t.Fatalf("runScenarioMoatCmd: %v", err) + } +} + +func repoRootForEvalCmd(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + return filepath.Join(filepath.Dir(file), "..", "..", "..") +} diff --git a/cli/docs/COMMANDS.md b/cli/docs/COMMANDS.md index 21af9142a..3151cc21a 100644 --- a/cli/docs/COMMANDS.md +++ b/cli/docs/COMMANDS.md @@ -1697,6 +1697,22 @@ ao eval scenario-ab [flags] --token-budget int Fail the gate if summed arm token cost exceeds this (0 = default 200000) ``` +#### `ao eval scenario-moat` + +Render a moat positive/null/inconclusive verdict over one or more + +``` +ao eval scenario-moat [flags] +``` + +**Flags:** + +``` + -h, --help help for scenario-moat + --output string Write the MoatClaimResult JSON to this path + --scorecard stringArray Path to a ScenarioDeltaScorecard JSON (repeatable) +``` + #### `ao eval scorecard` Build an eval scorecard from run records diff --git a/cli/internal/eval/moat_claim.go b/cli/internal/eval/moat_claim.go new file mode 100644 index 000000000..b0d01c787 --- /dev/null +++ b/cli/internal/eval/moat_claim.go @@ -0,0 +1,167 @@ +package eval + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" +) + +// MoatClaimVerdict is the publication-tier verdict for the gold/corpus axis when +// aggregating moat-eligible ScenarioDeltaScorecards. See +// docs/evals/applied-ood-claim-rule.md. +type MoatClaimVerdict string + +const ( + // MoatVerdictPositive means eligible scorecards collectively support a moat + // positive claim under the locked publication rule. + MoatVerdictPositive MoatClaimVerdict = "moat_positive" + // MoatVerdictHonestNull means eligible scorecards had headroom but the corpus + // did not improve work — a valid null result. + MoatVerdictHonestNull MoatClaimVerdict = "honest_null" + // MoatVerdictInconclusive means the inputs cannot support positive or null. + MoatVerdictInconclusive MoatClaimVerdict = "inconclusive" +) + +// MoatClaimResult is the persisted output of the moat claim aggregation surface +// (age-sb0). It renders a moat positive/null/inconclusive verdict over one or +// more scenario A/B scorecards. +type MoatClaimResult struct { + SchemaVersion int `json:"schema_version"` + GeneratedAt time.Time `json:"generated_at"` + Verdict MoatClaimVerdict `json:"verdict"` + Reason string `json:"reason"` + ScenarioCount int `json:"scenario_count"` + MeanAggregateDelta float64 `json:"mean_aggregate_delta"` + ScorecardScenarioIDs []string `json:"scorecard_scenario_ids"` + ExcludedCeiling []string `json:"excluded_ceiling_violation,omitempty"` + ExcludedGateFail []string `json:"excluded_gate_fail,omitempty"` +} + +// ErrMoatIneligibleScorecard is returned when aggregation is asked to include a +// scorecard with moat_eligible=false. The claim surface fail-closes rather than +// silently mixing plumbing into a moat verdict (age-6ys/age-sb0). +type ErrMoatIneligibleScorecard struct { + ScenarioID string + VerdictClass string +} + +func (e ErrMoatIneligibleScorecard) Error() string { + class := e.VerdictClass + if class == "" { + class = "unknown" + } + return fmt.Sprintf( + "moat claim aggregation refused: scorecard %q has moat_eligible=false (verdict_class=%q, NOT-moat-evidence/plumbing); cannot aggregate into a moat verdict (age-6ys/age-sb0)", + e.ScenarioID, class, + ) +} + +// LoadScenarioDeltaScorecard reads a ScenarioDeltaScorecard JSON artifact. +func LoadScenarioDeltaScorecard(path string) (ScenarioDeltaScorecard, error) { + data, err := os.ReadFile(path) + if err != nil { + return ScenarioDeltaScorecard{}, fmt.Errorf("read scenario delta scorecard %s: %w", path, err) + } + var card ScenarioDeltaScorecard + if err := json.Unmarshal(data, &card); err != nil { + return ScenarioDeltaScorecard{}, fmt.Errorf("decode scenario delta scorecard %s: %w", path, err) + } + if strings.TrimSpace(card.ScenarioID) == "" { + return ScenarioDeltaScorecard{}, fmt.Errorf("scenario delta scorecard %s has no scenario_id", path) + } + return card, nil +} + +// AggregateMoatClaim renders a moat positive/null/inconclusive verdict over the +// provided scorecards. It fail-closes on any moat_eligible=false input — plumbing +// scorecards must never be aggregated into a moat claim (age-sb0). +func AggregateMoatClaim(cards []ScenarioDeltaScorecard) (MoatClaimResult, error) { + if len(cards) == 0 { + return MoatClaimResult{}, fmt.Errorf("moat claim aggregation requires at least one scorecard") + } + for _, card := range cards { + if !card.MoatEligible { + return MoatClaimResult{}, ErrMoatIneligibleScorecard{ + ScenarioID: card.ScenarioID, + VerdictClass: card.VerdictClass, + } + } + } + + now := time.Now().UTC() + result := MoatClaimResult{ + SchemaVersion: 1, + GeneratedAt: now, + ScenarioCount: len(cards), + } + for _, card := range cards { + result.ScorecardScenarioIDs = append(result.ScorecardScenarioIDs, card.ScenarioID) + } + + var admissible []ScenarioDeltaScorecard + var positiveEligible []ScenarioDeltaScorecard + for _, card := range cards { + if card.CeilingViolation { + result.ExcludedCeiling = append(result.ExcludedCeiling, card.ScenarioID) + continue + } + admissible = append(admissible, card) + if card.Gate.Pass && card.AggregateDelta > 0 { + positiveEligible = append(positiveEligible, card) + } else if !card.Gate.Pass { + result.ExcludedGateFail = append(result.ExcludedGateFail, card.ScenarioID) + } + } + + if len(admissible) == 0 { + result.Verdict = MoatVerdictInconclusive + result.Reason = "no admissible scorecards after excluding ceiling violations" + return result, nil + } + + var deltaSum float64 + allHadHeadroom := true + for _, card := range admissible { + deltaSum += card.AggregateDelta + if card.Without.Score >= card.SatisfactionThreshold { + allHadHeadroom = false + } + } + result.MeanAggregateDelta = roundDelta(deltaSum / float64(len(admissible))) + + switch { + case len(positiveEligible) == len(admissible): + result.Verdict = MoatVerdictPositive + result.Reason = fmt.Sprintf( + "all %d admissible scorecards have positive delta and passed gate; mean aggregate_delta=%.4f", + len(admissible), result.MeanAggregateDelta, + ) + case allHadHeadroom && len(positiveEligible) == 0: + result.Verdict = MoatVerdictHonestNull + result.Reason = fmt.Sprintf( + "admissible scorecards had headroom but corpus did not beat control; mean aggregate_delta=%.4f", + result.MeanAggregateDelta, + ) + default: + result.Verdict = MoatVerdictInconclusive + result.Reason = "mixed headroom or delta signals across admissible scorecards — cannot claim positive or honest null" + } + return result, nil +} + +// WriteMoatClaimResult persists a MoatClaimResult JSON artifact. +func WriteMoatClaimResult(path string, result MoatClaimResult) error { + if strings.TrimSpace(path) == "" { + return fmt.Errorf("output path is required") + } + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + return fmt.Errorf("marshal moat claim result: %w", err) + } + if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil { + return fmt.Errorf("write moat claim result: %w", err) + } + return nil +} diff --git a/cli/internal/eval/moat_claim_test.go b/cli/internal/eval/moat_claim_test.go new file mode 100644 index 000000000..9d77d1b47 --- /dev/null +++ b/cli/internal/eval/moat_claim_test.go @@ -0,0 +1,123 @@ +package eval + +import ( + "errors" + "path/filepath" + "runtime" + "testing" +) + +func TestAggregateMoatClaimRejectsIneligibleScorecard(t *testing.T) { + plumbing := ScenarioDeltaScorecard{ + ScenarioID: "s-plumbing-001", + VerdictClass: VerdictClassFactRecall, + MoatEligible: false, + AggregateDelta: 1.0, + SatisfactionThreshold: 0.8, + Without: ScenarioArmResult{Score: 0.1}, + With: ScenarioArmResult{Score: 1.0}, + Gate: ScenarioGate{Pass: true}, + } + _, err := AggregateMoatClaim([]ScenarioDeltaScorecard{plumbing}) + if err == nil { + t.Fatal("expected error for moat_eligible=false scorecard") + } + var ineligible ErrMoatIneligibleScorecard + if !errors.As(err, &ineligible) { + t.Fatalf("expected ErrMoatIneligibleScorecard, got %T: %v", err, err) + } + if ineligible.ScenarioID != "s-plumbing-001" { + t.Errorf("ScenarioID = %q, want s-plumbing-001", ineligible.ScenarioID) + } +} + +func TestAggregateMoatClaimPositive(t *testing.T) { + card := ScenarioDeltaScorecard{ + ScenarioID: "s-applied-001", + VerdictClass: VerdictClassAppliedOOD, + MoatEligible: true, + AggregateDelta: 0.57, + SatisfactionThreshold: 0.8, + Without: ScenarioArmResult{Score: 0.35}, + With: ScenarioArmResult{Score: 0.92}, + Gate: ScenarioGate{Pass: true}, + } + result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card}) + if err != nil { + t.Fatalf("AggregateMoatClaim: %v", err) + } + if result.Verdict != MoatVerdictPositive { + t.Errorf("Verdict = %q, want %q", result.Verdict, MoatVerdictPositive) + } +} + +func TestAggregateMoatClaimHonestNull(t *testing.T) { + card := ScenarioDeltaScorecard{ + ScenarioID: "s-applied-002", + VerdictClass: VerdictClassAppliedOOD, + MoatEligible: true, + AggregateDelta: -0.07, + SatisfactionThreshold: 0.8, + Without: ScenarioArmResult{Score: 0.35}, + With: ScenarioArmResult{Score: 0.28}, + Gate: ScenarioGate{Pass: false, Reasons: []string{"aggregate_delta <= 0", "with-gold score < threshold"}}, + } + result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card}) + if err != nil { + t.Fatalf("AggregateMoatClaim: %v", err) + } + if result.Verdict != MoatVerdictHonestNull { + t.Errorf("Verdict = %q, want %q", result.Verdict, MoatVerdictHonestNull) + } +} + +func TestAggregateMoatClaimExcludesCeilingViolation(t *testing.T) { + card := ScenarioDeltaScorecard{ + ScenarioID: "s-ceiling-001", + VerdictClass: VerdictClassAppliedOOD, + MoatEligible: true, + CeilingViolation: true, + AggregateDelta: 0, + SatisfactionThreshold: 0.8, + Without: ScenarioArmResult{Score: 0.95}, + With: ScenarioArmResult{Score: 0.95}, + Gate: ScenarioGate{Pass: false}, + } + result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card}) + if err != nil { + t.Fatalf("AggregateMoatClaim: %v", err) + } + if result.Verdict != MoatVerdictInconclusive { + t.Errorf("Verdict = %q, want %q", result.Verdict, MoatVerdictInconclusive) + } + if len(result.ExcludedCeiling) != 1 { + t.Errorf("ExcludedCeiling = %v, want [s-ceiling-001]", result.ExcludedCeiling) + } +} + +func TestLoadScenarioDeltaScorecardFromFixture(t *testing.T) { + fixture := filepath.Join(repoRoot(t), "evals/scenarios/fixtures/scenario-ab-valid-redacted.scorecard.json") + card, err := LoadScenarioDeltaScorecard(fixture) + if err != nil { + t.Fatalf("LoadScenarioDeltaScorecard: %v", err) + } + if !card.MoatEligible { + t.Error("fixture scorecard should be moat_eligible") + } + result, err := AggregateMoatClaim([]ScenarioDeltaScorecard{card}) + if err != nil { + t.Fatalf("AggregateMoatClaim: %v", err) + } + if result.Verdict != MoatVerdictPositive { + t.Errorf("fixture Verdict = %q, want %q", result.Verdict, MoatVerdictPositive) + } +} + +func repoRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + return filepath.Join(filepath.Dir(file), "..", "..", "..") +} diff --git a/docs/cli-surface.json b/docs/cli-surface.json index 843050a33..e424e9e6f 100644 --- a/docs/cli-surface.json +++ b/docs/cli-surface.json @@ -561,6 +561,13 @@ "kind": "leaf", "reason": "Covered by release smoke tests, direct command tests, or command handler tests." }, + { + "category": "public-tested", + "command": "eval scenario-moat", + "coverage_status": "covered", + "kind": "leaf", + "reason": "Covered by release smoke tests, direct command tests, or command handler tests." + }, { "category": "public-tested", "command": "eval scorecard", diff --git a/docs/cli-surface.md b/docs/cli-surface.md index 879a97fb3..34819f02d 100644 --- a/docs/cli-surface.md +++ b/docs/cli-surface.md @@ -84,6 +84,7 @@ | `ao eval outcomes ingest` | `public-stateful-fixture-needed` | `allowlisted` | Maps an Outcomes score to the council verdict record; core logic unit-tested (eval_outcomes_ingest_test.go ingestOutcomesScore); CLI smoke needs a score.json fixture (follow-up). | | `ao eval run` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. | | `ao eval scenario-ab` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. | +| `ao eval scenario-moat` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. | | `ao eval scorecard` | `public-tested` | `covered` | Covered by release smoke tests, direct command tests, or command handler tests. | | `ao eval suite n-required` | `public-stateful-fixture-needed` | `allowlisted` | Computes eval-suite thresholds and needs a representative suite fixture. | | `ao eval suite verdict` | `public-stateful-fixture-needed` | `allowlisted` | Projects eval-suite verdicts and needs a representative suite fixture. | diff --git a/docs/evals/applied-ood-claim-rule.md b/docs/evals/applied-ood-claim-rule.md index 3713f7e7f..33706abf4 100644 --- a/docs/evals/applied-ood-claim-rule.md +++ b/docs/evals/applied-ood-claim-rule.md @@ -118,3 +118,6 @@ partly done: as moat evidence; defined the applied-OOD class + validity checklist + locked publication rule. Mechanical enforcement landed as `verdict_class` / `moat_eligible` on `ScenarioDeltaScorecard`. +- 2026-06-18 (age-sb0): moat claim aggregation surface (`ao eval scenario-moat`) + fail-closes on `moat_eligible=false` inputs; renders moat_positive/honest_null/ + inconclusive over eligible scorecards only. diff --git a/evals/agentops-core/cli-command-surface-matrix.json b/evals/agentops-core/cli-command-surface-matrix.json index 8d79d645b..6d2de235b 100644 --- a/evals/agentops-core/cli-command-surface-matrix.json +++ b/evals/agentops-core/cli-command-surface-matrix.json @@ -41,7 +41,7 @@ }, "expectations": [ {"type": "exit_code", "value": 0}, - {"type": "stdout_contains", "value": "cli-command-headings: top=88 sub=219 all=307"}, + {"type": "stdout_contains", "value": "cli-command-headings: top=88 sub=220 all=308"}, {"type": "stdout_contains", "value": "cli-help-matrix-ok"} ], "dimensions": ["correctness", "runtime_compatibility", "artifact_quality"], diff --git a/evals/agentops-core/fixtures/cli-command-surface-smoke.sh b/evals/agentops-core/fixtures/cli-command-surface-smoke.sh index 909f9369e..f80ffb9e2 100755 --- a/evals/agentops-core/fixtures/cli-command-surface-smoke.sh +++ b/evals/agentops-core/fixtures/cli-command-surface-smoke.sh @@ -17,7 +17,7 @@ top_count="$(rg -c '^### `ao ' "$DOCS_PATH")" sub_count="$(rg -c '^#### `ao ' "$DOCS_PATH")" all_count="$(rg -c '^#{3,4} `ao ' "$DOCS_PATH")" -if [[ "$top_count" != "88" || "$sub_count" != "219" || "$all_count" != "307" ]]; then +if [[ "$top_count" != "88" || "$sub_count" != "220" || "$all_count" != "308" ]]; then printf 'unexpected command heading counts: top=%s sub=%s all=%s\n' "$top_count" "$sub_count" "$all_count" >&2 exit 1 fi @@ -25,7 +25,7 @@ fi # shellcheck disable=SC2016 # literal backticks delimit generated Markdown command headings. mapfile -t commands < <(rg '^#{3,4} `ao ' "$DOCS_PATH" | sed -E 's/^.*`([^`]+)`.*/\1/') -if [[ "${#commands[@]}" -ne 307 ]]; then +if [[ "${#commands[@]}" -ne 308 ]]; then printf 'unexpected command matrix size: %s\n' "${#commands[@]}" >&2 exit 1 fi diff --git a/evals/scenarios/fixtures/scenario-ab-fact-recall-plumbing.scorecard.json b/evals/scenarios/fixtures/scenario-ab-fact-recall-plumbing.scorecard.json new file mode 100644 index 000000000..88f42ddbc --- /dev/null +++ b/evals/scenarios/fixtures/scenario-ab-fact-recall-plumbing.scorecard.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "scenario_id": "s-fixture-plumbing-001", + "scenario_path": "evals/scenarios/fixtures/sentinel-recall-plumbing.json", + "generated_at": "2026-06-17T12:00:00Z", + "without_gold": { + "arm": "without_gold", + "score": 0.0, + "token_cost": 500 + }, + "with_gold": { + "arm": "with_gold", + "score": 1.0, + "token_cost": 600 + }, + "aggregate_delta": 1.0, + "satisfaction_threshold": 0.8, + "token_budget": 200000, + "ceiling_violation": false, + "verdict_class": "fact-recall", + "moat_eligible": false, + "gate": { + "pass": true, + "reasons": [] + }, + "_note": "Plumbing fixture for age-sb0: must be rejected by moat claim aggregation." +} diff --git a/registry.json b/registry.json index 56f8b828f..507c104a5 100644 --- a/registry.json +++ b/registry.json @@ -1,6 +1,6 @@ { "schema_version": 2, - "generated_at": "2026-06-17T11:58:35Z", + "generated_at": "2026-06-17T12:37:19Z", "summary": { "skills": 71, "hooks": 0, diff --git a/skills-codex/.agentops-manifest.json b/skills-codex/.agentops-manifest.json index d2389772f..be363ac4f 100644 --- a/skills-codex/.agentops-manifest.json +++ b/skills-codex/.agentops-manifest.json @@ -1366,8 +1366,8 @@ { "name": "plan", "source_skill": "skills/plan", - "source_hash": "129c2fec737a15f8b0abf10dbedc706ec52d94c74bc1bbbbaac28f90ca018275", - "generated_hash": "343d33512062b6232f30820a2f29d35a8a4e8b6e02e2194efb7032d5a90a5648" + "source_hash": "29465f3bacb12c1bc18a694b71c14987e217c3c0ee32f0123a617a27817fb260", + "generated_hash": "eb02068c559fdb80a0fada49a6438aad7ef6fd7f81a6e33024d7ecabcb77d00b" }, { "name": "post-mortem",