mirror of
https://github.com/Imbad0202/academic-research-skills.git
synced 2026-09-14 13:51:17 +08:00
Imbad0202
b8ab278df7
Empty array was silently passing — defeated "forces self-description" intent from the pattern doc. Now rejected with clear error. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
82 lines
2.7 KiB
JSON
82 lines
2.7 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://github.com/Imbad0202/academic-research-skills/shared/benchmark_report.schema.json",
|
|
"title": "ARS Benchmark Report",
|
|
"type": "object",
|
|
"required": [
|
|
"ars_version",
|
|
"task_definition",
|
|
"human_baseline",
|
|
"ars_run",
|
|
"metrics",
|
|
"caveats"
|
|
],
|
|
"properties": {
|
|
"ars_version": {
|
|
"type": "string",
|
|
"pattern": "^\\d+\\.\\d+\\.\\d+$",
|
|
"description": "Exact ARS version the benchmark was run against"
|
|
},
|
|
"task_definition": {
|
|
"type": "object",
|
|
"required": ["description", "task_type", "outcome_gradable"],
|
|
"properties": {
|
|
"description": { "type": "string", "minLength": 1 },
|
|
"task_type": { "enum": ["outcome-gradable", "open-ended"] },
|
|
"outcome_gradable": { "type": "boolean" }
|
|
}
|
|
},
|
|
"human_baseline": {
|
|
"type": "object",
|
|
"required": ["sample_size", "author_independence", "hours_spent", "recruitment", "tools_allowed"],
|
|
"properties": {
|
|
"sample_size": { "type": "integer", "minimum": 1 },
|
|
"author_independence": {
|
|
"enum": ["author-conducted", "author-blinded", "third-party-conducted"],
|
|
"description": "author-conducted means the benchmark creators also did the human baseline (downward bias risk)"
|
|
},
|
|
"hours_spent": { "type": "number", "minimum": 0 },
|
|
"recruitment": { "type": "string", "minLength": 1 },
|
|
"tools_allowed": {
|
|
"type": "array",
|
|
"items": { "type": "string", "minLength": 1 },
|
|
"minItems": 1
|
|
}
|
|
}
|
|
},
|
|
"ars_run": {
|
|
"type": "object",
|
|
"required": ["hours_spent", "cost_usd", "skills_used", "data_access_level_declared"],
|
|
"properties": {
|
|
"hours_spent": { "type": "number", "minimum": 0 },
|
|
"cost_usd": { "type": "number", "minimum": 0 },
|
|
"skills_used": {
|
|
"type": "array",
|
|
"items": { "type": "string" },
|
|
"minItems": 1
|
|
},
|
|
"data_access_level_declared": {
|
|
"enum": ["raw", "redacted", "verified_only"]
|
|
}
|
|
}
|
|
},
|
|
"metrics": {
|
|
"type": "object",
|
|
"required": ["primary_metric", "primary_metric_value", "scoring_independence"],
|
|
"properties": {
|
|
"primary_metric": { "type": "string", "minLength": 1 },
|
|
"primary_metric_value": { "type": "number" },
|
|
"scoring_independence": {
|
|
"enum": ["authors-scored", "third-party-scored", "blind-scored", "self-scored"]
|
|
}
|
|
}
|
|
},
|
|
"caveats": {
|
|
"type": "array",
|
|
"items": { "type": "string", "minLength": 1 },
|
|
"minItems": 1,
|
|
"description": "Known limitations. Empty array not permitted — honest disclosure required."
|
|
}
|
|
}
|
|
}
|