mirror of
https://github.com/Egonex-AI/Understand-Anything.git
synced 2026-09-14 20:26:55 +08:00
feat(bench): add deterministic large-repo scale reports
This commit is contained in:
KumamuKuma
@@ -0,0 +1,575 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://raw.githubusercontent.com/Egonex-AI/Understand-Anything/main/docs/benchmarks/large-repo-report-1.0.0.schema.json",
|
||||
"title": "Understand Anything deterministic large-repository benchmark report",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"schemaUrl",
|
||||
"schemaVersion",
|
||||
"status",
|
||||
"mode",
|
||||
"run",
|
||||
"tool",
|
||||
"subject",
|
||||
"environment",
|
||||
"configuration",
|
||||
"scale",
|
||||
"stages",
|
||||
"integrity",
|
||||
"determinism",
|
||||
"llm",
|
||||
"warnings",
|
||||
"error"
|
||||
],
|
||||
"properties": {
|
||||
"schemaUrl": {
|
||||
"const": "https://raw.githubusercontent.com/Egonex-AI/Understand-Anything/main/docs/benchmarks/large-repo-report-1.0.0.schema.json"
|
||||
},
|
||||
"schemaVersion": { "const": "1.0.0" },
|
||||
"status": { "enum": ["ok", "degraded", "failed"] },
|
||||
"mode": { "const": "deterministic" },
|
||||
"run": { "$ref": "#/$defs/run" },
|
||||
"tool": { "$ref": "#/$defs/tool" },
|
||||
"subject": { "$ref": "#/$defs/subject" },
|
||||
"environment": { "$ref": "#/$defs/environment" },
|
||||
"configuration": { "$ref": "#/$defs/configuration" },
|
||||
"scale": {
|
||||
"oneOf": [
|
||||
{ "$ref": "#/$defs/scale" },
|
||||
{ "type": "null" }
|
||||
]
|
||||
},
|
||||
"stages": { "$ref": "#/$defs/stages" },
|
||||
"integrity": {
|
||||
"oneOf": [
|
||||
{ "$ref": "#/$defs/integrity" },
|
||||
{ "type": "null" }
|
||||
]
|
||||
},
|
||||
"determinism": {
|
||||
"oneOf": [
|
||||
{ "$ref": "#/$defs/determinism" },
|
||||
{ "type": "null" }
|
||||
]
|
||||
},
|
||||
"llm": { "$ref": "#/$defs/llm" },
|
||||
"warnings": {
|
||||
"type": "array",
|
||||
"items": { "$ref": "#/$defs/warning" }
|
||||
},
|
||||
"error": { "type": ["string", "null"] }
|
||||
},
|
||||
"allOf": [
|
||||
{
|
||||
"if": {
|
||||
"properties": { "status": { "enum": ["ok", "degraded"] } },
|
||||
"required": ["status"]
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"scale": { "$ref": "#/$defs/scale" },
|
||||
"stages": {
|
||||
"type": "object",
|
||||
"required": ["scan", "imports", "batching", "structure"],
|
||||
"properties": {
|
||||
"scan": {
|
||||
"type": "object",
|
||||
"properties": { "status": { "const": "ok" } }
|
||||
},
|
||||
"imports": {
|
||||
"type": "object",
|
||||
"properties": { "status": { "const": "ok" } }
|
||||
},
|
||||
"batching": {
|
||||
"type": "object",
|
||||
"properties": { "status": { "const": "ok" } }
|
||||
},
|
||||
"structure": {
|
||||
"type": "object",
|
||||
"properties": { "status": { "const": "ok" } }
|
||||
}
|
||||
}
|
||||
},
|
||||
"integrity": { "$ref": "#/$defs/successIntegrity" },
|
||||
"determinism": { "$ref": "#/$defs/determinism" },
|
||||
"error": { "type": "null" }
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": { "status": { "const": "ok" } },
|
||||
"required": ["status"]
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"warnings": { "type": "array", "maxItems": 0 },
|
||||
"integrity": {
|
||||
"type": "object",
|
||||
"properties": { "filesSkipped": { "const": 0 } }
|
||||
},
|
||||
"stages": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"scan": { "$ref": "#/$defs/noWarningsStage" },
|
||||
"imports": { "$ref": "#/$defs/noWarningsStage" },
|
||||
"batching": { "$ref": "#/$defs/noWarningsStage" },
|
||||
"structure": {
|
||||
"allOf": [
|
||||
{ "$ref": "#/$defs/noWarningsStage" },
|
||||
{
|
||||
"type": "object",
|
||||
"properties": { "filesSkipped": { "const": 0 } }
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": { "status": { "const": "failed" } },
|
||||
"required": ["status"]
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"error": { "type": "string", "minLength": 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"$defs": {
|
||||
"nullableString": { "type": ["string", "null"] },
|
||||
"nullableBoolean": { "type": ["boolean", "null"] },
|
||||
"nullableNumber": { "type": ["number", "null"], "minimum": 0 },
|
||||
"count": { "type": "integer", "minimum": 0 },
|
||||
"status": { "enum": ["ok", "failed"] },
|
||||
"noWarningsStage": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"warningCount": { "const": 0 },
|
||||
"warningMessages": { "type": "array", "maxItems": 0 },
|
||||
"warningMessagesTruncated": { "const": false }
|
||||
}
|
||||
},
|
||||
"run": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["startedAt", "finishedAt", "durationMs"],
|
||||
"properties": {
|
||||
"startedAt": { "type": "string", "format": "date-time" },
|
||||
"finishedAt": { "type": ["string", "null"], "format": "date-time" },
|
||||
"durationMs": { "$ref": "#/$defs/nullableNumber" }
|
||||
}
|
||||
},
|
||||
"tool": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["commit", "dirty", "packageVersion"],
|
||||
"properties": {
|
||||
"commit": { "$ref": "#/$defs/nullableString" },
|
||||
"dirty": { "$ref": "#/$defs/nullableBoolean" },
|
||||
"packageVersion": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"subject": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["label", "commit", "dirty"],
|
||||
"properties": {
|
||||
"label": { "type": "string" },
|
||||
"commit": { "$ref": "#/$defs/nullableString" },
|
||||
"dirty": { "$ref": "#/$defs/nullableBoolean" }
|
||||
}
|
||||
},
|
||||
"environment": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"platform",
|
||||
"release",
|
||||
"arch",
|
||||
"nodeVersion",
|
||||
"cpuModel",
|
||||
"logicalCores",
|
||||
"totalMemoryBytes"
|
||||
],
|
||||
"properties": {
|
||||
"platform": { "type": "string" },
|
||||
"release": { "type": "string" },
|
||||
"arch": { "type": "string" },
|
||||
"nodeVersion": { "type": "string" },
|
||||
"cpuModel": { "type": "string" },
|
||||
"logicalCores": { "type": "integer", "minimum": 1 },
|
||||
"totalMemoryBytes": { "type": "integer", "minimum": 0 }
|
||||
}
|
||||
},
|
||||
"configuration": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["concurrency", "stages"],
|
||||
"properties": {
|
||||
"concurrency": { "type": "integer", "minimum": 1, "maximum": 32 },
|
||||
"stages": {
|
||||
"const": ["scan", "imports", "batching", "structure"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"stringCountMap": {
|
||||
"type": "object",
|
||||
"additionalProperties": { "$ref": "#/$defs/count" }
|
||||
},
|
||||
"scale": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"files",
|
||||
"lines",
|
||||
"bytes",
|
||||
"missingFiles",
|
||||
"filteredByUserIgnore",
|
||||
"byCategory",
|
||||
"byLanguage"
|
||||
],
|
||||
"properties": {
|
||||
"files": { "$ref": "#/$defs/count" },
|
||||
"lines": { "$ref": "#/$defs/count" },
|
||||
"bytes": { "$ref": "#/$defs/count" },
|
||||
"missingFiles": { "$ref": "#/$defs/count" },
|
||||
"filteredByUserIgnore": { "$ref": "#/$defs/count" },
|
||||
"byCategory": { "$ref": "#/$defs/stringCountMap" },
|
||||
"byLanguage": { "$ref": "#/$defs/stringCountMap" }
|
||||
}
|
||||
},
|
||||
"scanStage": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"status",
|
||||
"durationMs",
|
||||
"peakRssBytes",
|
||||
"userCpuTimeMicros",
|
||||
"systemCpuTimeMicros",
|
||||
"warningCount",
|
||||
"warningMessages",
|
||||
"warningMessagesTruncated",
|
||||
"stdoutTruncated",
|
||||
"stderrTruncated",
|
||||
"outputBytes"
|
||||
],
|
||||
"properties": {
|
||||
"status": { "$ref": "#/$defs/status" },
|
||||
"durationMs": { "type": "number", "minimum": 0 },
|
||||
"peakRssBytes": { "$ref": "#/$defs/nullableNumber" },
|
||||
"userCpuTimeMicros": { "$ref": "#/$defs/nullableNumber" },
|
||||
"systemCpuTimeMicros": { "$ref": "#/$defs/nullableNumber" },
|
||||
"warningCount": { "$ref": "#/$defs/count" },
|
||||
"warningMessages": { "$ref": "#/$defs/warningMessages" },
|
||||
"warningMessagesTruncated": { "type": "boolean" },
|
||||
"stdoutTruncated": { "type": "boolean" },
|
||||
"stderrTruncated": { "type": "boolean" },
|
||||
"outputBytes": { "$ref": "#/$defs/count" },
|
||||
"files": { "$ref": "#/$defs/count" },
|
||||
"lines": { "$ref": "#/$defs/count" },
|
||||
"bytes": { "$ref": "#/$defs/count" },
|
||||
"filteredByUserIgnore": { "$ref": "#/$defs/count" }
|
||||
},
|
||||
"allOf": [
|
||||
{
|
||||
"if": {
|
||||
"properties": { "status": { "const": "ok" } },
|
||||
"required": ["status"]
|
||||
},
|
||||
"then": {
|
||||
"required": [
|
||||
"files",
|
||||
"lines",
|
||||
"bytes",
|
||||
"filteredByUserIgnore"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"importsStage": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"status",
|
||||
"durationMs",
|
||||
"peakRssBytes",
|
||||
"userCpuTimeMicros",
|
||||
"systemCpuTimeMicros",
|
||||
"warningCount",
|
||||
"warningMessages",
|
||||
"warningMessagesTruncated",
|
||||
"stdoutTruncated",
|
||||
"stderrTruncated",
|
||||
"outputBytes"
|
||||
],
|
||||
"properties": {
|
||||
"status": { "$ref": "#/$defs/status" },
|
||||
"durationMs": { "type": "number", "minimum": 0 },
|
||||
"peakRssBytes": { "$ref": "#/$defs/nullableNumber" },
|
||||
"userCpuTimeMicros": { "$ref": "#/$defs/nullableNumber" },
|
||||
"systemCpuTimeMicros": { "$ref": "#/$defs/nullableNumber" },
|
||||
"warningCount": { "$ref": "#/$defs/count" },
|
||||
"warningMessages": { "$ref": "#/$defs/warningMessages" },
|
||||
"warningMessagesTruncated": { "type": "boolean" },
|
||||
"stdoutTruncated": { "type": "boolean" },
|
||||
"stderrTruncated": { "type": "boolean" },
|
||||
"outputBytes": { "$ref": "#/$defs/count" },
|
||||
"filesScanned": { "$ref": "#/$defs/count" },
|
||||
"filesWithImports": { "$ref": "#/$defs/count" },
|
||||
"edges": { "$ref": "#/$defs/count" }
|
||||
},
|
||||
"allOf": [
|
||||
{
|
||||
"if": {
|
||||
"properties": { "status": { "const": "ok" } },
|
||||
"required": ["status"]
|
||||
},
|
||||
"then": {
|
||||
"required": ["filesScanned", "filesWithImports", "edges"]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"distribution": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["min", "p50", "p95", "max", "mean"],
|
||||
"properties": {
|
||||
"min": { "type": "number", "minimum": 0 },
|
||||
"p50": { "type": "number", "minimum": 0 },
|
||||
"p95": { "type": "number", "minimum": 0 },
|
||||
"max": { "type": "number", "minimum": 0 },
|
||||
"mean": { "type": "number", "minimum": 0 }
|
||||
}
|
||||
},
|
||||
"batchingStage": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"status",
|
||||
"durationMs",
|
||||
"peakRssBytes",
|
||||
"userCpuTimeMicros",
|
||||
"systemCpuTimeMicros",
|
||||
"warningCount",
|
||||
"warningMessages",
|
||||
"warningMessagesTruncated",
|
||||
"stdoutTruncated",
|
||||
"stderrTruncated",
|
||||
"outputBytes"
|
||||
],
|
||||
"properties": {
|
||||
"status": { "$ref": "#/$defs/status" },
|
||||
"durationMs": { "type": "number", "minimum": 0 },
|
||||
"peakRssBytes": { "$ref": "#/$defs/nullableNumber" },
|
||||
"userCpuTimeMicros": { "$ref": "#/$defs/nullableNumber" },
|
||||
"systemCpuTimeMicros": { "$ref": "#/$defs/nullableNumber" },
|
||||
"warningCount": { "$ref": "#/$defs/count" },
|
||||
"warningMessages": { "$ref": "#/$defs/warningMessages" },
|
||||
"warningMessagesTruncated": { "type": "boolean" },
|
||||
"stdoutTruncated": { "type": "boolean" },
|
||||
"stderrTruncated": { "type": "boolean" },
|
||||
"outputBytes": { "$ref": "#/$defs/count" },
|
||||
"algorithm": { "type": "string" },
|
||||
"totalBatches": { "$ref": "#/$defs/count" },
|
||||
"batchSizes": { "$ref": "#/$defs/distribution" },
|
||||
"estimatedAgentInputBytes": { "$ref": "#/$defs/count" }
|
||||
},
|
||||
"allOf": [
|
||||
{
|
||||
"if": {
|
||||
"properties": { "status": { "const": "ok" } },
|
||||
"required": ["status"]
|
||||
},
|
||||
"then": {
|
||||
"required": [
|
||||
"algorithm",
|
||||
"totalBatches",
|
||||
"batchSizes",
|
||||
"estimatedAgentInputBytes"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"entities": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"functions",
|
||||
"classes",
|
||||
"exports",
|
||||
"callGraph",
|
||||
"definitions",
|
||||
"services",
|
||||
"endpoints",
|
||||
"steps",
|
||||
"resources"
|
||||
],
|
||||
"properties": {
|
||||
"functions": { "$ref": "#/$defs/count" },
|
||||
"classes": { "$ref": "#/$defs/count" },
|
||||
"exports": { "$ref": "#/$defs/count" },
|
||||
"callGraph": { "$ref": "#/$defs/count" },
|
||||
"definitions": { "$ref": "#/$defs/count" },
|
||||
"services": { "$ref": "#/$defs/count" },
|
||||
"endpoints": { "$ref": "#/$defs/count" },
|
||||
"steps": { "$ref": "#/$defs/count" },
|
||||
"resources": { "$ref": "#/$defs/count" }
|
||||
}
|
||||
},
|
||||
"structureStage": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"status",
|
||||
"durationMs",
|
||||
"maxWorkerPeakRssBytes",
|
||||
"userCpuTimeMicros",
|
||||
"systemCpuTimeMicros",
|
||||
"warningCount",
|
||||
"warningMessages",
|
||||
"warningMessagesTruncated",
|
||||
"outputBytes"
|
||||
],
|
||||
"properties": {
|
||||
"status": { "$ref": "#/$defs/status" },
|
||||
"durationMs": { "type": "number", "minimum": 0 },
|
||||
"maxWorkerPeakRssBytes": { "$ref": "#/$defs/nullableNumber" },
|
||||
"userCpuTimeMicros": { "$ref": "#/$defs/nullableNumber" },
|
||||
"systemCpuTimeMicros": { "$ref": "#/$defs/nullableNumber" },
|
||||
"warningCount": { "$ref": "#/$defs/count" },
|
||||
"warningMessages": { "$ref": "#/$defs/warningMessages" },
|
||||
"warningMessagesTruncated": { "type": "boolean" },
|
||||
"outputBytes": { "$ref": "#/$defs/count" },
|
||||
"batchesSucceeded": { "$ref": "#/$defs/count" },
|
||||
"batchesFailed": { "$ref": "#/$defs/count" },
|
||||
"filesAnalyzed": { "$ref": "#/$defs/count" },
|
||||
"filesSkipped": { "$ref": "#/$defs/count" },
|
||||
"entities": { "$ref": "#/$defs/entities" },
|
||||
"batchDurationMs": { "$ref": "#/$defs/distribution" }
|
||||
},
|
||||
"allOf": [
|
||||
{
|
||||
"if": {
|
||||
"properties": { "status": { "const": "ok" } },
|
||||
"required": ["status"]
|
||||
},
|
||||
"then": {
|
||||
"required": [
|
||||
"batchesSucceeded",
|
||||
"batchesFailed",
|
||||
"filesAnalyzed",
|
||||
"filesSkipped",
|
||||
"entities",
|
||||
"batchDurationMs"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"stages": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"scan": { "$ref": "#/$defs/scanStage" },
|
||||
"imports": { "$ref": "#/$defs/importsStage" },
|
||||
"batching": { "$ref": "#/$defs/batchingStage" },
|
||||
"structure": { "$ref": "#/$defs/structureStage" }
|
||||
}
|
||||
},
|
||||
"integrity": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"allScannedFilesBatched",
|
||||
"missingBatchFiles",
|
||||
"duplicateBatchFiles",
|
||||
"unexpectedBatchFiles",
|
||||
"missingImportTargets",
|
||||
"structureCoverage",
|
||||
"filesSkipped",
|
||||
"failedBatches",
|
||||
"missingStructurePaths",
|
||||
"duplicateStructurePaths",
|
||||
"unexpectedStructurePaths",
|
||||
"malformedStructureBatches"
|
||||
],
|
||||
"properties": {
|
||||
"allScannedFilesBatched": { "type": "boolean" },
|
||||
"missingBatchFiles": { "$ref": "#/$defs/count" },
|
||||
"duplicateBatchFiles": { "$ref": "#/$defs/count" },
|
||||
"unexpectedBatchFiles": { "$ref": "#/$defs/count" },
|
||||
"missingImportTargets": { "$ref": "#/$defs/count" },
|
||||
"structureCoverage": { "type": "number", "minimum": 0, "maximum": 1 },
|
||||
"filesSkipped": { "$ref": "#/$defs/count" },
|
||||
"failedBatches": { "$ref": "#/$defs/count" },
|
||||
"missingStructurePaths": { "$ref": "#/$defs/count" },
|
||||
"duplicateStructurePaths": { "$ref": "#/$defs/count" },
|
||||
"unexpectedStructurePaths": { "$ref": "#/$defs/count" },
|
||||
"malformedStructureBatches": { "$ref": "#/$defs/count" }
|
||||
}
|
||||
},
|
||||
"successIntegrity": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"allScannedFilesBatched": { "const": true },
|
||||
"missingBatchFiles": { "const": 0 },
|
||||
"duplicateBatchFiles": { "const": 0 },
|
||||
"unexpectedBatchFiles": { "const": 0 },
|
||||
"missingImportTargets": { "const": 0 },
|
||||
"structureCoverage": { "const": 1 },
|
||||
"failedBatches": { "const": 0 },
|
||||
"missingStructurePaths": { "const": 0 },
|
||||
"duplicateStructurePaths": { "const": 0 },
|
||||
"unexpectedStructurePaths": { "const": 0 },
|
||||
"malformedStructureBatches": { "const": 0 }
|
||||
}
|
||||
},
|
||||
"determinism": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["algorithm", "inputDigest", "outputDigest"],
|
||||
"properties": {
|
||||
"algorithm": { "const": "sha256" },
|
||||
"inputDigest": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
|
||||
"outputDigest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }
|
||||
}
|
||||
},
|
||||
"llm": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["invoked", "inputTokens", "outputTokens", "costUsd"],
|
||||
"properties": {
|
||||
"invoked": { "const": false },
|
||||
"inputTokens": { "type": "null" },
|
||||
"outputTokens": { "type": "null" },
|
||||
"costUsd": { "type": "null" }
|
||||
}
|
||||
},
|
||||
"warning": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["stage", "count", "messages", "truncated"],
|
||||
"properties": {
|
||||
"stage": { "enum": ["scan", "imports", "batching", "structure"] },
|
||||
"count": { "$ref": "#/$defs/count" },
|
||||
"messages": { "$ref": "#/$defs/warningMessages" },
|
||||
"truncated": { "type": "boolean" }
|
||||
}
|
||||
},
|
||||
"warningMessages": {
|
||||
"type": "array",
|
||||
"maxItems": 5,
|
||||
"items": { "type": "string" }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -30,11 +30,13 @@
|
||||
"prepare": "pnpm --filter @understand-anything/core build",
|
||||
"build": "pnpm -r build",
|
||||
"test": "vitest run",
|
||||
"benchmark:large-repo": "node scripts/benchmark-large-repo.mjs",
|
||||
"dev:dashboard": "pnpm --filter @understand-anything/dashboard dev",
|
||||
"lint": "eslint ."
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.0.0",
|
||||
"ajv": "8.17.1",
|
||||
"eslint": "^9.0.0",
|
||||
"globals": "^17.6.0",
|
||||
"typescript": "^5.7.0",
|
||||
|
||||
Generated
+29
@@ -11,6 +11,9 @@ importers:
|
||||
'@eslint/js':
|
||||
specifier: ^9.0.0
|
||||
version: 9.39.4
|
||||
ajv:
|
||||
specifier: 8.17.1
|
||||
version: 8.17.1
|
||||
eslint:
|
||||
specifier: ^9.0.0
|
||||
version: 9.39.4(jiti@2.6.1)
|
||||
@@ -1363,6 +1366,9 @@ packages:
|
||||
ajv@6.15.0:
|
||||
resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==}
|
||||
|
||||
ajv@8.17.1:
|
||||
resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==}
|
||||
|
||||
ansi-regex@5.0.1:
|
||||
resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
|
||||
engines: {node: '>=8'}
|
||||
@@ -1794,6 +1800,9 @@ packages:
|
||||
fast-levenshtein@2.0.6:
|
||||
resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==}
|
||||
|
||||
fast-uri@3.1.3:
|
||||
resolution: {integrity: sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==}
|
||||
|
||||
fdir@6.5.0:
|
||||
resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==}
|
||||
engines: {node: '>=12.0.0'}
|
||||
@@ -2075,6 +2084,9 @@ packages:
|
||||
json-schema-traverse@0.4.1:
|
||||
resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
|
||||
|
||||
json-schema-traverse@1.0.0:
|
||||
resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==}
|
||||
|
||||
json-stable-stringify-without-jsonify@1.0.1:
|
||||
resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==}
|
||||
|
||||
@@ -2667,6 +2679,10 @@ packages:
|
||||
remark-stringify@11.0.0:
|
||||
resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==}
|
||||
|
||||
require-from-string@2.0.2:
|
||||
resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
resolve-from@4.0.0:
|
||||
resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==}
|
||||
engines: {node: '>=4'}
|
||||
@@ -4405,6 +4421,13 @@ snapshots:
|
||||
json-schema-traverse: 0.4.1
|
||||
uri-js: 4.4.1
|
||||
|
||||
ajv@8.17.1:
|
||||
dependencies:
|
||||
fast-deep-equal: 3.1.3
|
||||
fast-uri: 3.1.3
|
||||
json-schema-traverse: 1.0.0
|
||||
require-from-string: 2.0.2
|
||||
|
||||
ansi-regex@5.0.1: {}
|
||||
|
||||
ansi-regex@6.2.2: {}
|
||||
@@ -4928,6 +4951,8 @@ snapshots:
|
||||
|
||||
fast-levenshtein@2.0.6: {}
|
||||
|
||||
fast-uri@3.1.3: {}
|
||||
|
||||
fdir@6.5.0(picomatch@4.0.3):
|
||||
optionalDependencies:
|
||||
picomatch: 4.0.3
|
||||
@@ -5256,6 +5281,8 @@ snapshots:
|
||||
|
||||
json-schema-traverse@0.4.1: {}
|
||||
|
||||
json-schema-traverse@1.0.0: {}
|
||||
|
||||
json-stable-stringify-without-jsonify@1.0.1: {}
|
||||
|
||||
json5@2.2.3: {}
|
||||
@@ -6047,6 +6074,8 @@ snapshots:
|
||||
mdast-util-to-markdown: 2.1.2
|
||||
unified: 11.0.5
|
||||
|
||||
require-from-string@2.0.2: {}
|
||||
|
||||
resolve-from@4.0.0: {}
|
||||
|
||||
resolve-pkg-maps@1.0.0: {}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import {
|
||||
BenchmarkArtifactCleanupError,
|
||||
BenchmarkReportWriteError,
|
||||
CliUsageError,
|
||||
helpText,
|
||||
parseArgs,
|
||||
runBenchmark,
|
||||
} from './lib/large-repo-benchmark.mjs';
|
||||
|
||||
async function main() {
|
||||
let options;
|
||||
try {
|
||||
options = parseArgs(process.argv.slice(2));
|
||||
} catch (error) {
|
||||
if (error instanceof CliUsageError) {
|
||||
process.stderr.write(`Error: ${error.message}\n\n${helpText()}`);
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (options.help) {
|
||||
process.stdout.write(helpText());
|
||||
return;
|
||||
}
|
||||
|
||||
let result;
|
||||
try {
|
||||
result = await runBenchmark(options, {
|
||||
onProgress(stage) {
|
||||
process.stderr.write(`[benchmark] ${stage}\n`);
|
||||
},
|
||||
onBatchProgress(completed, total) {
|
||||
process.stderr.write(`[benchmark] structure ${completed}/${total}\n`);
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof BenchmarkArtifactCleanupError) {
|
||||
process.stderr.write(`Error: ${error.message}\n`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (error instanceof BenchmarkReportWriteError) {
|
||||
process.stderr.write(`Error: ${error.message}\n`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
process.stdout.write(
|
||||
`Benchmark ${result.report.status}. JSON report: ${options.outputPath}\n` +
|
||||
`Markdown report: ${options.markdownPath}\n`,
|
||||
);
|
||||
if (result.artifactRoot) {
|
||||
process.stdout.write(`Artifacts preserved at: ${result.artifactRoot}\n`);
|
||||
}
|
||||
process.exitCode = result.exitCode;
|
||||
}
|
||||
|
||||
await main();
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import { resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
export const STAGE_METRICS_PREFIX = '__UA_BENCHMARK_METRICS__';
|
||||
|
||||
const [scriptPath, ...scriptArgs] = process.argv.slice(2);
|
||||
if (!scriptPath) {
|
||||
process.stderr.write('benchmark-stage-worker: missing script path\n');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const resolvedScript = resolve(scriptPath);
|
||||
const startedAt = performance.now();
|
||||
const usageBefore = process.resourceUsage();
|
||||
let failure = null;
|
||||
|
||||
try {
|
||||
// The deterministic helpers use process.argv to identify their CLI entry
|
||||
// point. Recreate the argv shape they receive when launched directly, then
|
||||
// import them in this process so resourceUsage() measures the real stage.
|
||||
process.argv = [process.execPath, resolvedScript, ...scriptArgs];
|
||||
await import(pathToFileURL(resolvedScript).href);
|
||||
} catch (error) {
|
||||
failure = error instanceof Error ? error : new Error(String(error));
|
||||
process.stderr.write(`${failure.stack ?? failure.message}\n`);
|
||||
}
|
||||
|
||||
const usageAfter = process.resourceUsage();
|
||||
const metrics = {
|
||||
durationMs: Math.round((performance.now() - startedAt) * 100) / 100,
|
||||
// Node reports maxRSS in KiB on every supported platform.
|
||||
peakRssBytes: usageAfter.maxRSS * 1024,
|
||||
userCpuTimeMicros: usageAfter.userCPUTime - usageBefore.userCPUTime,
|
||||
systemCpuTimeMicros: usageAfter.systemCPUTime - usageBefore.systemCPUTime,
|
||||
};
|
||||
|
||||
process.stderr.write(`${STAGE_METRICS_PREFIX}${JSON.stringify(metrics)}\n`);
|
||||
if (failure) process.exitCode = 1;
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,263 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import Ajv2020 from 'ajv/dist/2020.js';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const schemaPath = resolve(
|
||||
__dirname,
|
||||
'../../docs/benchmarks/large-repo-report-1.0.0.schema.json',
|
||||
);
|
||||
const schema = JSON.parse(readFileSync(schemaPath, 'utf-8'));
|
||||
const validateReport = new Ajv2020({
|
||||
allErrors: true,
|
||||
formats: { 'date-time': true },
|
||||
}).compile(schema);
|
||||
|
||||
function expectValid(report) {
|
||||
expect(
|
||||
validateReport(report),
|
||||
JSON.stringify(validateReport.errors, null, 2),
|
||||
).toBe(true);
|
||||
}
|
||||
|
||||
function expectInvalid(report) {
|
||||
expect(validateReport(report)).toBe(false);
|
||||
}
|
||||
|
||||
function regularStage(overrides = {}) {
|
||||
return {
|
||||
status: 'ok',
|
||||
durationMs: 1,
|
||||
peakRssBytes: 1024,
|
||||
userCpuTimeMicros: 10,
|
||||
systemCpuTimeMicros: 5,
|
||||
warningCount: 0,
|
||||
warningMessages: [],
|
||||
warningMessagesTruncated: false,
|
||||
stdoutTruncated: false,
|
||||
stderrTruncated: false,
|
||||
outputBytes: 128,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function validReport() {
|
||||
return {
|
||||
schemaUrl: schema.$id,
|
||||
schemaVersion: '1.0.0',
|
||||
status: 'ok',
|
||||
mode: 'deterministic',
|
||||
run: {
|
||||
startedAt: '2026-01-01T00:00:00.000Z',
|
||||
finishedAt: '2026-01-01T00:00:01.000Z',
|
||||
durationMs: 1000,
|
||||
},
|
||||
tool: {
|
||||
commit: null,
|
||||
dirty: null,
|
||||
packageVersion: '0.1.0',
|
||||
},
|
||||
subject: {
|
||||
label: 'fixture',
|
||||
commit: null,
|
||||
dirty: null,
|
||||
},
|
||||
environment: {
|
||||
platform: 'linux',
|
||||
release: 'test',
|
||||
arch: 'x64',
|
||||
nodeVersion: 'v22.0.0',
|
||||
cpuModel: 'test',
|
||||
logicalCores: 1,
|
||||
totalMemoryBytes: 1024,
|
||||
},
|
||||
configuration: {
|
||||
concurrency: 1,
|
||||
stages: ['scan', 'imports', 'batching', 'structure'],
|
||||
},
|
||||
scale: {
|
||||
files: 1,
|
||||
lines: 1,
|
||||
bytes: 1,
|
||||
missingFiles: 0,
|
||||
filteredByUserIgnore: 0,
|
||||
byCategory: { code: 1 },
|
||||
byLanguage: { JavaScript: 1 },
|
||||
},
|
||||
stages: {
|
||||
scan: regularStage({
|
||||
files: 1,
|
||||
lines: 1,
|
||||
bytes: 1,
|
||||
filteredByUserIgnore: 0,
|
||||
}),
|
||||
imports: regularStage({
|
||||
filesScanned: 1,
|
||||
filesWithImports: 0,
|
||||
edges: 0,
|
||||
}),
|
||||
batching: regularStage({
|
||||
algorithm: 'semantic-v1',
|
||||
totalBatches: 1,
|
||||
batchSizes: { min: 1, p50: 1, p95: 1, max: 1, mean: 1 },
|
||||
estimatedAgentInputBytes: 1,
|
||||
}),
|
||||
structure: {
|
||||
status: 'ok',
|
||||
durationMs: 1,
|
||||
maxWorkerPeakRssBytes: 1024,
|
||||
userCpuTimeMicros: 10,
|
||||
systemCpuTimeMicros: 5,
|
||||
warningCount: 0,
|
||||
warningMessages: [],
|
||||
warningMessagesTruncated: false,
|
||||
outputBytes: 128,
|
||||
batchesSucceeded: 1,
|
||||
batchesFailed: 0,
|
||||
filesAnalyzed: 1,
|
||||
filesSkipped: 0,
|
||||
entities: {
|
||||
functions: 0,
|
||||
classes: 0,
|
||||
exports: 0,
|
||||
callGraph: 0,
|
||||
definitions: 0,
|
||||
services: 0,
|
||||
endpoints: 0,
|
||||
steps: 0,
|
||||
resources: 0,
|
||||
},
|
||||
batchDurationMs: { min: 1, p50: 1, p95: 1, max: 1, mean: 1 },
|
||||
},
|
||||
},
|
||||
integrity: {
|
||||
allScannedFilesBatched: true,
|
||||
missingBatchFiles: 0,
|
||||
duplicateBatchFiles: 0,
|
||||
unexpectedBatchFiles: 0,
|
||||
missingImportTargets: 0,
|
||||
structureCoverage: 1,
|
||||
filesSkipped: 0,
|
||||
failedBatches: 0,
|
||||
missingStructurePaths: 0,
|
||||
duplicateStructurePaths: 0,
|
||||
unexpectedStructurePaths: 0,
|
||||
malformedStructureBatches: 0,
|
||||
},
|
||||
determinism: {
|
||||
algorithm: 'sha256',
|
||||
inputDigest: '0'.repeat(64),
|
||||
outputDigest: '1'.repeat(64),
|
||||
},
|
||||
llm: {
|
||||
invoked: false,
|
||||
inputTokens: null,
|
||||
outputTokens: null,
|
||||
costUsd: null,
|
||||
},
|
||||
warnings: [],
|
||||
error: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe('large repository report schema 1.0.0', () => {
|
||||
it('accepts a complete ok report and a derived valid degraded report', () => {
|
||||
const ok = validReport();
|
||||
const degraded = structuredClone(ok);
|
||||
degraded.status = 'degraded';
|
||||
degraded.stages.structure.filesAnalyzed = 0;
|
||||
degraded.stages.structure.filesSkipped = 1;
|
||||
degraded.integrity.filesSkipped = 1;
|
||||
|
||||
expectValid(ok);
|
||||
expectValid(degraded);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
'an incomplete ok report',
|
||||
(report) => {
|
||||
delete report.stages.structure;
|
||||
},
|
||||
],
|
||||
[
|
||||
'the old warning shape',
|
||||
(report) => {
|
||||
report.status = 'degraded';
|
||||
report.warnings = [{ stage: 'scan', count: 1 }];
|
||||
},
|
||||
],
|
||||
[
|
||||
'the old structure memory field',
|
||||
(report) => {
|
||||
report.stages.structure.peakRssBytes =
|
||||
report.stages.structure.maxWorkerPeakRssBytes;
|
||||
delete report.stages.structure.maxWorkerPeakRssBytes;
|
||||
},
|
||||
],
|
||||
[
|
||||
'a failed report with a null error',
|
||||
(report) => {
|
||||
report.status = 'failed';
|
||||
},
|
||||
],
|
||||
])('rejects %s', (_name, mutate) => {
|
||||
const report = validReport();
|
||||
mutate(report);
|
||||
expectInvalid(report);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
'warnings',
|
||||
(report) => {
|
||||
report.warnings = [
|
||||
{ stage: 'scan', count: 1, messages: ['warning'], truncated: false },
|
||||
];
|
||||
},
|
||||
],
|
||||
[
|
||||
'skipped files',
|
||||
(report) => {
|
||||
report.integrity.filesSkipped = 1;
|
||||
},
|
||||
],
|
||||
[
|
||||
'a stage warning',
|
||||
(report) => {
|
||||
report.stages.scan.warningCount = 1;
|
||||
report.stages.scan.warningMessages = ['warning'];
|
||||
},
|
||||
],
|
||||
[
|
||||
'a stage-level skipped file',
|
||||
(report) => {
|
||||
report.stages.structure.filesSkipped = 1;
|
||||
},
|
||||
],
|
||||
])('rejects ok status with %s', (_name, mutate) => {
|
||||
const report = validReport();
|
||||
mutate(report);
|
||||
expectInvalid(report);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['allScannedFilesBatched', false],
|
||||
['missingBatchFiles', 1],
|
||||
['duplicateBatchFiles', 1],
|
||||
['unexpectedBatchFiles', 1],
|
||||
['missingImportTargets', 1],
|
||||
['structureCoverage', 0.5],
|
||||
['failedBatches', 1],
|
||||
['missingStructurePaths', 1],
|
||||
['duplicateStructurePaths', 1],
|
||||
['unexpectedStructurePaths', 1],
|
||||
['malformedStructureBatches', 1],
|
||||
])('rejects successful status with failing integrity %s', (field, value) => {
|
||||
const report = validReport();
|
||||
report.integrity[field] = value;
|
||||
expectInvalid(report);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user