Merge pull request #4678 from kunallanjewar/fix/issue-4642-scorecard-unverified-dimension

fix(cli): omit unscored dimensions from scorecard caveats

internal/cli/shipcheck_test.go

@@ -832,6 +832,47 @@ func TestShipcheck_HoldsOnUnverifiedScorecard(t *testing.T) {
 	}
 }
 
+func TestShipcheck_AllowsUnscoredLiveAPIVerification(t *testing.T) {
+	h := newShipcheckHarness(t)
+	cliDir := filepath.Join(h.dir, "internal", "cli")
+	if err := os.MkdirAll(cliDir, 0o755); err != nil {
+		t.Fatalf("creating CLI fixture: %v", err)
+	}
+	const cliSource = `package cli
+
+func widgetsPath() string { return "/widgets" }
+`
+	if err := os.WriteFile(filepath.Join(cliDir, "widgets.go"), []byte(cliSource), 0o644); err != nil {
+		t.Fatalf("writing CLI fixture: %v", err)
+	}
+	specPath := filepath.Join(h.dir, "spec.yaml")
+	const spec = `name: widgets
+version: "1.0.0"
+base_url: https://api.example.com
+resources:
+  widgets:
+    endpoints:
+      list:
+        method: GET
+        path: /widgets
+`
+	if err := os.WriteFile(specPath, []byte(spec), 0o644); err != nil {
+		t.Fatalf("writing spec fixture: %v", err)
+	}
+
+	sc, err := pipeline.RunScorecard(h.dir, t.TempDir(), specPath, nil)
+	if err != nil {
+		t.Fatalf("running scorecard: %v", err)
+	}
+	if _, err := pipeline.PersistScorecardToManifest(filepath.Join(h.dir, pipeline.CLIManifestFilename), sc, ""); err != nil {
+		t.Fatalf("persisting scorecard: %v", err)
+	}
+
+	if err := runShipcheckCmd(t, "--dir", h.dir); err != nil {
+		t.Fatalf("unscored live API verification should not hold shipping: %v", err)
+	}
+}
+
 func TestShipcheck_HoldsWithoutScorecardManifestEvidence(t *testing.T) {
 	h := newShipcheckHarness(t)
 	if err := os.Remove(filepath.Join(h.dir, pipeline.CLIManifestFilename)); err != nil {

internal/pipeline/scorecard.go

@@ -327,8 +327,6 @@ func scoreDomainDimensions(sc *Scorecard, outputDir string, spec *openAPISpecInf
 	// shipped CLI has never been exercised against the real API.
 	if liveScore, scored := scoreLiveAPIVerification(verifyReport); scored {
 		sc.Steinberger.LiveAPIVerification = liveScore
-	} else if !isDevice && !isLocalDatastoreCLIDir(outputDir) {
-		markUnverifiedDimension(sc, DimLiveAPIVerification)
 	} else {
 		sc.UnscoredDimensions = append(sc.UnscoredDimensions, DimLiveAPIVerification)
 	}

internal/pipeline/scorecard_live_api_test.go

@@ -181,7 +181,7 @@ func TestRunScorecard_LiveAPIVerificationWiring(t *testing.T) {
 	})
 }
 
-func TestRunScorecardCarriesUnverifiedDimensionsIntoGrade(t *testing.T) {
+func TestRunScorecardCarriesApplicableUnverifiedDimensionsIntoGrade(t *testing.T) {
 	dir := t.TempDir()
 	pipelineDir := t.TempDir()
 
@@ -190,15 +190,43 @@ func TestRunScorecardCarriesUnverifiedDimensionsIntoGrade(t *testing.T) {
 
 	assert.Contains(t, sc.UnverifiedDimensions, DimPathValidity)
 	assert.Contains(t, sc.UnverifiedDimensions, DimAuthProtocol)
-	assert.Contains(t, sc.UnverifiedDimensions, DimLiveAPIVerification)
+	assert.NotContains(t, sc.UnverifiedDimensions, DimLiveAPIVerification)
 	assert.Contains(t, sc.OverallGrade, "unverified")
-	assert.Contains(t, sc.OverallGrade, DimLiveAPIVerification)
+	assert.NotContains(t, sc.OverallGrade, DimLiveAPIVerification)
 
 	data, err := json.Marshal(sc)
 	require.NoError(t, err)
 	assert.Contains(t, string(data), `"unverified_dimensions"`)
 }
 
+func TestRunScorecardOmitsUnscoredLiveVerificationFromGradeCaveat(t *testing.T) {
+	dir := t.TempDir()
+	writeScorecardFixture(t, dir, "internal/cli/widgets.go", `
+package cli
+
+func widgetsPath() string { return "/widgets" }
+`)
+	specPath := filepath.Join(dir, "spec.yaml")
+	writeScorecardFixture(t, dir, "spec.yaml", `
+name: widgets
+version: "1.0.0"
+base_url: https://api.example.com
+resources:
+  widgets:
+    endpoints:
+      list:
+        method: GET
+        path: /widgets
+`)
+
+	sc, err := RunScorecard(dir, t.TempDir(), specPath, nil)
+	require.NoError(t, err)
+
+	assert.Contains(t, sc.UnscoredDimensions, DimLiveAPIVerification)
+	assert.Empty(t, sc.UnverifiedDimensions)
+	assert.NotContains(t, sc.OverallGrade, "unverified")
+}
+
 func TestScorecardGradeLeavesFullyVerifiedGradeUnqualified(t *testing.T) {
 	sc := &Scorecard{
 		UnscoredDimensions: []string{DimMCPDescriptionQuality},

testdata/golden/expected/generate-golden-api/scorecard.json

@@ -5,7 +5,7 @@
     "gap_report": [
       "MCP: 11 tools (1 public, 10 auth-required) — readiness: full"
     ],
-    "overall_grade": "A (3 of 25 dimensions unverified: path_validity, auth_protocol, live_api_verification)",
+    "overall_grade": "A (2 of 24 dimensions unverified: path_validity, auth_protocol)",
     "steinberger": {
       "agent_native": 10,
       "agent_workflow_readiness": 9,
@@ -46,8 +46,7 @@
     ],
     "unverified_dimensions": [
       "path_validity",
-      "auth_protocol",
-      "live_api_verification"
+      "auth_protocol"
     ]
   }
 }