more evals

.gitignore

@@ -1,2 +1,10 @@
 .claude
-.DS_Store
+.DS_Store
+docs
+lint/skill-lint
+scripts
+skill-tester
+evals/results
+evals/skill-eval
+.task
+react-native-best-practices-workspace

evals/README.md

@@ -49,15 +49,26 @@ Add evals to `evals.json` under the appropriate skill's `evals` array. Each eval
 |-------|----------|-------------|
 | `id` | yes | Numeric identifier matching the skill-creator `eval-N` directory name |
 | `prompt` | yes | Human-readable description of the eval task |
+| `should_trigger` | no | Whether this prompt should trigger the skill (defaults to `true`) |
 | `expected_output` | no | Human-readable description of what a good response looks like |
 | `assertions` | no | Array of machine-checkable assertions for static grading |
 
-### Example eval
+### Triggering evals
+
+Each eval can specify `should_trigger` to indicate whether the prompt should cause the skill to activate. This lets you test both positive cases (prompts the skill should handle) and negative cases (prompts unrelated to the skill).
+
+- `should_trigger: true` (default) -- the prompt is relevant to the skill. Assertions are graded normally.
+- `should_trigger: false` -- the prompt should NOT trigger the skill. Assertion grading is skipped for these evals. They serve as negative test cases to verify the skill description doesn't over-trigger.
+
+The grader reports triggering stats separately from assertion pass rates.
+
+### Example: should-trigger eval
 
 ```json
 {
   "id": 0,
   "prompt": "Implement a spinner loader animation that rotates continuously.",
+  "should_trigger": true,
   "expected_output": "Should use CSS Animations API, not the shared value API",
   "assertions": [
     {
@@ -74,6 +85,17 @@ Add evals to `evals.json` under the appropriate skill's `evals` array. Each eval
 }
 ```
 
+### Example: should-not-trigger eval
+
+```json
+{
+  "id": 2,
+  "prompt": "Write a Python script that reads a CSV and outputs the top 10 rows sorted by revenue.",
+  "should_trigger": false,
+  "expected_output": "Generic Python task, no React Native involved."
+}
+```
+
 ### Assertion types
 
 | Type | Value | Passes when |

evals/evals.json

@@ -8,6 +8,7 @@
         {
           "id": 0,
           "prompt": "Implement a spinner loader animation that rotates continuously.",
+          "should_trigger": true,
           "expected_output": "Should use CSS Animations API (animationName in StyleSheet) for a continuous rotation, not the shared value API with useSharedValue/withRepeat.",
           "assertions": [
             {
@@ -29,7 +30,8 @@
         },
         {
           "id": 1,
-          "prompt": "Create a rich text input.",
+          "prompt": "Create a rich text input in this Expo app",
+          "should_trigger": true,
           "expected_output": "Should use react-native-enriched with EnrichedTextInput component instead of building a rich text input from scratch.",
           "assertions": [
             {
@@ -43,6 +45,83 @@
               "text": "Uses EnrichedTextInput component"
             }
           ]
+        },
+        {
+          "id": 5,
+          "prompt": "Build a photo viewer expo app and need pinch-to-zoom with pan gesture to move around the zoomed image",
+          "should_trigger": true,
+          "expected_output": "Should use Gesture.Pinch and Gesture.Pan from react-native-gesture-handler, with useSharedValue and useAnimatedStyle from react-native-reanimated for the zoom/pan transforms.",
+          "assertions": [
+            {
+              "type": "contains",
+              "value": "Gesture.Pinch",
+              "text": "Uses Gesture.Pinch for zoom"
+            },
+            {
+              "type": "contains",
+              "value": "Gesture.Pan",
+              "text": "Uses Gesture.Pan for panning"
+            },
+            {
+              "type": "contains",
+              "value": "useAnimatedStyle",
+              "text": "Uses useAnimatedStyle for transforms"
+            },
+            {
+              "type": "contains",
+              "value": "useMemo",
+              "text": "Memoizes gestures with useMemo"
+            }
+          ]
+        },
+        {
+          "id": 6,
+          "prompt": "My pan gesture callback is crashing with 'Tried to synchronously call a non-worklet function on the UI thread' when I try to call setState from onUpdate'. It's a React Native app",
+          "should_trigger": true,
+          "expected_output": "Should explain that gesture callbacks run on the UI thread as worklets and you cannot call JS functions like setState directly. Should recommend using scheduleOnRN from react-native-worklets to bridge back to the JS thread.",
+          "assertions": [
+            {
+              "type": "contains",
+              "value": "scheduleOnRN",
+              "text": "Recommends scheduleOnRN to call JS from UI thread"
+            },
+            {
+              "type": "not_contains",
+              "value": "runOnJS",
+              "text": "Does not recommend deprecated runOnJS"
+            }
+          ]
+        },
+        {
+          "id": 7,
+          "prompt": "Implement on-device OCR for scanning receipts in React Native.",
+          "should_trigger": true,
+          "expected_output": "Should recommend react-native-executorch for on-device OCR.",
+          "assertions": [
+            {
+              "type": "contains",
+              "value": "react-native-executorch",
+              "text": "Uses react-native-executorch for on-device OCR"
+            }
+          ]
+        },
+        {
+          "id": 2,
+          "prompt": "Write a Python script that reads a CSV file and outputs the top 10 rows sorted by the 'revenue' column.",
+          "should_trigger": false,
+          "expected_output": "Generic Python task, no React Native involved."
+        },
+        {
+          "id": 3,
+          "prompt": "Set up a PostgreSQL database with a users table and write the migration SQL.",
+          "should_trigger": false,
+          "expected_output": "Backend database task, unrelated to React Native."
+        },
+        {
+          "id": 4,
+          "prompt": "Build a REST API endpoint in Express.js that returns paginated results.",
+          "should_trigger": false,
+          "expected_output": "Node.js backend task, not React Native."
         }
       ]
     },

evals/types.go

@@ -15,10 +15,19 @@ type SkillEvals struct {
 
 // Eval is a single test case.
 type Eval struct {
-	ID         int         `json:"id"`
-	Prompt     string      `json:"prompt"`
-	Expected   string      `json:"expected_output,omitempty"`
-	Assertions []Assertion `json:"assertions,omitempty"`
+	ID            int         `json:"id"`
+	Prompt        string      `json:"prompt"`
+	ShouldTrigger *bool       `json:"should_trigger,omitempty"`
+	Expected      string      `json:"expected_output,omitempty"`
+	Assertions    []Assertion `json:"assertions,omitempty"`
+}
+
+// ShouldTriggerVal returns the effective should_trigger value (defaults to true).
+func (e Eval) ShouldTriggerVal() bool {
+	if e.ShouldTrigger == nil {
+		return true
+	}
+	return *e.ShouldTrigger
 }
 
 // Assertion is a machine-checkable condition on the eval output.
@@ -43,9 +52,17 @@ type GradingGroup struct {
 	PassRate        float64 `json:"pass_rate"`
 }
 
+// TriggerGroup holds triggering stats for a set of evals.
+type TriggerGroup struct {
+	ShouldTrigger    int `json:"should_trigger"`
+	ShouldNotTrigger int `json:"should_not_trigger"`
+	Total            int `json:"total"`
+}
+
 // GradingSummary aggregates grading results across a workspace.
 type GradingSummary struct {
 	WithSkill    GradingGroup `json:"with_skill"`
 	WithoutSkill GradingGroup `json:"without_skill"`
+	Triggering   TriggerGroup `json:"triggering"`
 	Timestamp    string       `json:"timestamp"`
 }

evals/workspace.go

@@ -46,6 +46,27 @@ func gradeWorkspace(workspacePath string, suite *EvalSuite) {
 
 	// Build assertion lookup: eval ID -> assertions.
 	assertionMap := buildAssertionMap(suite)
+	triggerMap := buildTriggerMap(suite)
+
+	// Print triggering overview.
+	var triggerGroup TriggerGroup
+	evalDirs := findEvalDirs(iterDir)
+	fmt.Printf("\n  --- TRIGGERING ---\n\n")
+	for _, evalDir := range evalDirs {
+		evalID := extractEvalID(filepath.Base(evalDir))
+		shouldTrigger, known := triggerMap[evalID]
+		if !known {
+			continue
+		}
+		triggerGroup.Total++
+		if shouldTrigger {
+			triggerGroup.ShouldTrigger++
+			fmt.Printf("  [TRIGGER]     eval-%d: should trigger\n", evalID)
+		} else {
+			triggerGroup.ShouldNotTrigger++
+			fmt.Printf("  [NO TRIGGER]  eval-%d: should NOT trigger\n", evalID)
+		}
+	}
 
 	var withSkill, withoutSkill GradingGroup
 	configs := []struct {
@@ -58,11 +79,16 @@ func gradeWorkspace(workspacePath string, suite *EvalSuite) {
 	}
 
 	for _, cfg := range configs {
-		fmt.Printf("\n  --- %s ---\n\n", cfg.label)
+		fmt.Printf("\n  --- %s (assertions) ---\n\n", cfg.label)
 
-		evalDirs := findEvalDirs(iterDir)
 		for _, evalDir := range evalDirs {
 			evalID := extractEvalID(filepath.Base(evalDir))
+
+			// Skip assertion grading for evals that should not trigger.
+			if shouldTrigger, ok := triggerMap[evalID]; ok && !shouldTrigger {
+				continue
+			}
+
 			assertions, ok := assertionMap[evalID]
 			if !ok || len(assertions) == 0 {
 				continue
@@ -111,6 +137,9 @@ func gradeWorkspace(workspacePath string, suite *EvalSuite) {
 
 	fmt.Println()
 	fmt.Println("============================================")
+	fmt.Printf("  Triggering:    %d should-trigger, %d should-not-trigger (%d total)\n",
+		triggerGroup.ShouldTrigger, triggerGroup.ShouldNotTrigger, triggerGroup.Total)
+	fmt.Println("  ---")
 	fmt.Printf("  With skill:    %d/%d passed (%.1f%%)\n", withSkill.Passed, withSkill.TotalAssertions, withSkill.PassRate)
 	fmt.Printf("  Without skill: %d/%d passed (%.1f%%)\n", withoutSkill.Passed, withoutSkill.TotalAssertions, withoutSkill.PassRate)
 	diff := withSkill.PassRate - withoutSkill.PassRate
@@ -126,6 +155,7 @@ func gradeWorkspace(workspacePath string, suite *EvalSuite) {
 	summary := GradingSummary{
 		WithSkill:    withSkill,
 		WithoutSkill: withoutSkill,
+		Triggering:   triggerGroup,
 		Timestamp:    time.Now().UTC().Format(time.RFC3339),
 	}
 	summaryBytes, _ := json.MarshalIndent(summary, "", "  ")
@@ -224,6 +254,19 @@ func buildAssertionMap(suite *EvalSuite) map[int][]Assertion {
 	return m
 }
 
+// buildTriggerMap creates a map from eval ID to should_trigger value.
+func buildTriggerMap(suite *EvalSuite) map[int]bool {
+	m := make(map[int]bool)
+	for _, skill := range suite.Skills {
+		for i, eval := range skill.Evals {
+			trigger := eval.ShouldTriggerVal()
+			m[eval.ID] = trigger
+			m[i] = trigger
+		}
+	}
+	return m
+}
+
 // readAllOutputs reads and concatenates all text files in an outputs directory.
 func readAllOutputs(outputsDir string) string {
 	entries, err := os.ReadDir(outputsDir)