feat: add skillgym tests (#453)

This commit is contained in:
Michał Pierzchała
2026-04-26 20:49:59 -04:00
committed by GitHub
parent 40fe5e23cc
commit 7c5b7670c8
33 changed files with 9508 additions and 10 deletions
+2 -8
View File
@@ -21,6 +21,7 @@
"website/docs/404.mdx",
"website/rspress.config.ts"
],
"ignorePatterns": ["examples/test-app/**"],
"ignoreDependencies": ["@theme"],
"ignoreExports": [
{
@@ -69,14 +70,7 @@
]
}
],
"usedClassMembers": [
"name",
"listActiveLeases",
"delete",
"values",
"elapsedMs",
"isExpired"
],
"usedClassMembers": ["name", "listActiveLeases", "delete", "values", "elapsedMs", "isExpired"],
"rules": {
"unused-types": "off",
"duplicate-exports": "off"
+1
View File
@@ -28,3 +28,4 @@ xcuserdata/
*.app
*.xctestrun
*.xcarchive
.skillgym-results/
+1 -1
View File
@@ -4,5 +4,5 @@
"trailingComma": "all",
"printWidth": 100,
"sortPackageJson": false,
"ignorePatterns": ["dist/**", "node_modules/**"]
"ignorePatterns": ["dist/**", "node_modules/**", "**/.skillgym-results/**"]
}
+5
View File
@@ -87,6 +87,11 @@ For people:
- [Website](https://agent-device.dev/)
- [Docs](https://incubator.callstack.com/agent-device/docs/introduction)
- [Skillgym starter](test/skillgym/README.md)
Local benchmark starter:
- `pnpm test:skillgym`
For agents:
+2
View File
@@ -0,0 +1,2 @@
.expo/
node_modules/
+78
View File
@@ -0,0 +1,78 @@
# Agent Device Tester
`Agent Device Tester` is a minimal Expo Router fixture app for `agent-device` and `skillgym` experiments.
It is intentionally small, but each surface is dense with durable accessibility targets so a few screens cover a large share of the workflows we care about.
## Why this app exists
- It gives `agent-device` a stable React Native target that we control.
- It makes `skillgym` prompts concrete: the agent can inspect real app files instead of answering against an imagined UI.
- It keeps the number of screens low while still covering roughly 50 practical interaction and verification cases.
## Screens
- `Home`: visible-text checks, dismissible banner, modal open/close, async loading, status badge, switch state
- `Catalog`: search debounce, filter chips, long-list scroll, favorite toggles, cart updates, drill-in navigation
- `Product detail`: back navigation, quantity stepper, multiline notes, save action
- `Checkout form`: required-field validation, fill vs type, checkbox state, choice groups, keyboard dismiss, success summary
- `Settings`: switch rows, accordion content, loading and error states, retry flow, destructive-confirm modal
Navigation uses Expo Router native bottom tabs, so the tab bar itself is also part of the test surface.
## Coverage map
These are the main case families this app can support without adding more screens:
- app open and close
- visible text verification with plain `snapshot`
- interactive discovery with `snapshot -i`
- `press` on stable buttons, pills, and rows
- `fill` on single-line and multiline fields
- `type` after focus for append flows
- `get text` on headings, badges, summaries, and accordion content
- `is visible` and `is exists` assertions
- `wait` for async loading and success states
- `diff snapshot` after dismissals and submits
- long-list scrolling and `scrollintoview`
- selector-based navigation across repeated cards
- modal open, cancel, and confirm flows
- switch and checkbox state changes
- validation-error and recovery loops
- retryable error banners
- cart counters and quantity changes
- screenshot and recording proof capture
## Run locally
From the repo root:
```bash
pnpm test-app:install
pnpm test-app:ios
```
Or on Android:
```bash
pnpm test-app:install
pnpm test-app:android
```
If you prefer to work from inside the app folder:
```bash
cd examples/test-app
pnpm install --ignore-workspace
pnpm ios
```
Or on Android:
```bash
cd examples/test-app
pnpm install --ignore-workspace
pnpm android
```
Once the app is running, use `agent-device` against `Agent Device Tester` like any other target app.
+20
View File
@@ -0,0 +1,20 @@
{
"expo": {
"name": "Agent Device Tester",
"slug": "agent-device-test-app",
"version": "1.0.0",
"orientation": "portrait",
"userInterfaceStyle": "automatic",
"newArchEnabled": true,
"plugins": ["expo-router"],
"ios": {
"supportsTablet": true,
"bundleIdentifier": "com.callstack.agentdevicelab"
},
"android": {
"package": "com.callstack.agentdevicelab",
"edgeToEdgeEnabled": true,
"predictiveBackGestureEnabled": false
}
}
}
+43
View File
@@ -0,0 +1,43 @@
import { NativeTabs } from 'expo-router/unstable-native-tabs';
import { useLabState } from '../../src/lab-state';
import { useAppColors } from '../../src/theme';
export default function TabsLayout() {
const colors = useAppColors();
const { cartCount, diagnosticsState } = useLabState();
return (
<NativeTabs
backgroundColor={colors.tabBar}
badgeBackgroundColor={colors.accent}
iconColor={{ default: colors.textSoft, selected: colors.text }}
labelStyle={{
default: { color: colors.textSoft, fontSize: 11, fontWeight: '600' },
selected: { color: colors.text, fontSize: 11, fontWeight: '700' },
}}
tintColor={colors.text}
>
<NativeTabs.Trigger name="index">
<NativeTabs.Trigger.Icon md="home" sf="house.fill" />
<NativeTabs.Trigger.Label>Home</NativeTabs.Trigger.Label>
</NativeTabs.Trigger>
<NativeTabs.Trigger name="catalog">
<NativeTabs.Trigger.Icon md="storefront" sf="square.grid.2x2.fill" />
<NativeTabs.Trigger.Label>Catalog</NativeTabs.Trigger.Label>
<NativeTabs.Trigger.Badge hidden={cartCount === 0}>
{String(cartCount)}
</NativeTabs.Trigger.Badge>
</NativeTabs.Trigger>
<NativeTabs.Trigger name="form">
<NativeTabs.Trigger.Icon md="fact_check" sf="doc.text.fill" />
<NativeTabs.Trigger.Label>Form</NativeTabs.Trigger.Label>
</NativeTabs.Trigger>
<NativeTabs.Trigger name="settings">
<NativeTabs.Trigger.Icon md="settings" sf="gearshape.fill" />
<NativeTabs.Trigger.Label>Settings</NativeTabs.Trigger.Label>
<NativeTabs.Trigger.Badge hidden={diagnosticsState !== 'error'}>!</NativeTabs.Trigger.Badge>
</NativeTabs.Trigger>
</NativeTabs>
);
}
+27
View File
@@ -0,0 +1,27 @@
import { useRouter } from 'expo-router';
import { AppFrame } from '../../src/components';
import { useLabState } from '../../src/lab-state';
import { CatalogScreen } from '../../src/screens/CatalogScreen';
export default function CatalogRoute() {
const router = useRouter();
const state = useLabState();
return (
<AppFrame>
<CatalogScreen
activeCategory={state.activeCategory}
cart={state.cartCounts}
favorites={new Set(state.favoriteIds)}
onAddToCart={state.addToCart}
onOpenDetails={(productId) => router.push(`/product/${productId}`)}
onSearchDraftChange={state.setSearchDraft}
onSelectCategory={state.setActiveCategory}
onToggleFavorite={state.toggleFavorite}
products={state.catalogProducts}
searchDraft={state.searchDraft}
/>
</AppFrame>
);
}
+20
View File
@@ -0,0 +1,20 @@
import { AppFrame } from '../../src/components';
import { useLabState } from '../../src/lab-state';
import { FormScreen } from '../../src/screens/FormScreen';
export default function FormRoute() {
const state = useLabState();
return (
<AppFrame>
<FormScreen
errors={state.formErrors}
form={state.form}
onChange={state.updateForm}
onReset={state.resetForm}
onSubmit={state.submitOrder}
submittedSummary={state.submittedSummary}
/>
</AppFrame>
);
}
+28
View File
@@ -0,0 +1,28 @@
import { useRouter } from 'expo-router';
import { AppFrame } from '../../src/components';
import { useLabState } from '../../src/lab-state';
import { HomeScreen } from '../../src/screens/HomeScreen';
export default function HomeRoute() {
const router = useRouter();
const state = useLabState();
return (
<AppFrame>
<HomeScreen
cartCount={state.cartCount}
isOnline={state.isOnline}
isRefreshing={state.isRefreshing}
lastSyncLabel={state.lastSyncLabel}
noticeVisible={state.noticeVisible}
onDismissNotice={state.dismissNotice}
onOpenCatalog={() => router.navigate('/catalog')}
onOpenForm={() => router.navigate('/form')}
onOpenSettings={() => router.navigate('/settings')}
onRefresh={state.refreshMetrics}
onSetOnline={state.setIsOnline}
/>
</AppFrame>
);
}
+25
View File
@@ -0,0 +1,25 @@
import { AppFrame } from '../../src/components';
import { useLabState } from '../../src/lab-state';
import { SettingsScreen } from '../../src/screens/SettingsScreen';
export default function SettingsRoute() {
const state = useLabState();
return (
<AppFrame>
<SettingsScreen
diagnosticsExpanded={state.diagnosticsExpanded}
diagnosticsLoading={state.diagnosticsLoading}
diagnosticsState={state.diagnosticsState}
notificationsEnabled={state.notificationsEnabled}
onConfirmReset={state.resetLabState}
onLoadDiagnostics={state.loadDiagnostics}
onRetryDiagnostics={state.retryDiagnostics}
onSetNotificationsEnabled={state.setNotificationsEnabled}
onSetReducedMotionEnabled={state.setReducedMotionEnabled}
onToggleDiagnostics={() => state.setDiagnosticsExpanded(!state.diagnosticsExpanded)}
reducedMotionEnabled={state.reducedMotionEnabled}
/>
</AppFrame>
);
}
+39
View File
@@ -0,0 +1,39 @@
import { ThemeProvider } from '@react-navigation/native';
import { Stack } from 'expo-router';
import { StatusBar } from 'expo-status-bar';
import { SafeAreaProvider } from 'react-native-safe-area-context';
import { ToastViewport } from '../src/components';
import { LabStateProvider, useLabState } from '../src/lab-state';
import { getNavigationTheme, useAppColors } from '../src/theme';
function RootLayoutContent() {
const colors = useAppColors();
const { toastMessage } = useLabState();
return (
<ThemeProvider value={getNavigationTheme(colors)}>
<StatusBar style={colors.mode === 'light' ? 'dark' : 'light'} />
<Stack
screenOptions={{
contentStyle: { backgroundColor: colors.surface },
headerShown: false,
}}
>
<Stack.Screen name="(tabs)" />
<Stack.Screen name="product/[productId]" />
</Stack>
{toastMessage ? <ToastViewport message={toastMessage} /> : null}
</ThemeProvider>
);
}
export default function RootLayout() {
return (
<SafeAreaProvider>
<LabStateProvider>
<RootLayoutContent />
</LabStateProvider>
</SafeAreaProvider>
);
}
@@ -0,0 +1,34 @@
import { useLocalSearchParams, useRouter } from 'expo-router';
import { AppFrame } from '../../src/components';
import { LAB_PRODUCTS } from '../../src/data';
import { useLabState } from '../../src/lab-state';
import { ProductScreen } from '../../src/screens/ProductScreen';
export default function ProductRoute() {
const { productId } = useLocalSearchParams<{ productId?: string }>();
const router = useRouter();
const state = useLabState();
const product = LAB_PRODUCTS.find((entry) => entry.id === productId) ?? LAB_PRODUCTS[0]!;
const draft = state.detailDrafts[product.id] ?? { note: '', quantity: 1 };
return (
<AppFrame>
<ProductScreen
detailNote={draft.note}
isFavorite={state.favoriteIds.includes(product.id)}
onBack={() => router.replace('/catalog')}
onChangeDetailNote={(value) => state.setProductNote(product.id, value)}
onDecreaseQuantity={() => state.decreaseProductQuantity(product.id)}
onIncreaseQuantity={() => state.increaseProductQuantity(product.id)}
onSave={() => {
state.saveProductToCart(product.id);
router.replace('/catalog');
}}
onToggleFavorite={() => state.toggleFavorite(product.id)}
product={product}
quantity={draft.quantity}
/>
</AppFrame>
);
}
+29
View File
@@ -0,0 +1,29 @@
{
"name": "agent-device-test-app",
"version": "1.0.0",
"private": true,
"main": "expo-router/entry",
"scripts": {
"start": "expo start",
"ios": "expo start --ios",
"android": "expo start --android",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@expo/metro-runtime": "~55.0.9",
"@react-navigation/native": "^7.2.2",
"expo": "~55.0.12",
"expo-constants": "55.0.12",
"expo-linking": "55.0.11",
"expo-router": "~55.0.11",
"expo-status-bar": "~55.0.5",
"react": "19.2.0",
"react-native": "0.83.4",
"react-native-safe-area-context": "~5.6.2",
"react-native-screens": "~4.23.0"
},
"devDependencies": {
"@types/react": "~19.2.2",
"typescript": "~5.9.2"
}
}
+6176
View File
File diff suppressed because it is too large Load Diff
+460
View File
@@ -0,0 +1,460 @@
import type { ReactNode } from 'react';
import {
Pressable,
StyleSheet,
Switch,
Text,
TextInput,
View,
type TextInputProps,
} from 'react-native';
import { useSafeAreaInsets } from 'react-native-safe-area-context';
import { useAppColors, type AppColors } from './theme';
export function ScreenTitle(props: {
title: string;
subtitle: string;
badge?: string;
testID?: string;
}) {
const colors = useAppColors();
const styles = createStyles(colors);
return (
<View style={styles.header} testID={props.testID}>
<View style={styles.headerText}>
<Text style={styles.title}>{props.title}</Text>
<Text style={styles.subtitle}>{props.subtitle}</Text>
</View>
{props.badge ? <InlineBadge label={props.badge} tone="accent" /> : null}
</View>
);
}
export function SectionCard(props: {
title: string;
subtitle?: string;
children: ReactNode;
tone?: 'base' | 'accent' | 'danger';
testID?: string;
}) {
const colors = useAppColors();
const styles = createStyles(colors);
return (
<View
style={[
styles.card,
props.tone === 'accent' ? styles.cardAccent : null,
props.tone === 'danger' ? styles.cardDanger : null,
]}
testID={props.testID}
>
<View style={styles.cardHeader}>
<Text style={styles.cardTitle}>{props.title}</Text>
{props.subtitle ? <Text style={styles.cardSubtitle}>{props.subtitle}</Text> : null}
</View>
{props.children}
</View>
);
}
export function ActionButton(props: {
label: string;
onPress: () => void;
kind?: 'primary' | 'secondary' | 'danger';
testID?: string;
}) {
const colors = useAppColors();
const styles = createStyles(colors);
return (
<Pressable
accessibilityLabel={props.label}
accessibilityRole="button"
onPress={props.onPress}
style={({ pressed }) => [
styles.button,
props.kind === 'secondary' ? styles.buttonSecondary : null,
props.kind === 'danger' ? styles.buttonDanger : null,
pressed ? styles.buttonPressed : null,
]}
testID={props.testID}
>
<Text
style={[
styles.buttonLabel,
props.kind === 'secondary' ? styles.buttonLabelSecondary : null,
props.kind === 'danger' ? styles.buttonLabelDanger : null,
]}
>
{props.label}
</Text>
</Pressable>
);
}
export function ChoiceChip(props: {
label: string;
selected: boolean;
onPress: () => void;
testID?: string;
}) {
const colors = useAppColors();
const styles = createStyles(colors);
return (
<Pressable
accessibilityLabel={props.label}
accessibilityRole="button"
accessibilityState={{ selected: props.selected }}
onPress={props.onPress}
style={({ pressed }) => [
styles.chip,
props.selected ? styles.chipSelected : null,
pressed ? styles.buttonPressed : null,
]}
testID={props.testID}
>
<Text style={[styles.chipLabel, props.selected ? styles.chipLabelSelected : null]}>
{props.label}
</Text>
</Pressable>
);
}
export function ToggleRow(props: {
label: string;
value: boolean;
onValueChange: (value: boolean) => void;
description?: string;
testID?: string;
}) {
const colors = useAppColors();
const styles = createStyles(colors);
return (
<View style={styles.toggleRow} testID={props.testID}>
<View style={styles.toggleText}>
<Text style={styles.toggleLabel}>{props.label}</Text>
{props.description ? (
<Text style={styles.toggleDescription}>{props.description}</Text>
) : null}
</View>
<Switch
accessibilityLabel={props.label}
accessibilityRole="switch"
accessibilityState={{ checked: props.value }}
ios_backgroundColor={colors.lineStrong}
thumbColor={props.value ? colors.accent : colors.card}
trackColor={{ false: colors.lineStrong, true: colors.accentSoft }}
value={props.value}
onValueChange={props.onValueChange}
/>
</View>
);
}
export function TextField(
props: TextInputProps & {
label: string;
testID?: string;
},
) {
const { accessibilityLabel, label, multiline, style, testID, ...inputProps } = props;
const colors = useAppColors();
const styles = createStyles(colors);
return (
<View style={styles.field}>
<Text style={styles.fieldLabel}>{label}</Text>
<TextInput
{...inputProps}
accessibilityLabel={accessibilityLabel ?? label}
multiline={multiline}
placeholderTextColor={colors.textSoft}
style={[styles.fieldInput, multiline ? styles.fieldInputMultiline : null, style]}
testID={testID}
/>
</View>
);
}
export function InlineBadge(props: {
label: string;
tone: 'accent' | 'success' | 'info' | 'neutral' | 'danger';
}) {
const colors = useAppColors();
const styles = createStyles(colors);
return (
<View
style={[
styles.badge,
props.tone === 'accent' ? styles.badgeAccent : null,
props.tone === 'success' ? styles.badgeSuccess : null,
props.tone === 'info' ? styles.badgeInfo : null,
props.tone === 'danger' ? styles.badgeDanger : null,
]}
>
<Text style={[styles.badgeLabel, props.tone === 'danger' ? styles.badgeLabelDanger : null]}>
{props.label}
</Text>
</View>
);
}
export function AppFrame(props: { children: ReactNode }) {
const colors = useAppColors();
const insets = useSafeAreaInsets();
const styles = createStyles(colors);
return (
<View
style={[
styles.frame,
{
paddingBottom: Math.max(insets.bottom, 12),
paddingTop: Math.max(insets.top, 12),
},
]}
>
{props.children}
</View>
);
}
export function ToastViewport(props: { message: string }) {
const colors = useAppColors();
const insets = useSafeAreaInsets();
const styles = createStyles(colors);
return (
<View
pointerEvents="none"
style={[
styles.toastViewport,
{
bottom: Math.max(insets.bottom, 16),
},
]}
>
<View style={styles.toast} testID="global-toast">
<Text style={styles.toastLabel}>{props.message}</Text>
</View>
</View>
);
}
function createStyles(colors: AppColors) {
return StyleSheet.create({
header: {
alignItems: 'flex-start',
flexDirection: 'row',
gap: 12,
justifyContent: 'space-between',
marginBottom: 24,
paddingBottom: 20,
borderBottomColor: colors.line,
borderBottomWidth: StyleSheet.hairlineWidth,
},
headerText: {
flex: 1,
gap: 6,
},
title: {
color: colors.text,
fontSize: 32,
fontWeight: '500',
letterSpacing: 0,
lineHeight: 36,
},
subtitle: {
color: colors.textSoft,
fontSize: 15,
lineHeight: 23,
},
card: {
backgroundColor: colors.card,
borderColor: colors.line,
borderRadius: 4,
borderWidth: StyleSheet.hairlineWidth,
gap: 14,
marginBottom: 12,
padding: 16,
},
cardAccent: {
borderColor: colors.accentSoft,
},
cardDanger: {
backgroundColor: colors.cardStrong,
borderColor: colors.danger,
},
cardHeader: {
gap: 6,
},
cardTitle: {
color: colors.text,
fontSize: 21,
fontWeight: '500',
letterSpacing: 0,
lineHeight: 25,
},
cardSubtitle: {
color: colors.textSoft,
fontSize: 14,
lineHeight: 20,
},
button: {
alignItems: 'center',
backgroundColor: colors.text,
borderColor: colors.text,
borderRadius: 4,
borderWidth: StyleSheet.hairlineWidth,
paddingHorizontal: 16,
paddingVertical: 13,
},
buttonSecondary: {
backgroundColor: 'transparent',
borderColor: colors.lineStrong,
},
buttonDanger: {
backgroundColor: colors.danger,
borderColor: colors.danger,
},
buttonPressed: {
opacity: 0.85,
},
buttonLabel: {
color: colors.surface,
fontSize: 15,
fontWeight: '600',
},
buttonLabelSecondary: {
color: colors.text,
},
buttonLabelDanger: {
color: colors.dangerContrast,
},
chip: {
backgroundColor: 'transparent',
borderColor: colors.line,
borderRadius: 4,
borderWidth: StyleSheet.hairlineWidth,
paddingHorizontal: 14,
paddingVertical: 10,
},
chipSelected: {
backgroundColor: colors.text,
borderColor: colors.text,
},
chipLabel: {
color: colors.text,
fontSize: 14,
fontWeight: '600',
},
chipLabelSelected: {
color: colors.surface,
},
toggleRow: {
alignItems: 'center',
flexDirection: 'row',
gap: 12,
justifyContent: 'space-between',
borderTopColor: colors.line,
borderTopWidth: StyleSheet.hairlineWidth,
paddingTop: 14,
},
toggleText: {
flex: 1,
gap: 4,
},
toggleLabel: {
color: colors.text,
fontSize: 16,
fontWeight: '600',
},
toggleDescription: {
color: colors.textSoft,
fontSize: 13,
lineHeight: 18,
},
field: {
gap: 8,
},
fieldLabel: {
color: colors.text,
fontSize: 14,
fontWeight: '700',
},
fieldInput: {
backgroundColor: colors.field,
borderColor: colors.line,
borderRadius: 4,
borderWidth: StyleSheet.hairlineWidth,
color: colors.text,
fontSize: 16,
minHeight: 52,
paddingHorizontal: 14,
paddingVertical: 12,
},
fieldInputMultiline: {
minHeight: 110,
textAlignVertical: 'top',
},
badge: {
alignSelf: 'flex-start',
backgroundColor: 'transparent',
borderColor: colors.line,
borderRadius: 4,
borderWidth: StyleSheet.hairlineWidth,
paddingHorizontal: 10,
paddingVertical: 6,
},
badgeAccent: {
borderColor: colors.accentSoft,
},
badgeSuccess: {
backgroundColor: colors.cardStrong,
},
badgeInfo: {
backgroundColor: colors.cardStrong,
},
badgeDanger: {
backgroundColor: colors.danger,
},
badgeLabel: {
color: colors.text,
fontSize: 12,
fontWeight: '600',
},
badgeLabelDanger: {
color: colors.dangerContrast,
},
frame: {
backgroundColor: colors.surface,
flex: 1,
paddingHorizontal: 18,
},
toastViewport: {
left: 16,
position: 'absolute',
right: 16,
},
toast: {
backgroundColor: colors.cardStrong,
borderColor: colors.lineStrong,
borderRadius: 4,
borderWidth: StyleSheet.hairlineWidth,
paddingHorizontal: 16,
paddingVertical: 14,
},
toastLabel: {
color: colors.text,
fontSize: 14,
fontWeight: '700',
textAlign: 'center',
},
});
}
+111
View File
@@ -0,0 +1,111 @@
export const PRODUCT_CATEGORIES = ['All', 'Starter Kits', 'Produce', 'Bakery', 'Pantry'] as const;
export type ProductCategory = (typeof PRODUCT_CATEGORIES)[number];
export interface LabProduct {
id: string;
name: string;
category: Exclude<ProductCategory, 'All'>;
price: string;
subtitle: string;
badge: string;
}
export const LAB_PRODUCTS: LabProduct[] = [
{
id: 'citrus-kit',
name: 'Citrus Starter Kit',
category: 'Starter Kits',
price: '$12',
subtitle: 'A bright bundle for quick search, detail, and quantity flows.',
badge: 'Popular',
},
{
id: 'morning-box',
name: 'Morning Prep Box',
category: 'Starter Kits',
price: '$18',
subtitle: 'Tests multi-step navigation with badges and action buttons.',
badge: 'New',
},
{
id: 'avocado-stack',
name: 'Avocado Stack',
category: 'Produce',
price: '$7',
subtitle: 'Short card copy with favorite and add-to-cart actions.',
badge: 'Fresh',
},
{
id: 'pepper-mix',
name: 'Pepper Mix',
category: 'Produce',
price: '$9',
subtitle: 'Useful for filters, scroll, and selector durability checks.',
badge: 'Crisp',
},
{
id: 'herb-bundle',
name: 'Herb Bundle',
category: 'Produce',
price: '$6',
subtitle: 'A compact row for visible-text and existence assertions.',
badge: 'Seasonal',
},
{
id: 'pretzel-bites',
name: 'Pretzel Bites',
category: 'Bakery',
price: '$8',
subtitle: 'Helps exercise off-screen discovery and scoped snapshots.',
badge: 'Snack',
},
{
id: 'sourdough-loaf',
name: 'Sourdough Loaf',
category: 'Bakery',
price: '$11',
subtitle: 'Works well for favorite toggles and detail page assertions.',
badge: 'Warm',
},
{
id: 'berry-tart',
name: 'Berry Tart',
category: 'Bakery',
price: '$14',
subtitle: 'A longer list item to force scrolling on smaller devices.',
badge: 'Sweet',
},
{
id: 'tea-tins',
name: 'Tea Tins',
category: 'Pantry',
price: '$10',
subtitle: 'Good target for search debounce and cart state updates.',
badge: 'Calm',
},
{
id: 'olive-jar',
name: 'Olive Jar',
category: 'Pantry',
price: '$13',
subtitle: 'A stable card for replay maintenance and selector exercises.',
badge: 'Classic',
},
{
id: 'noodle-pack',
name: 'Noodle Pack',
category: 'Pantry',
price: '$15',
subtitle: 'Works well for detail notes and quantity edits.',
badge: 'Fast',
},
{
id: 'seasonal-footer',
name: 'Seasonal Footer Pick',
category: 'Pantry',
price: '$16',
subtitle: 'Placed last on purpose so scroll-into-view flows have a durable target.',
badge: 'Scroll Target',
},
];
+332
View File
@@ -0,0 +1,332 @@
import { createContext, useContext, useEffect, useState, type ReactNode } from 'react';
import { LAB_PRODUCTS, type ProductCategory } from './data';
import type { CheckoutFormState } from './screens/FormScreen';
const initialFormState: CheckoutFormState = {
name: '',
email: '',
phone: '',
notes: '',
shipping: 'Delivery',
payment: 'Card',
subscribe: true,
agree: false,
};
interface ProductDraft {
note: string;
quantity: number;
}
interface LabStateContextValue {
activeCategory: ProductCategory;
cartCount: number;
cartCounts: Record<string, number>;
catalogProducts: typeof LAB_PRODUCTS;
detailDrafts: Record<string, ProductDraft>;
diagnosticsExpanded: boolean;
diagnosticsLoading: boolean;
diagnosticsState: 'idle' | 'ready' | 'error';
favoriteIds: string[];
form: CheckoutFormState;
formErrors: string[];
isOnline: boolean;
isRefreshing: boolean;
lastSyncLabel: string;
noticeVisible: boolean;
notificationsEnabled: boolean;
reducedMotionEnabled: boolean;
searchDraft: string;
submittedSummary: string | null;
toastMessage: string | null;
addToCart: (productId: string, quantity?: number) => void;
decreaseProductQuantity: (productId: string) => void;
dismissNotice: () => void;
increaseProductQuantity: (productId: string) => void;
loadDiagnostics: () => void;
resetForm: () => void;
resetLabState: () => void;
refreshMetrics: () => void;
retryDiagnostics: () => void;
saveProductToCart: (productId: string) => void;
setActiveCategory: (value: ProductCategory) => void;
setDiagnosticsExpanded: (value: boolean) => void;
setIsOnline: (value: boolean) => void;
setNotificationsEnabled: (value: boolean) => void;
setProductNote: (productId: string, value: string) => void;
setReducedMotionEnabled: (value: boolean) => void;
setSearchDraft: (value: string) => void;
submitOrder: () => void;
toggleFavorite: (productId: string) => void;
updateForm: <K extends keyof CheckoutFormState>(field: K, value: CheckoutFormState[K]) => void;
}
const LabStateContext = createContext<LabStateContextValue | null>(null);
export function LabStateProvider(props: { children: ReactNode }) {
const [detailDrafts, setDetailDrafts] = useState<Record<string, ProductDraft>>({});
const [searchDraft, setSearchDraft] = useState('');
const [searchQuery, setSearchQuery] = useState('');
const [activeCategory, setActiveCategory] = useState<ProductCategory>('All');
const [favoriteIds, setFavoriteIds] = useState<string[]>([]);
const [cartCounts, setCartCounts] = useState<Record<string, number>>({});
const [noticeVisible, setNoticeVisible] = useState(true);
const [toastMessage, setToastMessage] = useState<string | null>(null);
const [isOnline, setIsOnline] = useState(true);
const [isRefreshing, setIsRefreshing] = useState(false);
const [lastSyncLabel, setLastSyncLabel] = useState('Never');
const [form, setForm] = useState<CheckoutFormState>(initialFormState);
const [formErrors, setFormErrors] = useState<string[]>([]);
const [submittedSummary, setSubmittedSummary] = useState<string | null>(null);
const [notificationsEnabled, setNotificationsEnabled] = useState(true);
const [reducedMotionEnabled, setReducedMotionEnabled] = useState(false);
const [diagnosticsExpanded, setDiagnosticsExpanded] = useState(false);
const [diagnosticsLoading, setDiagnosticsLoading] = useState(false);
const [diagnosticsState, setDiagnosticsState] = useState<'idle' | 'ready' | 'error'>('idle');
useEffect(() => {
const timeout = setTimeout(() => {
setSearchQuery(searchDraft.trim().toLowerCase());
}, 320);
return () => clearTimeout(timeout);
}, [searchDraft]);
useEffect(() => {
if (toastMessage === null) return undefined;
const timeout = setTimeout(() => {
setToastMessage(null);
}, 2200);
return () => clearTimeout(timeout);
}, [toastMessage]);
const catalogProducts = LAB_PRODUCTS.filter((product) => {
const matchesCategory = activeCategory === 'All' || product.category === activeCategory;
const matchesQuery =
searchQuery.length === 0 ||
product.name.toLowerCase().includes(searchQuery) ||
product.subtitle.toLowerCase().includes(searchQuery);
return matchesCategory && matchesQuery;
});
const cartCount = Object.values(cartCounts).reduce(
(sum: number, count: number) => sum + count,
0,
);
function showToast(message: string) {
setToastMessage(message);
}
function getProductDraft(productId: string): ProductDraft {
return detailDrafts[productId] ?? { note: '', quantity: 1 };
}
function toggleFavorite(productId: string) {
let nextAdded = false;
setFavoriteIds((current) => {
if (current.includes(productId)) {
return current.filter((entry) => entry !== productId);
}
nextAdded = true;
return [...current, productId];
});
showToast(nextAdded ? 'Saved favorite' : 'Removed favorite');
}
function addToCart(productId: string, quantity = 1) {
setCartCounts((current) => ({
...current,
[productId]: (current[productId] ?? 0) + quantity,
}));
showToast('Cart updated');
}
function setProductNote(productId: string, value: string) {
setDetailDrafts((current) => ({
...current,
[productId]: {
...(current[productId] ?? { note: '', quantity: 1 }),
note: value,
},
}));
}
function increaseProductQuantity(productId: string) {
setDetailDrafts((current) => ({
...current,
[productId]: {
...(current[productId] ?? { note: '', quantity: 1 }),
quantity: (current[productId]?.quantity ?? 1) + 1,
},
}));
}
function decreaseProductQuantity(productId: string) {
setDetailDrafts((current) => ({
...current,
[productId]: {
...(current[productId] ?? { note: '', quantity: 1 }),
quantity: Math.max(1, (current[productId]?.quantity ?? 1) - 1),
},
}));
}
function saveProductToCart(productId: string) {
addToCart(productId, getProductDraft(productId).quantity);
}
function resetLabState() {
setDetailDrafts({});
setSearchDraft('');
setSearchQuery('');
setActiveCategory('All');
setFavoriteIds([]);
setCartCounts({});
setNoticeVisible(true);
setIsOnline(true);
setIsRefreshing(false);
setLastSyncLabel('Never');
setForm(initialFormState);
setFormErrors([]);
setSubmittedSummary(null);
setNotificationsEnabled(true);
setReducedMotionEnabled(false);
setDiagnosticsExpanded(false);
setDiagnosticsLoading(false);
setDiagnosticsState('idle');
showToast('Agent Device Tester reset');
}
function refreshMetrics() {
setIsRefreshing(true);
setTimeout(() => {
setIsRefreshing(false);
setLastSyncLabel('Synced just now');
showToast('Metrics refreshed');
}, 1200);
}
function loadDiagnostics() {
setDiagnosticsLoading(true);
setDiagnosticsState('idle');
setTimeout(() => {
setDiagnosticsLoading(false);
setDiagnosticsState('error');
showToast('Diagnostics failed');
}, 1100);
}
function retryDiagnostics() {
setDiagnosticsLoading(true);
setTimeout(() => {
setDiagnosticsLoading(false);
setDiagnosticsState('ready');
showToast('Diagnostics recovered');
}, 900);
}
function updateForm<K extends keyof CheckoutFormState>(field: K, value: CheckoutFormState[K]) {
setForm((current) => ({
...current,
[field]: value,
}));
}
function resetForm() {
setForm(initialFormState);
setFormErrors([]);
setSubmittedSummary(null);
showToast('Form cleared');
}
function submitOrder() {
const nextErrors: string[] = [];
if (form.name.trim().length === 0) nextErrors.push('Full name is required.');
if (!form.email.includes('@')) nextErrors.push('A valid email is required.');
if (!form.agree) nextErrors.push('Order confirmation must be checked.');
setFormErrors(nextErrors);
if (nextErrors.length > 0) {
setSubmittedSummary(null);
showToast('Form needs attention');
return;
}
setSubmittedSummary(
`${form.name} chose ${form.shipping.toLowerCase()} with ${form.payment.toLowerCase()} payment.`,
);
showToast('Order submitted');
}
return (
<LabStateContext.Provider
value={{
activeCategory,
addToCart,
cartCount,
cartCounts,
catalogProducts,
decreaseProductQuantity,
detailDrafts,
diagnosticsExpanded,
diagnosticsLoading,
diagnosticsState,
dismissNotice: () => setNoticeVisible(false),
favoriteIds,
form,
formErrors,
increaseProductQuantity,
isOnline,
isRefreshing,
lastSyncLabel,
loadDiagnostics,
noticeVisible,
notificationsEnabled,
reducedMotionEnabled,
refreshMetrics,
resetForm,
resetLabState,
retryDiagnostics,
saveProductToCart,
searchDraft,
setActiveCategory,
setDiagnosticsExpanded,
setIsOnline,
setNotificationsEnabled,
setProductNote,
setReducedMotionEnabled,
setSearchDraft,
submitOrder,
submittedSummary,
toastMessage,
toggleFavorite,
updateForm,
}}
>
{props.children}
</LabStateContext.Provider>
);
}
export function useLabState(): LabStateContextValue {
const value = useContext(LabStateContext);
if (value === null) {
throw new Error('useLabState must be used within LabStateProvider.');
}
return value;
}
@@ -0,0 +1,171 @@
import { Pressable, ScrollView, StyleSheet, Text, View } from 'react-native';
import { PRODUCT_CATEGORIES, type LabProduct, type ProductCategory } from '../data';
import {
ActionButton,
ChoiceChip,
InlineBadge,
ScreenTitle,
SectionCard,
TextField,
} from '../components';
import { useAppColors, type AppColors } from '../theme';
export interface CatalogScreenProps {
activeCategory: ProductCategory;
cart: Record<string, number>;
favorites: Set<string>;
products: LabProduct[];
searchDraft: string;
onAddToCart: (productId: string) => void;
onOpenDetails: (productId: string) => void;
onSearchDraftChange: (value: string) => void;
onSelectCategory: (value: ProductCategory) => void;
onToggleFavorite: (productId: string) => void;
}
export function CatalogScreen(props: CatalogScreenProps) {
const colors = useAppColors();
const styles = createStyles(colors);
return (
<ScrollView contentContainerStyle={styles.content} showsVerticalScrollIndicator={false}>
<ScreenTitle
badge={`${props.products.length} results`}
subtitle="Search, filter, scroll, favorite, and drill into detail without extra dependencies."
title="Catalog"
testID="catalog-title"
/>
<SectionCard subtitle="Search updates after a short debounce." title="Search">
<TextField
accessibilityLabel="Search products"
label="Find a product"
onChangeText={props.onSearchDraftChange}
placeholder="Try: tart, kit, loaf"
testID="catalog-search"
value={props.searchDraft}
/>
<View style={styles.chipRow}>
{PRODUCT_CATEGORIES.map((category) => (
<ChoiceChip
key={category}
label={category}
onPress={() => props.onSelectCategory(category)}
selected={props.activeCategory === category}
testID={`category-${category.toLowerCase().replace(/\s+/g, '-')}`}
/>
))}
</View>
</SectionCard>
{props.products.map((product) => {
const favoriteLabel = props.favorites.has(product.id) ? 'Saved' : 'Save';
const cartCount = props.cart[product.id] ?? 0;
return (
<SectionCard
key={product.id}
subtitle={product.subtitle}
title={product.name}
testID={`product-card-${product.id}`}
>
<View style={styles.metaRow}>
<InlineBadge label={product.badge} tone="info" />
<Text style={styles.price}>{product.price}</Text>
</View>
<View style={styles.metaRow}>
<Pressable
accessibilityLabel={`${favoriteLabel} ${product.name}`}
accessibilityRole="button"
accessibilityState={{ selected: props.favorites.has(product.id) }}
onPress={() => props.onToggleFavorite(product.id)}
style={({ pressed }) => [styles.favoritePill, pressed ? styles.pressed : null]}
testID={`favorite-${product.id}`}
>
<Text style={styles.favoriteLabel}>{favoriteLabel}</Text>
</Pressable>
<Text style={styles.cartCount}>In cart: {cartCount}</Text>
</View>
<View style={styles.buttonRow}>
<ActionButton
kind="secondary"
label="View details"
onPress={() => props.onOpenDetails(product.id)}
testID={`details-${product.id}`}
/>
<ActionButton
label="Add to cart"
onPress={() => props.onAddToCart(product.id)}
testID={`add-${product.id}`}
/>
</View>
</SectionCard>
);
})}
<SectionCard
subtitle="This footer card sits at the end of the list to force scroll-into-view on smaller screens."
title="Seasonal footer target"
testID="catalog-footer"
>
<Text style={styles.footerText}>
If your run reaches this card, you already exercised long-list navigation. The durable
text here is "Seasonal footer target".
</Text>
</SectionCard>
</ScrollView>
);
}
function createStyles(colors: AppColors) {
return StyleSheet.create({
content: {
paddingBottom: 28,
},
chipRow: {
flexDirection: 'row',
flexWrap: 'wrap',
gap: 8,
},
metaRow: {
alignItems: 'center',
flexDirection: 'row',
justifyContent: 'space-between',
},
price: {
color: colors.text,
fontSize: 18,
fontWeight: '700',
},
favoritePill: {
backgroundColor: 'transparent',
borderColor: colors.line,
borderRadius: 4,
borderWidth: StyleSheet.hairlineWidth,
paddingHorizontal: 12,
paddingVertical: 8,
},
favoriteLabel: {
color: colors.text,
fontSize: 13,
fontWeight: '700',
},
cartCount: {
color: colors.textSoft,
fontSize: 13,
fontWeight: '600',
},
buttonRow: {
gap: 10,
},
footerText: {
color: colors.text,
fontSize: 15,
lineHeight: 22,
},
pressed: {
opacity: 0.85,
},
});
}
@@ -0,0 +1,271 @@
import { Keyboard, Pressable, ScrollView, StyleSheet, Text, View } from 'react-native';
import {
ActionButton,
ChoiceChip,
InlineBadge,
ScreenTitle,
SectionCard,
TextField,
} from '../components';
import { useAppColors, type AppColors } from '../theme';
export interface CheckoutFormState {
name: string;
email: string;
phone: string;
notes: string;
shipping: 'Delivery' | 'Pickup';
payment: 'Card' | 'Cash';
subscribe: boolean;
agree: boolean;
}
export interface FormScreenProps {
errors: string[];
form: CheckoutFormState;
submittedSummary: string | null;
onChange: <K extends keyof CheckoutFormState>(field: K, value: CheckoutFormState[K]) => void;
onReset: () => void;
onSubmit: () => void;
}
function CheckboxRow(props: {
label: string;
value: boolean;
onPress: () => void;
testID: string;
}) {
const colors = useAppColors();
const styles = createStyles(colors);
return (
<Pressable
accessibilityLabel={props.label}
accessibilityRole="checkbox"
accessibilityState={{ checked: props.value }}
onPress={props.onPress}
style={({ pressed }) => [styles.checkboxRow, pressed ? styles.pressed : null]}
testID={props.testID}
>
<View style={[styles.checkbox, props.value ? styles.checkboxChecked : null]}>
<Text style={styles.checkboxMark}>{props.value ? 'X' : ''}</Text>
</View>
<Text style={styles.checkboxLabel}>{props.label}</Text>
</Pressable>
);
}
export function FormScreen(props: FormScreenProps) {
const colors = useAppColors();
const styles = createStyles(colors);
return (
<ScrollView
contentContainerStyle={styles.content}
keyboardShouldPersistTaps="handled"
showsVerticalScrollIndicator={false}
>
<ScreenTitle
badge="Validation"
subtitle="Text inputs, choice groups, checkbox state, multiline notes, and submit feedback."
title="Checkout form"
testID="form-title"
/>
{props.errors.length > 0 ? (
<SectionCard
subtitle="These messages should disappear after a valid submit."
title="Validation errors"
tone="danger"
testID="form-errors"
>
{props.errors.map((error) => (
<Text key={error} style={styles.errorText}>
{error}
</Text>
))}
</SectionCard>
) : null}
{props.submittedSummary ? (
<SectionCard
subtitle="This card appears only after a valid submit."
title="Order summary"
testID="form-success"
>
<InlineBadge label="Submitted" tone="success" />
<Text style={styles.summaryText}>{props.submittedSummary}</Text>
</SectionCard>
) : null}
<SectionCard
subtitle="Use fill for replacement and keyboard dismiss when the next control is blocked."
title="Contact details"
>
<TextField
accessibilityLabel="Full name"
label="Full name"
onChangeText={(value) => props.onChange('name', value)}
placeholder="Ada Lovelace"
testID="field-name"
value={props.form.name}
/>
<TextField
accessibilityLabel="Email"
autoCapitalize="none"
keyboardType="email-address"
label="Email"
onChangeText={(value) => props.onChange('email', value)}
placeholder="ada@example.com"
testID="field-email"
value={props.form.email}
/>
<TextField
accessibilityLabel="Phone"
keyboardType="phone-pad"
label="Phone"
onChangeText={(value) => props.onChange('phone', value)}
placeholder="+48 555 010 010"
testID="field-phone"
value={props.form.phone}
/>
</SectionCard>
<SectionCard
subtitle="These button groups are stable selector targets."
title="Delivery choices"
>
<View style={styles.choiceRow}>
<ChoiceChip
label="Delivery"
onPress={() => props.onChange('shipping', 'Delivery')}
selected={props.form.shipping === 'Delivery'}
testID="shipping-delivery"
/>
<ChoiceChip
label="Pickup"
onPress={() => props.onChange('shipping', 'Pickup')}
selected={props.form.shipping === 'Pickup'}
testID="shipping-pickup"
/>
</View>
<View style={styles.choiceRow}>
<ChoiceChip
label="Card"
onPress={() => props.onChange('payment', 'Card')}
selected={props.form.payment === 'Card'}
testID="payment-card"
/>
<ChoiceChip
label="Cash"
onPress={() => props.onChange('payment', 'Cash')}
selected={props.form.payment === 'Cash'}
testID="payment-cash"
/>
</View>
</SectionCard>
<SectionCard
subtitle="The notes field is intentionally multiline for append-vs-fill tests."
title="Preferences"
>
<TextField
accessibilityLabel="Delivery notes"
label="Delivery notes"
multiline
onChangeText={(value) => props.onChange('notes', value)}
placeholder="Leave at the orange counter."
testID="field-notes"
value={props.form.notes}
/>
<CheckboxRow
label="Email me product updates"
onPress={() => props.onChange('subscribe', !props.form.subscribe)}
testID="checkbox-subscribe"
value={props.form.subscribe}
/>
<CheckboxRow
label="I confirm the order details"
onPress={() => props.onChange('agree', !props.form.agree)}
testID="checkbox-agree"
value={props.form.agree}
/>
<View style={styles.actionStack}>
<ActionButton label="Submit order" onPress={props.onSubmit} testID="submit-order" />
<ActionButton
kind="secondary"
label="Dismiss keyboard"
onPress={() => Keyboard.dismiss()}
testID="dismiss-keyboard"
/>
<ActionButton
kind="secondary"
label="Reset form"
onPress={props.onReset}
testID="reset-form"
/>
</View>
</SectionCard>
</ScrollView>
);
}
function createStyles(colors: AppColors) {
return StyleSheet.create({
content: {
paddingBottom: 28,
},
errorText: {
color: colors.danger,
fontSize: 15,
lineHeight: 22,
},
summaryText: {
color: colors.text,
fontSize: 15,
lineHeight: 22,
},
choiceRow: {
flexDirection: 'row',
flexWrap: 'wrap',
gap: 8,
},
checkboxRow: {
alignItems: 'center',
flexDirection: 'row',
gap: 12,
},
checkbox: {
alignItems: 'center',
backgroundColor: colors.cardStrong,
borderColor: colors.line,
borderRadius: 4,
borderWidth: StyleSheet.hairlineWidth,
height: 24,
justifyContent: 'center',
width: 24,
},
checkboxChecked: {
backgroundColor: colors.text,
borderColor: colors.text,
},
checkboxMark: {
color: colors.surface,
fontSize: 12,
fontWeight: '700',
},
checkboxLabel: {
color: colors.text,
flex: 1,
fontSize: 15,
lineHeight: 21,
},
actionStack: {
gap: 10,
},
pressed: {
opacity: 0.85,
},
});
}
@@ -0,0 +1,192 @@
import { Alert, ActivityIndicator, ScrollView, StyleSheet, Text, View } from 'react-native';
import { ActionButton, InlineBadge, ScreenTitle, SectionCard, ToggleRow } from '../components';
import { useAppColors, type AppColors } from '../theme';
export interface HomeScreenProps {
cartCount: number;
isOnline: boolean;
isRefreshing: boolean;
lastSyncLabel: string;
noticeVisible: boolean;
onDismissNotice: () => void;
onOpenCatalog: () => void;
onOpenForm: () => void;
onOpenSettings: () => void;
onRefresh: () => void;
onSetOnline: (value: boolean) => void;
}
export function HomeScreen(props: HomeScreenProps) {
const colors = useAppColors();
const styles = createStyles(colors);
function showConfirmationAlert() {
Alert.alert(
'Confirm catalog refresh',
'Use this alert for confirm, cancel, and system-alert handling tests. Nothing destructive happens here.',
[
{
style: 'cancel',
text: 'Keep browsing',
},
{
text: 'Confirm refresh',
},
],
{
cancelable: true,
},
);
}
return (
<ScrollView contentContainerStyle={styles.content} showsVerticalScrollIndicator={false}>
<ScreenTitle
badge={`${props.cartCount} in cart`}
subtitle="An app for testing all the functionality of agent-device."
title="Agent Device Tester"
testID="home-title"
/>
{props.noticeVisible ? (
<SectionCard
subtitle="Dismiss this to exercise nearby mutations and compact diff verification."
title="Release notice"
tone="accent"
testID="release-notice"
>
<Text style={styles.noticeText}>
The bakery list was refreshed this morning. Seasonal items moved to the bottom of the
catalog.
</Text>
<ActionButton
kind="secondary"
label="Dismiss notice"
onPress={props.onDismissNotice}
testID="dismiss-notice"
/>
</SectionCard>
) : null}
<SectionCard
subtitle="These actions intentionally branch into different surfaces without leaving the app."
title="Quick actions"
>
<View style={styles.buttonStack}>
<ActionButton
label="Browse catalog"
onPress={props.onOpenCatalog}
testID="home-open-catalog"
/>
<ActionButton
kind="secondary"
label="Open checkout form"
onPress={props.onOpenForm}
testID="home-open-form"
/>
<ActionButton
kind="secondary"
label="Open settings"
onPress={props.onOpenSettings}
testID="home-open-settings"
/>
<ActionButton
kind="secondary"
label="Open confirmation alert"
onPress={showConfirmationAlert}
testID="home-open-modal"
/>
</View>
</SectionCard>
<SectionCard
subtitle="Good for wait, get, and state assertions."
title="Live status"
testID="home-status-card"
>
<View style={styles.row}>
<Text style={styles.label}>Session health</Text>
<InlineBadge
label={props.isOnline ? 'Online' : 'Offline'}
tone={props.isOnline ? 'success' : 'neutral'}
/>
</View>
<ToggleRow
description="Flip this to simulate a reachable or unreachable session target."
label="Lab online"
onValueChange={props.onSetOnline}
testID="toggle-online"
value={props.isOnline}
/>
<View style={styles.row}>
<Text style={styles.label}>Last sync</Text>
<Text style={styles.value}>{props.lastSyncLabel}</Text>
</View>
{props.isRefreshing ? (
<View style={styles.loadingRow} testID="metrics-loading">
<ActivityIndicator color={colors.accent} />
<Text style={styles.value}>Refreshing metrics...</Text>
</View>
) : (
<ActionButton
label="Refresh metrics"
onPress={props.onRefresh}
testID="refresh-metrics"
/>
)}
</SectionCard>
<SectionCard
subtitle="These bullets are stable targets for plain snapshot and get-text flows."
title="Verification targets"
>
<Text style={styles.bullet}>Visible heading with a durable test id.</Text>
<Text style={styles.bullet}>A dismissible banner for diff snapshots.</Text>
<Text style={styles.bullet}>A native alert with confirm and cancel actions.</Text>
<Text style={styles.bullet}>A loading state that becomes a success toast.</Text>
</SectionCard>
</ScrollView>
);
}
function createStyles(colors: AppColors) {
return StyleSheet.create({
content: {
paddingBottom: 28,
},
noticeText: {
color: colors.text,
fontSize: 15,
lineHeight: 22,
},
buttonStack: {
gap: 10,
},
row: {
alignItems: 'center',
flexDirection: 'row',
justifyContent: 'space-between',
},
label: {
color: colors.text,
fontSize: 15,
fontWeight: '600',
},
value: {
color: colors.textSoft,
fontSize: 14,
fontWeight: '600',
},
loadingRow: {
alignItems: 'center',
flexDirection: 'row',
gap: 10,
},
bullet: {
color: colors.text,
fontSize: 15,
lineHeight: 22,
},
});
}
@@ -0,0 +1,137 @@
import { ScrollView, StyleSheet, Text, View } from 'react-native';
import type { LabProduct } from '../data';
import { ActionButton, InlineBadge, ScreenTitle, SectionCard, TextField } from '../components';
import { useAppColors, type AppColors } from '../theme';
export interface ProductScreenProps {
detailNote: string;
isFavorite: boolean;
product: LabProduct;
quantity: number;
onBack: () => void;
onChangeDetailNote: (value: string) => void;
onDecreaseQuantity: () => void;
onIncreaseQuantity: () => void;
onSave: () => void;
onToggleFavorite: () => void;
}
export function ProductScreen(props: ProductScreenProps) {
const colors = useAppColors();
const styles = createStyles(colors);
return (
<ScrollView contentContainerStyle={styles.content} showsVerticalScrollIndicator={false}>
<ScreenTitle
badge={props.product.badge}
subtitle="A focused detail page for back navigation, quantity edits, notes, and save actions."
title={props.product.name}
testID="product-title"
/>
<SectionCard subtitle={props.product.subtitle} title="Product detail">
<View style={styles.metaRow}>
<InlineBadge label={props.product.category} tone="info" />
<Text style={styles.price}>{props.product.price}</Text>
</View>
<Text style={styles.description}>
This detail view is intentionally simple so selectors stay stable while still covering the
most useful interaction patterns.
</Text>
<View style={styles.actionStack}>
<ActionButton
kind="secondary"
label="Back to catalog"
onPress={props.onBack}
testID="product-back"
/>
<ActionButton
kind="secondary"
label={props.isFavorite ? 'Remove favorite' : 'Save favorite'}
onPress={props.onToggleFavorite}
testID="product-favorite"
/>
</View>
</SectionCard>
<SectionCard
subtitle="Good for press, get text, and state-change assertions."
title="Quantity"
>
<View style={styles.quantityRow}>
<ActionButton
kind="secondary"
label="Decrease"
onPress={props.onDecreaseQuantity}
testID="quantity-decrease"
/>
<Text style={styles.quantityValue} testID="quantity-value">
{props.quantity}
</Text>
<ActionButton
kind="secondary"
label="Increase"
onPress={props.onIncreaseQuantity}
testID="quantity-increase"
/>
</View>
</SectionCard>
<SectionCard
subtitle="Use fill or type depending on whether you want replace or append semantics."
title="Detail note"
>
<TextField
accessibilityLabel="Product note"
label="Order note"
multiline
onChangeText={props.onChangeDetailNote}
placeholder="Pack this with the breakfast order."
testID="product-note"
value={props.detailNote}
/>
<ActionButton label="Save to cart" onPress={props.onSave} testID="product-save" />
</SectionCard>
</ScrollView>
);
}
function createStyles(colors: AppColors) {
return StyleSheet.create({
content: {
paddingBottom: 28,
},
metaRow: {
alignItems: 'center',
flexDirection: 'row',
justifyContent: 'space-between',
},
price: {
color: colors.text,
fontSize: 18,
fontWeight: '700',
},
description: {
color: colors.text,
fontSize: 15,
lineHeight: 22,
},
actionStack: {
gap: 10,
},
quantityRow: {
alignItems: 'center',
flexDirection: 'row',
gap: 12,
justifyContent: 'space-between',
},
quantityValue: {
color: colors.text,
fontSize: 32,
fontWeight: '700',
minWidth: 40,
textAlign: 'center',
},
});
}
@@ -0,0 +1,209 @@
import {
Alert,
ActivityIndicator,
Pressable,
ScrollView,
StyleSheet,
Text,
View,
} from 'react-native';
import { ActionButton, InlineBadge, ScreenTitle, SectionCard, ToggleRow } from '../components';
import { useAppColors, type AppColors } from '../theme';
export interface SettingsScreenProps {
diagnosticsExpanded: boolean;
diagnosticsLoading: boolean;
diagnosticsState: 'idle' | 'ready' | 'error';
notificationsEnabled: boolean;
reducedMotionEnabled: boolean;
onLoadDiagnostics: () => void;
onRetryDiagnostics: () => void;
onSetNotificationsEnabled: (value: boolean) => void;
onSetReducedMotionEnabled: (value: boolean) => void;
onToggleDiagnostics: () => void;
onConfirmReset: () => void;
}
export function SettingsScreen(props: SettingsScreenProps) {
const colors = useAppColors();
const styles = createStyles(colors);
function showResetAlert() {
Alert.alert(
'Reset Agent Device Tester?',
'This clears cart, favorites, validation messages, and diagnostic states so the next workflow starts from a known baseline.',
[
{
style: 'cancel',
text: 'Cancel reset',
},
{
style: 'destructive',
text: 'Confirm reset',
onPress: props.onConfirmReset,
},
],
{
cancelable: true,
},
);
}
return (
<ScrollView contentContainerStyle={styles.content} showsVerticalScrollIndicator={false}>
<ScreenTitle
badge="Debug"
subtitle="Toggles, accordion content, loading states, retryable error banners, and native alerts."
title="Settings"
testID="settings-title"
/>
<SectionCard subtitle="Simple switch rows for durable selectors." title="Preferences">
<ToggleRow
description="Disabled notifications should remain visible in plain snapshots."
label="Push notifications"
onValueChange={props.onSetNotificationsEnabled}
testID="toggle-notifications"
value={props.notificationsEnabled}
/>
<ToggleRow
description="Useful when a test needs one more switch state without changing screens."
label="Reduced motion"
onValueChange={props.onSetReducedMotionEnabled}
testID="toggle-reduced-motion"
value={props.reducedMotionEnabled}
/>
</SectionCard>
<SectionCard
subtitle="Expand this section to surface long-form text and status details."
title="Diagnostics"
>
<Pressable
accessibilityLabel={props.diagnosticsExpanded ? 'Hide diagnostics' : 'Show diagnostics'}
accessibilityRole="button"
accessibilityState={{ expanded: props.diagnosticsExpanded }}
onPress={props.onToggleDiagnostics}
style={({ pressed }) => [styles.accordionButton, pressed ? styles.pressed : null]}
testID="toggle-diagnostics"
>
<Text style={styles.accordionLabel}>
{props.diagnosticsExpanded ? 'Hide diagnostics' : 'Show diagnostics'}
</Text>
</Pressable>
{props.diagnosticsExpanded ? (
<View style={styles.diagnosticsBody} testID="diagnostics-body">
<Text style={styles.diagnosticsText}>Build: expo-sdk-55 / lab-fixture-1</Text>
<Text style={styles.diagnosticsText}>API mode: mock network with retry simulation</Text>
<Text style={styles.diagnosticsText}>
Device target hint: use this accordion for get-text and exists assertions
</Text>
</View>
) : null}
{props.diagnosticsLoading ? (
<View style={styles.loadingRow} testID="diagnostics-loading">
<ActivityIndicator color={colors.accent} />
<Text style={styles.diagnosticsText}>Loading diagnostics...</Text>
</View>
) : null}
{props.diagnosticsState === 'ready' ? (
<View style={styles.statusRow} testID="diagnostics-ready">
<InlineBadge label="Ready" tone="success" />
<Text style={styles.diagnosticsText}>Last probe passed in 182 ms.</Text>
</View>
) : null}
{props.diagnosticsState === 'error' ? (
<View style={styles.errorBox} testID="diagnostics-error">
<InlineBadge label="Error" tone="danger" />
<Text style={styles.errorText}>
Catalog service timed out. Retry to restore the success state.
</Text>
<ActionButton
kind="secondary"
label="Retry diagnostics"
onPress={props.onRetryDiagnostics}
testID="retry-diagnostics"
/>
</View>
) : null}
<View style={styles.actionStack}>
<ActionButton
label="Load diagnostics"
onPress={props.onLoadDiagnostics}
testID="load-diagnostics"
/>
<ActionButton
kind="secondary"
label="Reset lab state"
onPress={showResetAlert}
testID="reset-lab"
/>
</View>
</SectionCard>
</ScrollView>
);
}
function createStyles(colors: AppColors) {
return StyleSheet.create({
content: {
paddingBottom: 28,
},
accordionButton: {
backgroundColor: colors.cardStrong,
borderColor: colors.line,
borderRadius: 4,
borderWidth: StyleSheet.hairlineWidth,
paddingHorizontal: 14,
paddingVertical: 14,
},
accordionLabel: {
color: colors.text,
fontSize: 15,
fontWeight: '700',
},
diagnosticsBody: {
gap: 8,
},
diagnosticsText: {
color: colors.text,
fontSize: 14,
lineHeight: 21,
},
loadingRow: {
alignItems: 'center',
flexDirection: 'row',
gap: 10,
},
statusRow: {
alignItems: 'center',
flexDirection: 'row',
gap: 10,
},
errorBox: {
backgroundColor: colors.cardStrong,
borderColor: colors.danger,
borderRadius: 4,
borderWidth: StyleSheet.hairlineWidth,
gap: 10,
padding: 14,
},
errorText: {
color: colors.text,
fontSize: 14,
lineHeight: 21,
},
actionStack: {
gap: 10,
},
pressed: {
opacity: 0.85,
},
});
}
+83
View File
@@ -0,0 +1,83 @@
import { DarkTheme, DefaultTheme, type Theme } from '@react-navigation/native';
import { useColorScheme, type ColorSchemeName } from 'react-native';
export interface AppColors {
accent: string;
accentSoft: string;
card: string;
cardStrong: string;
danger: string;
dangerContrast: string;
field: string;
line: string;
lineStrong: string;
mode: 'dark' | 'light';
overlay: string;
surface: string;
tabBar: string;
text: string;
textSoft: string;
}
const brand = '#8232FF';
const darkColors: AppColors = {
accent: brand,
accentSoft: '#8232ff40',
card: '#000000',
cardStrong: '#ffffff0a',
danger: '#b7354d',
dangerContrast: '#ffffff',
field: '#000000',
line: '#ffffff14',
lineStrong: '#ffffff29',
mode: 'dark',
overlay: 'rgba(0, 0, 0, 0.72)',
surface: '#000000',
tabBar: '#000000',
text: '#ffffff',
textSoft: '#ffffff99',
};
const lightColors: AppColors = {
accent: brand,
accentSoft: '#8232ff24',
card: '#ffffff',
cardStrong: '#0000000a',
danger: '#b3263e',
dangerContrast: '#ffffff',
field: '#ffffff',
line: '#00000014',
lineStrong: '#00000029',
mode: 'light',
overlay: 'rgba(16, 18, 27, 0.24)',
surface: '#ffffff',
tabBar: '#ffffff',
text: '#000000',
textSoft: '#00000099',
};
export function getAppColors(scheme?: ColorSchemeName): AppColors {
return scheme === 'light' ? lightColors : darkColors;
}
export function useAppColors(): AppColors {
return getAppColors(useColorScheme());
}
export function getNavigationTheme(colors: AppColors): Theme {
const baseTheme = colors.mode === 'light' ? DefaultTheme : DarkTheme;
return {
...baseTheme,
colors: {
...baseTheme.colors,
background: colors.surface,
border: colors.line,
card: colors.surface,
notification: colors.accent,
primary: colors.accent,
text: colors.text,
},
};
}
+6
View File
@@ -0,0 +1,6 @@
{
"extends": "expo/tsconfig.base",
"compilerOptions": {
"strict": true
}
}
+8 -1
View File
@@ -75,7 +75,7 @@
"build:all": "pnpm build:node && pnpm build:xcuitest",
"ad": "node bin/agent-device.mjs",
"lint": "oxlint . --deny-warnings",
"format": "oxfmt --write src test skills package.json tsconfig.json .oxlintrc.json .oxfmtrc.json",
"format": "oxfmt --write src test skills package.json tsconfig.json .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
"fallow": "fallow --summary",
"fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow dupes --save-baseline fallow-baselines/dupes.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)",
"check:fallow": "fallow audit",
@@ -85,8 +85,14 @@
"check": "pnpm check:tooling && pnpm check:fallow && pnpm check:unit",
"prepack": "pnpm build:all",
"typecheck": "tsc -p tsconfig.json",
"test-app:install": "pnpm install --dir examples/test-app --ignore-workspace",
"test-app:start": "pnpm --dir examples/test-app start",
"test-app:ios": "pnpm --dir examples/test-app ios",
"test-app:android": "pnpm --dir examples/test-app android",
"test-app:typecheck": "pnpm --dir examples/test-app typecheck",
"test": "vitest run",
"test:unit": "vitest run",
"test:skillgym": "skillgym run ./test/skillgym/suites/agent-device-smoke-suite.ts --config ./test/skillgym/skillgym.config.ts",
"test:smoke": "node --test test/integration/smoke-*.test.ts",
"test:integration": "node --test test/integration/*.test.ts",
"test:replay:ios": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator",
@@ -146,6 +152,7 @@
"fallow": "^2.52.0",
"oxfmt": "^0.42.0",
"oxlint": "^1.57.0",
"skillgym": "^0.5.0",
"typescript": "^6.0.2",
"vite": "^8.0.7"
}
+71
View File
@@ -36,6 +36,9 @@ importers:
oxlint:
specifier: ^1.57.0
version: 1.57.0
skillgym:
specifier: ^0.5.0
version: 0.5.0
typescript:
specifier: ^6.0.2
version: 6.0.2
@@ -942,6 +945,10 @@ packages:
ajv@8.18.0:
resolution: {integrity: sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==}
ansi-regex@6.2.2:
resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==}
engines: {node: '>=12'}
anymatch@3.1.3:
resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==}
engines: {node: '>= 8'}
@@ -1006,6 +1013,10 @@ packages:
resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==}
engines: {node: '>= 8.10.0'}
cli-spinners@3.4.0:
resolution: {integrity: sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw==}
engines: {node: '>=18.20'}
clsx@2.1.1:
resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==}
engines: {node: '>=6'}
@@ -1062,6 +1073,9 @@ packages:
resolution: {integrity: sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==}
engines: {node: '>=0.3.1'}
emoji-regex@10.6.0:
resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==}
entities@6.0.1:
resolution: {integrity: sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==}
engines: {node: '>=0.12'}
@@ -1593,6 +1607,10 @@ packages:
ms@2.1.3:
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
nano-spawn@2.1.0:
resolution: {integrity: sha512-yTW+2okrElHiH4fsiz/+/zc0EDo9BDDoC3iKk8dpv1GeRc9nUWzUZHx6TofMWErchhUQR8hY9/Eu1Uja9x1nqA==}
engines: {node: '>=20.17'}
nanoid@3.3.11:
resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==}
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
@@ -1632,6 +1650,10 @@ packages:
parse-entities@4.0.2:
resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==}
parse-ms@4.0.0:
resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==}
engines: {node: '>=18'}
parse5@7.3.0:
resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==}
@@ -1664,6 +1686,10 @@ packages:
resolution: {integrity: sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==}
engines: {node: ^10 || ^12 || >=14}
pretty-ms@9.3.0:
resolution: {integrity: sha512-gjVS5hOP+M3wMm5nmNOucbIrqudzs9v/57bWRHQWLYklXqoXKrVfYW2W9+glfGsqtPgpiz5WwyEEB+ksXIx3gQ==}
engines: {node: '>=18'}
property-information@7.1.0:
resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==}
@@ -1845,6 +1871,11 @@ packages:
siginfo@2.0.0:
resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==}
skillgym@0.5.0:
resolution: {integrity: sha512-+wl5DHbTta4zJAKbjVIdjI/QrRU+9HPofYBsNwekeICCFF/l364AnF17mdzIs6mK7pskhmnh+6dQ2aT37J07uQ==}
engines: {node: '>=22.18.0'}
hasBin: true
source-map-js@1.2.1:
resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==}
engines: {node: '>=0.10.0'}
@@ -1876,9 +1907,17 @@ packages:
resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==}
engines: {node: '>=0.6.19'}
string-width@7.2.0:
resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==}
engines: {node: '>=18'}
stringify-entities@4.0.4:
resolution: {integrity: sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==}
strip-ansi@7.2.0:
resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==}
engines: {node: '>=12'}
strip-bom-string@1.0.0:
resolution: {integrity: sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==}
engines: {node: '>=0.10.0'}
@@ -2911,6 +2950,8 @@ snapshots:
require-from-string: 2.0.2
optional: true
ansi-regex@6.2.2: {}
anymatch@3.1.3:
dependencies:
normalize-path: 3.0.0
@@ -2968,6 +3009,8 @@ snapshots:
optionalDependencies:
fsevents: 2.3.3
cli-spinners@3.4.0: {}
clsx@2.1.1: {}
collapse-white-space@2.1.0: {}
@@ -3008,6 +3051,8 @@ snapshots:
diff@8.0.4:
optional: true
emoji-regex@10.6.0: {}
entities@6.0.1: {}
error-stack-parser@2.1.4:
@@ -3878,6 +3923,8 @@ snapshots:
ms@2.1.3: {}
nano-spawn@2.1.0: {}
nanoid@3.3.11: {}
normalize-path@3.0.0: {}
@@ -3950,6 +3997,8 @@ snapshots:
is-decimal: 2.0.1
is-hexadecimal: 2.0.1
parse-ms@4.0.0: {}
parse5@7.3.0:
dependencies:
entities: 6.0.1
@@ -3975,6 +4024,10 @@ snapshots:
picocolors: 1.1.1
source-map-js: 1.2.1
pretty-ms@9.3.0:
dependencies:
parse-ms: 4.0.0
property-information@7.1.0: {}
react-dom@19.2.4(react@19.2.4):
@@ -4213,6 +4266,14 @@ snapshots:
siginfo@2.0.0: {}
skillgym@0.5.0:
dependencies:
cli-spinners: 3.4.0
nano-spawn: 2.1.0
picocolors: 1.1.1
pretty-ms: 9.3.0
string-width: 7.2.0
source-map-js@1.2.1: {}
source-map@0.6.1:
@@ -4233,11 +4294,21 @@ snapshots:
string-argv@0.3.2:
optional: true
string-width@7.2.0:
dependencies:
emoji-regex: 10.6.0
get-east-asian-width: 1.5.0
strip-ansi: 7.2.0
stringify-entities@4.0.4:
dependencies:
character-entities-html4: 2.1.0
character-entities-legacy: 3.0.0
strip-ansi@7.2.0:
dependencies:
ansi-regex: 6.2.2
strip-bom-string@1.0.0: {}
strip-json-comments@3.1.1:
+85
View File
@@ -0,0 +1,85 @@
# Skillgym For agent-device
This folder is a starter `skillgym` setup for benchmarking the `agent-device` skill with a controlled Expo target app.
## Why `skillgym` fits here
`skillgym` is useful for `agent-device` in three layers:
1. Skill-routing checks: verify that the runner loads `skills/agent-device/SKILL.md` and its required references before it answers.
2. Workflow-planning checks: verify that the agent describes the right `agent-device` loop for a known fixture app.
3. Optional live-device smoke runs: locally, you can extend prompts so the agent actually drives `agent-device` against a simulator or device.
The included suite focuses on the first two layers so it stays stable and CI-safe.
## Included files
- `../../examples/test-app/`: minimal Expo SDK 55 fixture app for broad UI coverage
- `skillgym.config.ts`: starter config that runs Codex and Claude Haiku against this repo
- `suites/agent-device-smoke-suite.ts`: 48-case suite for skill routing, fixture-aware planning, and skill-guidance regressions
## Current coverage
The suite keeps the app small while separating coverage into two non-overlapping groups.
Fixture smoke cases cover concrete app surfaces:
- open/snapshot/close defaults with Expo Go
- banners, alerts, toggles, and quick actions on Home
- search debounce, filters, long-list scroll, favorites, and cart updates in Catalog
- detail navigation, quantity edits, note append, and save-to-cart on Product
- form validation, success submit, keyboard dismiss, and reset on Checkout form
- diagnostics load/error/retry plus reset alert handling in Settings
- accessibility audit via screenshot + snapshot
Skill-guidance regression cases cover distinct command-planning habits:
- read-only inspection versus mutation
- fresh `@ref` targeting, durable selectors, and off-screen scroll recovery
- text replacement, append semantics, keyboard status, and keyboard dismiss
- install/open setup, app discovery, session scoping, and in-app back navigation
- Metro reload, logs, network dump, alert fallback, and screenshot evidence
- performance metrics, React DevTools profiling, gestures, settings, and trace capture
- remote config, macOS menu bar surfaces, replay update, and batch during recording
`assertAgentDeviceEvidence` is intentionally soft when a runner does not expose skill-detection telemetry. When telemetry exists, the suite asserts that `agent-device` was loaded; when it is absent, the cases still judge command-planning output instead of failing on missing runner metadata.
The `codex-main` baseline is a benchmark signal, not a required all-green gate. Its failures should map to command-planning regressions called out by individual case IDs; do not treat the historical pass/fail count as a fixed threshold.
## Suggested workflow
1. Start with the included smoke suite to benchmark routing and default guidance.
2. Extend the suite with app-specific prompts that cover a new command-planning category rather than duplicating an existing one.
3. Add local-only cases that expect real `agent-device` shell commands once you are ready to involve a running simulator.
## Running the suite
`skillgym` is installed as a repo dev dependency, so run the starter suite from the project root:
```bash
cd /absolute/path/to/agent-device
pnpm install
pnpm test:skillgym
```
If you want to run `skillgym` directly instead of using the convenience script:
```bash
cd /absolute/path/to/agent-device
pnpm exec skillgym run \
./test/skillgym/suites/agent-device-smoke-suite.ts \
--config ./test/skillgym/skillgym.config.ts
```
Prerequisites:
- `codex` CLI installed and authenticated, because the starter config uses the Codex runner
- `claude` CLI installed and authenticated, because the same cases also run against Claude Haiku
- repo dependencies installed with `pnpm install`
- if you want the fixture app running locally, use `pnpm test-app:install` and then `pnpm test-app:ios` or `pnpm test-app:android`
## Where to extend next
- Add suite cases that ask for selector-based plans against `Agent Device Tester`.
- Add local-only prompts that expect `agent-device open`, `snapshot`, `snapshot -i`, `get`, and `wait`.
- Add regression snapshots once the prompt set stabilizes.
+30
View File
@@ -0,0 +1,30 @@
import type { SkillGymConfig } from 'skillgym';
const config: SkillGymConfig = {
run: {
// Relative to this config file; points SkillGym at the repository root.
cwd: '../..',
outputDir: './.skillgym-results',
reporter: 'standard',
schedule: 'parallel',
},
defaults: {
timeoutMs: 120_000,
},
runners: {
'codex-main': {
agent: {
type: 'codex',
model: 'gpt-5.4-mini',
},
},
'claude-haiku': {
agent: {
type: 'claude-code',
model: 'haiku',
},
},
},
};
export default config;
@@ -0,0 +1,725 @@
import { assert, type TestCase } from 'skillgym';
type SessionReport = Parameters<typeof assert.skills.has>[0];
const APP_SOURCE = /(?:^|\/)examples\/test-app\//;
const REPO_SOURCE = /(?:^|\/)src\//;
const COMMAND_DOCS = /website\/docs\/docs\/commands\.md$/;
const SUITE_FILE = /test\/skillgym\/suites\/agent-device-smoke-suite\.ts$/;
const BASE_INSTRUCTIONS = `
You are benchmarking agent-device command planning for a known fixture app.
Do not read project source files or project docs.
Do not inspect examples/test-app, src/, README.md, or website/docs.
Use only the app contract provided in this prompt and your existing agent-device knowledge.
If you need command syntax, rely on known agent-device usage patterns instead of reading repository code.
Output only the requested commands, one per line, with no explanation.
`.trim();
function buildPrompt(options: { contract: string[]; task: string }) {
const contractLines = options.contract.map((line) => `- ${line}`).join('\n');
return `${BASE_INSTRUCTIONS}\n\nApp contract:\n${contractLines}\n\nTask:\n${options.task}`;
}
function assertAgentDeviceEvidence(report: SessionReport) {
const hasDetectedSkills = (report.detectedSkills?.length ?? 0) > 0;
// Some SkillGym runners do not expose skill telemetry. Keep this as a conditional routing
// assertion instead of failing otherwise valid command-planning runs on missing metadata.
if (hasDetectedSkills) {
assert.skills.has(report, 'agent-device');
}
}
function assertNoProjectSourceReads(report: SessionReport) {
assert.fileReads.notIncludes(report, APP_SOURCE);
assert.fileReads.notIncludes(report, REPO_SOURCE);
assert.fileReads.notIncludes(report, COMMAND_DOCS);
}
function commandPattern(command: string) {
// The suite asks agents for one command per line, so command-name assertions stay line anchored.
return new RegExp(`(?:^|\\n)(?:agent-device\\s+)?${command}(?:\\s|$)`, 'i');
}
function commandAlternativesPattern(commands: string[]) {
const alternatives = commands.join('|');
return new RegExp(`(?:^|\\n)(?:agent-device\\s+)?(?:${alternatives})(?:\\s|$)`, 'i');
}
function assertOutputs(report: SessionReport, matchers: Array<string | RegExp>) {
for (const matcher of matchers) {
assert.output.includes(report, matcher);
}
}
function assertNoOutputs(report: SessionReport, matchers: Array<string | RegExp>) {
for (const matcher of matchers) {
if (typeof matcher === 'string') {
assert.ok(
!report.finalOutput.includes(matcher),
`Expected final output not to include ${JSON.stringify(matcher)}. Observed final output: ${report.finalOutput}`,
);
continue;
}
assert.doesNotMatch(report.finalOutput, matcher);
}
}
function assertExpectedOutput(report: SessionReport, matchers: Array<string | RegExp> = []) {
if (matchers.length === 0) {
assert.output.notEmpty(report);
return;
}
assertOutputs(report, matchers);
}
const RAW_COORDINATE_TARGET =
/(?:^|\n)(?:agent-device\s+)?(?:click|fill|press)\s+-?\d+(?:\.\d+)?\s+-?\d+(?:\.\d+)?/i;
const PSEUDO_ASSERTION_COMMAND = /(?:^|\n)\s*(?:assert|assertVisible|waitFor|waitForText)\b/i;
function makeCase(options: {
id: string;
contract: string[];
task: string;
outputs?: Array<string | RegExp>;
forbiddenOutputs?: Array<string | RegExp>;
}): TestCase {
return {
id: options.id,
prompt: buildPrompt({ contract: options.contract, task: options.task }),
assert(report) {
assertAgentDeviceEvidence(report);
assertNoProjectSourceReads(report);
assert.fileReads.notIncludes(report, SUITE_FILE);
assertExpectedOutput(report, options.outputs);
assertNoOutputs(report, options.forbiddenOutputs ?? []);
},
};
}
const FIXTURE_SMOKE_CASES: TestCase[] = [
makeCase({
id: 'open-and-snapshot',
contract: ['App name: Agent Device Tester', 'Platform: iOS', 'Launch context: Expo Go'],
task: 'Plan the commands to open Agent Device Tester in Expo Go on iOS, take a snapshot -i, then close.',
outputs: [commandPattern('open'), /snapshot -i/i, commandPattern('close')],
}),
makeCase({
id: 'home-dismiss-notice',
contract: [
'App name: Agent Device Tester',
'Current screen: Home tab',
'testID=dismiss-notice',
'visible text: Release notice',
],
task: 'Assume Agent Device Tester is already open on the Home tab. Plan the commands to dismiss the Release notice using the dismiss-notice testID, verify it is gone with diff snapshot -i, then close.',
outputs: [/dismiss-notice/i, /diff snapshot -i/i, commandPattern('close')],
}),
makeCase({
id: 'home-confirm-alert',
contract: [
'App name: Agent Device Tester',
'Current screen: Home tab',
'testID=home-open-modal',
'Opening it shows a native confirmation alert',
],
task: 'Assume Agent Device Tester is already open on the Home tab. Plan the commands to open the confirmation alert and dismiss it using alert wait + alert dismiss.',
outputs: [/home-open-modal/i, commandPattern('alert wait'), commandPattern('alert dismiss')],
}),
makeCase({
id: 'home-refresh-metrics',
contract: [
'App name: Agent Device Tester',
'Current screen: Home tab',
'testID=refresh-metrics',
'visible loading text: Refreshing metrics...',
],
task: 'Assume Agent Device Tester is already open on Home. Plan the commands to tap Refresh metrics, wait for "Refreshing metrics..." to appear, then verify the loading state is gone.',
outputs: [/refresh-metrics/i, commandPattern('wait'), /Refreshing metrics/i],
}),
makeCase({
id: 'home-toggle-online',
contract: [
'App name: Agent Device Tester',
'Current screen: Home tab',
'testID=toggle-online',
'visible badge text after disabling: Offline',
],
task: 'Assume Agent Device Tester is open on Home. Plan the commands to toggle Lab online off and verify the Offline badge is visible.',
outputs: [/toggle-online/i, /Offline/i],
}),
makeCase({
id: 'catalog-search-debounce',
contract: [
'App name: Agent Device Tester',
'Current screen: Catalog tab',
'testID=catalog-search',
'Search should respect debounce timing',
],
task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to fill the search field with "tart" using --delay-ms to respect the debounce, then wait for results to update.',
outputs: [/catalog-search/i, /--delay-ms/i, commandPattern('wait')],
}),
makeCase({
id: 'catalog-filter-bakery',
contract: [
'App name: Agent Device Tester',
'Current screen: Catalog tab',
'category chip: category-bakery',
'visible product after filtering: Berry Tart',
],
task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to select the Bakery category and verify Berry Tart is visible.',
outputs: [/category-bakery/i, /Berry Tart/i],
}),
makeCase({
id: 'catalog-favorite-toggle',
contract: [
'App name: Agent Device Tester',
'Current screen: Catalog tab',
'testID=favorite-citrus-kit',
'label after toggling favorite: Saved',
],
task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to toggle favorite for Citrus Starter Kit and verify the label changes to Saved.',
outputs: [/favorite-citrus-kit/i, /Saved/i],
}),
makeCase({
id: 'catalog-add-to-cart',
contract: [
'App name: Agent Device Tester',
'Current screen: Catalog tab',
'testID=add-pepper-mix',
'visible text after add: In cart: 1',
],
task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to add Pepper Mix to the cart and verify the card shows In cart: 1.',
outputs: [/add-pepper-mix/i, /In cart: 1/i],
}),
makeCase({
id: 'catalog-scroll-footer',
contract: [
'App name: Agent Device Tester',
'Current screen: Catalog tab',
'testID=catalog-footer',
'footer visible text: Seasonal footer target',
],
task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to scroll to the Seasonal footer target card using the scroll command.',
outputs: [commandPattern('scroll'), /(?:catalog-footer|Seasonal footer|down)/i],
forbiddenOutputs: [/scrollintoview/i],
}),
makeCase({
id: 'product-open-details',
contract: [
'App name: Agent Device Tester',
'Current screen: Catalog tab',
'testID=details-citrus-kit',
'Product detail screen has testID=product-title',
],
task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to open Citrus Starter Kit details and verify the product title is visible.',
outputs: [/details-citrus-kit/i, /product-title/i],
}),
makeCase({
id: 'product-quantity',
contract: [
'App name: Agent Device Tester',
'Current screen: product detail',
'testID=quantity-increase',
'testID=quantity-decrease',
'testID=quantity-value',
],
task: 'Assume Agent Device Tester is already on a product detail screen. Plan the commands to increase quantity once, decrease it once, and get the quantity value.',
outputs: [/quantity-increase/i, /quantity-decrease/i, /quantity-value/i],
}),
makeCase({
id: 'product-note-append',
contract: [
'App name: Agent Device Tester',
'Current screen: product detail',
'testID=product-note',
'Use append semantics rather than replacement',
],
task: 'Assume Agent Device Tester is already on a product detail screen. Plan the commands to append "Handle with care" to the product note using press + type (not fill).',
outputs: [/product-note/i, commandPattern('press'), commandPattern('type')],
forbiddenOutputs: [commandPattern('fill'), /(?:^|\n)(?:agent-device\s+)?type\s+@/i],
}),
makeCase({
id: 'product-save-to-cart',
contract: [
'App name: Agent Device Tester',
'Current screen: product detail',
'testID=product-save',
'toast text after saving: Cart updated',
],
task: 'Assume Agent Device Tester is already on a product detail screen. Plan the commands to press Save to cart and verify the Cart updated toast appears.',
outputs: [/product-save/i, /Cart updated/i],
}),
makeCase({
id: 'form-validation-errors',
contract: [
'App name: Agent Device Tester',
'Current screen: Checkout form tab',
'testID=submit-order',
'validation errors card uses testID=form-errors',
],
task: 'Assume Agent Device Tester is on the Checkout form tab. Plan the commands to submit with empty fields and verify the validation errors card is visible.',
outputs: [/submit-order/i, /form-errors/i],
}),
makeCase({
id: 'form-success-submit',
contract: [
'App name: Agent Device Tester',
'Current screen: Checkout form tab',
'testID=field-name',
'testID=field-email',
'testID=checkbox-agree',
'success card uses testID=form-success',
],
task: 'Assume Agent Device Tester is on the Checkout form tab. Plan the commands to fill name and email, check order confirmation, submit, and verify the Order summary card is visible.',
outputs: [/field-name/i, /field-email/i, /checkbox-agree/i, /form-success/i],
}),
makeCase({
id: 'form-keyboard-dismiss',
contract: [
'App name: Agent Device Tester',
'Current screen: Checkout form tab',
'testID=field-name',
'keyboard can be dismissed after focusing the field',
],
task: 'Assume Agent Device Tester is on the Checkout form tab. Plan the commands to focus the Full name field and dismiss the keyboard using keyboard dismiss.',
outputs: [/field-name/i, /keyboard dismiss/i],
forbiddenOutputs: [commandPattern('back')],
}),
makeCase({
id: 'form-reset',
contract: [
'App name: Agent Device Tester',
'Current screen: Checkout form tab',
'testID=reset-form',
'toast text after reset: Form cleared',
],
task: 'Assume Agent Device Tester is on the Checkout form tab. Plan the commands to press Reset form and verify the Form cleared toast appears.',
outputs: [/reset-form/i, /Form cleared/i],
}),
makeCase({
id: 'settings-toggle-preferences',
contract: [
'App name: Agent Device Tester',
'Current screen: Settings tab',
'testID=toggle-notifications',
'testID=toggle-reduced-motion',
],
task: 'Assume Agent Device Tester is on the Settings tab. Plan the commands to toggle Push notifications and Reduced motion.',
outputs: [/toggle-notifications/i, /toggle-reduced-motion/i],
}),
makeCase({
id: 'settings-diagnostics-error',
contract: [
'App name: Agent Device Tester',
'Current screen: Settings tab',
'testID=load-diagnostics',
'error panel uses testID=diagnostics-error',
],
task: 'Assume Agent Device Tester is on the Settings tab. Plan the commands to load diagnostics, wait for the error state, and verify the diagnostics error panel is visible.',
outputs: [/load-diagnostics/i, /diagnostics-error/i],
}),
makeCase({
id: 'settings-diagnostics-retry',
contract: [
'App name: Agent Device Tester',
'Current screen: Settings tab',
'testID=load-diagnostics',
'testID=retry-diagnostics',
'ready state uses testID=diagnostics-ready',
],
task: 'Assume Agent Device Tester is on the Settings tab. Plan the commands to load diagnostics, wait for the error state, retry diagnostics, then verify the Ready badge is visible.',
outputs: [/load-diagnostics/i, /retry-diagnostics/i, /diagnostics-ready/i],
}),
makeCase({
id: 'settings-reset-alert',
contract: [
'App name: Agent Device Tester',
'Current screen: Settings tab',
'testID=reset-lab',
'native alert title: Reset Agent Device Tester?',
],
task: 'Assume Agent Device Tester is on the Settings tab. Plan the commands to trigger Reset lab state, then accept the native alert using alert wait + alert accept.',
outputs: [/reset-lab/i, commandPattern('alert wait'), commandPattern('alert accept')],
}),
makeCase({
id: 'home-accessibility-audit',
contract: [
'App name: Agent Device Tester',
'Current screen: Home tab',
'Compare visible UI with the accessibility tree',
],
task: 'Assume Agent Device Tester is on Home. Plan the commands to capture a screenshot and a snapshot to compare visible UI vs accessibility tree.',
outputs: [/screenshot/i, /snapshot/i],
}),
];
const SKILL_GUIDANCE_CASES: TestCase[] = [
makeCase({
id: 'inspect-visible-text-readonly',
contract: [
'App name: Agent Device Tester',
'Current screen: Home tab',
'visible status badge text: Online',
'No interaction is needed to answer this task',
],
task: 'Plan the minimal read-only command to verify whether the Online badge is visible. Do not request interactive refs or mutate the UI.',
outputs: [/(?:^|\n)(?:agent-device\s+)?(?:snapshot|is)(?:\s|$)/i, /Online/i],
forbiddenOutputs: [
/snapshot -i/i,
commandPattern('click'),
commandPattern('fill'),
commandPattern('press'),
],
}),
makeCase({
id: 'target-ref-after-interactive-snapshot',
contract: [
'App name: Agent Device Tester',
'Current screen: Home tab',
'Control label: Lab online',
'The current @ref is unknown until a fresh interactive snapshot is captured',
],
task: 'Plan the commands to capture fresh interactive refs, press the Lab online control by @ref, then verify the nearby change with diff snapshot -i.',
outputs: [
/snapshot -i/i,
/(?:^|\n)(?:agent-device\s+)?(?:click|press)\s+@(?:e\d+|ref)\b/i,
/(?:diff snapshot -i|snapshot\b.*(?:-i\b.*--diff|--diff\b.*-i\b))/i,
],
forbiddenOutputs: [RAW_COORDINATE_TARGET, /\btestID=/i],
}),
makeCase({
id: 'target-selector-for-durable-field',
contract: [
'App name: Agent Device Tester',
'Current screen: Catalog tab',
'Durable selector: id="catalog-search"',
'Search should respect debounce timing',
],
task: 'Plan the commands to fill the catalog search field through the durable id selector with "tart" using --delay-ms, then wait for results.',
outputs: [
commandPattern('fill'),
/id=(?:["']catalog-search["']|catalog-search)/i,
/--delay-ms/i,
commandPattern('wait'),
],
forbiddenOutputs: [
RAW_COORDINATE_TARGET,
/(?:^|\n)(?:agent-device\s+)?type\s+@/i,
/--selector\b/i,
/--text\b/i,
],
}),
makeCase({
id: 'text-replace-uses-fill',
contract: [
'App name: Agent Device Tester',
'Current screen: Checkout form tab',
'Field selector: id="field-email"',
'Existing field value must be replaced',
],
task: 'Plan the command to replace the Email field value with "qa@example.com".',
outputs: [
commandPattern('fill'),
/id=(?:["']field-email["']|field-email)/i,
/qa@example\.com/i,
],
forbiddenOutputs: [commandPattern('type'), /(?:^|\n)(?:agent-device\s+)?fill\s+\d+\s+\d+/i],
}),
makeCase({
id: 'offscreen-target-scroll-resnapshot',
contract: [
'App name: Agent Device Tester',
'Current screen: Catalog tab',
'Visible-first snapshot says [off-screen below] "Seasonal footer target"',
'Off-screen refs are discovery hints, not actionable refs',
],
task: 'Plan the commands to reach the Seasonal footer target from the off-screen summary, then refresh interactive refs before acting or verifying.',
outputs: [commandPattern('scroll'), /down/i, /snapshot -i/i],
forbiddenOutputs: [
/scrollintoview/i,
/(?:^|\n)(?:agent-device\s+)?(?:click|press)\s+@(?:e\d+|ref)/i,
],
}),
makeCase({
id: 'navigation-back-in-app',
contract: [
'App name: Agent Device Tester',
'Current screen: product detail',
'Goal: return to the Catalog tab through normal app navigation',
],
task: 'Plan the command to go back to Catalog using app-owned navigation semantics.',
outputs: [commandPattern('back')],
forbiddenOutputs: [/back\s+--system/i],
}),
makeCase({
id: 'setup-unknown-app-discover-first',
contract: [
'Platform: Android',
'Target app display name is known: Agent Device Tester',
'Package id is unknown',
'No app session is open yet',
],
task: 'Plan the bootstrap commands to discover the correct Android device and app identifier before opening the app in a named session.',
outputs: [
commandPattern('devices'),
commandPattern('apps'),
commandPattern('open'),
/--session/i,
],
forbiddenOutputs: [/com\.agent\.device\.tester/i, /com\.example/i],
}),
makeCase({
id: 'install-artifact-before-open',
contract: [
'Platform: Android',
'Known artifact path: ./dist/agent-device-tester.apk',
'Known package after install: com.callstack.agentdevicetester',
'The task requires installing the artifact',
],
task: 'Plan the commands to install the APK artifact, then open the installed package in a fresh runtime state.',
outputs: [
commandPattern('install'),
/\.\/dist\/agent-device-tester\.apk/i,
commandPattern('open'),
/--relaunch/i,
],
forbiddenOutputs: [/open\s+\.\/dist\/agent-device-tester\.apk/i],
}),
makeCase({
id: 'metro-reload-dev-loop',
contract: [
'App name: Agent Device Tester',
'React Native dev build is already open and connected to Metro',
'Only JavaScript changed',
],
task: 'Plan the commands to reload the running app after the JS change, then verify the Home screen is visible.',
outputs: [/(?:^|\n)(?:agent-device\s+)?metro\s+reload(?:\s|$)/i, commandPattern('snapshot')],
forbiddenOutputs: [/open\b.*--relaunch/i],
}),
makeCase({
id: 'debug-logs-short-window',
contract: [
'App name: Agent Device Tester',
'Current screen: Settings tab',
'Repro button selector: id="load-diagnostics"',
'Need app logs only for the retry failure window',
],
task: 'Plan the commands to clear and restart logs, mark the repro window, trigger diagnostics, and inspect the log path without dumping a whole stale log into context.',
outputs: [/logs clear --restart/i, /logs mark/i, /load-diagnostics/i, /logs path/i],
forbiddenOutputs: [/cat .*log/i, /tail -n \+1/i],
}),
makeCase({
id: 'debug-network-session-dump',
contract: [
'App name: Agent Device Tester',
'Current screen: Settings tab',
'Diagnostics load triggers HTTP traffic logged by the app',
'Need request and response headers',
],
task: 'Plan the commands to reproduce the diagnostics request and inspect recent session network traffic with headers.',
outputs: [commandPattern('network'), /dump/i, /--include headers/i],
forbiddenOutputs: [/logs path/i, /cat .*log/i],
}),
makeCase({
id: 'evidence-screenshot-overlay-refs',
contract: [
'App name: Agent Device Tester',
'Current screen: Catalog tab',
'The bug report needs visual proof and tappable-region context for icon-only controls',
],
task: 'Plan the command to capture screenshot evidence with current interactive ref overlays.',
outputs: [commandPattern('screenshot'), /--overlay-refs/i],
forbiddenOutputs: [/snapshot --raw/i],
}),
makeCase({
id: 'perf-session-metrics',
contract: [
'App name: Agent Device Tester',
'Platform: iOS simulator',
'No startup sample exists until the app is opened',
'Need session startup, memory, and CPU data as JSON',
],
task: 'Plan the commands to open the app first if needed, then collect session performance metrics as JSON.',
outputs: [commandPattern('open'), commandAlternativesPattern(['perf', 'metrics']), /--json/i],
forbiddenOutputs: [commandPattern('logs'), commandPattern('network')],
}),
makeCase({
id: 'react-devtools-profile-search',
contract: [
'App name: Agent Device Tester',
'React Native DevTools can connect to the running app',
'Interaction to profile: type in the Catalog search field',
'Need slow components and rerender counts',
],
task: 'Plan the commands to verify React DevTools is connected, profile the Catalog search interaction, then list slow components and rerenders.',
outputs: [
commandPattern('react-devtools status'),
commandPattern('react-devtools wait'),
commandPattern('react-devtools profile start'),
/catalog-search/i,
commandPattern('react-devtools profile stop'),
commandPattern('react-devtools profile slow'),
commandPattern('react-devtools profile rerenders'),
],
forbiddenOutputs: [commandPattern('snapshot'), commandPattern('perf')],
}),
makeCase({
id: 'gesture-swipe-carousel',
contract: [
'Platform: iOS simulator',
'Current screen: onboarding carousel',
'Need to advance and return across pages repeatedly',
'Gesture should use a swipe series, not scroll',
],
task: 'Plan the gesture command to swipe horizontally across the carousel eight times with a short pause and ping-pong pattern.',
outputs: [
commandPattern('swipe'),
/--count\s+8/i,
/--pause-ms\s+30/i,
/--pattern\s+ping-pong/i,
],
forbiddenOutputs: [commandPattern('scroll'), RAW_COORDINATE_TARGET],
}),
makeCase({
id: 'gesture-longpress-context-menu',
contract: [
'Platform: Android',
'Current screen: Catalog tab',
'Target center is x=300 y=500',
'Need to open a native context menu with an 800ms long press',
],
task: 'Plan the gesture command to long-press the target center for 800ms.',
outputs: [commandPattern('longpress'), /300\s+500\s+800/i],
forbiddenOutputs: [/--hold-ms/i, commandPattern('click')],
}),
makeCase({
id: 'gesture-pinch-zoom',
contract: [
'Platform: iOS simulator',
'Current screen: image preview',
'Pinch is supported on Apple simulators',
'Need to zoom out around x=200 y=400',
],
task: 'Plan the gesture command to pinch zoom out at the specified center.',
outputs: [commandPattern('pinch'), /0\.5/i, /200\s+400/i],
forbiddenOutputs: [commandPattern('scroll'), commandPattern('swipe')],
}),
makeCase({
id: 'settings-animation-stabilizer',
contract: [
'Platform: Android',
'App name: Agent Device Tester',
'Animations make this flow flaky',
'Animations should be restored after the check',
],
task: 'Plan the commands to disable platform animations before the app check, run a snapshot, then restore animations.',
outputs: [/settings animations off/i, commandPattern('snapshot'), /settings animations on/i],
forbiddenOutputs: [/--platform macos/i, /settings appearance/i],
}),
makeCase({
id: 'trace-capture-session',
contract: [
'App name: Agent Device Tester',
'An app session is already open',
'Need low-level session diagnostics for one diagnostics-button repro',
'Trace artifact path: ./traces/diagnostics.trace',
],
task: 'Plan the commands to start trace capture, trigger diagnostics, then stop the trace into the requested artifact path.',
outputs: [
/trace start \.\/traces\/diagnostics\.trace/i,
/load-diagnostics/i,
/trace stop \.\/traces\/diagnostics\.trace/i,
],
forbiddenOutputs: [commandPattern('record'), /logs clear --restart/i],
}),
makeCase({
id: 'alert-visible-ui-fallback',
contract: [
'App name: Agent Device Tester',
'Current screen: Home tab',
'A visible permission sheet contains the button text "Allow"',
'alert accept already returned no alert found',
],
task: 'Plan the fallback commands to handle the visible sheet as normal tappable UI instead of looping on alert accept.',
outputs: [
/(?:^|\n)(?:agent-device\s+)?(?:find\b.*\bpress\b|press\b.*Allow|snapshot -i)/is,
/Allow/i,
],
forbiddenOutputs: [/alert accept.*\n.*alert accept/is, RAW_COORDINATE_TARGET],
}),
makeCase({
id: 'android-keyboard-readonly-status',
contract: [
'Platform: Android',
'App name: Agent Device Tester',
'Current screen: Checkout form tab',
'Question: is the keyboard visible and what input type is active?',
],
task: 'Plan the read-only command to inspect Android keyboard visibility and input type.',
outputs: [/(?:^|\n)(?:agent-device\s+)?keyboard\s+(?:status|get)(?:\s|$)/i],
forbiddenOutputs: [commandPattern('fill'), commandPattern('type'), /keyboard dismiss/i],
}),
makeCase({
id: 'remote-config-connect-flow',
contract: [
'Remote config path: ./remote-config.json',
'App package: com.callstack.agentdevicetester',
'The remote profile owns tenant, run, lease, and Metro hints',
],
task: 'Plan a remote flow that connects through the remote config, opens the app, captures a snapshot, and disconnects cleanly.',
outputs: [
/connect --remote-config \.\/remote-config\.json/i,
commandPattern('open'),
commandPattern('snapshot'),
commandPattern('disconnect'),
],
forbiddenOutputs: [/--session\s+\w+/i, /--daemon-base-url/i, /--tenant/i, /--run-id/i],
}),
makeCase({
id: 'macos-menubar-surface',
contract: [
'Platform: macOS',
'App name: Agent Device Tester Menu',
'The app lives entirely as a menu bar extra',
'Normal app snapshots can be sparse or empty',
],
task: 'Plan the commands to inspect the menu bar app surface and capture interactive refs.',
outputs: [/--platform macos/i, /--surface menubar/i, /snapshot -i/i],
forbiddenOutputs: [/--surface app/i, /snapshot --raw/i],
}),
makeCase({
id: 'replay-maintenance-update',
contract: [
'Replay path: ./replays/catalog-checkout.ad',
'Selectors drifted after a UI label change',
'Goal: maintain the replay script in place',
],
task: 'Plan the command to maintain the existing replay script after selector drift.',
outputs: [commandPattern('replay'), /-u|--update/i, /\.\/replays\/catalog-checkout\.ad/i],
forbiddenOutputs: [/sed\s+-i/i, /open .*\.ad/i],
}),
makeCase({
id: 'batch-known-stable-flow',
contract: [
'App name: Agent Device Tester',
'The full checkout flow is already known and stable',
'Need fewer round trips while recording evidence',
],
task: 'Plan the commands to start a recording, execute the known checkout steps as one batch, and stop the recording.',
outputs: [
/(?:^|\n)(?:agent-device\s+)?record\s+start/i,
commandPattern('batch'),
/(?:^|\n)(?:agent-device\s+)?record\s+stop/i,
],
forbiddenOutputs: [PSEUDO_ASSERTION_COMMAND],
}),
];
const suite: TestCase[] = [...FIXTURE_SMOKE_CASES, ...SKILL_GUIDANCE_CASES];
export default suite;
+5
View File
@@ -59,6 +59,11 @@
"type": "file",
"label": "Snapshots"
},
{
"name": "skillgym",
"type": "file",
"label": "Skillgym"
},
{
"name": "known-limitations",
"type": "file",
+82
View File
@@ -0,0 +1,82 @@
# Skillgym
`agent-device` works well with [`skillgym`](https://github.com/callstackincubator/skillgym) when you want to benchmark skill routing and workflow quality before paying the cost of full live-device runs.
## What `skillgym` gives us
- repeatable agent sessions against the real repo
- assertions on detected skills, file reads, tool calls, commands, and final output
- artifact capture and token regression snapshots
For `agent-device`, that makes it a strong fit for:
- verifying that the `agent-device` skill is selected for simulator and device tasks
- verifying that the skill loads its mandatory references before normal interactions
- checking that planning guidance mentions the right `agent-device` loop for a known fixture app
## Included starter
This repo now includes a starter setup under `test/skillgym` plus a fixture app under `examples/test-app`:
- `examples/test-app`: a minimal Expo fixture app
- `test/skillgym/skillgym.config.ts`: starter config
- `test/skillgym/suites/agent-device-smoke-suite.ts`: CI-safe smoke suite
## Recommended rollout
1. Start with skill-routing suites that assert `agent-device` is loaded in the right prompts.
2. Add fixture-aware planning suites against `Agent Device Tester` to keep prompts concrete.
3. Add local-only cases that expect real `agent-device` command usage when a simulator or device is available.
## Fixture app coverage
`Agent Device Tester` keeps the screen count low while still covering a wide range of cases:
- visible-text verification
- interactive refs and selector targeting
- form fill and multiline notes
- search debounce and filter chips
- long-list scroll and detail drill-in
- modals, toggles, checkboxes, validation errors, and retryable async states
The default suite now covers 48 cases in two MECE groups.
Fixture smoke cases cover concrete app behavior:
- Expo Go open/snapshot/close
- Home banner dismissal, confirmation alerts, and refresh waits
- Catalog search debounce, category filters, favorites, add-to-cart, and scroll
- Product detail navigation, quantity edits, note append, and save-to-cart
- Form validation errors, successful submit, keyboard dismiss, and reset
- Settings diagnostics error/retry, preference toggles, and reset alert handling
- Accessibility audit (screenshot vs snapshot)
Skill-guidance regression cases cover command-planning habits:
- read-only inspection versus mutation
- fresh `@ref` targeting, durable selectors, and off-screen scroll recovery
- text replacement, append semantics, keyboard status, and keyboard dismiss
- install/open setup, app discovery, session scoping, and in-app back navigation
- Metro reload, logs, network dump, alert fallback, and screenshot evidence
- performance metrics, React DevTools profiling, gestures, settings, and trace capture
- remote config, macOS menu bar surfaces, replay update, and batch during recording
Runner skill telemetry is treated as optional. When a runner reports detected skills, the suite asserts that `agent-device` was selected; otherwise the suite still evaluates the final command plan.
## Run it
`skillgym` is installed as a repo dev dependency. From the repo root:
```bash
cd /absolute/path/to/agent-device
pnpm install
pnpm test:skillgym
```
Equivalent direct command:
```bash
pnpm exec skillgym run \
./test/skillgym/suites/agent-device-smoke-suite.ts \
--config ./test/skillgym/skillgym.config.ts
```