diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a817cf56b2..dfd55f1a0b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -12,7 +12,9 @@ on: urls: description: "urls" required: true - + pr_number: + description: "Pull Request number" + required: true jobs: e2e_tests: name: E2E Tests @@ -31,12 +33,23 @@ jobs: - name: Save E2E URLs working-directory: examples/e2e run: | - echo "${{ inputs.urls }}" | base64 -d > urls.json + echo "${{ inputs.urls }}" | base64 -d > app-configs.json - - name: Print E2E URLs - working-directory: examples/e2e + - name: Setup pnpm + uses: pnpm/action-setup@v2 + with: + version: 8 + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: "22" + + - name: Install Dependencies + working-directory: ./examples/e2e run: | - cat urls.json + pnpm install + pnpm exec playwright install --with-deps - name: Create Status Check uses: actions/github-script@v6 @@ -51,33 +64,50 @@ jobs: started_at: new Date().toISOString() }) - # - name: Run Tests - # id: run_tests - # run: | - # cat tests/.env.test - # echo "Running E2E tests..." - # # Your actual test commands here + - name: Print app configs + working-directory: examples/e2e + run: | + cat app-configs.json - - name: Wait 1 minute - run: sleep 60 - - # - name: Update Status Check - # if: always() - # uses: actions/github-script@v6 - # with: - # script: | - # const conclusion = '${{ steps.run_tests.outcome }}' === 'success' ? 'success' : 'failure'; - # await github.rest.checks.create({ - # owner: context.repo.owner, - # repo: context.repo.repo, - # name: 'E2E Tests', - # head_sha: '${{ github.event.client_payload.sha }}', - # status: 'completed', - # conclusion: conclusion, - # completed_at: new Date().toISOString(), - # output: { - # title: conclusion === 'success' ? 'E2E Tests Passed' : 'E2E Tests Failed', - # summary: conclusion === 'success' ? 'All tests passed successfully' : 'Some tests failed' - # } - # }); - + - name: Run Tests + id: run_tests + working-directory: ./examples/e2e + continue-on-error: true + run: | + pnpm playwright test + + - name: Comment post-deployment + uses: thollander/actions-comment-pull-request@v3 + with: + pr-number: ${{ inputs.pr_number }} + comment-tag: test-run-status-update + file-path: ./examples/e2e/test-results/test-run-comment.md + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report + path: | + examples/e2e/playwright-report/ + examples/e2e/test-results/ + retention-days: 30 + + - name: Update Status Check + if: always() + uses: actions/github-script@v6 + with: + script: | + const conclusion = '${{ steps.run_tests.outcome }}' === 'success' ? 'success' : 'failure'; + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'E2E Tests', + head_sha: '${{ inputs.sha }}', + status: 'completed', + conclusion: conclusion, + completed_at: new Date().toISOString(), + output: { + title: conclusion === 'success' ? 'E2E Tests Passed' : 'E2E Tests Failed', + summary: conclusion === 'success' ? 'All tests passed successfully' : 'Some tests failed' + } + }); diff --git a/.github/workflows/preview-envs-deploy.yml b/.github/workflows/preview-envs-deploy.yml index 4f576be1e0..ab5773a1f2 100644 --- a/.github/workflows/preview-envs-deploy.yml +++ b/.github/workflows/preview-envs-deploy.yml @@ -16,8 +16,16 @@ jobs: strategy: matrix: app: - - { project: "coagents-research-canvas", type: "agent", deps: "remote-deps" } - - { project: "coagents-research-canvas", type: "agent", deps: "local-deps" } + - { + project: "coagents-research-canvas", + type: "agent", + deps: "remote-deps", + } + - { + project: "coagents-research-canvas", + type: "agent", + deps: "local-deps", + } - { project: "coagents-research-canvas", type: "ui" } - { project: "coagents-routing", type: "agent", deps: "remote-deps" } - { project: "coagents-routing", type: "agent", deps: "local-deps" } @@ -25,7 +33,11 @@ jobs: - { project: "coagents-qa-text", type: "agent", deps: "remote-deps" } - { project: "coagents-qa-text", type: "agent", deps: "local-deps" } - { project: "coagents-qa-text", type: "ui" } - - { project: "coagents-qa-native", type: "agent", deps: "remote-deps" } + - { + project: "coagents-qa-native", + type: "agent", + deps: "remote-deps", + } - { project: "coagents-qa-native", type: "agent", deps: "local-deps" } - { project: "coagents-qa-native", type: "ui" } steps: @@ -51,7 +63,7 @@ jobs: - name: AWS ECR Login run: | aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 058264219587.dkr.ecr.us-east-1.amazonaws.com - + - name: (UI) Setup pnpm if: matrix.app.type == 'ui' uses: pnpm/action-setup@v4 @@ -63,8 +75,8 @@ jobs: uses: actions/setup-node@v2 with: node-version: 21.x - cache: 'pnpm' - cache-dependency-path: '**/pnpm-lock.yaml' + cache: "pnpm" + cache-dependency-path: "**/pnpm-lock.yaml" - name: (UI) Install dependencies if: matrix.app.type == 'ui' @@ -147,8 +159,8 @@ jobs: uses: actions/setup-node@v2 with: node-version: 21.x - cache: 'pnpm' - cache-dependency-path: '**/pnpm-lock.yaml' + cache: "pnpm" + cache-dependency-path: "**/pnpm-lock.yaml" - name: Infra install dependencies working-directory: infra @@ -174,10 +186,10 @@ jobs: working-directory: ./infra run: cdk deploy --profile default --all --require-approval never --outputs-file=cdk_outputs.json --concurrency 10 env: - UNIQUE_ENV_ID: 'PR${{ github.event.number }}' + UNIQUE_ENV_ID: "PR${{ github.event.number }}" GITHUB_ACTIONS_RUN_ID: "${{ github.run_id }}" GITHUB_PR_NUMBER: "${{ github.event.number }}" - + - name: Generate table from outputs id: generate-table working-directory: infra @@ -207,11 +219,11 @@ jobs: working-directory: infra run: | echo "e2e-urls=$(node get-e2e-tests-urls.js | base64 -w 0)" >> "$GITHUB_OUTPUT" - + - name: Trigger E2E Tests uses: benc-uk/workflow-dispatch@v1 with: token: ${{ secrets.GITHUB_TOKEN }} workflow: "e2e.yml" ref: "${{ github.event.pull_request.head.ref }}" - inputs: '{"ref": "${{ github.event.pull_request.head.ref }}", "sha": "${{ github.event.pull_request.head.sha }}", "urls": "${{ steps.export-e2e-urls.outputs.e2e-urls }}"}' \ No newline at end of file + inputs: '{"ref": "${{ github.event.pull_request.head.ref }}", "sha": "${{ github.event.pull_request.head.sha }}", "urls": "${{ steps.export-e2e-urls.outputs.e2e-urls }}", "pr_number": "${{ github.event.pull_request.number }}"}' diff --git a/.gitignore b/.gitignore index 2f718731aa..c3d225cd8b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ __snapshots__ node_modules .idea/ cdk_outputs.json -.turbo \ No newline at end of file +.turbo +test-run-comment.md \ No newline at end of file diff --git a/examples/e2e/.gitignore b/examples/e2e/.gitignore index 68c5d18f00..4c3f3816a1 100644 --- a/examples/e2e/.gitignore +++ b/examples/e2e/.gitignore @@ -3,3 +3,4 @@ node_modules/ /playwright-report/ /blob-report/ /playwright/.cache/ +app-configs.json \ No newline at end of file diff --git a/examples/e2e/lib/config-helper.ts b/examples/e2e/lib/config-helper.ts new file mode 100644 index 0000000000..783eae27b5 --- /dev/null +++ b/examples/e2e/lib/config-helper.ts @@ -0,0 +1,67 @@ +// lib/config-helper.ts +import configs from "../app-configs.json"; + +// Project name constants with type safety +export const PROJECT_NAMES = { + AI_RESEARCHER: "coagents-ai-researcher", + QA_NATIVE: "coagents-qa-native", + QA_TEXT: "coagents-qa-text", + RESEARCH_CANVAS: "coagents-research-canvas", +} as const; + +export type ProjectName = (typeof PROJECT_NAMES)[keyof typeof PROJECT_NAMES]; + +export interface ConfigItem { + url: string; + description: string; + projectName: ProjectName; +} + +export interface ConfigMap { + [key: string]: ConfigItem; +} + +/** + * Returns raw config map + */ +export const getConfigs = (): ConfigMap => { + return configs as ConfigMap; +}; + +/** + * Groups configuration items by project name and description + */ +export const groupConfigsByDescription = ( + configs: ConfigMap +): Record> => { + return Object.entries(configs).reduce((acc, [key, value]) => { + const { projectName, description } = value; + + if (!acc[projectName]) { + acc[projectName] = {}; + } + + if (!acc[projectName][description]) { + acc[projectName][description] = []; + } + + acc[projectName][description].push({ ...value, key }); + + return acc; + }, {} as Record>); +}; + +/** + * Filter configs by project name + */ +export const filterConfigsByProject = ( + configs: ConfigMap, + projectName: ProjectName +): ConfigMap => { + return Object.entries(configs).reduce((acc, [key, value]) => { + if (value.projectName === projectName) { + acc[key] = value; + } + return acc; + }, {} as ConfigMap); +}; diff --git a/examples/e2e/package.json b/examples/e2e/package.json index 41e974a6c9..d901beff18 100644 --- a/examples/e2e/package.json +++ b/examples/e2e/package.json @@ -17,6 +17,7 @@ "license": "ISC", "devDependencies": { "@playwright/test": "^1.48.2", - "@types/node": "^22.8.4" + "@types/node": "^22.8.4", + "json2md": "^2.0.1" } } diff --git a/examples/e2e/playwright.config.ts b/examples/e2e/playwright.config.ts index 965b03eccd..24d95362f7 100644 --- a/examples/e2e/playwright.config.ts +++ b/examples/e2e/playwright.config.ts @@ -1,4 +1,4 @@ -import { defineConfig, devices } from '@playwright/test'; +import { defineConfig, devices } from "@playwright/test"; /** * Read environment variables from file. @@ -12,8 +12,7 @@ import { defineConfig, devices } from '@playwright/test'; * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ - timeout: 340000, - testDir: './tests', + testDir: "./tests", /* Run tests in files in parallel */ fullyParallel: true, /* Fail the build on CI if you accidentally left test.only in the source code. */ @@ -23,52 +22,29 @@ export default defineConfig({ /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', + reporter: [ + [ + "./reporters/structured-reporter.ts", + { + outputFile: "test-results/test-run-comment.md", + }, + ], + ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ // baseURL: 'http://127.0.0.1:3000', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', + trace: "on-first-retry", }, /* Configure projects for major browsers */ projects: [ { - name: 'chromium', - use: { ...devices['Desktop Chrome'] }, + name: "chromium", + use: { ...devices["Desktop Chrome"] }, }, - - { - name: 'firefox', - use: { ...devices['Desktop Firefox'] }, - }, - - { - name: 'webkit', - use: { ...devices['Desktop Safari'] }, - }, - - /* Test against mobile viewports. */ - // { - // name: 'Mobile Chrome', - // use: { ...devices['Pixel 5'] }, - // }, - // { - // name: 'Mobile Safari', - // use: { ...devices['iPhone 12'] }, - // }, - - /* Test against branded browsers. */ - // { - // name: 'Microsoft Edge', - // use: { ...devices['Desktop Edge'], channel: 'msedge' }, - // }, - // { - // name: 'Google Chrome', - // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, - // }, ], /* Run your local dev server before starting the tests */ diff --git a/examples/e2e/pnpm-lock.yaml b/examples/e2e/pnpm-lock.yaml index 36d36f3b76..1a25c7b50f 100644 --- a/examples/e2e/pnpm-lock.yaml +++ b/examples/e2e/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: '@types/node': specifier: ^22.8.4 version: 22.8.4 + json2md: + specifier: ^2.0.1 + version: 2.0.1 packages: @@ -30,6 +33,12 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + indento@1.1.13: + resolution: {integrity: sha512-YZWk3mreBEM7sBPddsiQnW9Z8SGg/gNpFfscJq00HCDS7pxcQWWWMSVKJU7YkTRyDu1Zv2s8zaK8gQWKmCXHlg==} + + json2md@2.0.1: + resolution: {integrity: sha512-VbwmZ83qmUfKBS2pUOHlzNKEZFPBeJSbzEok3trMYyboZUgdHNn1XZfc1uT8UZs1GHCrmRUBXCfqw4YmmQuOhw==} + playwright-core@1.48.2: resolution: {integrity: sha512-sjjw+qrLFlriJo64du+EK0kJgZzoQPsabGF4lBvsid+3CNIZIYLgnMj9V6JY5VhM2Peh20DJWIVpVljLLnlawA==} engines: {node: '>=18'} @@ -56,6 +65,12 @@ snapshots: fsevents@2.3.2: optional: true + indento@1.1.13: {} + + json2md@2.0.1: + dependencies: + indento: 1.1.13 + playwright-core@1.48.2: {} playwright@1.48.2: diff --git a/examples/e2e/reporters/structured-reporter.ts b/examples/e2e/reporters/structured-reporter.ts new file mode 100644 index 0000000000..7ebd6f061c --- /dev/null +++ b/examples/e2e/reporters/structured-reporter.ts @@ -0,0 +1,131 @@ +import type { + FullConfig, + FullResult, + Reporter, + Suite, + TestCase, + TestResult, +} from "@playwright/test/reporter"; +import fs from "fs"; +import path from "path"; +import { ConfigMap, getConfigs } from "../lib/config-helper"; + +export default class StructuredReporter implements Reporter { + private groupedResults: { + [projectName: string]: { + [description: string]: { + [browser: string]: { + total: number; + passed: number; + failed: number; + skipped: number; + }; + }; + }; + } = {}; + private outputFile: string; + private configs: ConfigMap; + + constructor(options: { outputFile?: string } = {}) { + this.outputFile = options.outputFile || "test-results/test-run-comment.md"; + this.configs = getConfigs(); + } + + private getTestInfo(testTitle: string, testPath: string[]) { + if (testPath.length < 6) return null; + + const projectName = testPath[3]; + const description = testPath[4]; + const browser = testPath[1]; + + return { + projectName, + description, + browser, + }; + } + + onBegin(config: FullConfig, suite: Suite) { + suite.allTests().forEach((test) => { + const testInfo = this.getTestInfo(test.title, test.titlePath()); + if (!testInfo) return; + + const { projectName, description, browser } = testInfo; + + if (!this.groupedResults[projectName]) { + this.groupedResults[projectName] = {}; + } + if (!this.groupedResults[projectName][description]) { + this.groupedResults[projectName][description] = {}; + } + if (!this.groupedResults[projectName][description][browser]) { + this.groupedResults[projectName][description][browser] = { + total: 0, + passed: 0, + failed: 0, + skipped: 0, + }; + } + }); + } + + onTestEnd(test: TestCase, result: TestResult) { + const testInfo = this.getTestInfo(test.title, test.titlePath()); + if (!testInfo) return; + + const { projectName, description, browser } = testInfo; + const stats = this.groupedResults[projectName]?.[description]?.[browser]; + if (!stats) return; + + stats.total++; + switch (result.status) { + case "passed": + stats.passed++; + break; + case "skipped": + stats.skipped++; + break; + case "failed": + case "timedOut": + stats.failed++; + break; + } + } + + onEnd(result: FullResult) { + const parts = ["# Test Results\n"]; + + // Add overall status + parts.push(`\nStatus: ${result.status}\n`); + parts.push(`\nDuration: ${(result.duration / 1000).toFixed(1)}s\n\n`); + + const sortedProjects = Object.entries(this.groupedResults).sort( + ([nameA], [nameB]) => nameA.localeCompare(nameB) + ); + + for (const [projectName, descriptions] of sortedProjects) { + parts.push(`## ${projectName}\n`); + + for (const [description, browsers] of Object.entries(descriptions)) { + parts.push(`### ${description}\n`); + + for (const [browser, stats] of Object.entries(browsers)) { + const resultParts: string[] = []; + resultParts.push(`${stats.passed}/${stats.total} passed`); + if (stats.failed > 0) resultParts.push(`${stats.failed} failed`); + if (stats.skipped > 0) resultParts.push(`${stats.skipped} skipped`); + + parts.push(`${browser} → ${resultParts.join(" • ")}\n\n`); + } + parts.push("\n"); + } + } + + const outputDir = path.dirname(this.outputFile); + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + fs.writeFileSync(this.outputFile, parts.join(""), "utf8"); + } +} diff --git a/examples/e2e/tests/coagents-canvas-researcher-demo.spec.ts b/examples/e2e/tests/coagents-canvas-researcher-demo.spec.ts index 1ae3a80e3d..c7bf6a3ef9 100644 --- a/examples/e2e/tests/coagents-canvas-researcher-demo.spec.ts +++ b/examples/e2e/tests/coagents-canvas-researcher-demo.spec.ts @@ -1,58 +1,81 @@ import { test, expect } from "@playwright/test"; import { waitForSteps, waitForResponse, sendChatMessage } from "../lib/helpers"; +import { + getConfigs, + filterConfigsByProject, + groupConfigsByDescription, + PROJECT_NAMES, +} from "../lib/config-helper"; -const models = [ - { name: "OpenAI", value: "openai" }, -] +const models = [{ name: "OpenAI", value: "openai" }]; test.beforeAll(async () => { - // Simple HTTP calls to eliminate cold start in the apps - const uiFetch = fetch("https://zvmhdot5bsszoxog7hm3jt37ju0vaoah.lambda-url.us-east-1.on.aws/"); - const agentFetch = fetch("https://tc553nczoocujifiqugfqj4ukm0msijo.lambda-url.us-east-1.on.aws/copilotkit/info") + const uiFetch = fetch( + "https://zvmhdot5bsszoxog7hm3jt37ju0vaoah.lambda-url.us-east-1.on.aws/" + ); + const agentFetch = fetch( + "https://tc553nczoocujifiqugfqj4ukm0msijo.lambda-url.us-east-1.on.aws/copilotkit/info" + ); await Promise.all([uiFetch, agentFetch]); console.log("Warmed up all endpoints"); }); -test.describe("Canvas Researcher Demo", () => { - models.forEach(model => { - test(`End-to-end test with model ${model.name}`, async ({ page }) => { - - await page.goto(`http://localhost:3000?coAgentsModel=${model.value}`) - // Type in research question - await page.getByPlaceholder('Enter your research question').fill('Lifespan of penguins'); - - // Ask AI to conduct research - await sendChatMessage(page, "Conduct research based on my research question, please") - - // Ensure intermediate state (steps) is rendered - await waitForSteps(page); - - // Wait for response - await waitForResponse(page); - - // Count number of resources - const resourceCount = await page.locator('[data-test-id="resource"]').count(); - - // Ask AI to delete a resource - await sendChatMessage(page, `Delete the first resource, please`); - - // Expect delete confirmation Generative UI component in chat - await page.locator('[data-test-id="delete-resource-generative-ui-container"]'); - - // Click "Delete" to confirm deletion - await page.locator('button:has-text("Delete")').click(); - - // Wait for response completion - await waitForResponse(page); - - // Count for number of resources, it should be -1 - const newResourceCount = await page.locator('[data-test-id="resource"]').count(); - expect(newResourceCount).toBe(resourceCount - 1); - - await page.keyboard.press('Enter'); +// Get configurations for Research Canvas project +const allConfigs = getConfigs(); +const researchCanvasConfigs = filterConfigsByProject( + allConfigs, + PROJECT_NAMES.RESEARCH_CANVAS +); +const groupedConfigs = groupConfigsByDescription(researchCanvasConfigs); + +Object.entries(groupedConfigs).forEach(([projectName, descriptions]) => { + test.describe(`${projectName}`, () => { + Object.entries(descriptions).forEach(([description, configs]) => { + test.describe(`${description}`, () => { + configs.forEach((config) => { + models.forEach((model) => { + test(`Test ${config.description} with model ${model.name}`, async ({ + page, + }) => { + await page.goto(`${config.url}?coAgentsModel=${model.value}`); + + const researchQuestion = "Lifespan of penguins"; + await page + .getByPlaceholder("Enter your research question") + .fill(researchQuestion); + + await sendChatMessage( + page, + "Conduct research based on my research question, please" + ); + + await waitForSteps(page); + await waitForResponse(page); + + const resourceCount = await page + .locator('[data-test-id="resource"]') + .count(); + + await sendChatMessage(page, `Delete the first resource, please`); + + const deleteContainer = await page.locator( + '[data-test-id="delete-resource-generative-ui-container"]' + ); + expect(deleteContainer).toBeTruthy(); + + await page.locator('button:has-text("Delete")').click(); + await waitForResponse(page); + + const newResourceCount = await page + .locator('[data-test-id="resource"]') + .count(); + expect(newResourceCount).toBe(resourceCount - 1); + + await page.keyboard.press("Enter"); + }); + }); + }); + }); }); - - }) - -}) - + }); +}); diff --git a/infra/get-e2e-tests-urls.js b/infra/get-e2e-tests-urls.js index 009120ff04..0b8c559977 100644 --- a/infra/get-e2e-tests-urls.js +++ b/infra/get-e2e-tests-urls.js @@ -13,7 +13,11 @@ function generateTable() { // Group entries by ProjectName Object.values(json).forEach((value) => { if (value.EndToEndProjectKey) { - e2eUrls[value.EndToEndProjectKey] = value.FunctionUrl; + e2eUrls[value.EndToEndProjectKey] = { + url: value.FunctionUrl, + description: value.ProjectDescription, + projectName: value.ProjectName + }; } }, {}); diff --git a/infra/lib/utils.ts b/infra/lib/utils.ts index 7dc01b6711..5bb6728a88 100644 --- a/infra/lib/utils.ts +++ b/infra/lib/utils.ts @@ -92,7 +92,7 @@ export function createUIProjectStack({ return new PreviewProjectStack(app, cdkStackName, { projectName: project, - projectDescription: `${description} (Dependencies: ${dependencies})`, + projectDescription: `${description}`, demoDir: `examples/${project}/ui`, overrideDockerfile: dockerfile, environmentVariablesFromSecrets: ["OPENAI_API_KEY"],