skills/copilotkit__copilotkit
1
0
Fork 0
mirror of https://github.com/CopilotKit/CopilotKit.git synced 2026-09-14 16:26:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

connect the flow (#943)

Browse Source
This commit is contained in:
Suhas Deshpande
2024-11-20 09:13:42 -08:00
committed by GitHub
parent ec2c5ab7fa
commit ae695b4fc0
12 changed files with 396 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/e2e.yml
+64 -34
View File
@@ -12,7 +12,9 @@ on:
urls:
description: "urls"
required: true
pr_number:
description: "Pull Request number"
required: true
jobs:
e2e_tests:
name: E2E Tests
@@ -31,12 +33,23 @@ jobs:
- name: Save E2E URLs
working-directory: examples/e2e
run: |
echo "${{ inputs.urls }}" | base64 -d > urls.json
echo "${{ inputs.urls }}" | base64 -d > app-configs.json
- name: Print E2E URLs
working-directory: examples/e2e
- name: Setup pnpm
uses: pnpm/action-setup@v2
with:
version: 8
- name: Setup Node.js
uses: actions/setup-node@v3
with:
node-version: "22"
- name: Install Dependencies
working-directory: ./examples/e2e
run: |
cat urls.json
pnpm install
pnpm exec playwright install --with-deps
- name: Create Status Check
uses: actions/github-script@v6
@@ -51,33 +64,50 @@ jobs:
started_at: new Date().toISOString()
})
# - name: Run Tests
# id: run_tests
# run: |
# cat tests/.env.test
# echo "Running E2E tests..."
# # Your actual test commands here
- name: Print app configs
working-directory: examples/e2e
run: |
cat app-configs.json
- name: Wait 1 minute
run: sleep 60
# - name: Update Status Check
# if: always()
# uses: actions/github-script@v6
# with:
# script: |
# const conclusion = '${{ steps.run_tests.outcome }}' === 'success' ? 'success' : 'failure';
# await github.rest.checks.create({
# owner: context.repo.owner,
# repo: context.repo.repo,
# name: 'E2E Tests',
# head_sha: '${{ github.event.client_payload.sha }}',
# status: 'completed',
# conclusion: conclusion,
# completed_at: new Date().toISOString(),
# output: {
# title: conclusion === 'success' ? 'E2E Tests Passed' : 'E2E Tests Failed',
# summary: conclusion === 'success' ? 'All tests passed successfully' : 'Some tests failed'
# }
# });
- name: Run Tests
id: run_tests
working-directory: ./examples/e2e
continue-on-error: true
run: |
pnpm playwright test
- name: Comment post-deployment
uses: thollander/actions-comment-pull-request@v3
with:
pr-number: ${{ inputs.pr_number }}
comment-tag: test-run-status-update
file-path: ./examples/e2e/test-results/test-run-comment.md
- uses: actions/upload-artifact@v4
if: always()
with:
name: playwright-report
path: |
examples/e2e/playwright-report/
examples/e2e/test-results/
retention-days: 30
- name: Update Status Check
if: always()
uses: actions/github-script@v6
with:
script: |
const conclusion = '${{ steps.run_tests.outcome }}' === 'success' ? 'success' : 'failure';
await github.rest.checks.create({
owner: context.repo.owner,
repo: context.repo.repo,
name: 'E2E Tests',
head_sha: '${{ inputs.sha }}',
status: 'completed',
conclusion: conclusion,
completed_at: new Date().toISOString(),
output: {
title: conclusion === 'success' ? 'E2E Tests Passed' : 'E2E Tests Failed',
summary: conclusion === 'success' ? 'All tests passed successfully' : 'Some tests failed'
}
});
.github/workflows/preview-envs-deploy.yml
+24 -12
View File
@@ -16,8 +16,16 @@ jobs:
strategy:
matrix:
app:
- { project: "coagents-research-canvas", type: "agent", deps: "remote-deps" }
- { project: "coagents-research-canvas", type: "agent", deps: "local-deps" }
- {
project: "coagents-research-canvas",
type: "agent",
deps: "remote-deps",
}
- {
project: "coagents-research-canvas",
type: "agent",
deps: "local-deps",
}
- { project: "coagents-research-canvas", type: "ui" }
- { project: "coagents-routing", type: "agent", deps: "remote-deps" }
- { project: "coagents-routing", type: "agent", deps: "local-deps" }
@@ -25,7 +33,11 @@ jobs:
- { project: "coagents-qa-text", type: "agent", deps: "remote-deps" }
- { project: "coagents-qa-text", type: "agent", deps: "local-deps" }
- { project: "coagents-qa-text", type: "ui" }
- { project: "coagents-qa-native", type: "agent", deps: "remote-deps" }
- {
project: "coagents-qa-native",
type: "agent",
deps: "remote-deps",
}
- { project: "coagents-qa-native", type: "agent", deps: "local-deps" }
- { project: "coagents-qa-native", type: "ui" }
steps:
@@ -51,7 +63,7 @@ jobs:
- name: AWS ECR Login
run: |
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 058264219587.dkr.ecr.us-east-1.amazonaws.com
- name: (UI) Setup pnpm
if: matrix.app.type == 'ui'
uses: pnpm/action-setup@v4
@@ -63,8 +75,8 @@ jobs:
uses: actions/setup-node@v2
with:
node-version: 21.x
cache: 'pnpm'
cache-dependency-path: '**/pnpm-lock.yaml'
cache: "pnpm"
cache-dependency-path: "**/pnpm-lock.yaml"
- name: (UI) Install dependencies
if: matrix.app.type == 'ui'
@@ -147,8 +159,8 @@ jobs:
uses: actions/setup-node@v2
with:
node-version: 21.x
cache: 'pnpm'
cache-dependency-path: '**/pnpm-lock.yaml'
cache: "pnpm"
cache-dependency-path: "**/pnpm-lock.yaml"
- name: Infra install dependencies
working-directory: infra
@@ -174,10 +186,10 @@ jobs:
working-directory: ./infra
run: cdk deploy --profile default --all --require-approval never --outputs-file=cdk_outputs.json --concurrency 10
env:
UNIQUE_ENV_ID: 'PR${{ github.event.number }}'
UNIQUE_ENV_ID: "PR${{ github.event.number }}"
GITHUB_ACTIONS_RUN_ID: "${{ github.run_id }}"
GITHUB_PR_NUMBER: "${{ github.event.number }}"
- name: Generate table from outputs
id: generate-table
working-directory: infra
@@ -207,11 +219,11 @@ jobs:
working-directory: infra
run: |
echo "e2e-urls=$(node get-e2e-tests-urls.js | base64 -w 0)" >> "$GITHUB_OUTPUT"
- name: Trigger E2E Tests
uses: benc-uk/workflow-dispatch@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
workflow: "e2e.yml"
ref: "${{ github.event.pull_request.head.ref }}"
inputs: '{"ref": "${{ github.event.pull_request.head.ref }}", "sha": "${{ github.event.pull_request.head.sha }}", "urls": "${{ steps.export-e2e-urls.outputs.e2e-urls }}"}'
inputs: '{"ref": "${{ github.event.pull_request.head.ref }}", "sha": "${{ github.event.pull_request.head.sha }}", "urls": "${{ steps.export-e2e-urls.outputs.e2e-urls }}", "pr_number": "${{ github.event.pull_request.number }}"}'
.gitignore
+2 -1
View File
@@ -8,4 +8,5 @@ __snapshots__
node_modules
.idea/
cdk_outputs.json
.turbo
.turbo
test-run-comment.md
examples/e2e/.gitignore
+1
View File
@@ -3,3 +3,4 @@ node_modules/
/playwright-report/
/blob-report/
/playwright/.cache/
app-configs.json
examples/e2e/lib/config-helper.ts
+67
View File
@@ -0,0 +1,67 @@
// lib/config-helper.ts
import configs from "../app-configs.json";
// Project name constants with type safety
export const PROJECT_NAMES = {
AI_RESEARCHER: "coagents-ai-researcher",
QA_NATIVE: "coagents-qa-native",
QA_TEXT: "coagents-qa-text",
RESEARCH_CANVAS: "coagents-research-canvas",
} as const;
export type ProjectName = (typeof PROJECT_NAMES)[keyof typeof PROJECT_NAMES];
export interface ConfigItem {
url: string;
description: string;
projectName: ProjectName;
}
export interface ConfigMap {
[key: string]: ConfigItem;
}
/**
* Returns raw config map
*/
export const getConfigs = (): ConfigMap => {
return configs as ConfigMap;
};
/**
* Groups configuration items by project name and description
*/
export const groupConfigsByDescription = (
configs: ConfigMap
): Record<ProjectName, Record<string, ConfigItem[]>> => {
return Object.entries(configs).reduce((acc, [key, value]) => {
const { projectName, description } = value;
if (!acc[projectName]) {
acc[projectName] = {};
}
if (!acc[projectName][description]) {
acc[projectName][description] = [];
}
acc[projectName][description].push({ ...value, key });
return acc;
}, {} as Record<ProjectName, Record<string, ConfigItem[]>>);
};
/**
* Filter configs by project name
*/
export const filterConfigsByProject = (
configs: ConfigMap,
projectName: ProjectName
): ConfigMap => {
return Object.entries(configs).reduce((acc, [key, value]) => {
if (value.projectName === projectName) {
acc[key] = value;
}
return acc;
}, {} as ConfigMap);
};
examples/e2e/package.json
+2 -1
View File
@@ -17,6 +17,7 @@
"license": "ISC",
"devDependencies": {
"@playwright/test": "^1.48.2",
"@types/node": "^22.8.4"
"@types/node": "^22.8.4",
"json2md": "^2.0.1"
}
}
examples/e2e/playwright.config.ts
+13 -37
View File
@@ -1,4 +1,4 @@
import { defineConfig, devices } from '@playwright/test';
import { defineConfig, devices } from "@playwright/test";
/**
* Read environment variables from file.
@@ -12,8 +12,7 @@ import { defineConfig, devices } from '@playwright/test';
* See https://playwright.dev/docs/test-configuration.
*/
export default defineConfig({
timeout: 340000,
testDir: './tests',
testDir: "./tests",
/* Run tests in files in parallel */
fullyParallel: true,
/* Fail the build on CI if you accidentally left test.only in the source code. */
@@ -23,52 +22,29 @@ export default defineConfig({
/* Opt out of parallel tests on CI. */
workers: process.env.CI ? 1 : undefined,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: 'html',
reporter: [
[
"./reporters/structured-reporter.ts",
{
outputFile: "test-results/test-run-comment.md",
},
],
],
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {
/* Base URL to use in actions like `await page.goto('/')`. */
// baseURL: 'http://127.0.0.1:3000',
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
trace: 'on-first-retry',
trace: "on-first-retry",
},
/* Configure projects for major browsers */
projects: [
{
name: 'chromium',
use: { ...devices['Desktop Chrome'] },
name: "chromium",
use: { ...devices["Desktop Chrome"] },
},
{
name: 'firefox',
use: { ...devices['Desktop Firefox'] },
},
{
name: 'webkit',
use: { ...devices['Desktop Safari'] },
},
/* Test against mobile viewports. */
// {
// name: 'Mobile Chrome',
// use: { ...devices['Pixel 5'] },
// },
// {
// name: 'Mobile Safari',
// use: { ...devices['iPhone 12'] },
// },
/* Test against branded browsers. */
// {
// name: 'Microsoft Edge',
// use: { ...devices['Desktop Edge'], channel: 'msedge' },
// },
// {
// name: 'Google Chrome',
// use: { ...devices['Desktop Chrome'], channel: 'chrome' },
// },
],
/* Run your local dev server before starting the tests */
examples/e2e/pnpm-lock.yaml
Generated
+15
View File
@@ -14,6 +14,9 @@ importers:
'@types/node':
specifier: ^22.8.4
version: 22.8.4
json2md:
specifier: ^2.0.1
version: 2.0.1
packages:
@@ -30,6 +33,12 @@ packages:
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
os: [darwin]
indento@1.1.13:
resolution: {integrity: sha512-YZWk3mreBEM7sBPddsiQnW9Z8SGg/gNpFfscJq00HCDS7pxcQWWWMSVKJU7YkTRyDu1Zv2s8zaK8gQWKmCXHlg==}
json2md@2.0.1:
resolution: {integrity: sha512-VbwmZ83qmUfKBS2pUOHlzNKEZFPBeJSbzEok3trMYyboZUgdHNn1XZfc1uT8UZs1GHCrmRUBXCfqw4YmmQuOhw==}
playwright-core@1.48.2:
resolution: {integrity: sha512-sjjw+qrLFlriJo64du+EK0kJgZzoQPsabGF4lBvsid+3CNIZIYLgnMj9V6JY5VhM2Peh20DJWIVpVljLLnlawA==}
engines: {node: '>=18'}
@@ -56,6 +65,12 @@ snapshots:
fsevents@2.3.2:
optional: true
indento@1.1.13: {}
json2md@2.0.1:
dependencies:
indento: 1.1.13
playwright-core@1.48.2: {}
playwright@1.48.2:
examples/e2e/reporters/structured-reporter.ts
+131
View File
@@ -0,0 +1,131 @@
import type {
FullConfig,
FullResult,
Reporter,
Suite,
TestCase,
TestResult,
} from "@playwright/test/reporter";
import fs from "fs";
import path from "path";
import { ConfigMap, getConfigs } from "../lib/config-helper";
export default class StructuredReporter implements Reporter {
private groupedResults: {
[projectName: string]: {
[description: string]: {
[browser: string]: {
total: number;
passed: number;
failed: number;
skipped: number;
};
};
};
} = {};
private outputFile: string;
private configs: ConfigMap;
constructor(options: { outputFile?: string } = {}) {
this.outputFile = options.outputFile || "test-results/test-run-comment.md";
this.configs = getConfigs();
}
private getTestInfo(testTitle: string, testPath: string[]) {
if (testPath.length < 6) return null;
const projectName = testPath[3];
const description = testPath[4];
const browser = testPath[1];
return {
projectName,
description,
browser,
};
}
onBegin(config: FullConfig, suite: Suite) {
suite.allTests().forEach((test) => {
const testInfo = this.getTestInfo(test.title, test.titlePath());
if (!testInfo) return;
const { projectName, description, browser } = testInfo;
if (!this.groupedResults[projectName]) {
this.groupedResults[projectName] = {};
}
if (!this.groupedResults[projectName][description]) {
this.groupedResults[projectName][description] = {};
}
if (!this.groupedResults[projectName][description][browser]) {
this.groupedResults[projectName][description][browser] = {
total: 0,
passed: 0,
failed: 0,
skipped: 0,
};
}
});
}
onTestEnd(test: TestCase, result: TestResult) {
const testInfo = this.getTestInfo(test.title, test.titlePath());
if (!testInfo) return;
const { projectName, description, browser } = testInfo;
const stats = this.groupedResults[projectName]?.[description]?.[browser];
if (!stats) return;
stats.total++;
switch (result.status) {
case "passed":
stats.passed++;
break;
case "skipped":
stats.skipped++;
break;
case "failed":
case "timedOut":
stats.failed++;
break;
}
}
onEnd(result: FullResult) {
const parts = ["# Test Results\n"];
// Add overall status
parts.push(`\nStatus: ${result.status}\n`);
parts.push(`\nDuration: ${(result.duration / 1000).toFixed(1)}s\n\n`);
const sortedProjects = Object.entries(this.groupedResults).sort(
([nameA], [nameB]) => nameA.localeCompare(nameB)
);
for (const [projectName, descriptions] of sortedProjects) {
parts.push(`## ${projectName}\n`);
for (const [description, browsers] of Object.entries(descriptions)) {
parts.push(`### ${description}\n`);
for (const [browser, stats] of Object.entries(browsers)) {
const resultParts: string[] = [];
resultParts.push(`${stats.passed}/${stats.total} passed`);
if (stats.failed > 0) resultParts.push(`${stats.failed} failed`);
if (stats.skipped > 0) resultParts.push(`${stats.skipped} skipped`);
parts.push(`${browser} → ${resultParts.join(" • ")}\n\n`);
}
parts.push("\n");
}
}
const outputDir = path.dirname(this.outputFile);
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
fs.writeFileSync(this.outputFile, parts.join(""), "utf8");
}
}
examples/e2e/tests/coagents-canvas-researcher-demo.spec.ts
+71 -48
View File
@@ -1,58 +1,81 @@
import { test, expect } from "@playwright/test";
import { waitForSteps, waitForResponse, sendChatMessage } from "../lib/helpers";
import {
getConfigs,
filterConfigsByProject,
groupConfigsByDescription,
PROJECT_NAMES,
} from "../lib/config-helper";
const models = [
{ name: "OpenAI", value: "openai" },
]
const models = [{ name: "OpenAI", value: "openai" }];
test.beforeAll(async () => {
// Simple HTTP calls to eliminate cold start in the apps
const uiFetch = fetch("https://zvmhdot5bsszoxog7hm3jt37ju0vaoah.lambda-url.us-east-1.on.aws/");
const agentFetch = fetch("https://tc553nczoocujifiqugfqj4ukm0msijo.lambda-url.us-east-1.on.aws/copilotkit/info")
const uiFetch = fetch(
"https://zvmhdot5bsszoxog7hm3jt37ju0vaoah.lambda-url.us-east-1.on.aws/"
);
const agentFetch = fetch(
"https://tc553nczoocujifiqugfqj4ukm0msijo.lambda-url.us-east-1.on.aws/copilotkit/info"
);
await Promise.all([uiFetch, agentFetch]);
console.log("Warmed up all endpoints");
});
test.describe("Canvas Researcher Demo", () => {
models.forEach(model => {
test(`End-to-end test with model ${model.name}`, async ({ page }) => {
await page.goto(`http://localhost:3000?coAgentsModel=${model.value}`)
// Type in research question
await page.getByPlaceholder('Enter your research question').fill('Lifespan of penguins');
// Ask AI to conduct research
await sendChatMessage(page, "Conduct research based on my research question, please")
// Ensure intermediate state (steps) is rendered
await waitForSteps(page);
// Wait for response
await waitForResponse(page);
// Count number of resources
const resourceCount = await page.locator('[data-test-id="resource"]').count();
// Ask AI to delete a resource
await sendChatMessage(page, `Delete the first resource, please`);
// Expect delete confirmation Generative UI component in chat
await page.locator('[data-test-id="delete-resource-generative-ui-container"]');
// Click "Delete" to confirm deletion
await page.locator('button:has-text("Delete")').click();
// Wait for response completion
await waitForResponse(page);
// Count for number of resources, it should be -1
const newResourceCount = await page.locator('[data-test-id="resource"]').count();
expect(newResourceCount).toBe(resourceCount - 1);
await page.keyboard.press('Enter');
// Get configurations for Research Canvas project
const allConfigs = getConfigs();
const researchCanvasConfigs = filterConfigsByProject(
allConfigs,
PROJECT_NAMES.RESEARCH_CANVAS
);
const groupedConfigs = groupConfigsByDescription(researchCanvasConfigs);
Object.entries(groupedConfigs).forEach(([projectName, descriptions]) => {
test.describe(`${projectName}`, () => {
Object.entries(descriptions).forEach(([description, configs]) => {
test.describe(`${description}`, () => {
configs.forEach((config) => {
models.forEach((model) => {
test(`Test ${config.description} with model ${model.name}`, async ({
page,
}) => {
await page.goto(`${config.url}?coAgentsModel=${model.value}`);
const researchQuestion = "Lifespan of penguins";
await page
.getByPlaceholder("Enter your research question")
.fill(researchQuestion);
await sendChatMessage(
page,
"Conduct research based on my research question, please"
);
await waitForSteps(page);
await waitForResponse(page);
const resourceCount = await page
.locator('[data-test-id="resource"]')
.count();
await sendChatMessage(page, `Delete the first resource, please`);
const deleteContainer = await page.locator(
'[data-test-id="delete-resource-generative-ui-container"]'
);
expect(deleteContainer).toBeTruthy();
await page.locator('button:has-text("Delete")').click();
await waitForResponse(page);
const newResourceCount = await page
.locator('[data-test-id="resource"]')
.count();
expect(newResourceCount).toBe(resourceCount - 1);
await page.keyboard.press("Enter");
});
});
});
});
});
})
})
});
});
infra/get-e2e-tests-urls.js
+5 -1
View File
@@ -13,7 +13,11 @@ function generateTable() {
// Group entries by ProjectName
Object.values(json).forEach((value) => {
if (value.EndToEndProjectKey) {
e2eUrls[value.EndToEndProjectKey] = value.FunctionUrl;
e2eUrls[value.EndToEndProjectKey] = {
url: value.FunctionUrl,
description: value.ProjectDescription,
projectName: value.ProjectName
};
}
}, {});
infra/lib/utils.ts
+1 -1
View File
@@ -92,7 +92,7 @@ export function createUIProjectStack({
return new PreviewProjectStack(app, cdkStackName, {
projectName: project,
projectDescription: `${description} (Dependencies: ${dependencies})`,
projectDescription: `${description}`,
demoDir: `examples/${project}/ui`,
overrideDockerfile: dockerfile,
environmentVariablesFromSecrets: ["OPENAI_API_KEY"],