mirror of
https://github.com/angular/angular.git
synced 2026-09-14 13:54:52 +08:00
build: create script to run benchmarks and perform comparisons (#50745)
This commit creates a new script that solves the following use-cases: - Running benchmarks. It's not trivial to figure out the benchmark target names, and it's also easy to mess up the right Bazel flags. - Performing comparisons. When e.g. working on a runtime senstive change, it should be trivial to run benchmarks between the current working stage, and a base revision (e.g. `main`). The script takes care of both these use-cases and comes with a prompt-based command line tool experience. The script will also be used by a future GitHub action that can run comparisons triggered via GitHub PR comment (by trusted team members). PR Close #50745
This commit is contained in:
Paul Gschwendtner
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import yargs from 'yargs';
|
||||
import {bold, yellow, GitClient, green, Log} from '@angular/ng-dev';
|
||||
import inquirer from 'inquirer';
|
||||
import {exec} from './utils.mjs';
|
||||
import {ResolvedTarget, findBenchmarkTargets, getTestlogPath, resolveTarget} from './targets.mjs';
|
||||
import {collectBenchmarkResults} from './results.mjs';
|
||||
import {setOutput} from '@actions/core';
|
||||
|
||||
const benchmarkTestFlags = [
|
||||
'--cache_test_results=no',
|
||||
'--color=yes',
|
||||
'--curses=no',
|
||||
// We may have RBE set up, but test should run locally on the same machine to
|
||||
// reduce fluctuation. Output streamed ensures that deps can build with RBE, but
|
||||
// tests run locally while also providing useful output for debugging.
|
||||
'--test_output=streamed',
|
||||
];
|
||||
|
||||
await yargs(process.argv.slice(2))
|
||||
.command(
|
||||
'run-compare <compare-ref> [bazel-target]',
|
||||
'Runs a benchmark between two SHAs',
|
||||
(argv) =>
|
||||
argv
|
||||
.positional('compare-ref', {description: 'Comparison SHA', type: 'string', demandOption: true})
|
||||
.positional('bazel-target', {description: 'Bazel target', type: 'string'}),
|
||||
(args) => runCompare(args.bazelTarget, args.compareRef)
|
||||
)
|
||||
.command(
|
||||
'run [bazel-target]',
|
||||
'Runs a benchmark',
|
||||
(argv) => argv.positional('bazel-target', {description: 'Bazel target', type: 'string'}),
|
||||
(args) => runBenchmarkCmd(args.bazelTarget)
|
||||
)
|
||||
.command(
|
||||
'extract-compare-comment <comment-body>',
|
||||
false, // Do not show in help.
|
||||
(argv) => argv.positional('comment-body', {demandOption: true, type: 'string'}),
|
||||
(args) => extractCompareComment(args.commentBody)
|
||||
)
|
||||
.demandCommand()
|
||||
.scriptName('$0')
|
||||
.help()
|
||||
.strict()
|
||||
.parseAsync();
|
||||
|
||||
/** Prompts for a benchmark target. */
|
||||
async function promptForBenchmarkTarget(): Promise<string> {
|
||||
const targets = await findBenchmarkTargets();
|
||||
|
||||
return (
|
||||
await inquirer.prompt<{bazelTarget: string}>({
|
||||
name: 'bazelTarget',
|
||||
message: 'Select benchmark target to run:',
|
||||
type: 'list',
|
||||
choices: targets.map((t) => ({value: t, name: t})),
|
||||
})
|
||||
).bazelTarget;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts arguments from a benchmark compare comment.
|
||||
*
|
||||
* This is a helper used by the GitHub action to perform benchmark
|
||||
* comparisons. Commands follow the format of: `/benchmark-compare <sha> <target>`.
|
||||
*/
|
||||
async function extractCompareComment(commentBody: string): Promise<void> {
|
||||
const matches = /\/[^ ]+ ([^ ]+) ([^ ]+)/.exec(commentBody);
|
||||
if (matches === null) {
|
||||
Log.error('Could not extract information from comment', commentBody);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
setOutput('compareRef', matches[1]);
|
||||
setOutput('benchmarkTarget', matches[2]);
|
||||
}
|
||||
|
||||
/** Runs a specified benchmark, or a benchmark selected via prompt. */
|
||||
async function runBenchmarkCmd(bazelTargetRaw: string | undefined): Promise<void> {
|
||||
if (bazelTargetRaw === undefined) {
|
||||
bazelTargetRaw = await promptForBenchmarkTarget();
|
||||
}
|
||||
await runBenchmarkTarget(await resolveTarget(bazelTargetRaw));
|
||||
}
|
||||
|
||||
/** Runs a benchmark Bazel target. */
|
||||
async function runBenchmarkTarget(bazelTarget: ResolvedTarget): Promise<void> {
|
||||
await exec('bazel', ['test', bazelTarget, ...benchmarkTestFlags]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs a comparison of benchmark results between the current
|
||||
* working stage and the comparison Git reference.
|
||||
*/
|
||||
async function runCompare(bazelTargetRaw: string | undefined, compareRef: string): Promise<void> {
|
||||
const git = await GitClient.get();
|
||||
const initialRef = git.getCurrentBranchOrRevision();
|
||||
|
||||
if (git.hasUncommittedChanges()) {
|
||||
Log.warn(bold('You have uncommitted changes.'));
|
||||
Log.warn('The script will stash your changes and re-apply them so that');
|
||||
Log.warn('the comparison ref can be checked out.');
|
||||
Log.warn('');
|
||||
}
|
||||
|
||||
if (bazelTargetRaw === undefined) {
|
||||
bazelTargetRaw = await promptForBenchmarkTarget();
|
||||
}
|
||||
|
||||
const bazelTarget = await resolveTarget(bazelTargetRaw);
|
||||
const testlogPath = await getTestlogPath(bazelTarget);
|
||||
|
||||
Log.log(green('Test log path:', testlogPath));
|
||||
|
||||
// Run benchmark with the current working stage.
|
||||
await runBenchmarkTarget(bazelTarget);
|
||||
|
||||
const workingDirResults = await collectBenchmarkResults(testlogPath);
|
||||
|
||||
// Stash working directory as we might be in the middle of developing
|
||||
// and we wouldn't want to discard changes when checking out the compare SHA.
|
||||
git.run(['stash']);
|
||||
|
||||
try {
|
||||
Log.log(green('Fetching comparison revision.'));
|
||||
git.run(['fetch', '--depth=1', git.getRepoGitUrl(), compareRef]);
|
||||
Log.log(green('Checking out comparison revision.'));
|
||||
git.run(['checkout', 'FETCH_HEAD']);
|
||||
|
||||
await exec('yarn');
|
||||
await runBenchmarkTarget(bazelTarget);
|
||||
} finally {
|
||||
restoreWorkingStage(git, initialRef);
|
||||
}
|
||||
|
||||
// Re-install dependencies for `HEAD`.
|
||||
await exec('yarn');
|
||||
|
||||
const comparisonResults = await collectBenchmarkResults(testlogPath);
|
||||
|
||||
// If we are running in a GitHub action, expose the benchmark text
|
||||
// results as outputs. Useful if those are exposed as a GitHub comment then.
|
||||
if (process.env.GITHUB_ACTION !== undefined) {
|
||||
setOutput('comparisonResultsText', comparisonResults.textSummary);
|
||||
setOutput('workingStageResultsText', workingDirResults.textSummary);
|
||||
}
|
||||
|
||||
Log.info('\n\n\n');
|
||||
Log.info(bold(green('Results!')));
|
||||
|
||||
Log.info(bold(yellow('Comparison results')), '\n');
|
||||
Log.info(comparisonResults.textSummary);
|
||||
|
||||
Log.info(bold(yellow('Working stage results')), '\n');
|
||||
Log.info(workingDirResults.textSummary);
|
||||
}
|
||||
|
||||
function restoreWorkingStage(git: GitClient, initialRef: string) {
|
||||
Log.log(green('Restoring working stage'));
|
||||
git.run(['checkout', '-f', initialRef]);
|
||||
|
||||
// Stash apply could fail if there were not changes in the working stage.
|
||||
git.runGraceful(['stash', 'apply']);
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
import Zip from 'adm-zip';
|
||||
|
||||
import type {JsonReport} from '../../packages/benchpress/src/reporter/json_file_reporter_types.js';
|
||||
|
||||
/** Results of an individual benchmark scenario. */
|
||||
export interface ScenarioResult {
|
||||
id: string;
|
||||
data: JsonReport;
|
||||
textSummary: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Overall result of a benchmark target.
|
||||
* A benchmark target may contain multiple scenarios.
|
||||
*/
|
||||
export interface OverallResult {
|
||||
scenarios: ScenarioResult[];
|
||||
textSummary: string;
|
||||
}
|
||||
|
||||
/** Collects and parses the benchmark results of the given Bazel target testlog directory. */
|
||||
export function collectBenchmarkResults(testlogDir: string): OverallResult {
|
||||
const z = new Zip(path.join(testlogDir, 'test.outputs/outputs.zip'));
|
||||
const scenarioResults: ScenarioResult[] = [];
|
||||
|
||||
for (const e of z.getEntries()) {
|
||||
if (path.extname(e.entryName) !== '.json') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const data = JSON.parse(z.readAsText(e.entryName));
|
||||
|
||||
// Skip files that do not look like benchpress reports.
|
||||
if (!isJsonReport(data)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
scenarioResults.push({
|
||||
id: data.description.id,
|
||||
data,
|
||||
textSummary: `${data.metricsText}\n${data.validSampleTexts.join('\n')}\n${data.statsText}`,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
scenarios: scenarioResults,
|
||||
textSummary: scenarioResults.map((s) => `### ${s.id}\n\n${s.textSummary}`).join('`\n'),
|
||||
};
|
||||
}
|
||||
|
||||
/** Whether the object corresponds to a benchpress JSON report. */
|
||||
function isJsonReport(data: any): data is JsonReport {
|
||||
return data['completeSample'] !== undefined;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
import {exec} from './utils.mjs';
|
||||
|
||||
/** Branded string representing a resolved Bazel benchmark target. */
|
||||
export type ResolvedTarget = string & {
|
||||
__resolvedTarget: true;
|
||||
};
|
||||
|
||||
/** Finds all benchmark Bazel targets in the project. */
|
||||
export async function findBenchmarkTargets(): Promise<string[]> {
|
||||
return (
|
||||
await exec('bazel', [
|
||||
'query',
|
||||
'--output=label',
|
||||
`'kind("^web_test", //modules/...) intersect attr("name", "perf", //modules/...)'`,
|
||||
])
|
||||
)
|
||||
.split(/\r?\n/)
|
||||
.filter((t) => t !== '');
|
||||
}
|
||||
|
||||
/** Gets the testlog path of a given Bazel target. */
|
||||
export async function getTestlogPath(target: ResolvedTarget): Promise<string> {
|
||||
return path.join(await bazelTestlogDir(), target.substring(2).replace(':', '/'));
|
||||
}
|
||||
|
||||
/** Resolves a given benchmark Bazel target to the fully expanded label. */
|
||||
export async function resolveTarget(target: string): Promise<ResolvedTarget> {
|
||||
// If the target does not specify an explicit browser test target, we attempt
|
||||
// to automatically add the Chromium suffix. This is necessary for e.g.
|
||||
// resolving testlogs which would reside under the actual test target.
|
||||
if (!target.endsWith('_chromium')) {
|
||||
target = `${target}_chromium`;
|
||||
}
|
||||
|
||||
return (await exec('bazel', ['query', '--output=label', target])).trim() as ResolvedTarget;
|
||||
}
|
||||
|
||||
let testlogDir: string | null = null;
|
||||
async function bazelTestlogDir(): Promise<string> {
|
||||
return testlogDir ?? (testlogDir = (await exec('bazel', ['info', 'bazel-testlogs'])).trim());
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import {Log} from '@angular/ng-dev';
|
||||
import childProcess from 'child_process';
|
||||
import path from 'path';
|
||||
import url from 'url';
|
||||
|
||||
const scriptDir = path.dirname(url.fileURLToPath(import.meta.url));
|
||||
|
||||
/** Absolute disk path to the project directory. */
|
||||
export const projectDir = path.join(scriptDir, '../..');
|
||||
|
||||
/**
|
||||
* Executes the given command, forwarding stdin, stdout and stderr while
|
||||
* still capturing stdout in order to return it.
|
||||
*/
|
||||
export function exec(cmd: string, args: string[] = []): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
Log.info('Running command:', cmd, args.join(' '));
|
||||
|
||||
const proc = childProcess.spawn(cmd, args, {
|
||||
shell: true,
|
||||
cwd: projectDir,
|
||||
// Only capture `stdout`. Forward the rest to the parent TTY.
|
||||
stdio: ['inherit', 'pipe', 'inherit'],
|
||||
});
|
||||
let stdout = '';
|
||||
|
||||
proc.stdout.on('data', (chunk) => {
|
||||
stdout += chunk.toString('utf8');
|
||||
process.stdout.write(chunk);
|
||||
});
|
||||
|
||||
proc.on('close', (status, signal) => {
|
||||
if (status !== 0 || signal !== null) {
|
||||
reject(`Command failed. Status code: ${status}. Signal: ${signal}`);
|
||||
}
|
||||
resolve(stdout);
|
||||
});
|
||||
|
||||
proc.on('error', (err) => {
|
||||
reject(`Command failed: ${err}`);
|
||||
});
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user