From 5a9059be380853922f6b73daed56ab9f49dd8daa Mon Sep 17 00:00:00 2001 From: Greg Magolan Date: Mon, 20 Feb 2023 12:37:34 -0800 Subject: [PATCH] build: share Saucelabs browsers between karma test targets using background Saucelabs daemon and custom karma launcher (#49200) This upgrades the Saucelabs Bazel step on CI to use the more efficient Saucelabs daemon PR Close #49200 --- .circleci/config.yml | 14 +- .pullapprove.yml | 1 + BUILD.bazel | 18 + WORKSPACE | 17 + browser-providers.conf.d.ts | 20 + karma-js.conf.js | 39 +- package.json | 2 + packages/zone.js/test/karma_test.bzl | 1 + tools/defaults.bzl | 6 +- tools/saucelabs-daemon/BUILD.bazel | 12 + tools/saucelabs-daemon/README.md | 88 +++++ .../background-service/BUILD.bazel | 28 ++ .../background-service/cli.ts | 70 ++++ .../background-service/ipc.ts | 79 ++++ .../background-service/saucelabs-daemon.ts | 356 ++++++++++++++++++ tools/saucelabs-daemon/browser.ts | 25 ++ tools/saucelabs-daemon/ipc-defaults.ts | 9 + tools/saucelabs-daemon/ipc-messages.ts | 35 ++ tools/saucelabs-daemon/launcher/BUILD.bazel | 26 ++ tools/saucelabs-daemon/launcher/index.ts | 13 + tools/saucelabs-daemon/launcher/launcher.ts | 83 ++++ tools/saucelabs/BUILD.bazel | 15 - tools/saucelabs/README.md | 114 +----- tools/saucelabs/karma-saucelabs.mjs | 102 ----- yarn.lock | 31 +- 25 files changed, 940 insertions(+), 264 deletions(-) create mode 100644 browser-providers.conf.d.ts create mode 100644 tools/saucelabs-daemon/BUILD.bazel create mode 100644 tools/saucelabs-daemon/README.md create mode 100644 tools/saucelabs-daemon/background-service/BUILD.bazel create mode 100644 tools/saucelabs-daemon/background-service/cli.ts create mode 100644 tools/saucelabs-daemon/background-service/ipc.ts create mode 100644 tools/saucelabs-daemon/background-service/saucelabs-daemon.ts create mode 100644 tools/saucelabs-daemon/browser.ts create mode 100644 tools/saucelabs-daemon/ipc-defaults.ts create mode 100644 tools/saucelabs-daemon/ipc-messages.ts create mode 100644 tools/saucelabs-daemon/launcher/BUILD.bazel create mode 100644 tools/saucelabs-daemon/launcher/index.ts create mode 100644 tools/saucelabs-daemon/launcher/launcher.ts delete mode 100644 tools/saucelabs/karma-saucelabs.mjs diff --git a/.circleci/config.yml b/.circleci/config.yml index 46cf202740d..040dc47f863 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -308,18 +308,21 @@ jobs: # container for this job. This is necessary because we launch a lot of browsers concurrently # and therefore the tunnel and Karma need to process a lot of file requests and tests. resource_class: xlarge + environment: + NUMBER_OF_PARALLEL_BROWSERS: 2 steps: - custom_attach_workspace - init_environment - init_saucelabs_environment + - run: + name: Start Saucelabs daemon service + command: yarn bazel run //tools/saucelabs-daemon/background-service -- ${NUMBER_OF_PARALLEL_BROWSERS} + background: true - run: name: Run Bazel tests on Saucelabs - # See /tools/saucelabs/README.md for more info command: | - yarn bazel run //tools/saucelabs:sauce_service_setup TESTS=$(./node_modules/.bin/bazelisk query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "fixme-saucelabs", ...)') - yarn bazel test --config=saucelabs ${TESTS} - yarn bazel run //tools/saucelabs:sauce_service_stop + yarn bazel test --config=saucelabs --jobs=${NUMBER_OF_PARALLEL_BROWSERS} ${TESTS} no_output_timeout: 40m - notify_webhook_on_fail: webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL @@ -674,6 +677,9 @@ workflows: - build-npm-packages: requires: - setup + - saucelabs: + requires: + - setup - legacy-unit-tests-saucelabs: requires: - setup diff --git a/.pullapprove.yml b/.pullapprove.yml index e31fe909e49..d9999cfbfd6 100644 --- a/.pullapprove.yml +++ b/.pullapprove.yml @@ -1166,6 +1166,7 @@ groups: 'tools/legacy-saucelabs/**/{*,.*}', 'tools/rxjs/**/{*,.*}', 'tools/saucelabs/**/{*,.*}', + 'tools/saucelabs-daemon/**/{*,.*}', 'tools/symbol-extractor/**/{*,.*}', 'tools/testing/**/{*,.*}', 'tools/tslint/**/{*,.*}', diff --git a/BUILD.bazel b/BUILD.bazel index 5b10a1d2048..16bbabc014e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,4 +1,5 @@ load("//tools:defaults.bzl", "nodejs_binary") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") load("//:yarn.bzl", "YARN_PATH") package(default_visibility = ["//visibility:public"]) @@ -26,6 +27,14 @@ alias( actual = "//packages:tsconfig-build.json", ) +js_library( + name = "browser-providers", + srcs = [ + "browser-providers.conf.d.ts", + "browser-providers.conf.js", + ], +) + filegroup( name = "angularjs_scripts", srcs = [ @@ -61,3 +70,12 @@ nodejs_binary( "//integration:__subpackages__", ], ) + +alias( + name = "sauce_connect", + actual = select({ + "@npm//@angular/build-tooling/bazel/constraints:linux_x64": "@sauce_connect_linux_amd64//:bin/sc", + "@npm//@angular/build-tooling/bazel/constraints:macos_x64": "@sauce_connect_mac//:bin/sc", + "@npm//@angular/build-tooling/bazel/constraints:macos_arm64": "@sauce_connect_mac//:bin/sc", + }), +) diff --git a/WORKSPACE b/WORKSPACE index 6a2f3cd32cf..a50e9cfac9a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -215,3 +215,20 @@ register_toolchains( "@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_arm64_toolchain", "@npm//@angular/build-tooling/bazel/git-toolchain:git_windows_toolchain", ) + +# Fetch sauce connect (tool to open Saucelabs tunnel for Saucelabs browser tests) +http_archive( + name = "sauce_connect_linux_amd64", + build_file_content = """exports_files(["bin/sc"], visibility = ["//visibility:public"])""", + sha256 = "26b9c3630f441b47854b6032f7eca6f1d88d3f62e50ee44c27015d71a5155c36", + strip_prefix = "sc-4.8.2-linux", + url = "https://saucelabs.com/downloads/sc-4.8.2-linux.tar.gz", +) + +http_archive( + name = "sauce_connect_mac", + build_file_content = """exports_files(["bin/sc"], visibility = ["//visibility:public"])""", + sha256 = "28277ce81ef9ab84f5b87b526258920a8ead44789a5034346e872629bbf38089", + strip_prefix = "sc-4.8.2-osx", + url = "https://saucelabs.com/downloads/sc-4.8.2-osx.zip", +) diff --git a/browser-providers.conf.d.ts b/browser-providers.conf.d.ts new file mode 100644 index 00000000000..eae2795e973 --- /dev/null +++ b/browser-providers.conf.d.ts @@ -0,0 +1,20 @@ +type CustomLauncher = { + base: string; + browserName: string; + platformName: string; + platformVersion: string; + deviceName: string; + appiumVersion: string; + extendedDebugging: boolean; +} + +type CustomLaunchers = { + [string]: CustomLauncher; +}; + +type SauceAliases = { + [string]: string[]; +}; + +export const customLaunchers: CustomLaunchers; +export const sauceAliases: SauceAliases; diff --git a/karma-js.conf.js b/karma-js.conf.js index a469cb056d6..4c62eafdd0d 100644 --- a/karma-js.conf.js +++ b/karma-js.conf.js @@ -12,6 +12,13 @@ const {hostname} = require('os'); const seed = process.env.JASMINE_RANDOM_SEED || String(Math.random()).slice(-5); console.info(`Jasmine random seed: ${seed}`); +const isBazel = !!process.env.TEST_TARGET; + +if (!process.env.KARMA_WEB_TEST_MODE && isBazel && process.env.TEST_TARGET.includes('_saucelabs')) { + console.info(`Saucelabs target detected: ${process.env.TEST_TARGET}`); + process.env.KARMA_WEB_TEST_MODE = 'SL_REQUIRED'; +} + module.exports = function(config) { const conf = { frameworks: ['jasmine'], @@ -50,7 +57,6 @@ module.exports = function(config) { plugins: [ 'karma-jasmine', - 'karma-sauce-launcher', 'karma-chrome-launcher', 'karma-sourcemap-loader', ], @@ -99,20 +105,29 @@ module.exports = function(config) { set: () => {}, }); - if (process.env['SAUCE_TUNNEL_IDENTIFIER']) { - console.log(`SAUCE_TUNNEL_IDENTIFIER: ${process.env.SAUCE_TUNNEL_IDENTIFIER}`); + if (isBazel) { + // Add the custom Saucelabs daemon to the plugins + const saucelabsDaemonLauncher = require('./tools/saucelabs-daemon/launcher/index.cjs').default; + conf.plugins.push(saucelabsDaemonLauncher); + } else { + conf.plugins.push('karma-sauce-launcher'); - const tunnelIdentifier = process.env['SAUCE_TUNNEL_IDENTIFIER']; + if (process.env['SAUCE_TUNNEL_IDENTIFIER']) { + console.log(`SAUCE_TUNNEL_IDENTIFIER: ${process.env.SAUCE_TUNNEL_IDENTIFIER}`); - // Setup the Saucelabs plugin so that it can launch browsers using the proper tunnel. - conf.sauceLabs.build = tunnelIdentifier; - conf.sauceLabs.tunnelIdentifier = tunnelIdentifier; + const tunnelIdentifier = process.env['SAUCE_TUNNEL_IDENTIFIER']; - // Patch the `saucelabs` package so that `karma-sauce-launcher` does not attempt downloading - // the test logs from upstream and tries re-uploading them with the Karma enhanced details. - // This slows-down tests/browser restarting and can decrease stability. - // https://github.com/karma-runner/karma-sauce-launcher/blob/59b0c5c877448e064ad56449cd906743721c6b62/src/launcher/launcher.ts#L72-L79. - require('saucelabs').default.prototype.downloadJobAsset = () => Promise.resolve(''); + // Setup the Saucelabs plugin so that it can launch browsers using the proper tunnel. + conf.sauceLabs.build = tunnelIdentifier; + conf.sauceLabs.tunnelIdentifier = tunnelIdentifier; + + // Patch the `saucelabs` package so that `karma-sauce-launcher` does not attempt downloading + // the test logs from upstream and tries re-uploading them with the Karma enhanced details. + // This slows-down tests/browser restarting and can decrease stability. + // https://github.com/karma-runner/karma-sauce-launcher/blob/59b0c5c877448e064ad56449cd906743721c6b62/src/launcher/launcher.ts#L72-L79. + require('saucelabs').default.prototype.downloadJobAsset = () => + Promise.resolve(''); + } } // For SauceLabs jobs, we set up a domain which resolves to the machine which launched diff --git a/package.json b/package.json index 2d6ae8e9b89..783d1ee90bc 100644 --- a/package.json +++ b/package.json @@ -88,6 +88,7 @@ "@types/jasminewd2": "^2.0.8", "@types/node": "^16.11.7", "@types/selenium-webdriver": "3.0.7", + "@types/selenium-webdriver4": "npm:@types/selenium-webdriver@4.1.12", "@types/semver": "^7.3.4", "@types/shelljs": "^0.8.6", "@types/systemjs": "0.19.32", @@ -135,6 +136,7 @@ "rollup-plugin-sourcemaps": "^0.6.3", "rxjs": "^6.6.7", "selenium-webdriver": "3.5.0", + "selenium-webdriver4": "npm:selenium-webdriver@4.8.1", "semver-dsl": "^1.0.1", "shelljs": "^0.8.5", "source-map": "0.7.4", diff --git a/packages/zone.js/test/karma_test.bzl b/packages/zone.js/test/karma_test.bzl index 16dcd94d65c..18f660fe5a3 100644 --- a/packages/zone.js/test/karma_test.bzl +++ b/packages/zone.js/test/karma_test.bzl @@ -100,6 +100,7 @@ def karma_test(name, env_srcs, env_deps, env_entry_point, test_srcs, test_deps, configuration_env_vars = ["KARMA_WEB_TEST_MODE"], data = [ "//:browser-providers.conf.js", + "//tools/saucelabs-daemon/launcher:launcher_cjs", ], static_files = [ ":assets/sample.json", diff --git a/tools/defaults.bzl b/tools/defaults.bzl index 60e3d460aa3..8ddcd9c4354 100644 --- a/tools/defaults.bzl +++ b/tools/defaults.bzl @@ -321,7 +321,7 @@ def karma_web_test_suite( # Add a saucelabs target for Karma tests in `//packages/`. if native.package_name().startswith("packages/"): _karma_web_test( - name = "saucelabs_%s" % name, + name = "{}_saucelabs".format(name), # Default timeout is moderate (5min). This causes the test to be terminated while # Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers # unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use @@ -329,15 +329,13 @@ def karma_web_test_suite( timeout = "long", config_file = "//:karma-js.conf.js", deps = [ - "@npm//karma-sauce-launcher", ":%s_bundle" % name, ], data = data + [ "//:browser-providers.conf.js", + "//tools/saucelabs-daemon/launcher:launcher_cjs", ], - karma = "//tools/saucelabs:karma-saucelabs", tags = tags + [ - "exclusive", "manual", "no-remote-exec", "saucelabs", diff --git a/tools/saucelabs-daemon/BUILD.bazel b/tools/saucelabs-daemon/BUILD.bazel new file mode 100644 index 00000000000..1e8817ca543 --- /dev/null +++ b/tools/saucelabs-daemon/BUILD.bazel @@ -0,0 +1,12 @@ +load("//tools:defaults.bzl", "ts_library") + +package(default_visibility = ["//visibility:public"]) + +ts_library( + name = "saucelabs-daemon", + srcs = [ + "browser.ts", + "ipc-defaults.ts", + "ipc-messages.ts", + ], +) diff --git a/tools/saucelabs-daemon/README.md b/tools/saucelabs-daemon/README.md new file mode 100644 index 00000000000..04d78a1d43d --- /dev/null +++ b/tools/saucelabs-daemon/README.md @@ -0,0 +1,88 @@ +# Saucelabs testing with Bazel + +## Local testing + +1. Set up your `SAUCE_USERNAME`, `SAUCE_ACCESS_KEY` & `SAUCE_TUNNEL_IDENTIFIER` environment variables. +These are required. You can find the values for `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` in `/.circleci/env.sh`. `SAUCE_TUNNEL_IDENTIFIER` can be set to any unique value. + +If you are having trouble running Saucelabs tests locally you can contact [Joey Perrott](https://github.com/josephperrott) or [Paul Gschwendtner](https://github.com/devversion) for support. + +2. Once you have your environment variables set up, run the setup task in the root of the repo: + +``` bash +yarn bazel run //tools/saucelabs-daemon/background-service -- +``` + +This will start a daemon process that will connect to Saucelabs and provision browsers +once you start running your first test target. + +3. In another terminal, you can run a particular test target through SauceLabs by suffixing the target name with "_saucelabs". + +For example, `packages/core/test:test_web` becomes `packages/core/test:test_web_saucelabs`. + +``` +yarn bazel test //packages/core/test:test_web_saucelabs +``` + +## Additional test features + +To see the test output while the tests are running (as these are long tests), add the `--test_output=streamed` option. +Note, this option will also prevent bazel from using the test cache and will force the test to run. + +For running all Saucelabs tests in the project, `bazel query` is used to gather up all karma Saucelabs test labels because they are otherwise hidden by the `manual` tag. + +Running all karma tests in Saucelabs: + +Start the saucelabs-daemon background service in one terminal window: + +``` bash +yarn bazel run //tools/saucelabs-daemon/background-service -- +``` + +In a second terminal window, run all of the saucelabs test targets: + +``` bash +TESTS=$(./node_modules/.bin/bazelisk query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "fixme-saucelabs", ...)') +yarn bazel test --config=saucelabs --jobs= ${TESTS} +``` + +NB: The number of parallel Bazel tests specified by `--jobs=` must not exceed the number parallel browsers requested when starting the daemon. + +## Under the hood + +The `//tools/saucelabs-daemon/background-service` target does not start the Sauce Connect proxy at start-up, but instead listens for the start signal from the saucelabs karma launcher. +This signal is sent by saucelabs-daemon custom karma launcher `tools/saucelabs-daemon/launcher/launcher.ts`. +This is necessary as the Sauce Connect Proxy process must be started outside of `bazel test` as Bazel will automatically kill any processes spawned during a test when that tests completes, which would prevent the tunnel & provisioned browsers from being shared by multiple tests. + +The karma_web_test rule for saucelabs must have a few important tags: + +* `no-remote-exec` as they cannot be executed remotely since tests need to communicate with the daemon. +* `manual` so they are not automatically tested with `//...` +* `saucelabs` so that they can be easily gathered up for testing in a `bazel query` + +These are added automatically the by `karma_web_test_suite` macro in `tools/defaults.bzl`. + +## Debugging + +**Q: How do I get the tests to run on IE? I only see Chromium.** + +If you see something like this at the end of your test output, it means you're not actually running SauceLabs: + +``` +INFO: Build completed successfully, 43 total actions +/packages/core/test:test_web_chromium +``` + +This is a common error caused by forgetting to suffix your test target with "_saucelabs". + +For example, `/packages/core/test:test_web` becomes `/packages/core/test:test_web_saucelabs`. + +**Q: How can I tell that the SauceLabs connection was successfully made?** + +There is a dashboard at saucelabs.com where you can see active tunnel connections (Angular has an account). +As soon as you actually run the test target (not after the setup task), you should see an active tunnel connection under the SAUCE_TUNNEL_IDENTIFICATION_KEY you entered. +If a tunnel connection is not there, you are not actually connecting with SauceLabs. + +Note: It may *look* like the tests are running because of the Bazel output. +The progress Bazel is showing does not mean that SauceLabs is connected. +If the tests are actually running, you should see the "..." test report for passing tests. diff --git a/tools/saucelabs-daemon/background-service/BUILD.bazel b/tools/saucelabs-daemon/background-service/BUILD.bazel new file mode 100644 index 00000000000..c877cbc36c1 --- /dev/null +++ b/tools/saucelabs-daemon/background-service/BUILD.bazel @@ -0,0 +1,28 @@ +load("//tools:defaults.bzl", "nodejs_binary", "ts_library") + +package(default_visibility = ["//visibility:public"]) + +ts_library( + name = "background-service_lib", + srcs = glob(["*.ts"]), + deps = [ + "//:browser-providers", + "//tools/saucelabs-daemon", + "@npm//@types/node", + "@npm//@types/selenium-webdriver4", + "@npm//chalk", + "@npm//selenium-webdriver4", + ], +) + +nodejs_binary( + name = "background-service", + data = [ + ":background-service_lib", + "//:sauce_connect", + ], + entry_point = ":cli.ts", + templated_args = [ + "$(rootpath //:sauce_connect)", + ], +) diff --git a/tools/saucelabs-daemon/background-service/cli.ts b/tools/saucelabs-daemon/background-service/cli.ts new file mode 100644 index 00000000000..45ba352172b --- /dev/null +++ b/tools/saucelabs-daemon/background-service/cli.ts @@ -0,0 +1,70 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {customLaunchers} from '../../../browser-providers.conf'; +import {Browser} from '../browser'; + +import {SaucelabsDaemon} from './saucelabs-daemon'; + +const args = process.argv.slice(2); +const username = process.env.SAUCE_USERNAME; +const accessKey = process.env.SAUCE_ACCESS_KEY; +const tunnelIdentifier = process.env.SAUCE_TUNNEL_IDENTIFIER; + +const buildName = process.env.CIRCLECI ? `circleci-${process.env.CIRCLE_BUILD_NUM}` : 'localdev'; + +if (!username || !accessKey) { + throw Error('Please set the `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` variables.'); +} + +if (!tunnelIdentifier) { + throw Error('No tunnel set up. Please set the `SAUCE_TUNNEL_IDENTIFIER` variable.'); +} + +if (!buildName) { + throw Error('No build name specified.'); +} + +// First argument is the path to the sauce connect binary. This argument is templated into the bazel +// binary. +if (args.length < 1) { + throw Error(`Path to the sauce connect binary expected as first argument`); +} +const sauceConnect = args[0]; + +// Second argument is the number of parallel browsers to start. This argument is user supplied and +// required. +if (args.length != 2) { + throw Error(`Please specify the number of parallel browsers to start on the command line.`); +} +const parallelExecutions = parseInt(args[1]); +if (!parallelExecutions) { + throw Error(`Please specify a non-zero number of parallel browsers to start.`); +} + +const browserInstances: Browser[] = []; +for (let i = 0; i < parallelExecutions; i++) { + browserInstances.push(...Object.values(customLaunchers) as any); +} + +// Start the daemon and launch the given browser +const daemon = new SaucelabsDaemon( + username, + accessKey, + process.env.CIRCLE_BUILD_NUM!, + browserInstances, + sauceConnect, + {tunnelIdentifier}, +); + +if (args.includes('--connect')) { + daemon.connect().catch((err) => { + console.error(`Failed to connect to Saucelabs: ${err}`); + process.exit(1); + }); +} diff --git a/tools/saucelabs-daemon/background-service/ipc.ts b/tools/saucelabs-daemon/background-service/ipc.ts new file mode 100644 index 00000000000..a62fbd48aad --- /dev/null +++ b/tools/saucelabs-daemon/background-service/ipc.ts @@ -0,0 +1,79 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {createServer, Server, Socket} from 'net'; + +import {IPC_PORT} from '../ipc-defaults'; +import {BackgroundServiceReceiveMessages, InternalErrorMessage, NoAvailableBrowserMessage} from '../ipc-messages'; + +import {SaucelabsDaemon} from './saucelabs-daemon'; + +let nextSocketId = 0; + +export class IpcServer { + private readonly _server: Server; + private _connections = new Map(); + + constructor(private _service: SaucelabsDaemon) { + this._server = createServer(this._connectionHandler.bind(this)); + this._server.listen(IPC_PORT, () => console.info('Daemon IPC server listening.')); + } + + private _connectionHandler(socket: Socket) { + const socketId = nextSocketId++; + this._connections.set(socketId, socket); + socket.on('data', b => { + this._processMessage( + socket, + socketId, + JSON.parse(b.toString()) as BackgroundServiceReceiveMessages, + ) + .catch((err) => { + console.error(err); + this._sendInternalError(socket, err.toString()); + }); + }); + } + + private async _processMessage( + socket: Socket, + socketId: number, + message: BackgroundServiceReceiveMessages, + ) { + switch (message.type) { + case 'start-test': + console.debug(`Requesting test browser: SID#${socketId}: ${message.testDescription}`); + const started = await this._service.startTest({ + testId: socketId, + pageUrl: message.url, + requestedBrowserId: message.browserId, + }); + if (!started) { + console.debug(' > Browser not available.'); + this._sendUnavailableBrowserMessage(socket); + } else { + console.debug(' > Browser available. Test can start.'); + } + break; + case 'end-test': + console.debug(`Ending tests for SID#${socketId}`); + this._service.endTest(socketId); + break; + default: + throw new Error(`Unsupported msg type: ${(message as any).type}`); + } + } + + private _sendUnavailableBrowserMessage(socket: Socket) { + socket.write(JSON.stringify(new NoAvailableBrowserMessage())); + } + + private _sendInternalError(socket: Socket, msg: string) { + socket.write(JSON.stringify(new InternalErrorMessage(msg))); + } +} diff --git a/tools/saucelabs-daemon/background-service/saucelabs-daemon.ts b/tools/saucelabs-daemon/background-service/saucelabs-daemon.ts new file mode 100644 index 00000000000..02d2bb2b943 --- /dev/null +++ b/tools/saucelabs-daemon/background-service/saucelabs-daemon.ts @@ -0,0 +1,356 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import chalk from 'chalk'; +import {spawn} from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import {Builder, WebDriver} from 'selenium-webdriver4'; + +import {Browser, getUniqueId} from '../browser'; + +import {IpcServer} from './ipc'; + +const defaultCapabilities = { + recordVideo: false, + recordScreenshots: false, + idleTimeout: 90, + // These represent the maximum values supported by Saucelabs. + // See: https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options + commandTimeout: 600, + maxDuration: 10800, + extendedDebugging: true, +}; + +interface RemoteBrowser { + id: string; + state: 'acquired'|'free'|'launching'; + driver: WebDriver|null; +} + +interface BrowserTest { + testId: number; + pageUrl: string; + requestedBrowserId: string; +} + +export class SaucelabsDaemon { + /** + * Map of browsers and their pending tests. If a browser is acquired on the + * remote selenium server, the browser is not immediately ready. If the browser + * becomes active, the pending tests will be started. + */ + private _pendingTests = new Map(); + + /** List of active browsers that are managed by the daemon. */ + private _activeBrowsers = new Set(); + + /** Map that contains test ids with their acquired browser. */ + private _runningTests = new Map(); + + /** Server used for communication with the Karma launcher. */ + private _server = new IpcServer(this); + + /** Base selenium capabilities that will be added to each browser. */ + private _baseCapabilities = {...defaultCapabilities, ...this._userCapabilities}; + + /** Id of the keep alive interval that ensures no remote browsers time out. */ + private _keepAliveIntervalId: NodeJS.Timeout|null = null; + + /* Have we connected to Saucelabs or are in the process of connecting? */ + private _connection: Promise|undefined = undefined; + + constructor( + private _username: string, + private _accessKey: string, + private _buildName: string, + private _browsers: Browser[], + private _sauceConnect: string, + private _userCapabilities: object = {}, + ) { + // Starts the keep alive loop for all active browsers, running every 15 seconds. + this._keepAliveIntervalId = setInterval(() => this._keepAliveBrowsers(), 15_000); + } + + /** + * Connects the daemon to Saucelabs. + * This is typically done when the first test is started so that no connection is made + * if all tests are cache hits. + */ + async connect() { + if (!this._connection) { + this._connection = this._connect(); + } + return this._connection; + } + + /** + * Quits all active browsers. + */ + async quitAllBrowsers() { + let quitBrowsers: Promise[] = []; + this._activeBrowsers.forEach(b => { + if (b.driver) { + quitBrowsers.push(b.driver.quit()); + } + }); + await Promise.all(quitBrowsers); + this._activeBrowsers.clear(); + this._runningTests.clear(); + this._pendingTests.clear(); + } + + /** + * Shutdown the daemon. + * + * Awaits the shutdown of browsers. + */ + async shutdown() { + await this.quitAllBrowsers(); + if (this._keepAliveIntervalId !== null) { + clearInterval(this._keepAliveIntervalId); + } + } + + /** + * End a browser test if it is running. + */ + endTest(testId: number) { + if (!this._runningTests.has(testId)) { + return; + } + + const browser = this._runningTests.get(testId)!; + browser.state = 'free'; + this._runningTests.delete(testId); + } + + /** + * Start a test on a remote browser. + * + * If the daemon has not yet initiated the saucelabs tunnel creation and browser launching then + * this initiates that process and awaits until it succeeds or fails. + * + * If the daemon has already initiated the saucelabs tunnel creation and browser launching then + * but it is not yet complete then this until it succeeds or fails. + * + * If all matching browsers are occupied with other tests then test is not run. Promise returns + * false. + * + * If there is a matching browser that are still launching then the test is scheduled to run + * on the browser when it is ready. Promise returns true. + * + * If there is a matching browser that is available the test it started. Promise returns true. + */ + async startTest(test: BrowserTest): Promise { + await this.connect(); + + const browsers = this._findMatchingBrowsers(test.requestedBrowserId); + if (!browsers.length) { + return false; + } + + // Find the first available browser and start the test. + for (const browser of browsers) { + // If the browser is acquired, continue searching. + if (browser.state === 'acquired') { + continue; + } + + // If the browser is launching, check if it can be pre-claimed so that + // the test starts once the browser is ready. If it's already claimed, + // continue searching. + if (browser.state === 'launching') { + if (this._pendingTests.has(browser)) { + continue; + } else { + this._pendingTests.set(browser, test); + return true; + } + } + + // TS21225: [tsetse] All Promises in async functions must either be awaited or used in an + // expression + const _ = this._startBrowserTest(browser, test); + return true; + } + + return false; + } + + /** + * @internal + * Connects the daemon to Saucelabs. + * This is typically done when the first test is started so that no connection is made + * if all tests are cache hits. + **/ + async _connect() { + await this._openSauceConnectTunnel(); + await this._launchBrowsers(); + } + + /** + * @internal + * Establishes the Saucelabs connect tunnel. + **/ + async _openSauceConnectTunnel() { + console.debug('Starting sauce connect tunnel...'); + + const tmpFolder = await fs.mkdtemp('saucelabs-daemon-'); + + await new Promise((resolve, reject) => { + // First we need to start the sauce connect tunnel + const sauceConnectArgs = [ + '--readyfile', + `${tmpFolder}/readyfile`, + '--pidfile', + `${tmpFolder}/pidfile`, + '--tunnel-identifier', + (this._userCapabilities as any).tunnelIdentifier || path.basename(tmpFolder), + ]; + const sc = spawn(this._sauceConnect, sauceConnectArgs); + + sc.stdout!.on('data', (data) => { + if (data.includes('Sauce Connect is up, you may start your tests.')) { + resolve(); + } + }); + + sc.on('close', (code) => { + reject(new Error(`sauce connect closed all stdio with code ${code}`)); + }); + + sc.on('exit', (code) => { + reject(new Error(`sauce connect exited with code ${code}`)); + }); + }); + + console.debug('Starting sauce connect tunnel established'); + } + + /** + * @internal + * Launches all browsers. If there are pending tests waiting for a particular browser to launch + * before they can start, those tests are started once the browser is launched. + **/ + async _launchBrowsers() { + console.debug('Launching browsers...'); + + // Once the tunnel is established we can launch browsers + await Promise.all( + this._browsers.map(async (browser, id) => { + const browserId = getUniqueId(browser); + const launched: RemoteBrowser = {state: 'launching', driver: null, id: browserId}; + const browserDescription = `${this._buildName} - ${browser.browserName} - #${id + 1}`; + + const capabilities: any = { + 'browserName': browser.browserName, + 'sauce:options': {...this._baseCapabilities, ...browser}, + }; + + // Set `sauce:options` to provide a build name for the remote browser instances. + // This helps with debugging. Also ensures the W3C protocol is used. + // See. https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options + capabilities['sauce:options']['name'] = browserDescription; + capabilities['sauce:options']['build'] = browserDescription; + + console.debug( + `Capabilities for ${browser.browserName}:`, JSON.stringify(capabilities, null, 2)); + console.debug(` > Browser-ID: `, browserId); + console.debug(` > Browser-Description: `, browserDescription); + + // Keep track of the launched browser. We do this before it even completed the + // launch as we can then handle scheduled tests when the browser is still launching. + this._activeBrowsers.add(launched); + + // See the following link for public API of the selenium server. + // https://wiki.saucelabs.com/display/DOCS/Instant+Selenium+Node.js+Tests + const driver = await new Builder() + .withCapabilities(capabilities) + .usingServer( + `http://${this._username}:${ + this._accessKey}@ondemand.saucelabs.com:80/wd/hub`, + ) + .build(); + + // Only wait 30 seconds to load a test page. + await driver.manage().setTimeouts({pageLoad: 30000}); + + const sessionId = (await driver.getSession()).getId(); + console.info( + chalk.yellow( + `Started browser ${browser.browserName} on Saucelabs: ` + + `https://saucelabs.com/tests/${sessionId}`, + ), + ); + + // Mark the browser as available after launch completion. + launched.state = 'free'; + launched.driver = driver; + + // If a test has been scheduled before the browser completed launching, run + // it now given that the browser is ready now. + if (this._pendingTests.has(launched)) { + // TS21225: [tsetse] All Promises in async functions must either be awaited or used in + // an expression + const _ = this._startBrowserTest(launched, this._pendingTests.get(launched)!); + } + }), + ); + } + + /** + * @internal + * Starts a browser test on a browser. + * This sets the browser's state to "acquired" and navigates the browser to the test URL. + **/ + private async _startBrowserTest(browser: RemoteBrowser, test: BrowserTest) { + this._runningTests.set(test.testId, browser); + browser.state = 'acquired'; + + try { + console.debug(`Opening test url for #${test.testId}: ${test.pageUrl}`); + await browser.driver!.get(test.pageUrl); + const pageTitle = await browser.driver!.getTitle(); + console.debug(`Test page loaded for #${test.testId}: "${pageTitle}".`); + } catch (e) { + console.error('Could not start browser test with id', test.testId, test.pageUrl); + } + } + + /** + * @internal + * Given a browserId, returns a list of matching browsers from the list of active browsers. + **/ + private _findMatchingBrowsers(browserId: string): RemoteBrowser[] { + const browsers: RemoteBrowser[] = []; + this._activeBrowsers.forEach(b => { + if (b.id === browserId) { + browsers.push(b); + } + }); + return browsers; + } + + /** + * @internal + * Implements a heartbeat for Saucelabs browsers as they could end up not receiving any + * commands when the daemon is unused (i.e. Bazel takes a while to start tests). + * https://saucelabs.com/blog/selenium-tips-how-to-coordinate-multiple-browsers-in-sauce-ondemand. + **/ + private async _keepAliveBrowsers() { + const pendingCommands: Promise[] = []; + this._activeBrowsers.forEach(b => { + if (b.driver !== null) { + pendingCommands.push(b.driver.getTitle() as Promise); + } + }); + await Promise.all(pendingCommands); + console.debug(`${Date().toLocaleString()}: Refreshed ${pendingCommands.length} browsers.`); + } +} diff --git a/tools/saucelabs-daemon/browser.ts b/tools/saucelabs-daemon/browser.ts new file mode 100644 index 00000000000..e8c172de304 --- /dev/null +++ b/tools/saucelabs-daemon/browser.ts @@ -0,0 +1,25 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +/** Definition of a test browser. */ +export interface Browser { + browserName: string; + browserVersion?: string; + platformName?: string; + platformVersion?: string; + deviceName?: string; +} + +/** + * Gets a unique id for the specified browser. This id can be shared + * across the background service and launcher using IPC. + */ +export function getUniqueId(browser: Browser): string { + let result = Object.keys(browser).sort().map(key => `${key}=${browser[key as keyof Browser]}`); + return result.join(':'); +} diff --git a/tools/saucelabs-daemon/ipc-defaults.ts b/tools/saucelabs-daemon/ipc-defaults.ts new file mode 100644 index 00000000000..63fd8a60baa --- /dev/null +++ b/tools/saucelabs-daemon/ipc-defaults.ts @@ -0,0 +1,9 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +export const IPC_PORT = 5324; diff --git a/tools/saucelabs-daemon/ipc-messages.ts b/tools/saucelabs-daemon/ipc-messages.ts new file mode 100644 index 00000000000..26184698efc --- /dev/null +++ b/tools/saucelabs-daemon/ipc-messages.ts @@ -0,0 +1,35 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +/** Message that can be sent to the daemon to start a given test. */ +export class StartTestMessage { + readonly type = 'start-test'; + constructor(public url: string, public browserId: string, public testDescription: string) {} +} + +/** Message that can be sent to the daemon if a test completed. */ +export class EndTestMessage { + readonly type = 'end-test'; +} + +/** Message being sent from the daemon if a request browser is not available. */ +export class NoAvailableBrowserMessage { + readonly type = 'browser-not-ready'; +} + +/** Message that indicates an internal error in background service. */ +export class InternalErrorMessage { + readonly type = 'internal-error'; + constructor(public msg: string) {} +} + +/** Type of messages the background service can receive. */ +export type BackgroundServiceReceiveMessages = StartTestMessage|EndTestMessage; + +/** Type of messages the background services can send to clients. */ +export type BackgroundServiceSendMessages = NoAvailableBrowserMessage|InternalErrorMessage; diff --git a/tools/saucelabs-daemon/launcher/BUILD.bazel b/tools/saucelabs-daemon/launcher/BUILD.bazel new file mode 100644 index 00000000000..dda2e14a036 --- /dev/null +++ b/tools/saucelabs-daemon/launcher/BUILD.bazel @@ -0,0 +1,26 @@ +load("//tools:defaults.bzl", "esbuild", "ts_library") + +package(default_visibility = ["//visibility:public"]) + +ts_library( + name = "launcher", + srcs = [ + "index.ts", + "launcher.ts", + ], + deps = [ + "//tools/saucelabs-daemon", + "@npm//@types/node", + ], +) + +# We need a commonjs version of the launcher that can be required from the +# root karma-js.conf.js. +esbuild( + name = "launcher_cjs", + entry_point = "index.ts", + format = "cjs", + output = "index.cjs", + platform = "node", + deps = [":launcher"], +) diff --git a/tools/saucelabs-daemon/launcher/index.ts b/tools/saucelabs-daemon/launcher/index.ts new file mode 100644 index 00000000000..69eceac95a2 --- /dev/null +++ b/tools/saucelabs-daemon/launcher/index.ts @@ -0,0 +1,13 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {SaucelabsLauncher} from './launcher'; + +export default { + 'launcher:SauceLabs': ['type', SaucelabsLauncher], +}; diff --git a/tools/saucelabs-daemon/launcher/launcher.ts b/tools/saucelabs-daemon/launcher/launcher.ts new file mode 100644 index 00000000000..848dadf6efe --- /dev/null +++ b/tools/saucelabs-daemon/launcher/launcher.ts @@ -0,0 +1,83 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {createConnection, Socket} from 'net'; + +import {Browser, getUniqueId} from '../browser'; +import {IPC_PORT} from '../ipc-defaults'; +import {BackgroundServiceSendMessages, EndTestMessage, StartTestMessage} from '../ipc-messages'; + +export function SaucelabsLauncher( + this: any, + args: Browser, + config: unknown, + logger: any, + baseLauncherDecorator: any, + captureTimeoutLauncherDecorator: any, + retryLauncherDecorator: any, +) { + // Apply base class mixins. This would be nice to have typed, but this is a low-priority now. + baseLauncherDecorator(this); + captureTimeoutLauncherDecorator(this); + retryLauncherDecorator(this); + + const log = logger.create('SaucelabsLauncher'); + const browserDisplayName = args.browserName + + (args.browserVersion ? ' ' + args.browserVersion : '') + + (args.platformName ? ' (' + args.platformName + ')' : ''); + const testSuiteDescription = process.env.TEST_TARGET ?? ''; + + let daemonConnection: Socket|null = null; + + // Setup Browser name that will be printed out by Karma. + this.name = browserDisplayName + ' on SauceLabs (daemon)'; + + this.on('start', (pageUrl: string) => { + daemonConnection = createConnection({port: IPC_PORT}, () => _startBrowserTest(pageUrl, args)); + + daemonConnection.on( + 'data', + b => _processMessage(JSON.parse(b.toString()) as BackgroundServiceSendMessages), + ); + daemonConnection.on('error', err => { + log.error(err); + + // Notify karma about the failure. + this._done('failure'); + }); + }); + + this.on('kill', async (doneFn: () => void) => { + _endBrowserTest(); + daemonConnection?.end(); + doneFn(); + }); + + const _processMessage = (message: BackgroundServiceSendMessages) => { + switch (message.type) { + case 'browser-not-ready': + log.error( + 'Browser %s is not ready in the Saucelabs background service.', + browserDisplayName, + ); + this._done('failure'); + } + }; + + const _startBrowserTest = (pageUrl: string, browser: Browser) => { + log.info('Starting browser %s test in daemon with URL: %s', browserDisplayName, pageUrl); + daemonConnection!.write( + JSON.stringify(new StartTestMessage(pageUrl, getUniqueId(browser), testSuiteDescription)), + ); + }; + + const _endBrowserTest = () => { + log.info('Test for browser %s completed', browserDisplayName); + daemonConnection!.write(JSON.stringify(new EndTestMessage())); + }; +} diff --git a/tools/saucelabs/BUILD.bazel b/tools/saucelabs/BUILD.bazel index fda3e22cd3b..b6af256e85f 100644 --- a/tools/saucelabs/BUILD.bazel +++ b/tools/saucelabs/BUILD.bazel @@ -1,5 +1,3 @@ -load("//tools:defaults.bzl", "nodejs_binary") - package(default_visibility = ["//visibility:public"]) sh_binary( @@ -50,16 +48,3 @@ sh_binary( args = ["log"], data = ["@npm//sauce-connect"], ) - -nodejs_binary( - name = "karma-saucelabs", - data = [ - "sauce-service.sh", - "@npm//@bazel/runfiles", - "@npm//karma", - "@npm//sauce-connect", - "@npm//shelljs", - ], - entry_point = "karma-saucelabs.mjs", - templated_args = ["$(rootpath sauce-service.sh)"], -) diff --git a/tools/saucelabs/README.md b/tools/saucelabs/README.md index dd591d6c0fd..d5df5af012c 100644 --- a/tools/saucelabs/README.md +++ b/tools/saucelabs/README.md @@ -1,115 +1,3 @@ # Saucelabs testing with Bazel -## Local testing - -1. Set up your `SAUCE_USERNAME`, `SAUCE_ACCESS_KEY` & `SAUCE_TUNNEL_IDENTIFIER` environment variables. -These are required. You can find the values for `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` in `/.circleci/env.sh`. `SAUCE_TUNNEL_IDENTIFIER` can be set to any unique value. - -If you are having trouble running Saucelabs tests locally you can contact [Joey Perrott](https://github.com/josephperrott) or [Greg Magolan](https://github.com/gregmagolan) for support. - -1. On OSX and Windows, you will also need to set `SAUCE_CONNECT` to the path of your `sc` binary (Sauce Connect Proxy). -You will have to download Sauce Connect Proxy if you don't already have it downloaded. -It's available on the SauceLabs website [here](https://wiki.saucelabs.com/display/DOCS/Downloading+Sauce+Connect+Proxy). -Unzip it and point the SAUCE_CONNECT env variable to the `sc` binary. - -``` -export SAUCE_CONNECT=/{path_to_sc}/bin/sc -``` - -Note: it will not work to use the Sauce Connect that's already in node_modules unless you are using Linux. -Download the one above for other platforms. - -3. Once you have your environment variables set up, run the setup task in the root of the repo: - -``` bash -yarn bazel run //tools/saucelabs:sauce_service_setup -``` - -4. You can run a particular test target through SauceLabs by prefixing the target name with "saucelabs_" and adding the `--config=saucelabs` option. -For example, `packages/core/test:test_web` becomes `packages/core/test:saucelabs_test_web`. - -``` -yarn bazel test //packages/core/test:saucelabs_test_web --config=saucelabs -``` - -5. Sauce service log may be tailed or dumped with the following targets: - -``` bash -yarn bazel run //tools/saucelabs:sauce_service_tail -yarn bazel run //tools/saucelabs:sauce_service_log -``` - - -## Additional test features - -To see the test output while the tests are running (as these are long tests), add the `--test_output=streamed` option. -Note, this option will also prevent bazel from using the test cache and will force the test to run. - -`bazel query` is required gather up all karma saucelabs test labels so they can be run in one command as they are tagged `manual`. - -Running all karma tests in Saucelabs: - -``` bash -yarn bazel run //tools/saucelabs:sauce_service_setup -TESTS=$(./node_modules/.bin/bazelisk query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "fixme-saucelabs", ...)') -yarn bazel test --config=saucelabs ${TESTS} -``` - -## Under the hood - -The `//tools/saucelabs:sauce_service_setup` target does not start the Sauce Connect proxy but it does start the process that then listens for the start signal from the service manager script. -This signal is sent by the karma wrapper script `//tools/saucelabs:karma-saucelabs` which calls `./tools/saucelabs/sauce-service.sh start`. -This is necessary as the Sauce Connect Proxy process must be started outside of `bazel test` as Bazel will automatically kill any processes spawned during a test when that tests completes, which would prevent the tunnel from being shared by multiple tests. - -The karma_web_test rule is to test with saucelabs with a modified `karma` attribute set to -`//tools/saucelabs:karma-saucelabs`. This runs the `/tools/saucelabs/karma-saucelabs.js` wrapper -script which configures the saucelabs environment and starts Sauce Connect before running karma. - -For example, - -``` python -karma_web_test( - name = "saucelabs_core_acceptance_tests", - timeout = "long", - karma = "//tools/saucelabs:karma-saucelabs", - tags = [ - "exclusive", - "manual", - "no-remote-exec", - "saucelabs", - ], - deps = [ - "//packages/core/test/acceptance:acceptance_lib", - ], -) -``` - -These saucelabs targets must have a few important tags: -* `no-remote-exec` as they cannot be executed remotely since they require a local Sauce Connect process -* `manual` so they are not automatically tested with `//...` -* `exclusive` as they must be run serially in order to not over-provision Saucelabs browsers -* `saucelabs` so that they can be easily gathered up for testing in a `bazel query` - -## Debugging - -**Q: How do I get the tests to run on IE? I only see Chromium.** - -If you see something like this at the end of your test output, it means you're not actually running SauceLabs: - -``` -INFO: Build completed successfully, 43 total actions -/packages/core/test:test_web_chromium -``` - -This is a common error caused by forgetting to prefix your test target with "saucelabs_". -For example, `/packages/core/test:test_web` becomes `/packages/core/test:saucelabs_test_web`. - -**Q: How can I tell that the SauceLabs connection was successfully made?** - -There is a dashboard at saucelabs.com where you can see active tunnel connections (Angular has an account). -As soon as you actually run the test target (not after the setup task), you should see an active tunnel connection under the SAUCE_TUNNEL_IDENTIFICATION_KEY you entered. -If a tunnel connection is not there, you are not actually connecting with SauceLabs. - -Note: It may *look* like the tests are running because of the Bazel output. -The progress Bazel is showing does not mean that SauceLabs is connected. -If the tests are actually running, you should see the "..." test report for passing tests. +Moved to tools/saucelabs-daemon/README.md. diff --git a/tools/saucelabs/karma-saucelabs.mjs b/tools/saucelabs/karma-saucelabs.mjs deleted file mode 100644 index f51d91cc3c4..00000000000 --- a/tools/saucelabs/karma-saucelabs.mjs +++ /dev/null @@ -1,102 +0,0 @@ -/** - * @license - * Copyright Google LLC All Rights Reserved. - * - * Use of this source code is governed by an MIT-style license that can be - * found in the LICENSE file at https://angular.io/license - */ - -'use strict'; - -import shell from 'shelljs'; -import {runfiles} from '@bazel/runfiles'; -import fs from 'fs'; -import childProcess from 'child_process'; - -const karmaBin = runfiles.resolve('npm/node_modules/karma/bin/karma'); -const sauceService = runfiles.resolveWorkspaceRelative(process.argv[2]); - -process.argv = [process.argv[0], karmaBin, ...process.argv.splice(3)]; - -main().catch((e) => { - console.error(e); - process.exitCode = 1; -}); - -async function main() { - console.error(`Setting up environment for SauceLabs karma tests...`); - // KARMA_WEB_TEST_MODE is set which informs /karma-js.conf.js that it should - // run the test with the karma saucelabs launcher - process.env['KARMA_WEB_TEST_MODE'] = 'SL_REQUIRED'; - // Saucelabs parameters read from a temporary file that is created by the `sauce-service`. This - // will be `null` if the test runs locally without the `sauce-service` being started. - const saucelabsParams = readLocalSauceConnectParams(); - // Setup required SAUCE_* env if they are not already set - if ( - !process.env['SAUCE_USERNAME'] || - !process.env['SAUCE_ACCESS_KEY'] || - !process.env['SAUCE_TUNNEL_IDENTIFIER'] - ) { - // We print a helpful error message below if the required Saucelabs parameters have not - // been specified in test environment, and the `sauce-service` params file has not been - // created either. - if (saucelabsParams === null) { - console.error(` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!!! Make sure that you have run "yarn bazel run //tools/saucelabs:sauce_service_setup" -!!! (or "./tools/saucelabs/sauce-service.sh setup") before the test target. Alternately -!!! you can provide the required SAUCE_* environment variables (SAUCE_USERNAME, SAUCE_ACCESS_KEY & -!!! SAUCE_TUNNEL_IDENTIFIER) to the test with --test_env or --define but this may prevent bazel from -!!! using cached test results. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`); - process.exit(1); - } - process.env['SAUCE_USERNAME'] = saucelabsParams.SAUCE_USERNAME; - process.env['SAUCE_ACCESS_KEY'] = saucelabsParams.SAUCE_ACCESS_KEY; - process.env['SAUCE_TUNNEL_IDENTIFIER'] = saucelabsParams.SAUCE_TUNNEL_IDENTIFIER; - process.env['SAUCE_LOCALHOST_ALIAS_DOMAIN'] = saucelabsParams.SAUCE_LOCALHOST_ALIAS_DOMAIN; - } - - // Pass through the optional `SAUCE_LOCALHOST_ALIAS_DOMAIN` environment variable. The - // variable is usually specified on CI, but is not required for testing with Saucelabs. - if (!process.env['SAUCE_LOCALHOST_ALIAS_DOMAIN'] && saucelabsParams !== null) { - process.env['SAUCE_LOCALHOST_ALIAS_DOMAIN'] = saucelabsParams.SAUCE_LOCALHOST_ALIAS_DOMAIN; - } - - const scStart = `${sauceService} start-ready-wait`; - console.error(`Starting SauceConnect (${scStart})...`); - const result = shell.exec(scStart).code; - if (result !== 0) { - throw new Error(`Starting SauceConnect failed with code ${result}`); - } - - console.error(`Launching karma ${karmaBin}...`); - - await launchNodeBinaryAndWait(karmaBin, process.argv.slice(2)); -} - -function launchNodeBinaryAndWait(entryPointPath, args) { - return new Promise((resolve, reject) => { - const proc = childProcess.fork(entryPointPath, args, {stdio: 'inherit'}); - proc.on('error', (err) => reject(err)); - proc.on('close', (code) => - code !== 0 - ? reject(new Error(`Script "${entryPointPath}" exited with non-zero status code: ${code}`)) - : resolve() - ); - }); -} - -function readLocalSauceConnectParams() { - try { - // The following path comes from /tools/saucelabs/sauce-service.sh. - // We setup the required saucelabs environment variables here for the karma test - // from a json file under /tmp/angular/sauce-service so that we don't break the - // test cache with a changing SAUCE_TUNNEL_IDENTIFIER provided through --test_env - return JSON.parse( - fs.readFileSync('/tmp/angular/sauce-service/sauce-connect-params.json', 'utf8') - ); - } catch { - return null; - } -} diff --git a/yarn.lock b/yarn.lock index 82910cdd8cb..70c4811f899 100644 --- a/yarn.lock +++ b/yarn.lock @@ -237,7 +237,6 @@ "@angular/build-tooling@https://github.com/angular/dev-infra-private-build-tooling-builds.git#7dad055464ea9847e4870b9e3baad1f0c417bdf7": version "0.0.0-5f06c4774df908ed69e1441f4ec63b898acf0c68" - uid "7dad055464ea9847e4870b9e3baad1f0c417bdf7" resolved "https://github.com/angular/dev-infra-private-build-tooling-builds.git#7dad055464ea9847e4870b9e3baad1f0c417bdf7" dependencies: "@angular-devkit/build-angular" "16.0.0-rc.0" @@ -371,7 +370,6 @@ "@angular/ng-dev@https://github.com/angular/dev-infra-private-ng-dev-builds.git#231bc6ee07c0f799d28f201a4e6d68ae31d88cfa": version "0.0.0-5f06c4774df908ed69e1441f4ec63b898acf0c68" - uid "231bc6ee07c0f799d28f201a4e6d68ae31d88cfa" resolved "https://github.com/angular/dev-infra-private-ng-dev-builds.git#231bc6ee07c0f799d28f201a4e6d68ae31d88cfa" dependencies: "@yarnpkg/lockfile" "^1.1.0" @@ -3672,6 +3670,13 @@ "@types/glob" "*" "@types/node" "*" +"@types/selenium-webdriver4@npm:@types/selenium-webdriver@4.1.12": + version "4.1.12" + resolved "https://registry.yarnpkg.com/@types/selenium-webdriver/-/selenium-webdriver-4.1.12.tgz#a2042f6bef044c48ca7a8b4caede4928242271a9" + integrity sha512-3DxuYHEbzIyUtuICRtxXtJomdOOkBeCbkXAlAPm/h8ArIn/ArWNnVSC6tIj/ECZ/sdxBoI0F/wuwcpxBew3hGw== + dependencies: + "@types/ws" "*" + "@types/selenium-webdriver@3.0.7": version "3.0.7" resolved "https://registry.yarnpkg.com/@types/selenium-webdriver/-/selenium-webdriver-3.0.7.tgz#5d3613d1ab3ca08b74d19683a3a7c573129ab18f" @@ -7244,8 +7249,7 @@ domhandler@^4.2.0, domhandler@^4.3.1: domelementtype "^2.2.0" "domino@https://github.com/angular/domino.git#aa8de3486307f57a518b4b0d9e5e16d9fbd998d1": - version "2.1.6+git" - uid aa8de3486307f57a518b4b0d9e5e16d9fbd998d1 + version "2.1.6" resolved "https://github.com/angular/domino.git#aa8de3486307f57a518b4b0d9e5e16d9fbd998d1" domutils@^2.8.0: @@ -14038,7 +14042,6 @@ sass@1.62.1: "sauce-connect@https://saucelabs.com/downloads/sc-4.8.1-linux.tar.gz": version "0.0.0" - uid "9c16682e4c9716734432789884f868212f95f563" resolved "https://saucelabs.com/downloads/sc-4.8.1-linux.tar.gz#9c16682e4c9716734432789884f868212f95f563" saucelabs@7.2.1, saucelabs@^1.5.0, saucelabs@^4.6.3: @@ -14095,6 +14098,15 @@ select-hose@^2.0.0: resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" integrity sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg== +"selenium-webdriver4@npm:selenium-webdriver@4.8.1", selenium-webdriver@4.8.1: + version "4.8.1" + resolved "https://registry.yarnpkg.com/selenium-webdriver/-/selenium-webdriver-4.8.1.tgz#4b0a546c4ea747c44e9688c108f7a46b8d8244ab" + integrity sha512-p4MtfhCQdcV6xxkS7eI0tQN6+WNReRULLCAuT4RDGkrjfObBNXMJ3WT8XdK+aXTr5nnBKuh+PxIevM0EjJgkxA== + dependencies: + jszip "^3.10.0" + tmp "^0.2.1" + ws ">=8.11.0" + selenium-webdriver@3.5.0: version "3.5.0" resolved "https://registry.yarnpkg.com/selenium-webdriver/-/selenium-webdriver-3.5.0.tgz#9036c82874e6c0f5cbff0a0f18223bc31c99cb77" @@ -14115,15 +14127,6 @@ selenium-webdriver@3.6.0, selenium-webdriver@^3.0.1: tmp "0.0.30" xml2js "^0.4.17" -selenium-webdriver@4.8.1: - version "4.8.1" - resolved "https://registry.yarnpkg.com/selenium-webdriver/-/selenium-webdriver-4.8.1.tgz#4b0a546c4ea747c44e9688c108f7a46b8d8244ab" - integrity sha512-p4MtfhCQdcV6xxkS7eI0tQN6+WNReRULLCAuT4RDGkrjfObBNXMJ3WT8XdK+aXTr5nnBKuh+PxIevM0EjJgkxA== - dependencies: - jszip "^3.10.0" - tmp "^0.2.1" - ws ">=8.11.0" - selfsigned@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-2.1.1.tgz#18a7613d714c0cd3385c48af0075abf3f266af61"