mirror of
https://github.com/angular/angular.git
synced 2026-09-14 13:54:52 +08:00
build: share Saucelabs browsers between karma test targets using background Saucelabs daemon and custom karma launcher (#49200)
This upgrades the Saucelabs Bazel step on CI to use the more efficient Saucelabs daemon PR Close #49200
This commit is contained in:
Greg Magolan
committed by
Andrew Kushnir
Andrew Kushnir
parent
85b4941be1
commit
5a9059be38
+10
-4
@@ -308,18 +308,21 @@ jobs:
|
||||
# container for this job. This is necessary because we launch a lot of browsers concurrently
|
||||
# and therefore the tunnel and Karma need to process a lot of file requests and tests.
|
||||
resource_class: xlarge
|
||||
environment:
|
||||
NUMBER_OF_PARALLEL_BROWSERS: 2
|
||||
steps:
|
||||
- custom_attach_workspace
|
||||
- init_environment
|
||||
- init_saucelabs_environment
|
||||
- run:
|
||||
name: Start Saucelabs daemon service
|
||||
command: yarn bazel run //tools/saucelabs-daemon/background-service -- ${NUMBER_OF_PARALLEL_BROWSERS}
|
||||
background: true
|
||||
- run:
|
||||
name: Run Bazel tests on Saucelabs
|
||||
# See /tools/saucelabs/README.md for more info
|
||||
command: |
|
||||
yarn bazel run //tools/saucelabs:sauce_service_setup
|
||||
TESTS=$(./node_modules/.bin/bazelisk query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "fixme-saucelabs", ...)')
|
||||
yarn bazel test --config=saucelabs ${TESTS}
|
||||
yarn bazel run //tools/saucelabs:sauce_service_stop
|
||||
yarn bazel test --config=saucelabs --jobs=${NUMBER_OF_PARALLEL_BROWSERS} ${TESTS}
|
||||
no_output_timeout: 40m
|
||||
- notify_webhook_on_fail:
|
||||
webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL
|
||||
@@ -674,6 +677,9 @@ workflows:
|
||||
- build-npm-packages:
|
||||
requires:
|
||||
- setup
|
||||
- saucelabs:
|
||||
requires:
|
||||
- setup
|
||||
- legacy-unit-tests-saucelabs:
|
||||
requires:
|
||||
- setup
|
||||
|
||||
@@ -1166,6 +1166,7 @@ groups:
|
||||
'tools/legacy-saucelabs/**/{*,.*}',
|
||||
'tools/rxjs/**/{*,.*}',
|
||||
'tools/saucelabs/**/{*,.*}',
|
||||
'tools/saucelabs-daemon/**/{*,.*}',
|
||||
'tools/symbol-extractor/**/{*,.*}',
|
||||
'tools/testing/**/{*,.*}',
|
||||
'tools/tslint/**/{*,.*}',
|
||||
|
||||
+18
@@ -1,4 +1,5 @@
|
||||
load("//tools:defaults.bzl", "nodejs_binary")
|
||||
load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
|
||||
load("//:yarn.bzl", "YARN_PATH")
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
@@ -26,6 +27,14 @@ alias(
|
||||
actual = "//packages:tsconfig-build.json",
|
||||
)
|
||||
|
||||
js_library(
|
||||
name = "browser-providers",
|
||||
srcs = [
|
||||
"browser-providers.conf.d.ts",
|
||||
"browser-providers.conf.js",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "angularjs_scripts",
|
||||
srcs = [
|
||||
@@ -61,3 +70,12 @@ nodejs_binary(
|
||||
"//integration:__subpackages__",
|
||||
],
|
||||
)
|
||||
|
||||
alias(
|
||||
name = "sauce_connect",
|
||||
actual = select({
|
||||
"@npm//@angular/build-tooling/bazel/constraints:linux_x64": "@sauce_connect_linux_amd64//:bin/sc",
|
||||
"@npm//@angular/build-tooling/bazel/constraints:macos_x64": "@sauce_connect_mac//:bin/sc",
|
||||
"@npm//@angular/build-tooling/bazel/constraints:macos_arm64": "@sauce_connect_mac//:bin/sc",
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -215,3 +215,20 @@ register_toolchains(
|
||||
"@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_arm64_toolchain",
|
||||
"@npm//@angular/build-tooling/bazel/git-toolchain:git_windows_toolchain",
|
||||
)
|
||||
|
||||
# Fetch sauce connect (tool to open Saucelabs tunnel for Saucelabs browser tests)
|
||||
http_archive(
|
||||
name = "sauce_connect_linux_amd64",
|
||||
build_file_content = """exports_files(["bin/sc"], visibility = ["//visibility:public"])""",
|
||||
sha256 = "26b9c3630f441b47854b6032f7eca6f1d88d3f62e50ee44c27015d71a5155c36",
|
||||
strip_prefix = "sc-4.8.2-linux",
|
||||
url = "https://saucelabs.com/downloads/sc-4.8.2-linux.tar.gz",
|
||||
)
|
||||
|
||||
http_archive(
|
||||
name = "sauce_connect_mac",
|
||||
build_file_content = """exports_files(["bin/sc"], visibility = ["//visibility:public"])""",
|
||||
sha256 = "28277ce81ef9ab84f5b87b526258920a8ead44789a5034346e872629bbf38089",
|
||||
strip_prefix = "sc-4.8.2-osx",
|
||||
url = "https://saucelabs.com/downloads/sc-4.8.2-osx.zip",
|
||||
)
|
||||
|
||||
Vendored
+20
@@ -0,0 +1,20 @@
|
||||
type CustomLauncher = {
|
||||
base: string;
|
||||
browserName: string;
|
||||
platformName: string;
|
||||
platformVersion: string;
|
||||
deviceName: string;
|
||||
appiumVersion: string;
|
||||
extendedDebugging: boolean;
|
||||
}
|
||||
|
||||
type CustomLaunchers = {
|
||||
[string]: CustomLauncher;
|
||||
};
|
||||
|
||||
type SauceAliases = {
|
||||
[string]: string[];
|
||||
};
|
||||
|
||||
export const customLaunchers: CustomLaunchers;
|
||||
export const sauceAliases: SauceAliases;
|
||||
+27
-12
@@ -12,6 +12,13 @@ const {hostname} = require('os');
|
||||
const seed = process.env.JASMINE_RANDOM_SEED || String(Math.random()).slice(-5);
|
||||
console.info(`Jasmine random seed: ${seed}`);
|
||||
|
||||
const isBazel = !!process.env.TEST_TARGET;
|
||||
|
||||
if (!process.env.KARMA_WEB_TEST_MODE && isBazel && process.env.TEST_TARGET.includes('_saucelabs')) {
|
||||
console.info(`Saucelabs target detected: ${process.env.TEST_TARGET}`);
|
||||
process.env.KARMA_WEB_TEST_MODE = 'SL_REQUIRED';
|
||||
}
|
||||
|
||||
module.exports = function(config) {
|
||||
const conf = {
|
||||
frameworks: ['jasmine'],
|
||||
@@ -50,7 +57,6 @@ module.exports = function(config) {
|
||||
|
||||
plugins: [
|
||||
'karma-jasmine',
|
||||
'karma-sauce-launcher',
|
||||
'karma-chrome-launcher',
|
||||
'karma-sourcemap-loader',
|
||||
],
|
||||
@@ -99,20 +105,29 @@ module.exports = function(config) {
|
||||
set: () => {},
|
||||
});
|
||||
|
||||
if (process.env['SAUCE_TUNNEL_IDENTIFIER']) {
|
||||
console.log(`SAUCE_TUNNEL_IDENTIFIER: ${process.env.SAUCE_TUNNEL_IDENTIFIER}`);
|
||||
if (isBazel) {
|
||||
// Add the custom Saucelabs daemon to the plugins
|
||||
const saucelabsDaemonLauncher = require('./tools/saucelabs-daemon/launcher/index.cjs').default;
|
||||
conf.plugins.push(saucelabsDaemonLauncher);
|
||||
} else {
|
||||
conf.plugins.push('karma-sauce-launcher');
|
||||
|
||||
const tunnelIdentifier = process.env['SAUCE_TUNNEL_IDENTIFIER'];
|
||||
if (process.env['SAUCE_TUNNEL_IDENTIFIER']) {
|
||||
console.log(`SAUCE_TUNNEL_IDENTIFIER: ${process.env.SAUCE_TUNNEL_IDENTIFIER}`);
|
||||
|
||||
// Setup the Saucelabs plugin so that it can launch browsers using the proper tunnel.
|
||||
conf.sauceLabs.build = tunnelIdentifier;
|
||||
conf.sauceLabs.tunnelIdentifier = tunnelIdentifier;
|
||||
const tunnelIdentifier = process.env['SAUCE_TUNNEL_IDENTIFIER'];
|
||||
|
||||
// Patch the `saucelabs` package so that `karma-sauce-launcher` does not attempt downloading
|
||||
// the test logs from upstream and tries re-uploading them with the Karma enhanced details.
|
||||
// This slows-down tests/browser restarting and can decrease stability.
|
||||
// https://github.com/karma-runner/karma-sauce-launcher/blob/59b0c5c877448e064ad56449cd906743721c6b62/src/launcher/launcher.ts#L72-L79.
|
||||
require('saucelabs').default.prototype.downloadJobAsset = () => Promise.resolve('<FAKE-LOGS>');
|
||||
// Setup the Saucelabs plugin so that it can launch browsers using the proper tunnel.
|
||||
conf.sauceLabs.build = tunnelIdentifier;
|
||||
conf.sauceLabs.tunnelIdentifier = tunnelIdentifier;
|
||||
|
||||
// Patch the `saucelabs` package so that `karma-sauce-launcher` does not attempt downloading
|
||||
// the test logs from upstream and tries re-uploading them with the Karma enhanced details.
|
||||
// This slows-down tests/browser restarting and can decrease stability.
|
||||
// https://github.com/karma-runner/karma-sauce-launcher/blob/59b0c5c877448e064ad56449cd906743721c6b62/src/launcher/launcher.ts#L72-L79.
|
||||
require('saucelabs').default.prototype.downloadJobAsset = () =>
|
||||
Promise.resolve('<FAKE-LOGS>');
|
||||
}
|
||||
}
|
||||
|
||||
// For SauceLabs jobs, we set up a domain which resolves to the machine which launched
|
||||
|
||||
@@ -88,6 +88,7 @@
|
||||
"@types/jasminewd2": "^2.0.8",
|
||||
"@types/node": "^16.11.7",
|
||||
"@types/selenium-webdriver": "3.0.7",
|
||||
"@types/selenium-webdriver4": "npm:@types/selenium-webdriver@4.1.12",
|
||||
"@types/semver": "^7.3.4",
|
||||
"@types/shelljs": "^0.8.6",
|
||||
"@types/systemjs": "0.19.32",
|
||||
@@ -135,6 +136,7 @@
|
||||
"rollup-plugin-sourcemaps": "^0.6.3",
|
||||
"rxjs": "^6.6.7",
|
||||
"selenium-webdriver": "3.5.0",
|
||||
"selenium-webdriver4": "npm:selenium-webdriver@4.8.1",
|
||||
"semver-dsl": "^1.0.1",
|
||||
"shelljs": "^0.8.5",
|
||||
"source-map": "0.7.4",
|
||||
|
||||
@@ -100,6 +100,7 @@ def karma_test(name, env_srcs, env_deps, env_entry_point, test_srcs, test_deps,
|
||||
configuration_env_vars = ["KARMA_WEB_TEST_MODE"],
|
||||
data = [
|
||||
"//:browser-providers.conf.js",
|
||||
"//tools/saucelabs-daemon/launcher:launcher_cjs",
|
||||
],
|
||||
static_files = [
|
||||
":assets/sample.json",
|
||||
|
||||
+2
-4
@@ -321,7 +321,7 @@ def karma_web_test_suite(
|
||||
# Add a saucelabs target for Karma tests in `//packages/`.
|
||||
if native.package_name().startswith("packages/"):
|
||||
_karma_web_test(
|
||||
name = "saucelabs_%s" % name,
|
||||
name = "{}_saucelabs".format(name),
|
||||
# Default timeout is moderate (5min). This causes the test to be terminated while
|
||||
# Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
|
||||
# unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
|
||||
@@ -329,15 +329,13 @@ def karma_web_test_suite(
|
||||
timeout = "long",
|
||||
config_file = "//:karma-js.conf.js",
|
||||
deps = [
|
||||
"@npm//karma-sauce-launcher",
|
||||
":%s_bundle" % name,
|
||||
],
|
||||
data = data + [
|
||||
"//:browser-providers.conf.js",
|
||||
"//tools/saucelabs-daemon/launcher:launcher_cjs",
|
||||
],
|
||||
karma = "//tools/saucelabs:karma-saucelabs",
|
||||
tags = tags + [
|
||||
"exclusive",
|
||||
"manual",
|
||||
"no-remote-exec",
|
||||
"saucelabs",
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
load("//tools:defaults.bzl", "ts_library")
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
ts_library(
|
||||
name = "saucelabs-daemon",
|
||||
srcs = [
|
||||
"browser.ts",
|
||||
"ipc-defaults.ts",
|
||||
"ipc-messages.ts",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,88 @@
|
||||
# Saucelabs testing with Bazel
|
||||
|
||||
## Local testing
|
||||
|
||||
1. Set up your `SAUCE_USERNAME`, `SAUCE_ACCESS_KEY` & `SAUCE_TUNNEL_IDENTIFIER` environment variables.
|
||||
These are required. You can find the values for `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` in `/.circleci/env.sh`. `SAUCE_TUNNEL_IDENTIFIER` can be set to any unique value.
|
||||
|
||||
If you are having trouble running Saucelabs tests locally you can contact [Joey Perrott](https://github.com/josephperrott) or [Paul Gschwendtner](https://github.com/devversion) for support.
|
||||
|
||||
2. Once you have your environment variables set up, run the setup task in the root of the repo:
|
||||
|
||||
``` bash
|
||||
yarn bazel run //tools/saucelabs-daemon/background-service -- <number_of_browsers>
|
||||
```
|
||||
|
||||
This will start a daemon process that will connect to Saucelabs and provision browsers
|
||||
once you start running your first test target.
|
||||
|
||||
3. In another terminal, you can run a particular test target through SauceLabs by suffixing the target name with "_saucelabs".
|
||||
|
||||
For example, `packages/core/test:test_web` becomes `packages/core/test:test_web_saucelabs`.
|
||||
|
||||
```
|
||||
yarn bazel test //packages/core/test:test_web_saucelabs
|
||||
```
|
||||
|
||||
## Additional test features
|
||||
|
||||
To see the test output while the tests are running (as these are long tests), add the `--test_output=streamed` option.
|
||||
Note, this option will also prevent bazel from using the test cache and will force the test to run.
|
||||
|
||||
For running all Saucelabs tests in the project, `bazel query` is used to gather up all karma Saucelabs test labels because they are otherwise hidden by the `manual` tag.
|
||||
|
||||
Running all karma tests in Saucelabs:
|
||||
|
||||
Start the saucelabs-daemon background service in one terminal window:
|
||||
|
||||
``` bash
|
||||
yarn bazel run //tools/saucelabs-daemon/background-service -- <number_of_browsers>
|
||||
```
|
||||
|
||||
In a second terminal window, run all of the saucelabs test targets:
|
||||
|
||||
``` bash
|
||||
TESTS=$(./node_modules/.bin/bazelisk query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "fixme-saucelabs", ...)')
|
||||
yarn bazel test --config=saucelabs --jobs=<number_of_browsers> ${TESTS}
|
||||
```
|
||||
|
||||
NB: The number of parallel Bazel tests specified by `--jobs=<number_of_browsers>` must not exceed the number parallel browsers requested when starting the daemon.
|
||||
|
||||
## Under the hood
|
||||
|
||||
The `//tools/saucelabs-daemon/background-service` target does not start the Sauce Connect proxy at start-up, but instead listens for the start signal from the saucelabs karma launcher.
|
||||
This signal is sent by saucelabs-daemon custom karma launcher `tools/saucelabs-daemon/launcher/launcher.ts`.
|
||||
This is necessary as the Sauce Connect Proxy process must be started outside of `bazel test` as Bazel will automatically kill any processes spawned during a test when that tests completes, which would prevent the tunnel & provisioned browsers from being shared by multiple tests.
|
||||
|
||||
The karma_web_test rule for saucelabs must have a few important tags:
|
||||
|
||||
* `no-remote-exec` as they cannot be executed remotely since tests need to communicate with the daemon.
|
||||
* `manual` so they are not automatically tested with `//...`
|
||||
* `saucelabs` so that they can be easily gathered up for testing in a `bazel query`
|
||||
|
||||
These are added automatically the by `karma_web_test_suite` macro in `tools/defaults.bzl`.
|
||||
|
||||
## Debugging
|
||||
|
||||
**Q: How do I get the tests to run on IE? I only see Chromium.**
|
||||
|
||||
If you see something like this at the end of your test output, it means you're not actually running SauceLabs:
|
||||
|
||||
```
|
||||
INFO: Build completed successfully, 43 total actions
|
||||
/packages/core/test:test_web_chromium
|
||||
```
|
||||
|
||||
This is a common error caused by forgetting to suffix your test target with "_saucelabs".
|
||||
|
||||
For example, `/packages/core/test:test_web` becomes `/packages/core/test:test_web_saucelabs`.
|
||||
|
||||
**Q: How can I tell that the SauceLabs connection was successfully made?**
|
||||
|
||||
There is a dashboard at saucelabs.com where you can see active tunnel connections (Angular has an account).
|
||||
As soon as you actually run the test target (not after the setup task), you should see an active tunnel connection under the SAUCE_TUNNEL_IDENTIFICATION_KEY you entered.
|
||||
If a tunnel connection is not there, you are not actually connecting with SauceLabs.
|
||||
|
||||
Note: It may *look* like the tests are running because of the Bazel output.
|
||||
The progress Bazel is showing does not mean that SauceLabs is connected.
|
||||
If the tests are actually running, you should see the "..." test report for passing tests.
|
||||
@@ -0,0 +1,28 @@
|
||||
load("//tools:defaults.bzl", "nodejs_binary", "ts_library")
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
ts_library(
|
||||
name = "background-service_lib",
|
||||
srcs = glob(["*.ts"]),
|
||||
deps = [
|
||||
"//:browser-providers",
|
||||
"//tools/saucelabs-daemon",
|
||||
"@npm//@types/node",
|
||||
"@npm//@types/selenium-webdriver4",
|
||||
"@npm//chalk",
|
||||
"@npm//selenium-webdriver4",
|
||||
],
|
||||
)
|
||||
|
||||
nodejs_binary(
|
||||
name = "background-service",
|
||||
data = [
|
||||
":background-service_lib",
|
||||
"//:sauce_connect",
|
||||
],
|
||||
entry_point = ":cli.ts",
|
||||
templated_args = [
|
||||
"$(rootpath //:sauce_connect)",
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import {customLaunchers} from '../../../browser-providers.conf';
|
||||
import {Browser} from '../browser';
|
||||
|
||||
import {SaucelabsDaemon} from './saucelabs-daemon';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const username = process.env.SAUCE_USERNAME;
|
||||
const accessKey = process.env.SAUCE_ACCESS_KEY;
|
||||
const tunnelIdentifier = process.env.SAUCE_TUNNEL_IDENTIFIER;
|
||||
|
||||
const buildName = process.env.CIRCLECI ? `circleci-${process.env.CIRCLE_BUILD_NUM}` : 'localdev';
|
||||
|
||||
if (!username || !accessKey) {
|
||||
throw Error('Please set the `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` variables.');
|
||||
}
|
||||
|
||||
if (!tunnelIdentifier) {
|
||||
throw Error('No tunnel set up. Please set the `SAUCE_TUNNEL_IDENTIFIER` variable.');
|
||||
}
|
||||
|
||||
if (!buildName) {
|
||||
throw Error('No build name specified.');
|
||||
}
|
||||
|
||||
// First argument is the path to the sauce connect binary. This argument is templated into the bazel
|
||||
// binary.
|
||||
if (args.length < 1) {
|
||||
throw Error(`Path to the sauce connect binary expected as first argument`);
|
||||
}
|
||||
const sauceConnect = args[0];
|
||||
|
||||
// Second argument is the number of parallel browsers to start. This argument is user supplied and
|
||||
// required.
|
||||
if (args.length != 2) {
|
||||
throw Error(`Please specify the number of parallel browsers to start on the command line.`);
|
||||
}
|
||||
const parallelExecutions = parseInt(args[1]);
|
||||
if (!parallelExecutions) {
|
||||
throw Error(`Please specify a non-zero number of parallel browsers to start.`);
|
||||
}
|
||||
|
||||
const browserInstances: Browser[] = [];
|
||||
for (let i = 0; i < parallelExecutions; i++) {
|
||||
browserInstances.push(...Object.values(customLaunchers) as any);
|
||||
}
|
||||
|
||||
// Start the daemon and launch the given browser
|
||||
const daemon = new SaucelabsDaemon(
|
||||
username,
|
||||
accessKey,
|
||||
process.env.CIRCLE_BUILD_NUM!,
|
||||
browserInstances,
|
||||
sauceConnect,
|
||||
{tunnelIdentifier},
|
||||
);
|
||||
|
||||
if (args.includes('--connect')) {
|
||||
daemon.connect().catch((err) => {
|
||||
console.error(`Failed to connect to Saucelabs: ${err}`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import {createServer, Server, Socket} from 'net';
|
||||
|
||||
import {IPC_PORT} from '../ipc-defaults';
|
||||
import {BackgroundServiceReceiveMessages, InternalErrorMessage, NoAvailableBrowserMessage} from '../ipc-messages';
|
||||
|
||||
import {SaucelabsDaemon} from './saucelabs-daemon';
|
||||
|
||||
let nextSocketId = 0;
|
||||
|
||||
export class IpcServer {
|
||||
private readonly _server: Server;
|
||||
private _connections = new Map<number, Socket>();
|
||||
|
||||
constructor(private _service: SaucelabsDaemon) {
|
||||
this._server = createServer(this._connectionHandler.bind(this));
|
||||
this._server.listen(IPC_PORT, () => console.info('Daemon IPC server listening.'));
|
||||
}
|
||||
|
||||
private _connectionHandler(socket: Socket) {
|
||||
const socketId = nextSocketId++;
|
||||
this._connections.set(socketId, socket);
|
||||
socket.on('data', b => {
|
||||
this._processMessage(
|
||||
socket,
|
||||
socketId,
|
||||
JSON.parse(b.toString()) as BackgroundServiceReceiveMessages,
|
||||
)
|
||||
.catch((err) => {
|
||||
console.error(err);
|
||||
this._sendInternalError(socket, err.toString());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async _processMessage(
|
||||
socket: Socket,
|
||||
socketId: number,
|
||||
message: BackgroundServiceReceiveMessages,
|
||||
) {
|
||||
switch (message.type) {
|
||||
case 'start-test':
|
||||
console.debug(`Requesting test browser: SID#${socketId}: ${message.testDescription}`);
|
||||
const started = await this._service.startTest({
|
||||
testId: socketId,
|
||||
pageUrl: message.url,
|
||||
requestedBrowserId: message.browserId,
|
||||
});
|
||||
if (!started) {
|
||||
console.debug(' > Browser not available.');
|
||||
this._sendUnavailableBrowserMessage(socket);
|
||||
} else {
|
||||
console.debug(' > Browser available. Test can start.');
|
||||
}
|
||||
break;
|
||||
case 'end-test':
|
||||
console.debug(`Ending tests for SID#${socketId}`);
|
||||
this._service.endTest(socketId);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unsupported msg type: ${(message as any).type}`);
|
||||
}
|
||||
}
|
||||
|
||||
private _sendUnavailableBrowserMessage(socket: Socket) {
|
||||
socket.write(JSON.stringify(new NoAvailableBrowserMessage()));
|
||||
}
|
||||
|
||||
private _sendInternalError(socket: Socket, msg: string) {
|
||||
socket.write(JSON.stringify(new InternalErrorMessage(msg)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import chalk from 'chalk';
|
||||
import {spawn} from 'node:child_process';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import {Builder, WebDriver} from 'selenium-webdriver4';
|
||||
|
||||
import {Browser, getUniqueId} from '../browser';
|
||||
|
||||
import {IpcServer} from './ipc';
|
||||
|
||||
const defaultCapabilities = {
|
||||
recordVideo: false,
|
||||
recordScreenshots: false,
|
||||
idleTimeout: 90,
|
||||
// These represent the maximum values supported by Saucelabs.
|
||||
// See: https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options
|
||||
commandTimeout: 600,
|
||||
maxDuration: 10800,
|
||||
extendedDebugging: true,
|
||||
};
|
||||
|
||||
interface RemoteBrowser {
|
||||
id: string;
|
||||
state: 'acquired'|'free'|'launching';
|
||||
driver: WebDriver|null;
|
||||
}
|
||||
|
||||
interface BrowserTest {
|
||||
testId: number;
|
||||
pageUrl: string;
|
||||
requestedBrowserId: string;
|
||||
}
|
||||
|
||||
export class SaucelabsDaemon {
|
||||
/**
|
||||
* Map of browsers and their pending tests. If a browser is acquired on the
|
||||
* remote selenium server, the browser is not immediately ready. If the browser
|
||||
* becomes active, the pending tests will be started.
|
||||
*/
|
||||
private _pendingTests = new Map<RemoteBrowser, BrowserTest>();
|
||||
|
||||
/** List of active browsers that are managed by the daemon. */
|
||||
private _activeBrowsers = new Set<RemoteBrowser>();
|
||||
|
||||
/** Map that contains test ids with their acquired browser. */
|
||||
private _runningTests = new Map<number, RemoteBrowser>();
|
||||
|
||||
/** Server used for communication with the Karma launcher. */
|
||||
private _server = new IpcServer(this);
|
||||
|
||||
/** Base selenium capabilities that will be added to each browser. */
|
||||
private _baseCapabilities = {...defaultCapabilities, ...this._userCapabilities};
|
||||
|
||||
/** Id of the keep alive interval that ensures no remote browsers time out. */
|
||||
private _keepAliveIntervalId: NodeJS.Timeout|null = null;
|
||||
|
||||
/* Have we connected to Saucelabs or are in the process of connecting? */
|
||||
private _connection: Promise<void>|undefined = undefined;
|
||||
|
||||
constructor(
|
||||
private _username: string,
|
||||
private _accessKey: string,
|
||||
private _buildName: string,
|
||||
private _browsers: Browser[],
|
||||
private _sauceConnect: string,
|
||||
private _userCapabilities: object = {},
|
||||
) {
|
||||
// Starts the keep alive loop for all active browsers, running every 15 seconds.
|
||||
this._keepAliveIntervalId = setInterval(() => this._keepAliveBrowsers(), 15_000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Connects the daemon to Saucelabs.
|
||||
* This is typically done when the first test is started so that no connection is made
|
||||
* if all tests are cache hits.
|
||||
*/
|
||||
async connect() {
|
||||
if (!this._connection) {
|
||||
this._connection = this._connect();
|
||||
}
|
||||
return this._connection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Quits all active browsers.
|
||||
*/
|
||||
async quitAllBrowsers() {
|
||||
let quitBrowsers: Promise<void>[] = [];
|
||||
this._activeBrowsers.forEach(b => {
|
||||
if (b.driver) {
|
||||
quitBrowsers.push(b.driver.quit());
|
||||
}
|
||||
});
|
||||
await Promise.all(quitBrowsers);
|
||||
this._activeBrowsers.clear();
|
||||
this._runningTests.clear();
|
||||
this._pendingTests.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown the daemon.
|
||||
*
|
||||
* Awaits the shutdown of browsers.
|
||||
*/
|
||||
async shutdown() {
|
||||
await this.quitAllBrowsers();
|
||||
if (this._keepAliveIntervalId !== null) {
|
||||
clearInterval(this._keepAliveIntervalId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* End a browser test if it is running.
|
||||
*/
|
||||
endTest(testId: number) {
|
||||
if (!this._runningTests.has(testId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const browser = this._runningTests.get(testId)!;
|
||||
browser.state = 'free';
|
||||
this._runningTests.delete(testId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a test on a remote browser.
|
||||
*
|
||||
* If the daemon has not yet initiated the saucelabs tunnel creation and browser launching then
|
||||
* this initiates that process and awaits until it succeeds or fails.
|
||||
*
|
||||
* If the daemon has already initiated the saucelabs tunnel creation and browser launching then
|
||||
* but it is not yet complete then this until it succeeds or fails.
|
||||
*
|
||||
* If all matching browsers are occupied with other tests then test is not run. Promise returns
|
||||
* false.
|
||||
*
|
||||
* If there is a matching browser that are still launching then the test is scheduled to run
|
||||
* on the browser when it is ready. Promise returns true.
|
||||
*
|
||||
* If there is a matching browser that is available the test it started. Promise returns true.
|
||||
*/
|
||||
async startTest(test: BrowserTest): Promise<boolean> {
|
||||
await this.connect();
|
||||
|
||||
const browsers = this._findMatchingBrowsers(test.requestedBrowserId);
|
||||
if (!browsers.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Find the first available browser and start the test.
|
||||
for (const browser of browsers) {
|
||||
// If the browser is acquired, continue searching.
|
||||
if (browser.state === 'acquired') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// If the browser is launching, check if it can be pre-claimed so that
|
||||
// the test starts once the browser is ready. If it's already claimed,
|
||||
// continue searching.
|
||||
if (browser.state === 'launching') {
|
||||
if (this._pendingTests.has(browser)) {
|
||||
continue;
|
||||
} else {
|
||||
this._pendingTests.set(browser, test);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// TS21225: [tsetse] All Promises in async functions must either be awaited or used in an
|
||||
// expression
|
||||
const _ = this._startBrowserTest(browser, test);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal
|
||||
* Connects the daemon to Saucelabs.
|
||||
* This is typically done when the first test is started so that no connection is made
|
||||
* if all tests are cache hits.
|
||||
**/
|
||||
async _connect() {
|
||||
await this._openSauceConnectTunnel();
|
||||
await this._launchBrowsers();
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal
|
||||
* Establishes the Saucelabs connect tunnel.
|
||||
**/
|
||||
async _openSauceConnectTunnel() {
|
||||
console.debug('Starting sauce connect tunnel...');
|
||||
|
||||
const tmpFolder = await fs.mkdtemp('saucelabs-daemon-');
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
// First we need to start the sauce connect tunnel
|
||||
const sauceConnectArgs = [
|
||||
'--readyfile',
|
||||
`${tmpFolder}/readyfile`,
|
||||
'--pidfile',
|
||||
`${tmpFolder}/pidfile`,
|
||||
'--tunnel-identifier',
|
||||
(this._userCapabilities as any).tunnelIdentifier || path.basename(tmpFolder),
|
||||
];
|
||||
const sc = spawn(this._sauceConnect, sauceConnectArgs);
|
||||
|
||||
sc.stdout!.on('data', (data) => {
|
||||
if (data.includes('Sauce Connect is up, you may start your tests.')) {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
|
||||
sc.on('close', (code) => {
|
||||
reject(new Error(`sauce connect closed all stdio with code ${code}`));
|
||||
});
|
||||
|
||||
sc.on('exit', (code) => {
|
||||
reject(new Error(`sauce connect exited with code ${code}`));
|
||||
});
|
||||
});
|
||||
|
||||
console.debug('Starting sauce connect tunnel established');
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal
|
||||
* Launches all browsers. If there are pending tests waiting for a particular browser to launch
|
||||
* before they can start, those tests are started once the browser is launched.
|
||||
**/
|
||||
async _launchBrowsers() {
|
||||
console.debug('Launching browsers...');
|
||||
|
||||
// Once the tunnel is established we can launch browsers
|
||||
await Promise.all(
|
||||
this._browsers.map(async (browser, id) => {
|
||||
const browserId = getUniqueId(browser);
|
||||
const launched: RemoteBrowser = {state: 'launching', driver: null, id: browserId};
|
||||
const browserDescription = `${this._buildName} - ${browser.browserName} - #${id + 1}`;
|
||||
|
||||
const capabilities: any = {
|
||||
'browserName': browser.browserName,
|
||||
'sauce:options': {...this._baseCapabilities, ...browser},
|
||||
};
|
||||
|
||||
// Set `sauce:options` to provide a build name for the remote browser instances.
|
||||
// This helps with debugging. Also ensures the W3C protocol is used.
|
||||
// See. https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options
|
||||
capabilities['sauce:options']['name'] = browserDescription;
|
||||
capabilities['sauce:options']['build'] = browserDescription;
|
||||
|
||||
console.debug(
|
||||
`Capabilities for ${browser.browserName}:`, JSON.stringify(capabilities, null, 2));
|
||||
console.debug(` > Browser-ID: `, browserId);
|
||||
console.debug(` > Browser-Description: `, browserDescription);
|
||||
|
||||
// Keep track of the launched browser. We do this before it even completed the
|
||||
// launch as we can then handle scheduled tests when the browser is still launching.
|
||||
this._activeBrowsers.add(launched);
|
||||
|
||||
// See the following link for public API of the selenium server.
|
||||
// https://wiki.saucelabs.com/display/DOCS/Instant+Selenium+Node.js+Tests
|
||||
const driver = await new Builder()
|
||||
.withCapabilities(capabilities)
|
||||
.usingServer(
|
||||
`http://${this._username}:${
|
||||
this._accessKey}@ondemand.saucelabs.com:80/wd/hub`,
|
||||
)
|
||||
.build();
|
||||
|
||||
// Only wait 30 seconds to load a test page.
|
||||
await driver.manage().setTimeouts({pageLoad: 30000});
|
||||
|
||||
const sessionId = (await driver.getSession()).getId();
|
||||
console.info(
|
||||
chalk.yellow(
|
||||
`Started browser ${browser.browserName} on Saucelabs: ` +
|
||||
`https://saucelabs.com/tests/${sessionId}`,
|
||||
),
|
||||
);
|
||||
|
||||
// Mark the browser as available after launch completion.
|
||||
launched.state = 'free';
|
||||
launched.driver = driver;
|
||||
|
||||
// If a test has been scheduled before the browser completed launching, run
|
||||
// it now given that the browser is ready now.
|
||||
if (this._pendingTests.has(launched)) {
|
||||
// TS21225: [tsetse] All Promises in async functions must either be awaited or used in
|
||||
// an expression
|
||||
const _ = this._startBrowserTest(launched, this._pendingTests.get(launched)!);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal
|
||||
* Starts a browser test on a browser.
|
||||
* This sets the browser's state to "acquired" and navigates the browser to the test URL.
|
||||
**/
|
||||
private async _startBrowserTest(browser: RemoteBrowser, test: BrowserTest) {
|
||||
this._runningTests.set(test.testId, browser);
|
||||
browser.state = 'acquired';
|
||||
|
||||
try {
|
||||
console.debug(`Opening test url for #${test.testId}: ${test.pageUrl}`);
|
||||
await browser.driver!.get(test.pageUrl);
|
||||
const pageTitle = await browser.driver!.getTitle();
|
||||
console.debug(`Test page loaded for #${test.testId}: "${pageTitle}".`);
|
||||
} catch (e) {
|
||||
console.error('Could not start browser test with id', test.testId, test.pageUrl);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal
|
||||
* Given a browserId, returns a list of matching browsers from the list of active browsers.
|
||||
**/
|
||||
private _findMatchingBrowsers(browserId: string): RemoteBrowser[] {
|
||||
const browsers: RemoteBrowser[] = [];
|
||||
this._activeBrowsers.forEach(b => {
|
||||
if (b.id === browserId) {
|
||||
browsers.push(b);
|
||||
}
|
||||
});
|
||||
return browsers;
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal
|
||||
* Implements a heartbeat for Saucelabs browsers as they could end up not receiving any
|
||||
* commands when the daemon is unused (i.e. Bazel takes a while to start tests).
|
||||
* https://saucelabs.com/blog/selenium-tips-how-to-coordinate-multiple-browsers-in-sauce-ondemand.
|
||||
**/
|
||||
private async _keepAliveBrowsers() {
|
||||
const pendingCommands: Promise<string>[] = [];
|
||||
this._activeBrowsers.forEach(b => {
|
||||
if (b.driver !== null) {
|
||||
pendingCommands.push(b.driver.getTitle() as Promise<string>);
|
||||
}
|
||||
});
|
||||
await Promise.all(pendingCommands);
|
||||
console.debug(`${Date().toLocaleString()}: Refreshed ${pendingCommands.length} browsers.`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/** Definition of a test browser. */
|
||||
export interface Browser {
|
||||
browserName: string;
|
||||
browserVersion?: string;
|
||||
platformName?: string;
|
||||
platformVersion?: string;
|
||||
deviceName?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a unique id for the specified browser. This id can be shared
|
||||
* across the background service and launcher using IPC.
|
||||
*/
|
||||
export function getUniqueId(browser: Browser): string {
|
||||
let result = Object.keys(browser).sort().map(key => `${key}=${browser[key as keyof Browser]}`);
|
||||
return result.join(':');
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
export const IPC_PORT = 5324;
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
/** Message that can be sent to the daemon to start a given test. */
|
||||
export class StartTestMessage {
|
||||
readonly type = 'start-test';
|
||||
constructor(public url: string, public browserId: string, public testDescription: string) {}
|
||||
}
|
||||
|
||||
/** Message that can be sent to the daemon if a test completed. */
|
||||
export class EndTestMessage {
|
||||
readonly type = 'end-test';
|
||||
}
|
||||
|
||||
/** Message being sent from the daemon if a request browser is not available. */
|
||||
export class NoAvailableBrowserMessage {
|
||||
readonly type = 'browser-not-ready';
|
||||
}
|
||||
|
||||
/** Message that indicates an internal error in background service. */
|
||||
export class InternalErrorMessage {
|
||||
readonly type = 'internal-error';
|
||||
constructor(public msg: string) {}
|
||||
}
|
||||
|
||||
/** Type of messages the background service can receive. */
|
||||
export type BackgroundServiceReceiveMessages = StartTestMessage|EndTestMessage;
|
||||
|
||||
/** Type of messages the background services can send to clients. */
|
||||
export type BackgroundServiceSendMessages = NoAvailableBrowserMessage|InternalErrorMessage;
|
||||
@@ -0,0 +1,26 @@
|
||||
load("//tools:defaults.bzl", "esbuild", "ts_library")
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
ts_library(
|
||||
name = "launcher",
|
||||
srcs = [
|
||||
"index.ts",
|
||||
"launcher.ts",
|
||||
],
|
||||
deps = [
|
||||
"//tools/saucelabs-daemon",
|
||||
"@npm//@types/node",
|
||||
],
|
||||
)
|
||||
|
||||
# We need a commonjs version of the launcher that can be required from the
|
||||
# root karma-js.conf.js.
|
||||
esbuild(
|
||||
name = "launcher_cjs",
|
||||
entry_point = "index.ts",
|
||||
format = "cjs",
|
||||
output = "index.cjs",
|
||||
platform = "node",
|
||||
deps = [":launcher"],
|
||||
)
|
||||
@@ -0,0 +1,13 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import {SaucelabsLauncher} from './launcher';
|
||||
|
||||
export default {
|
||||
'launcher:SauceLabs': ['type', SaucelabsLauncher],
|
||||
};
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
import {createConnection, Socket} from 'net';
|
||||
|
||||
import {Browser, getUniqueId} from '../browser';
|
||||
import {IPC_PORT} from '../ipc-defaults';
|
||||
import {BackgroundServiceSendMessages, EndTestMessage, StartTestMessage} from '../ipc-messages';
|
||||
|
||||
export function SaucelabsLauncher(
|
||||
this: any,
|
||||
args: Browser,
|
||||
config: unknown,
|
||||
logger: any,
|
||||
baseLauncherDecorator: any,
|
||||
captureTimeoutLauncherDecorator: any,
|
||||
retryLauncherDecorator: any,
|
||||
) {
|
||||
// Apply base class mixins. This would be nice to have typed, but this is a low-priority now.
|
||||
baseLauncherDecorator(this);
|
||||
captureTimeoutLauncherDecorator(this);
|
||||
retryLauncherDecorator(this);
|
||||
|
||||
const log = logger.create('SaucelabsLauncher');
|
||||
const browserDisplayName = args.browserName +
|
||||
(args.browserVersion ? ' ' + args.browserVersion : '') +
|
||||
(args.platformName ? ' (' + args.platformName + ')' : '');
|
||||
const testSuiteDescription = process.env.TEST_TARGET ?? '<unknown>';
|
||||
|
||||
let daemonConnection: Socket|null = null;
|
||||
|
||||
// Setup Browser name that will be printed out by Karma.
|
||||
this.name = browserDisplayName + ' on SauceLabs (daemon)';
|
||||
|
||||
this.on('start', (pageUrl: string) => {
|
||||
daemonConnection = createConnection({port: IPC_PORT}, () => _startBrowserTest(pageUrl, args));
|
||||
|
||||
daemonConnection.on(
|
||||
'data',
|
||||
b => _processMessage(JSON.parse(b.toString()) as BackgroundServiceSendMessages),
|
||||
);
|
||||
daemonConnection.on('error', err => {
|
||||
log.error(err);
|
||||
|
||||
// Notify karma about the failure.
|
||||
this._done('failure');
|
||||
});
|
||||
});
|
||||
|
||||
this.on('kill', async (doneFn: () => void) => {
|
||||
_endBrowserTest();
|
||||
daemonConnection?.end();
|
||||
doneFn();
|
||||
});
|
||||
|
||||
const _processMessage = (message: BackgroundServiceSendMessages) => {
|
||||
switch (message.type) {
|
||||
case 'browser-not-ready':
|
||||
log.error(
|
||||
'Browser %s is not ready in the Saucelabs background service.',
|
||||
browserDisplayName,
|
||||
);
|
||||
this._done('failure');
|
||||
}
|
||||
};
|
||||
|
||||
const _startBrowserTest = (pageUrl: string, browser: Browser) => {
|
||||
log.info('Starting browser %s test in daemon with URL: %s', browserDisplayName, pageUrl);
|
||||
daemonConnection!.write(
|
||||
JSON.stringify(new StartTestMessage(pageUrl, getUniqueId(browser), testSuiteDescription)),
|
||||
);
|
||||
};
|
||||
|
||||
const _endBrowserTest = () => {
|
||||
log.info('Test for browser %s completed', browserDisplayName);
|
||||
daemonConnection!.write(JSON.stringify(new EndTestMessage()));
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
load("//tools:defaults.bzl", "nodejs_binary")
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
sh_binary(
|
||||
@@ -50,16 +48,3 @@ sh_binary(
|
||||
args = ["log"],
|
||||
data = ["@npm//sauce-connect"],
|
||||
)
|
||||
|
||||
nodejs_binary(
|
||||
name = "karma-saucelabs",
|
||||
data = [
|
||||
"sauce-service.sh",
|
||||
"@npm//@bazel/runfiles",
|
||||
"@npm//karma",
|
||||
"@npm//sauce-connect",
|
||||
"@npm//shelljs",
|
||||
],
|
||||
entry_point = "karma-saucelabs.mjs",
|
||||
templated_args = ["$(rootpath sauce-service.sh)"],
|
||||
)
|
||||
|
||||
+1
-113
@@ -1,115 +1,3 @@
|
||||
# Saucelabs testing with Bazel
|
||||
|
||||
## Local testing
|
||||
|
||||
1. Set up your `SAUCE_USERNAME`, `SAUCE_ACCESS_KEY` & `SAUCE_TUNNEL_IDENTIFIER` environment variables.
|
||||
These are required. You can find the values for `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` in `/.circleci/env.sh`. `SAUCE_TUNNEL_IDENTIFIER` can be set to any unique value.
|
||||
|
||||
If you are having trouble running Saucelabs tests locally you can contact [Joey Perrott](https://github.com/josephperrott) or [Greg Magolan](https://github.com/gregmagolan) for support.
|
||||
|
||||
1. On OSX and Windows, you will also need to set `SAUCE_CONNECT` to the path of your `sc` binary (Sauce Connect Proxy).
|
||||
You will have to download Sauce Connect Proxy if you don't already have it downloaded.
|
||||
It's available on the SauceLabs website [here](https://wiki.saucelabs.com/display/DOCS/Downloading+Sauce+Connect+Proxy).
|
||||
Unzip it and point the SAUCE_CONNECT env variable to the `sc` binary.
|
||||
|
||||
```
|
||||
export SAUCE_CONNECT=/{path_to_sc}/bin/sc
|
||||
```
|
||||
|
||||
Note: it will not work to use the Sauce Connect that's already in node_modules unless you are using Linux.
|
||||
Download the one above for other platforms.
|
||||
|
||||
3. Once you have your environment variables set up, run the setup task in the root of the repo:
|
||||
|
||||
``` bash
|
||||
yarn bazel run //tools/saucelabs:sauce_service_setup
|
||||
```
|
||||
|
||||
4. You can run a particular test target through SauceLabs by prefixing the target name with "saucelabs_" and adding the `--config=saucelabs` option.
|
||||
For example, `packages/core/test:test_web` becomes `packages/core/test:saucelabs_test_web`.
|
||||
|
||||
```
|
||||
yarn bazel test //packages/core/test:saucelabs_test_web --config=saucelabs
|
||||
```
|
||||
|
||||
5. Sauce service log may be tailed or dumped with the following targets:
|
||||
|
||||
``` bash
|
||||
yarn bazel run //tools/saucelabs:sauce_service_tail
|
||||
yarn bazel run //tools/saucelabs:sauce_service_log
|
||||
```
|
||||
|
||||
|
||||
## Additional test features
|
||||
|
||||
To see the test output while the tests are running (as these are long tests), add the `--test_output=streamed` option.
|
||||
Note, this option will also prevent bazel from using the test cache and will force the test to run.
|
||||
|
||||
`bazel query` is required gather up all karma saucelabs test labels so they can be run in one command as they are tagged `manual`.
|
||||
|
||||
Running all karma tests in Saucelabs:
|
||||
|
||||
``` bash
|
||||
yarn bazel run //tools/saucelabs:sauce_service_setup
|
||||
TESTS=$(./node_modules/.bin/bazelisk query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "fixme-saucelabs", ...)')
|
||||
yarn bazel test --config=saucelabs ${TESTS}
|
||||
```
|
||||
|
||||
## Under the hood
|
||||
|
||||
The `//tools/saucelabs:sauce_service_setup` target does not start the Sauce Connect proxy but it does start the process that then listens for the start signal from the service manager script.
|
||||
This signal is sent by the karma wrapper script `//tools/saucelabs:karma-saucelabs` which calls `./tools/saucelabs/sauce-service.sh start`.
|
||||
This is necessary as the Sauce Connect Proxy process must be started outside of `bazel test` as Bazel will automatically kill any processes spawned during a test when that tests completes, which would prevent the tunnel from being shared by multiple tests.
|
||||
|
||||
The karma_web_test rule is to test with saucelabs with a modified `karma` attribute set to
|
||||
`//tools/saucelabs:karma-saucelabs`. This runs the `/tools/saucelabs/karma-saucelabs.js` wrapper
|
||||
script which configures the saucelabs environment and starts Sauce Connect before running karma.
|
||||
|
||||
For example,
|
||||
|
||||
``` python
|
||||
karma_web_test(
|
||||
name = "saucelabs_core_acceptance_tests",
|
||||
timeout = "long",
|
||||
karma = "//tools/saucelabs:karma-saucelabs",
|
||||
tags = [
|
||||
"exclusive",
|
||||
"manual",
|
||||
"no-remote-exec",
|
||||
"saucelabs",
|
||||
],
|
||||
deps = [
|
||||
"//packages/core/test/acceptance:acceptance_lib",
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
These saucelabs targets must have a few important tags:
|
||||
* `no-remote-exec` as they cannot be executed remotely since they require a local Sauce Connect process
|
||||
* `manual` so they are not automatically tested with `//...`
|
||||
* `exclusive` as they must be run serially in order to not over-provision Saucelabs browsers
|
||||
* `saucelabs` so that they can be easily gathered up for testing in a `bazel query`
|
||||
|
||||
## Debugging
|
||||
|
||||
**Q: How do I get the tests to run on IE? I only see Chromium.**
|
||||
|
||||
If you see something like this at the end of your test output, it means you're not actually running SauceLabs:
|
||||
|
||||
```
|
||||
INFO: Build completed successfully, 43 total actions
|
||||
/packages/core/test:test_web_chromium
|
||||
```
|
||||
|
||||
This is a common error caused by forgetting to prefix your test target with "saucelabs_".
|
||||
For example, `/packages/core/test:test_web` becomes `/packages/core/test:saucelabs_test_web`.
|
||||
|
||||
**Q: How can I tell that the SauceLabs connection was successfully made?**
|
||||
|
||||
There is a dashboard at saucelabs.com where you can see active tunnel connections (Angular has an account).
|
||||
As soon as you actually run the test target (not after the setup task), you should see an active tunnel connection under the SAUCE_TUNNEL_IDENTIFICATION_KEY you entered.
|
||||
If a tunnel connection is not there, you are not actually connecting with SauceLabs.
|
||||
|
||||
Note: It may *look* like the tests are running because of the Bazel output.
|
||||
The progress Bazel is showing does not mean that SauceLabs is connected.
|
||||
If the tests are actually running, you should see the "..." test report for passing tests.
|
||||
Moved to tools/saucelabs-daemon/README.md.
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright Google LLC All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by an MIT-style license that can be
|
||||
* found in the LICENSE file at https://angular.io/license
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
import shell from 'shelljs';
|
||||
import {runfiles} from '@bazel/runfiles';
|
||||
import fs from 'fs';
|
||||
import childProcess from 'child_process';
|
||||
|
||||
const karmaBin = runfiles.resolve('npm/node_modules/karma/bin/karma');
|
||||
const sauceService = runfiles.resolveWorkspaceRelative(process.argv[2]);
|
||||
|
||||
process.argv = [process.argv[0], karmaBin, ...process.argv.splice(3)];
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
||||
async function main() {
|
||||
console.error(`Setting up environment for SauceLabs karma tests...`);
|
||||
// KARMA_WEB_TEST_MODE is set which informs /karma-js.conf.js that it should
|
||||
// run the test with the karma saucelabs launcher
|
||||
process.env['KARMA_WEB_TEST_MODE'] = 'SL_REQUIRED';
|
||||
// Saucelabs parameters read from a temporary file that is created by the `sauce-service`. This
|
||||
// will be `null` if the test runs locally without the `sauce-service` being started.
|
||||
const saucelabsParams = readLocalSauceConnectParams();
|
||||
// Setup required SAUCE_* env if they are not already set
|
||||
if (
|
||||
!process.env['SAUCE_USERNAME'] ||
|
||||
!process.env['SAUCE_ACCESS_KEY'] ||
|
||||
!process.env['SAUCE_TUNNEL_IDENTIFIER']
|
||||
) {
|
||||
// We print a helpful error message below if the required Saucelabs parameters have not
|
||||
// been specified in test environment, and the `sauce-service` params file has not been
|
||||
// created either.
|
||||
if (saucelabsParams === null) {
|
||||
console.error(`
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
!!! Make sure that you have run "yarn bazel run //tools/saucelabs:sauce_service_setup"
|
||||
!!! (or "./tools/saucelabs/sauce-service.sh setup") before the test target. Alternately
|
||||
!!! you can provide the required SAUCE_* environment variables (SAUCE_USERNAME, SAUCE_ACCESS_KEY &
|
||||
!!! SAUCE_TUNNEL_IDENTIFIER) to the test with --test_env or --define but this may prevent bazel from
|
||||
!!! using cached test results.
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`);
|
||||
process.exit(1);
|
||||
}
|
||||
process.env['SAUCE_USERNAME'] = saucelabsParams.SAUCE_USERNAME;
|
||||
process.env['SAUCE_ACCESS_KEY'] = saucelabsParams.SAUCE_ACCESS_KEY;
|
||||
process.env['SAUCE_TUNNEL_IDENTIFIER'] = saucelabsParams.SAUCE_TUNNEL_IDENTIFIER;
|
||||
process.env['SAUCE_LOCALHOST_ALIAS_DOMAIN'] = saucelabsParams.SAUCE_LOCALHOST_ALIAS_DOMAIN;
|
||||
}
|
||||
|
||||
// Pass through the optional `SAUCE_LOCALHOST_ALIAS_DOMAIN` environment variable. The
|
||||
// variable is usually specified on CI, but is not required for testing with Saucelabs.
|
||||
if (!process.env['SAUCE_LOCALHOST_ALIAS_DOMAIN'] && saucelabsParams !== null) {
|
||||
process.env['SAUCE_LOCALHOST_ALIAS_DOMAIN'] = saucelabsParams.SAUCE_LOCALHOST_ALIAS_DOMAIN;
|
||||
}
|
||||
|
||||
const scStart = `${sauceService} start-ready-wait`;
|
||||
console.error(`Starting SauceConnect (${scStart})...`);
|
||||
const result = shell.exec(scStart).code;
|
||||
if (result !== 0) {
|
||||
throw new Error(`Starting SauceConnect failed with code ${result}`);
|
||||
}
|
||||
|
||||
console.error(`Launching karma ${karmaBin}...`);
|
||||
|
||||
await launchNodeBinaryAndWait(karmaBin, process.argv.slice(2));
|
||||
}
|
||||
|
||||
function launchNodeBinaryAndWait(entryPointPath, args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const proc = childProcess.fork(entryPointPath, args, {stdio: 'inherit'});
|
||||
proc.on('error', (err) => reject(err));
|
||||
proc.on('close', (code) =>
|
||||
code !== 0
|
||||
? reject(new Error(`Script "${entryPointPath}" exited with non-zero status code: ${code}`))
|
||||
: resolve()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function readLocalSauceConnectParams() {
|
||||
try {
|
||||
// The following path comes from /tools/saucelabs/sauce-service.sh.
|
||||
// We setup the required saucelabs environment variables here for the karma test
|
||||
// from a json file under /tmp/angular/sauce-service so that we don't break the
|
||||
// test cache with a changing SAUCE_TUNNEL_IDENTIFIER provided through --test_env
|
||||
return JSON.parse(
|
||||
fs.readFileSync('/tmp/angular/sauce-service/sauce-connect-params.json', 'utf8')
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -237,7 +237,6 @@
|
||||
|
||||
"@angular/build-tooling@https://github.com/angular/dev-infra-private-build-tooling-builds.git#7dad055464ea9847e4870b9e3baad1f0c417bdf7":
|
||||
version "0.0.0-5f06c4774df908ed69e1441f4ec63b898acf0c68"
|
||||
uid "7dad055464ea9847e4870b9e3baad1f0c417bdf7"
|
||||
resolved "https://github.com/angular/dev-infra-private-build-tooling-builds.git#7dad055464ea9847e4870b9e3baad1f0c417bdf7"
|
||||
dependencies:
|
||||
"@angular-devkit/build-angular" "16.0.0-rc.0"
|
||||
@@ -371,7 +370,6 @@
|
||||
|
||||
"@angular/ng-dev@https://github.com/angular/dev-infra-private-ng-dev-builds.git#231bc6ee07c0f799d28f201a4e6d68ae31d88cfa":
|
||||
version "0.0.0-5f06c4774df908ed69e1441f4ec63b898acf0c68"
|
||||
uid "231bc6ee07c0f799d28f201a4e6d68ae31d88cfa"
|
||||
resolved "https://github.com/angular/dev-infra-private-ng-dev-builds.git#231bc6ee07c0f799d28f201a4e6d68ae31d88cfa"
|
||||
dependencies:
|
||||
"@yarnpkg/lockfile" "^1.1.0"
|
||||
@@ -3672,6 +3670,13 @@
|
||||
"@types/glob" "*"
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/selenium-webdriver4@npm:@types/selenium-webdriver@4.1.12":
|
||||
version "4.1.12"
|
||||
resolved "https://registry.yarnpkg.com/@types/selenium-webdriver/-/selenium-webdriver-4.1.12.tgz#a2042f6bef044c48ca7a8b4caede4928242271a9"
|
||||
integrity sha512-3DxuYHEbzIyUtuICRtxXtJomdOOkBeCbkXAlAPm/h8ArIn/ArWNnVSC6tIj/ECZ/sdxBoI0F/wuwcpxBew3hGw==
|
||||
dependencies:
|
||||
"@types/ws" "*"
|
||||
|
||||
"@types/selenium-webdriver@3.0.7":
|
||||
version "3.0.7"
|
||||
resolved "https://registry.yarnpkg.com/@types/selenium-webdriver/-/selenium-webdriver-3.0.7.tgz#5d3613d1ab3ca08b74d19683a3a7c573129ab18f"
|
||||
@@ -7244,8 +7249,7 @@ domhandler@^4.2.0, domhandler@^4.3.1:
|
||||
domelementtype "^2.2.0"
|
||||
|
||||
"domino@https://github.com/angular/domino.git#aa8de3486307f57a518b4b0d9e5e16d9fbd998d1":
|
||||
version "2.1.6+git"
|
||||
uid aa8de3486307f57a518b4b0d9e5e16d9fbd998d1
|
||||
version "2.1.6"
|
||||
resolved "https://github.com/angular/domino.git#aa8de3486307f57a518b4b0d9e5e16d9fbd998d1"
|
||||
|
||||
domutils@^2.8.0:
|
||||
@@ -14038,7 +14042,6 @@ sass@1.62.1:
|
||||
|
||||
"sauce-connect@https://saucelabs.com/downloads/sc-4.8.1-linux.tar.gz":
|
||||
version "0.0.0"
|
||||
uid "9c16682e4c9716734432789884f868212f95f563"
|
||||
resolved "https://saucelabs.com/downloads/sc-4.8.1-linux.tar.gz#9c16682e4c9716734432789884f868212f95f563"
|
||||
|
||||
saucelabs@7.2.1, saucelabs@^1.5.0, saucelabs@^4.6.3:
|
||||
@@ -14095,6 +14098,15 @@ select-hose@^2.0.0:
|
||||
resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca"
|
||||
integrity sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==
|
||||
|
||||
"selenium-webdriver4@npm:selenium-webdriver@4.8.1", selenium-webdriver@4.8.1:
|
||||
version "4.8.1"
|
||||
resolved "https://registry.yarnpkg.com/selenium-webdriver/-/selenium-webdriver-4.8.1.tgz#4b0a546c4ea747c44e9688c108f7a46b8d8244ab"
|
||||
integrity sha512-p4MtfhCQdcV6xxkS7eI0tQN6+WNReRULLCAuT4RDGkrjfObBNXMJ3WT8XdK+aXTr5nnBKuh+PxIevM0EjJgkxA==
|
||||
dependencies:
|
||||
jszip "^3.10.0"
|
||||
tmp "^0.2.1"
|
||||
ws ">=8.11.0"
|
||||
|
||||
selenium-webdriver@3.5.0:
|
||||
version "3.5.0"
|
||||
resolved "https://registry.yarnpkg.com/selenium-webdriver/-/selenium-webdriver-3.5.0.tgz#9036c82874e6c0f5cbff0a0f18223bc31c99cb77"
|
||||
@@ -14115,15 +14127,6 @@ selenium-webdriver@3.6.0, selenium-webdriver@^3.0.1:
|
||||
tmp "0.0.30"
|
||||
xml2js "^0.4.17"
|
||||
|
||||
selenium-webdriver@4.8.1:
|
||||
version "4.8.1"
|
||||
resolved "https://registry.yarnpkg.com/selenium-webdriver/-/selenium-webdriver-4.8.1.tgz#4b0a546c4ea747c44e9688c108f7a46b8d8244ab"
|
||||
integrity sha512-p4MtfhCQdcV6xxkS7eI0tQN6+WNReRULLCAuT4RDGkrjfObBNXMJ3WT8XdK+aXTr5nnBKuh+PxIevM0EjJgkxA==
|
||||
dependencies:
|
||||
jszip "^3.10.0"
|
||||
tmp "^0.2.1"
|
||||
ws ">=8.11.0"
|
||||
|
||||
selfsigned@^2.1.1:
|
||||
version "2.1.1"
|
||||
resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-2.1.1.tgz#18a7613d714c0cd3385c48af0075abf3f266af61"
|
||||
|
||||
Reference in New Issue
Block a user