build: use bazel from node modules (#16361)

The goal of https://github.com/angular/components/commit/2ce5ffd56b70c3791331ceedeadc1f1f6daf3924 was to avoid the use of Bazel through the `node_modules` because it adds unnecessary code overhead, slows down CI and means that some tools need to be installed multiple times (such as `yarn`). Unfortunately when building a release with Bazel right now, the bazel workspace status script depends on the global `node` binary. This means that we need to bring in Node when running with the `--config=release` flag.. Since that causes inconsistent behavior and just complicates the setup, we are using Bazel through Yarn for now. Similar to how we did it for `angular/angular` with https://github.com/angular/angular/commit/8fc4ae51fb66d14f6dc82712a536c3842205fc6d.
2026-09-14 13:54:52 +08:00 · 2019-06-22 00:08:30 +02:00
parent 94c84edd58
commit 10370ef659
5 changed files with 113 additions and 121 deletions
@@ -8,23 +8,22 @@
 # http://yaml-online-parser.appspot.com/

 var_1: &docker_image circleci/node:10.16
-var_2: &docker_bazel_image l.gcr.io/google/bazel:0.26.1

 # **Note**: When updating the beginning of the cache key, also update the cache key to match
 # the new cache key prefix. This allows us to take advantage of CircleCI's fallback caching.
 # Read more here: https://circleci.com/docs/2.0/caching/#restoring-cache.
-var_3: &cache_key v3-ng-mat-{{ checksum "WORKSPACE" }}-{{ checksum "yarn.lock" }}
-var_4: &cache_fallback_key v3-ng-mat-
+var_2: &cache_key v4-ng-mat-{{ checksum "WORKSPACE" }}-{{ checksum "yarn.lock" }}
+var_3: &cache_fallback_key v4-ng-mat-

 # Settings common to each job
-var_5: &job_defaults
+var_4: &job_defaults
  working_directory: ~/ng
  docker:
    - image: *docker_image

 # Job step for checking out the source code from GitHub. This also ensures that the source code
 # is rebased on top of master.
-var_6: &checkout_code
+var_5: &checkout_code
  checkout:
    # After checkout, rebase on top of master. By default, PRs are not rebased on top of master,
    # which we want. See https://discuss.circleci.com/t/1662
@@ -32,7 +31,7 @@ var_6: &checkout_code

 # Restores the cache that could be available for the current Yarn lock file. The cache usually
 # includes the node modules and the Bazel repository cache.
-var_7: &restore_cache
+var_6: &restore_cache
  restore_cache:
    keys:
      - *cache_key
@@ -40,7 +39,7 @@ var_7: &restore_cache

 # Saves the cache for the current Yarn lock file. We store the node modules and the Bazel
 # repository cache in order to make subsequent builds faster.
-var_8: &save_cache
+var_7: &save_cache
  save_cache:
    key: *cache_key
    paths:
@@ -48,19 +47,19 @@ var_8: &save_cache
      - "~/bazel_repository_cache"

 # Decryption token that is used to decode the GCP credentials file in ".circleci/gcp_token".
-var_9: &gcp_decrypt_token "angular"
+var_8: &gcp_decrypt_token "angular"

 # Job step that ensures that the node module dependencies are installed and up-to-date. We use
 # Yarn with the frozen lockfile option in order to make sure that lock file and package.json are
 # in sync. Unlike in Travis, we don't need to manually purge the node modules if stale because
 # CircleCI automatically discards the cache if the checksum of the lock file has changed.
-var_10: &yarn_install
+var_9: &yarn_install
  run:
    name: "Installing project dependencies"
    command: yarn install --frozen-lockfile --non-interactive

 # Anchor that can be used to download and install Yarn globally in the bash environment.
-var_11: &yarn_download
+var_10: &yarn_download
  run:
    name: "Downloading and installing Yarn"
    command: |
@@ -68,7 +67,7 @@ var_11: &yarn_download
      curl -o- -L https://yarnpkg.com/install.sh | PROFILE=$BASH_ENV bash -s -- --version "1.16.0"

 # Sets up the Bazel config which is specific for CircleCI builds.
-var_12: &setup_bazel_ci_config
+var_11: &setup_bazel_ci_config
  run:
    name: "Setting up Bazel configuration for CI"
    command: |
@@ -76,21 +75,21 @@ var_12: &setup_bazel_ci_config

 # Sets up a different Docker image that includes a moe recent Firefox version which
 # is needed for headless testing.
-var_13: &docker-firefox-image
+var_12: &docker-firefox-image
  # TODO(devversion): Temporarily use a image that includes Firefox 62 because the
  # ngcontainer image does include an old Firefox version that does not support headless.
  - image: circleci/node:11.4.0-browsers

 # Attaches the release output which has been stored in the workspace to the current job.
 # https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
-var_14: &attach_release_output
+var_13: &attach_release_output
  attach_workspace:
    at: dist/

 # Branch filter that we can specify for jobs that should only run on publish branches. This filter
 # is used to ensure that not all upstream branches will be published as Github builds
 # (e.g. revert branches, feature branches)
-var_15: &publish_branches_filter
+var_14: &publish_branches_filter
  branches:
    only:
      - master
@@ -104,7 +103,7 @@ var_15: &publish_branches_filter
 # In order to reduce duplication we use a YAML anchor that just always excludes the "_presubmit"
 # branch. We don't want to run Circle for the temporary "_presubmit" branch which is reserved
 # for the caretaker.
-var_16: &ignore_presubmit_branch_filter
+var_15: &ignore_presubmit_branch_filter
  branches:
    ignore:
      - "_presubmit"
@@ -112,7 +111,7 @@ var_16: &ignore_presubmit_branch_filter

 # Runs a script that sets up the Bazel remote execution. This will be used by jobs that run
 # Bazel primarily and should benefit from remote caching and execution.
-var_17: &setup_bazel_remote_execution
+var_16: &setup_bazel_remote_execution
  run:
    name: "Setup bazel RBE remote execution"
    command: ./scripts/circleci/bazel/setup-remote-execution.sh
@@ -132,8 +131,7 @@ jobs:
  # Build and test job that uses Bazel.
  # -----------------------------------
  bazel_build_test:
-    docker:
-      - image: *docker_bazel_image
+    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
@@ -142,17 +140,18 @@ jobs:
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
+      - *yarn_download
+      - *yarn_install

-      - run: bazel build src/... --build_tag_filters=-docs-package
-      - run: bazel test src/... --build_tag_filters=-docs-package --test_tag_filters=-e2e
+      - run: yarn bazel build src/... --build_tag_filters=-docs-package
+      - run: yarn bazel test src/... --build_tag_filters=-docs-package --test_tag_filters=-e2e

  # --------------------------------------------------------------------------------------------
  # Job that runs ts-api-guardian against our API goldens in "tools/public_api_guard".
  # This job fails whenever an API has been updated but not explicitly approved through goldens.
  # --------------------------------------------------------------------------------------------
  api_golden_checks:
-    docker:
-      - image: *docker_bazel_image
+    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
@@ -161,15 +160,16 @@ jobs:
    - *restore_cache
    - *setup_bazel_ci_config
    - *setup_bazel_remote_execution
+    - *yarn_download
+    - *yarn_install

-    - run: bazel test tools/public_api_guard/...
+    - run: yarn bazel test tools/public_api_guard/...

  # -----------------------------------------------------------------
  # Job that runs the e2e tests with Protractor and Chromium headless
  # -----------------------------------------------------------------
  e2e_tests:
-    docker:
-      - image: *docker_bazel_image
+    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
@@ -178,8 +178,10 @@ jobs:
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
+      - *yarn_download
+      - *yarn_install

-      - run: bazel test src/... --test_tag_filters=e2e
+      - run: yarn bazel test src/... --test_tag_filters=e2e

  # ------------------------------------------------------------------------------------------
  # Job that runs the unit tests on locally installed browsers (Chrome and Firefox headless).
@@ -330,8 +332,7 @@ jobs:
  # Job that publishes the build snapshots
  # ----------------------------------------
  publish_snapshots:
-    docker:
-      - image: *docker_bazel_image
+    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
@@ -341,6 +342,8 @@ jobs:
      - *attach_release_output
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
+      - *yarn_download
+      - *yarn_install

      # CircleCI has a config setting to enforce SSH for all github connections.
      # This is not compatible with our mechanism of using a Personal Access Token
@@ -349,7 +352,7 @@ jobs:

      # TODO(devversion): Ideally the "build_release_packages" job should build all packages with
      # Bazel, but for now we mix up the Gulp and bazel setup, so we need to build the package here.
-      - run: bazel build src/material-examples:npm_package --config=release
+      - run: yarn bazel build src/material-examples:npm_package --config=release

      - run: ./scripts/circleci/publish-snapshots.sh

@@ -365,8 +368,11 @@ jobs:
    - *restore_cache
    - *yarn_download

-    - run: python ./scripts/circleci/setup-angular-snapshots.py --tag master
-    - *yarn_install
+    - run: node ./scripts/circleci/setup-angular-snapshots.js --tag master
+      # Install yarn dependencies after the package.json has been updated. Note that
+      # we can't use frozen-lockfile since the setup snapshots script does not update
+      # the lock file.
+    - run: yarn install --non-interactive
    - run: ./scripts/circleci/run-local-browser-tests.sh


@@ -375,8 +381,7 @@ jobs:
  # specified in the project dev dependencies.
  # ----------------------------------------------------------------------------
  ivy_test:
-    docker:
-      - image: *docker_bazel_image
+    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
@@ -385,6 +390,8 @@ jobs:
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
+      - *yarn_download
+      - *yarn_install

      # Setup Angular ivy snapshots built with ngtsc but locked to a specific tag. We
      # cannot determine the tag automatically based on the Angular version specified in
@@ -392,21 +399,20 @@ jobs:
      # we use ngcc to apply the ivy switches because ngcc currently does not handle the
      # UMD format which is used by Bazel when running tests. UMD processing is in
      # progress and tracked with FW-85.
-      - run: python ./scripts/circleci/setup-angular-snapshots.py --tag 8.1.0-next.1-ivy-aot+82e0b4a
+      - run: node ./scripts/circleci/setup-angular-snapshots.js --tag 8.1.0-next.1-ivy-aot+82e0b4a
      # Disable type checking when building with Ivy. This is necessary because
      # type checking is not complete yet and can incorrectly break compilation.
      # Issue is tracked with FW-1004.
      - run: sed -i "s/\(_ENABLE_NG_TYPE_CHECKING = \)True/\1False/g" tools/defaults.bzl
      # Run project tests with ngtsc and the Ivy Angular packages.
-      - run: bazel build src/... --build_tag_filters=-docs-package --define=compile=aot
-      - run: bazel test src/... --build_tag_filters=-docs-package --define=compile=aot --test_tag_filters=-e2e
+      - run: yarn bazel build src/... --build_tag_filters=-docs-package --define=compile=aot
+      - run: yarn bazel test src/... --build_tag_filters=-docs-package --define=compile=aot --test_tag_filters=-e2e

  # ----------------------------------------------------------------------------
  # Job that runs all Bazel tests against Ivy from angular/angular#master.
  # ----------------------------------------------------------------------------
  ivy_snapshot_test_cronjob:
-    docker:
-      - image: *docker_bazel_image
+    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
@@ -415,16 +421,18 @@ jobs:
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
+      - *yarn_download
+      - *yarn_install

      # Setup Angular ivy snapshots built with ngtsc.
-      - run: python ./scripts/circleci/setup-angular-snapshots.py --tag master-ivy-aot
+      - run: node ./scripts/circleci/setup-angular-snapshots.js --tag master-ivy-aot
      # Disable type checking when building with Ivy. This is necessary because
      # type checking is not complete yet and can incorrectly break compilation.
      # Issue is tracked with FW-1004.
      - run: sed -i "s/\(_ENABLE_NG_TYPE_CHECKING = \)True/\1False/g" tools/defaults.bzl
      # Run project tests with ngtsc and the Ivy Angular packages.
-      - run: bazel build src/... --build_tag_filters=-docs-package --define=compile=aot
-      - run: bazel test src/... --build_tag_filters=-docs-package --define=compile=aot --test_tag_filters=-e2e
+      - run: yarn bazel build src/... --build_tag_filters=-docs-package --define=compile=aot
+      - run: yarn bazel test src/... --build_tag_filters=-docs-package --define=compile=aot --test_tag_filters=-e2e

 # ----------------------------------------------------------------------------------------
 # Workflow definitions. A workflow usually groups multiple jobs together. This is useful if
@@ -1,5 +1,8 @@
 #!/bin/bash

+# The script should immediately exit if any command in the script fails.
+set -e
+
 if [[ -z "${GCP_DECRYPT_TOKEN}" ]]; then
  echo "Please specify the \"GCP_DECRYPT_TOKEN\" environment variable when setting up remote " \
      "execution"
@@ -8,12 +11,12 @@ fi

 # Decode the GCP token that is needed to authenticate the Bazel remote execution.
 openssl aes-256-cbc -d -in .circleci/gcp_token -md md5 -k ${GCP_DECRYPT_TOKEN} \
-  -out /home/.gcp_credentials
+  -out $HOME/.gcp_credentials

 # Export the "GOOGLE_APPLICATION_CREDENTIALS" variable that should refer to the GCP credentials
 # file. Bazel automatically picks up the credentials from that variable.
 # https://github.com/bazelbuild/bazel/blob/master/third_party/grpc/include/grpc/grpc_security.h#L134-L137
-echo "export GOOGLE_APPLICATION_CREDENTIALS=/home/.gcp_credentials" >> $BASH_ENV
+echo "export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.gcp_credentials" >> $BASH_ENV

 # Update the CircleCI Bazel configuration to always use remote execution.
 echo 'build --config=remote' >> .circleci/bazel.rc
@@ -0,0 +1,58 @@
+/**
+ * Script that sets up the Angular snapshot github builds. We set up the snapshot builds by
+ * overwriting the versions in the "package.json" and taking advantage of Yarn's resolutions
+ * feature. Yarn resolutions will be used to flatten nested Angular packages because by default
+ * Yarn does not flatten any dependency. See:
+ *
+ *  node_modules/compiler@snapshot
+ *  node_modules/compiler-cli@snapshot
+ *    node_modules/compiler@7.0.1
+ *
+ * Note that we cannot just use Yarn's `--flat` option because that would mean that it tries
+ * to flatten **all** dependencies and could cause unexpected results. We **only** want to
+ * explicitly flatten out all `@angular/*` dependencies. This can be achieved with resolutions.
+ * Read more here: https://yarnpkg.com/lang/en/docs/package-json/#toc-resolutions
+ */
+
+const {yellow, green} = require('chalk');
+const {writeFileSync} = require('fs');
+const {join} = require('path');
+
+const {tag} = require('minimist')(process.argv.slice(2), {string: ['tag']});
+const projectDir = join(__dirname, '../../');
+const packageJsonPath = join(projectDir, 'package.json');
+const packageJson = require(packageJsonPath);
+
+// Initialize the "resolutions" property in case it is not present in the "package.json" yet.
+// See: https://yarnpkg.com/lang/en/docs/package-json/#toc-resolutions for the API.
+packageJson['resolutions'] = packageJson['resolutions'] || {};
+
+// List that contains the names of all installed Angular packages (e.g. "@angular/core")
+const angularPackages = Object.keys({...packageJson.dependencies, ...packageJson.devDependencies})
+  .filter(packageName => packageName.startsWith('@angular/'));
+const packageSuffix = tag ? ` (${tag})` : '';
+
+console.log(green('Setting up snapshot builds for:\n'));
+console.log(yellow(`  ${angularPackages.map(n => `${n}${packageSuffix}`).join('\n  ')}\n`));
+
+// Setup the snapshot version for each Angular package specified in the "package.json" file.
+angularPackages.forEach(packageName => {
+  let buildsUrl = `github:angular/${packageName.split('/')[1]}-builds${tag ? `#${tag}` : ''}`;
+
+  // Add resolutions for each package in the format "**/{PACKAGE}" so that all
+  // nested versions of that specific Angular package will have the same version.
+  packageJson.resolutions[`**/${packageName}`] = buildsUrl;
+
+  // Since the resolutions only cover the version of all nested installs, we also need
+  // to explicitly set the version for the package listed in the project "package.json".
+  packageJson.dependencies[packageName] = buildsUrl;
+
+  // In case this dependency was previously a dev dependency, just remove it because we
+  // re-added it as a normal dependency for simplicity.
+  delete packageJson.devDependencies[packageName];
+});
+
+// Write changes to the "packageJson", so that we can install the new versions afterwards.
+writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2));
+
+console.log(green('Successfully added the "resolutions" to the "package.json".'));
@@ -1,77 +0,0 @@
-"""
-    Script that sets up the Angular snapshot github builds. We set up the snapshot builds by
-    overwriting the versions in the "package.json" and taking advantage of Yarn's resolutions
-    feature. Yarn resolutions will be used to flatten nested Angular packages because by default
-    Yarn does not flatten any dependency. See:
-
-    node_modules/compiler@snapshot
-    node_modules/compiler-cli@snapshot
-      node_modules/compiler@7.0.1
-
-    Note that we cannot just use Yarn's `--flat` option because that would mean that it tries
-    to flatten **all** dependencies and could cause unexpected results. We **only** want to
-    explicitly flatten out all `@angular/*` dependencies. This can be achieved with resolutions.
-    Read more here: https://yarnpkg.com/lang/en/docs/package-json/#toc-resolutions
-
-    **Note** that this script has to be written in Python because the docker container which
-    is used for our CI does not come with NodeJS pre-installed. Since this script needs to run
-    before the Bazel analysis phase is started, we can only run this script only with tools that
-    are pre-installed in the image. Python is a pre-installed in the google/bazel official image
-    and therefore we use that for the "package.json" modification. Note that this script is only
-    running on CI and therefore there are no implications with writing this in python.
-"""
-
-import json
-import os
-import argparse
-
-def collect_angular_deps(obj, result):
-    for item in obj:
-        if (item.startswith('@angular/')):
-            result.append(item)
-
-project_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../'))
-package_json_path = os.path.join(project_dir, 'package.json')
-
-parser = argparse.ArgumentParser(description='Setup Angular snapshot packages')
-parser.add_argument('--tag', required = True)
-parsed_args = parser.parse_args()
-
-with open(package_json_path, 'r') as package_json:
-    data = json.load(package_json)
-    angular_pkgs = []
-
-    collect_angular_deps(data['dependencies'], angular_pkgs)
-    collect_angular_deps(data['devDependencies'], angular_pkgs)
-
-    if not 'resolutions' in data:
-        data['resolutions'] = {}
-
-    print('Setting up snapshot builds for:')
-
-    for pkg in angular_pkgs:
-        new_url = "github:angular/%s-builds#%s" % (pkg.split('/')[1], parsed_args.tag)
-
-        print("  %s -> %s" % (pkg, new_url))
-
-        # Add resolutions for each package in the format "**/{PACKAGE}" so that all
-        # nested versions of that specific Angular package will have the same version.
-        data['resolutions']['**/%s' % pkg] = new_url
-
-        # Since the resolutions only cover the version of all nested installs, we also need
-        # to explicitly set the version for the package listed in the project "package.json".
-        data['dependencies'][pkg] = new_url
-
-        # In case this dependency was previously a dev dependency, just remove it because we
-        # re-added it as a normal dependency for simplicity.
-        if pkg in data['devDependencies']:
-            del data['devDependencies'][pkg]
-
-    new_package_json = json.dumps(data, indent = 2)
-
-    with open(package_json_path, 'w') as package_json_out:
-        package_json_out.write(new_package_json)
-
-    print('')
-    print('Successfully added the "resolutions" to the "package.json".')
-    print('Please run "yarn install" in the project to update the lock file.')
@@ -24,7 +24,7 @@ docsDistPath="${projectPath}/dist/docs"
 docsContentPath="${projectPath}/tmp/material2-docs-content"

 # Path to the release output of the Bazel "@angular/material-examples" NPM package.
-examplesPackagePath="$(bazel info bazel-bin)/src/material-examples/npm_package"
+examplesPackagePath="$(yarn -s bazel info bazel-bin)/src/material-examples/npm_package"

 # Git clone URL for the material2-docs-content repository.
 docsContentRepoUrl="https://github.com/angular/material2-docs-content"