From 2a10050bea4ed69046945e2cabe0b69917fd1882 Mon Sep 17 00:00:00 2001 From: Joey Perrott Date: Thu, 28 Sep 2023 19:17:41 +0000 Subject: [PATCH] ci: migrate snapshot publishing from CircleCI to GHA (#51957) Migrate the snapshot publishing from CircleCI to GHA PR Close #51957 --- .circleci/config.yml | 346 +------------------------- .github/workflows/ci.yml | 19 ++ scripts/ci/publish-build-artifacts.sh | 2 +- 3 files changed, 26 insertions(+), 341 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3a07ff7faf9..9454be12fd0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,350 +1,16 @@ -# Configuration file for https://circleci.com/gh/angular/angular +# This config is remaining in place to prevent pull requests failing because of CircleCI config missing. -# Note: YAML anchors allow an object to be re-used, reducing duplication. -# The ampersand declares an alias for an object, then later the `<<: *name` -# syntax dereferences it. -# See https://blog.daemonl.com/2016/02/yaml.html -# To validate changes, use an online parser, eg. -# https://yaml-online-parser.appspot.com/ - -# CircleCI configuration version -# Version 2.1 allows for extra config reuse features -# https://circleci.com/docs/2.0/reusing-config/#getting-started-with-config-reuse version: 2.1 -# We don't want to include the current branch name in the cache key because that would prevent -# PRs from being able to restore the cache since the branch names are always different for PRs. -# The cache key should only consist of dynamic values that change whenever something in the -# cache changes. For example: -# 1) yarn lock file changes --> cached "node_modules" are different. -# 2) bazel repository definitions change --> cached bazel repositories are different. -# Windows needs its own cache key because binaries in node_modules are different. -# **NOTE 1 **: In order to avoid the cache from growing indefinitely and causing slow-downs, we invalidate the cache monthly. -# (See https://support.circleci.com/hc/en-us/articles/360012618473-Creating-a-daily-cache.) -# **NOTE 2 **: If you change the cache key prefix, also sync the cache_key_fallback to match. -# **NOTE 3 **: Keep the static part of the cache key as prefix to enable correct fallbacks. -# **NOTE 4 **: To make sure modified patches can be applied without needing to manually update the cache key, -# the fallback cache key will not match if patches have been modified. -# See https://circleci.com/docs/2.0/caching/#restoring-cache for how prefixes work in CircleCI. -var_3: &cache_key v8-angular-node-16-{{ checksum "month.txt" }}-{{ checksum "patches.hash"}}-{{ checksum ".bazelversion" }}-{{ checksum "yarn.lock" }}-{{ checksum "WORKSPACE" }}-{{ checksum "aio/yarn.lock" }} -# We invalidate the cache if the Bazel version changes because otherwise the `bazelisk` cache -# folder will contain all previously used versions and ultimately cause the cache restoring to -# be slower due to its growing size. -var_4: &cache_key_fallback v8-angular-node-16-{{ checksum "month.txt" }}-{{ checksum "patches.hash"}}-{{ checksum ".bazelversion" }} - -# Windows needs its own cache key because binaries in node_modules are different. -var_3_win: &cache_key_win v10-angular-win-node-16-{{ checksum "month.txt" }}-{{ checksum "patches.hash"}}-{{ checksum ".bazelversion" }}-{{ checksum "yarn.lock" }}-{{ checksum "WORKSPACE" }} -var_4_win: &cache_key_win_fallback v10-angular-win-node-16-{{ checksum "month.txt" }}-{{ checksum "patches.hash"}}-{{ checksum ".bazelversion" }} - -# Workspace initially persisted by the `setup` job, and then enhanced by `build-npm-packages`. -# https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs -# https://circleci.com/blog/deep-diving-into-circleci-workspaces/ -var_5: &workspace_location ~/ - -# Filter to run a job on builds for pull requests only. -var_6: &only_on_pull_requests - filters: - branches: - only: - - /pull\/\d+/ - -# Filter to skip a job on builds for pull requests. -var_7: &skip_on_pull_requests - filters: - branches: - ignore: - - /pull\/\d+/ - -# Filter to run a job on builds for the main branch only. -var_8: &only_on_main_branch - filters: - branches: - only: - - main - -# Filter to run a job on all releasable branches. -var_9: &only_release_branches - filters: - branches: - only: - - main - - /\d+\.\d+\.x/ - -# CircleCI orbs -# https://circleci.com/developer/orbs. -orbs: - node: circleci/node@5.0.1 - devinfra: angular/dev-infra@1.0.8 - -# Executor Definitions -# https://circleci.com/docs/2.0/reusing-config/#authoring-reusable-executors -# **NOTE 1**: Pin to exact images using an ID (SHA). See https://circleci.com/docs/2.0/circleci-images/#using-a-docker-image-id-to-pin-an-image-to-a-fixed-version. -# (Using the tag in not necessary when pinning by ID, but include it anyway for documentation purposes.) -# **NOTE 2**: If you change the version of the docker images, also change the `cache_key` suffix. -# **NOTE 3**: If you change the version of Node.js provided by the docker images, also update `.devcontainer/recommended-Dockerfile` to match the new version. -executors: - default-executor: - parameters: - resource_class: - type: string - default: medium - docker: - - image: cimg/node:18.13.0 - resource_class: << parameters.resource_class >> - working_directory: ~/ng - - test-browser-executor: - parameters: - resource_class: - type: string - default: medium - docker: - - image: cimg/node:18.13.0-browsers - resource_class: << parameters.resource_class >> - working_directory: ~/ng - - windows-executor: - working_directory: ~/ng - resource_class: windows.2xlarge - shell: bash - machine: - image: windows-server-2019-vs2019:201908-02 - -# Command Definitions -# https://circleci.com/docs/2.0/reusing-config/#authoring-reusable-commands -commands: - custom_attach_workspace: - description: Attach workspace at a predefined location - steps: - - attach_workspace: - at: *workspace_location - - # Install java runtime which is required by some integration tests such as - # //integration:hello_world__closure_test, //integration:i18n_test and - # //integration:ng_elements_test to run the closure compiler - install_java: - description: Install java - steps: - - run: - name: Install java - command: | - sudo apt-get update - # Install java runtime - sudo apt-get install default-jre - - # Initializes the CI environment by setting up common environment variables. - init_environment: - description: Initializing environment (setting up variables) - steps: - - run: - name: Set up environment - environment: - CIRCLE_GIT_BASE_REVISION: << pipeline.git.base_revision >> - CIRCLE_GIT_REVISION: << pipeline.git.revision >> - command: ./.circleci/env.sh - - devinfra/setup-bazel-remote-exec - - run: - # Configure git as the CircleCI `checkout` command does. - # This is needed because we only checkout on the setup job. - # Add GitHub to known hosts - name: Configure git - command: | - mkdir -p ~/.ssh - echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=' >> ~/.ssh/known_hosts - git config --global url."ssh://git@github.com".insteadOf "https://github.com" || true - git config --global gc.auto 0 || true - - init_saucelabs_environment: - description: Sets up a domain that resolves to the local host. - steps: - - run: - name: Preparing environment for running tests on Sauce Labs. - command: | - # For SauceLabs jobs, we set up a domain which resolves to the machine which launched - # the tunnel. We do this because devices are sometimes not able to properly resolve - # `localhost` or `127.0.0.1` through the SauceLabs tunnel. Using a domain that does not - # resolve to anything on SauceLabs VMs ensures that such requests are always resolved - # through the tunnel, and resolve to the actual tunnel host machine (i.e. the CircleCI VM). - # More context can be found in: https://github.com/angular/angular/pull/35171. - setPublicVar SAUCE_LOCALHOST_ALIAS_DOMAIN "angular-ci.local" - setSecretVar SAUCE_ACCESS_KEY $(echo $SAUCE_ACCESS_KEY | rev) - - run: - # Sets up a local domain in the machine's host file that resolves to the local - # host. This domain is helpful in Sauce Labs tests where devices are not able to - # properly resolve `localhost` or `127.0.0.1` through the sauce-connect tunnel. - name: Setting up alias domain for local host. - command: echo "127.0.0.1 $SAUCE_LOCALHOST_ALIAS_DOMAIN" | sudo tee -a /etc/hosts - - save_month_to_file: - description: Store the current year and month in a file, so that it can be used for computing the cache key. - steps: - - run: - name: Save month to file - # Note: Make sure this file is excluded in the `.gitignore` as otherwise the - # snapshot stamping would have the `-with-local-changes` suffix. - command: date +%Y-%m > month.txt - - capture_patches_for_cache_key: - description: Hashes all patches so that the cache can be reset upon changes. - steps: - - run: | - md5sum tools/esm-interop/patches/npm/* > patches.hash - - yarn_install: - description: Install dependencies - steps: - - run: - name: Running Yarn install - command: yarn install --frozen-lockfile --non-interactive --cache-folder ~/.cache/yarn - # Yarn's requests sometimes take more than 10mins to complete (especially on Windows). - no_output_timeout: 45m - - notify_webhook_on_fail: - description: Notify a webhook about failure - parameters: - # `webhook_url_env_var` are secret env vars defined in CircleCI project settings. - # The URLs come from https://angular-team.slack.com/apps/A0F7VRE7N-circleci. - webhook_url_env_var: - type: env_var_name - steps: - - run: - when: on_fail - command: | - notificationJson="{\"text\":\":x: \`$CIRCLE_JOB\` job for $CIRCLE_BRANCH branch failed on build $CIRCLE_BUILD_NUM: $CIRCLE_BUILD_URL :scream:\"}" - curl --request POST --header "Content-Type: application/json" --data "$notificationJson" ${<< parameters.webhook_url_env_var >>} - -# Job definitions -# Jobs can include parameters that are passed in the workflow job invocation. -# https://circleci.com/docs/2.0/reusing-config/#authoring-parameterized-jobs jobs: - setup: - executor: default-executor + pass: + docker: + - image: cimg/base:2022.05 steps: - - checkout - - save_month_to_file - - capture_patches_for_cache_key - - init_environment - - devinfra/rebase-pr-on-target-branch: - base_revision: << pipeline.git.base_revision >> - head_revision: << pipeline.git.revision >> - # This cache is saved in the build-npm-packages so that Bazel cache is also included. - - restore_cache: - keys: - - *cache_key - - *cache_key_fallback - - yarn_install - - run: yarn --cwd aio install --frozen-lockfile --non-interactive --cache-folder ~/.cache/yarn - # Make the bazel directories and add a file to them if they don't exist already so that - # persist_to_workspace does not fail. - - run: | - if [ ! -d ~/bazel_repository_cache ]; then - mkdir ~/bazel_repository_cache - touch ~/bazel_repository_cache/MARKER - fi - # Persist any changes at this point to be reused by further jobs. - # **NOTE**: To add new content to the workspace, always persist on the same root. - - persist_to_workspace: - root: *workspace_location - paths: - - ./ng - - ./bazel_repository_cache - - # The `build-npm-packages` tasks exist for backwards-compatibility with old scripts and - # tests that rely on the pre-Bazel `dist/packages-dist` output structure (build.sh). - # Having multiple jobs that independently build in this manner duplicates some work; we build - # the bazel packages more than once. Even though we have a remote cache, these jobs will - # typically run in parallel so up-to-date outputs will not be available at the time the build - # starts. - build-npm-packages: - executor: - name: default-executor - resource_class: xlarge - steps: - - custom_attach_workspace - - init_environment - - run: yarn build - - # Save the npm packages from //packages/... for other workflow jobs to read - - persist_to_workspace: - root: *workspace_location - paths: - - ng/dist/packages-dist - - ng/dist/angular-in-memory-web-api-dist - - ng/dist/zone.js-dist - - # Save dependencies and bazel repository cache to use on subsequent runs. - - save_cache: - key: *cache_key - paths: - - ~/.cache/yarn - - ~/bazel_repository_cache - - ~/.cache/bazelisk - - # This job updates the content of repos like github.com/angular/core-builds - # for every green build on angular/angular. - publish_snapshot: - executor: default-executor - steps: - - custom_attach_workspace - - init_environment - # CircleCI has a config setting to force SSH for all github connections - # This is not compatible with our mechanism of using a Personal Access Token - # Clear the global setting - - run: git config --global --unset "url.ssh://git@github.com.insteadof" - - run: - name: Prepare GitHub credentials - command: echo "https://${SNAPSHOT_BUILDS_GITHUB_TOKEN}:@github.com" > ${HOME}/.git_credentials - - run: ./scripts/ci/publish-build-artifacts.sh - - aio_monitoring_stable: - executor: test-browser-executor - steps: - - custom_attach_workspace - - init_environment - - run: setPublicVar_CI_STABLE_BRANCH - - run: - name: Check out `aio/` and yarn from the stable branch - command: | - git fetch origin $CI_STABLE_BRANCH - git checkout --force origin/$CI_STABLE_BRANCH -- aio/ .yarn/ .yarnrc - # Ignore yarn's engines check, because we checked out `aio/package.json` from the stable - # branch and there could be a node version skew, which is acceptable in this monitoring job. - - run: yarn config set ignore-engines true - - run: - name: Run tests against https://angular.io/ - command: ./aio/scripts/test-production.sh https://angular.io/ $CI_AIO_MIN_PWA_SCORE - - notify_webhook_on_fail: - webhook_url_env_var: SLACK_CARETAKER_WEBHOOK_URL - - notify_webhook_on_fail: - webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL - - aio_monitoring_next: - executor: test-browser-executor - steps: - - custom_attach_workspace - - init_environment - - run: - name: Run tests against https://next.angular.io/ - command: ./aio/scripts/test-production.sh https://next.angular.io/ $CI_AIO_MIN_PWA_SCORE - - notify_webhook_on_fail: - webhook_url_env_var: SLACK_CARETAKER_WEBHOOK_URL - - notify_webhook_on_fail: - webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL + - run: echo "This too shall pass (always)" workflows: version: 2 default_workflow: jobs: - - setup: - filters: - branches: - ignore: g3 - - build-npm-packages: - requires: - - setup - - publish_snapshot: - <<: *only_release_branches - requires: - # Get the artifacts to publish from the build-packages-dist job - # since the publishing script expects the legacy outputs layout. - - build-npm-packages + - pass diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f945f524f0..29d052ee850 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,6 +192,25 @@ jobs: env: SLACK_BOT_TOKEN: ${{ secrets.ANGULAR_ROBOT_SLACK_TOKEN }} + publish-snapshots: + if: github.event_name == 'push' + runs-on: + labels: ubuntu-latest + steps: + - name: Initialize environment + uses: angular/dev-infra/github-actions/npm/checkout-and-setup-node@ba9b4487ced515e5b4d87edd681a3bd9792444d6 + with: + cache-node-modules: true + - name: Setup Bazel + uses: angular/dev-infra/github-actions/bazel/setup@ba9b4487ced515e5b4d87edd681a3bd9792444d6 + - name: Setup Bazel RBE + uses: angular/dev-infra/github-actions/bazel/configure-remote@ba9b4487ced515e5b4d87edd681a3bd9792444d6 + - name: Install node modules + run: yarn install --frozen-lockfile + - run: echo "https://${{secrets.SNAPSHOT_BUILDS_GITHUB_TOKEN}}:@github.com" > ${HOME}/.git_credentials + - run: yarn build + - run: ./scripts/ci/publish-build-artifacts.sh + zone-js: runs-on: labels: ubuntu-latest-4core diff --git a/scripts/ci/publish-build-artifacts.sh b/scripts/ci/publish-build-artifacts.sh index 7ffd2a043b3..b89a74be2c8 100755 --- a/scripts/ci/publish-build-artifacts.sh +++ b/scripts/ci/publish-build-artifacts.sh @@ -152,7 +152,7 @@ function publishAllBuilds() { } # See docs/DEVELOPER.md for help -CUR_BRANCH=${CI_BRANCH:-$(git symbolic-ref --short HEAD)} +CUR_BRANCH=$(git symbolic-ref --short HEAD) if [ $# -gt 0 ]; then ORG=$1 publishAllBuilds "ssh"