Rename mirror-404-artifacts-for-bazel to mirror-bazel-external-deps and use tools/mirror_external_deps.py

[meteorcloudy]

Rename the mirror-404-artifacts-for-bazel pipeline to mirror-bazel-external-deps ("Mirror Bazel External Deps") and update it to check out bazelbuild/bazel and run ./tools/mirror_external_deps.py --upload to discover and upload missing external repository URLs to gs://bazel-mirror, replacing buildkite/mirror_404_downloads.py.

[github-actions]

Terraform Plan for bazel-trusted 📖

Plan: 1 to add, 0 to change, 1 to destroy.

Show Plan

Terraform used the selected providers to generate the following execution
plan. Resource actions are indicated with the following symbols:
  + create
  - destroy

Terraform will perform the following actions:

  # buildkite_pipeline.mirror-404-artifacts-for-bazel will be destroyed
  # (because buildkite_pipeline.mirror-404-artifacts-for-bazel is not in configuration)
  - resource "buildkite_pipeline" "mirror-404-artifacts-for-bazel" {
      - allow_rebuilds                           = true -> null
      - badge_url                                = "https://badge.buildkite.com/3f315c65aca18811c830f920e8e7aa88f64a1c9dcc59e73a80.svg" -> null
      - branch_configuration                     = "master" -> null
      - cancel_intermediate_builds               = false -> null
      - cancel_intermediate_builds_branch_filter = "" -> null
      - default_branch                           = "master" -> null
      - description                              = "See https://github.com/bazelbuild/continuous-integration/pull/2386" -> null
      - id                                       = "UGlwZWxpbmUtLS0wMTk5ZGQ3Ni0wMmNiLTQ4MDQtYTdjMS1iOGI5NTlhNmZlMTc=" -> null
      - name                                     = "Mirror 404 artifacts for Bazel" -> null
      - provider_settings                        = {
          - build_branches                                = false -> null
          - build_merge_group_checks_requested            = false -> null
          - build_pull_request_base_branch_changed        = false -> null
          - build_pull_request_forks                      = false -> null
          - build_pull_request_labels_changed             = false -> null
          - build_pull_request_ready_for_review           = false -> null
          - build_pull_requests                           = false -> null
          - build_tags                                    = false -> null
          - cancel_deleted_branch_builds                  = false -> null
          - cancel_when_merge_group_destroyed             = false -> null
          - filter_condition                              = "" -> null
          - filter_enabled                                = false -> null
          - ignore_default_branch_pull_requests           = false -> null
          - prefix_pull_request_fork_branch_names         = false -> null
          - publish_blocked_as_pending                    = false -> null
          - publish_commit_status                         = false -> null
          - publish_commit_status_per_step                = false -> null
          - pull_request_branch_filter_configuration      = "" -> null
          - pull_request_branch_filter_enabled            = false -> null
          - separate_pull_request_statuses                = false -> null
          - skip_builds_for_existing_commits              = false -> null
          - skip_pull_request_builds_for_existing_commits = false -> null
          - trigger_mode                                  = "none" -> null
          - use_merge_group_base_commit_for_git_diff_base = false -> null
        } -> null
      - repository                               = "https://github.com/bazelbuild/continuous-integration.git" -> null
      - skip_intermediate_builds                 = false -> null
      - skip_intermediate_builds_branch_filter   = "" -> null
      - slug                                     = "mirror-404-artifacts-for-bazel" -> null
      - steps                                    = <<-EOT
            ---
            env:
              GIT_HTTP_LOW_SPEED_LIMIT: "102400"
              GIT_HTTP_LOW_SPEED_TIME: "180"
            
            priority: 0
            steps:
              - command: |-
                  cd buildkite
                  python3 mirror_404_downloads.py
                label: ":mirror: Mirror missing artifacts"
                agents:
                  - "queue=default"
                plugins:
                  - docker#v3.8.0:
                      always-pull: true
                      environment:
                        - "ANDROID_HOME"
                        - "ANDROID_NDK_HOME"
                        - "BUILDKITE_ARTIFACT_UPLOAD_DESTINATION"
                      image: "gcr.io/bazel-public/ubuntu2204"
                      network: "host"
                      privileged: true
                      propagate-environment: true
                      propagate-uid-gid: true
                      volumes:
                        - "/etc/group:/etc/group:ro"
                        - "/etc/passwd:/etc/passwd:ro"
                        - "/etc/shadow:/etc/shadow:ro"
                        - "/opt/android-ndk-r15c:/opt/android-ndk-r15c:ro"
                        - "/opt/android-sdk-linux:/opt/android-sdk-linux:ro"
                        - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
                        - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
        EOT -> null
      - tags                                     = [] -> null
      - uuid                                     = "0199dd76-02cb-4804-a7c1-b8b959a6fe17" -> null
      - webhook_url                              = "https://webhook.buildkite.com/deliver/b819c42f169ebd51dcff92aa0f1c79fde5a61e6d78368e3480" -> null
    }

  # buildkite_pipeline.mirror-bazel-external-deps will be created
  + resource "buildkite_pipeline" "mirror-bazel-external-deps" {
      + allow_rebuilds                           = true
      + badge_url                                = (known after apply)
      + branch_configuration                     = "master"
      + cancel_intermediate_builds               = false
      + cancel_intermediate_builds_branch_filter = (known after apply)
      + cluster_name                             = (known after apply)
      + default_branch                           = "master"
      + default_timeout_in_minutes               = (known after apply)
      + description                              = "Mirror missing external repository artifacts for Bazel to gs://bazel-mirror"
      + id                                       = (known after apply)
      + maximum_timeout_in_minutes               = (known after apply)
      + name                                     = "Mirror Bazel External Deps"
      + provider_settings                        = {
          + build_branches                                = (known after apply)
          + build_merge_group_checks_requested            = (known after apply)
          + build_pull_request_base_branch_changed        = (known after apply)
          + build_pull_request_forks                      = (known after apply)
          + build_pull_request_labels_changed             = (known after apply)
          + build_pull_request_ready_for_review           = (known after apply)
          + build_pull_requests                           = (known after apply)
          + build_tags                                    = (known after apply)
          + cancel_deleted_branch_builds                  = (known after apply)
          + cancel_when_merge_group_destroyed             = (known after apply)
          + filter_condition                              = (known after apply)
          + filter_enabled                                = (known after apply)
          + ignore_default_branch_pull_requests           = (known after apply)
          + prefix_pull_request_fork_branch_names         = (known after apply)
          + publish_blocked_as_pending                    = (known after apply)
          + publish_commit_status                         = (known after apply)
          + publish_commit_status_per_step                = (known after apply)
          + pull_request_branch_filter_configuration      = (known after apply)
          + pull_request_branch_filter_enabled            = (known after apply)
          + separate_pull_request_statuses                = (known after apply)
          + skip_builds_for_existing_commits              = (known after apply)
          + skip_pull_request_builds_for_existing_commits = (known after apply)
          + trigger_mode                                  = "none"
          + use_merge_group_base_commit_for_git_diff_base = (known after apply)
        }
      + repository                               = "https://github.com/bazelbuild/bazel.git"
      + skip_intermediate_builds                 = false
      + skip_intermediate_builds_branch_filter   = (known after apply)
      + slug                                     = (known after apply)
      + steps                                    = <<-EOT
            ---
            env:
              GIT_HTTP_LOW_SPEED_LIMIT: "102400"
              GIT_HTTP_LOW_SPEED_TIME: "180"
            
            priority: 0
            steps:
              - command: |-
                  ./tools/mirror_external_deps.py --upload
                label: ":mirror: Mirror missing artifacts"
                agents:
                  - "queue=default"
                plugins:
                  - docker#v3.8.0:
                      always-pull: true
                      environment:
                        - "ANDROID_HOME"
                        - "ANDROID_NDK_HOME"
                        - "BUILDKITE_ARTIFACT_UPLOAD_DESTINATION"
                      image: "gcr.io/bazel-public/ubuntu2204"
                      network: "host"
                      privileged: true
                      propagate-environment: true
                      propagate-uid-gid: true
                      volumes:
                        - "/etc/group:/etc/group:ro"
                        - "/etc/passwd:/etc/passwd:ro"
                        - "/etc/shadow:/etc/shadow:ro"
                        - "/opt/android-ndk-r15c:/opt/android-ndk-r15c:ro"
                        - "/opt/android-sdk-linux:/opt/android-sdk-linux:ro"
                        - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
                        - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
        EOT
      + tags                                     = []
      + uuid                                     = (known after apply)
      + webhook_url                              = (known after apply)
    }

Plan: 1 to add, 0 to change, 1 to destroy.

[ted-xie]

hm, I don't see this file in the bazel repo: https://github.com/bazelbuild/bazel/tree/master/tools