diff --git a/.github/workflows/compute-matrix.yaml b/.github/workflows/compute-matrix.yaml index 8f0f3963..8d0ccd5b 100644 --- a/.github/workflows/compute-matrix.yaml +++ b/.github/workflows/compute-matrix.yaml @@ -2,7 +2,7 @@ on: workflow_call: inputs: build_type: - description: "One of: [branch, nightly, pull-request]" + description: "One of: [branch, nightly, pull-request, release-candidate]" required: true type: string matrix_name: @@ -200,8 +200,8 @@ jobs: env: MATRIX: ${{ steps.prepare-matrix.outputs.matrix }} run: | - if [[ "$BUILD_TYPE" != "branch" ]] && [[ "$BUILD_TYPE" != "nightly" ]] && [[ "$BUILD_TYPE" != "pull-request" ]]; then - echo "Invalid build_type! Must be one of 'branch', 'nightly', or 'pull-request'." + if [[ "$BUILD_TYPE" != "branch" ]] && [[ "$BUILD_TYPE" != "nightly" ]] && [[ "$BUILD_TYPE" != "pull-request" ]] && [[ "$BUILD_TYPE" != "release-candidate" ]]; then + echo "Invalid build_type! Must be one of 'branch', 'nightly', 'pull-request', or 'release-candidate'." exit 1 fi if [[ "$MATRIX_TYPE" != "auto" ]] && [[ "$MATRIX_TYPE" != "nightly" ]] && [[ "$MATRIX_TYPE" != "pull-request" ]]; then @@ -224,7 +224,7 @@ jobs: # only overwrite MATRIX_TYPE if it was set to 'auto' if [[ "${MATRIX_TYPE}" == "auto" ]]; then - if [[ "${BUILD_TYPE}" == "branch" ]]; then + if [[ "${BUILD_TYPE}" == "branch" || "${BUILD_TYPE}" == "release-candidate" ]]; then # Use the nightly matrix for branch tests MATRIX_TYPE="nightly" else diff --git a/.github/workflows/conda-cpp-build.yaml b/.github/workflows/conda-cpp-build.yaml index 4f8a4461..b8937fb3 100644 --- a/.github/workflows/conda-cpp-build.yaml +++ b/.github/workflows/conda-cpp-build.yaml @@ -2,7 +2,7 @@ on: workflow_call: inputs: build_type: - description: "One of: [branch, nightly, pull-request]" + description: "One of: [branch, nightly, pull-request, release-candidate]" required: true type: string branch: @@ -44,6 +44,10 @@ on: default: true required: false description: "One of [true, false], true if artifacts should be uploaded to GitHub's artifact store" + candidate-train-sha256: + description: "Canonical SHA-256 of the release train; required for release-candidate builds." + type: string + default: "" matrix_filter: description: | jq expression which modifies the matrix. @@ -200,7 +204,7 @@ jobs: MAMBA_USE_SHARDED_REPODATA: false RATTLER_SHARDED: false - name: Get Package Name and Location - if: ${{ inputs.upload-artifacts }} + if: ${{ inputs.upload-artifacts || inputs.build_type == 'release-candidate' }} env: # Pass RAPIDS_PACKAGE_NAME from cpp-build step if available RAPIDS_PACKAGE_NAME: ${{ steps.cpp-build.outputs.rapids-package-name }} @@ -214,20 +218,40 @@ jobs: echo "CONDA_OUTPUT_DIR=${RAPIDS_CONDA_BLD_OUTPUT_DIR}" >> "${GITHUB_OUTPUT}" id: package-name - name: Show files to be uploaded - if: ${{ inputs.upload-artifacts }} + if: ${{ inputs.upload-artifacts || inputs.build_type == 'release-candidate' }} env: CONDA_OUTPUT_DIR: ${{ steps.package-name.outputs.CONDA_OUTPUT_DIR }} run: | echo "Contents of directory to be uploaded:" ls -R "${CONDA_OUTPUT_DIR}" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: ${{ inputs.upload-artifacts }} + if: ${{ inputs.upload-artifacts && inputs.build_type != 'release-candidate' }} with: if-no-files-found: 'error' name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} path: ${{ steps.package-name.outputs.CONDA_OUTPUT_DIR }} + + - name: Configure release-candidate store credentials + if: ${{ inputs.build_type == 'release-candidate' }} + uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2 + with: + aws-region: us-east-2 + role-duration-seconds: 43200 # 12h + role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-rapids-release-candidates + - name: Create Conda release catalog companion + if: ${{ inputs.build_type == 'release-candidate' }} + uses: rapidsai/shared-actions/release-catalog-dispatch@d2f44b948645ee187102bf586d2ea44596a637c1 # shared-actions PR 136 candidate-store head + with: + config: >- + { + "release_catalog_key": ${{ toJSON(format('conda:{0}', github.event.repository.name)) }}, + "artifact_directory": ${{ toJSON(steps.package-name.outputs.CONDA_OUTPUT_DIR) }} + } + source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} + candidate-train-sha256: ${{ inputs.candidate-train-sha256 }} + upload-to-s3: 'true' - name: Upload additional artifacts - if: "!cancelled()" + if: ${{ !cancelled() && inputs.build_type != 'release-candidate' }} run: rapids-upload-artifacts-dir "cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)" - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main diff --git a/.github/workflows/conda-python-build.yaml b/.github/workflows/conda-python-build.yaml index 1a2bb291..ab9040c5 100644 --- a/.github/workflows/conda-python-build.yaml +++ b/.github/workflows/conda-python-build.yaml @@ -2,7 +2,7 @@ on: workflow_call: inputs: build_type: - description: "One of: [branch, nightly, pull-request]" + description: "One of: [branch, nightly, pull-request, release-candidate]" required: true type: string branch: @@ -44,6 +44,10 @@ on: default: true required: false description: "One of [true, false], true if artifacts should be uploaded to GitHub's artifact store" + candidate-train-sha256: + description: "Canonical SHA-256 of the release train; required for release-candidate builds." + type: string + default: "" matrix_filter: description: | jq expression which modifies the matrix. @@ -205,7 +209,7 @@ jobs: MAMBA_USE_SHARDED_REPODATA: false RATTLER_SHARDED: false - name: Get Package Name and Location - if: ${{ inputs.upload-artifacts }} + if: ${{ inputs.upload-artifacts || inputs.build_type == 'release-candidate' }} env: # Pass RAPIDS_PACKAGE_NAME from python-build step if available RAPIDS_PACKAGE_NAME: ${{ steps.python-build.outputs.rapids-package-name }} @@ -219,20 +223,40 @@ jobs: echo "CONDA_OUTPUT_DIR=${RAPIDS_CONDA_BLD_OUTPUT_DIR}" >> "${GITHUB_OUTPUT}" id: package-name - name: Show files to be uploaded - if: ${{ inputs.upload-artifacts }} + if: ${{ inputs.upload-artifacts || inputs.build_type == 'release-candidate' }} env: CONDA_OUTPUT_DIR: ${{ steps.package-name.outputs.CONDA_OUTPUT_DIR }} run: | echo "Contents of directory to be uploaded:" ls -R "${CONDA_OUTPUT_DIR}" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: ${{ inputs.upload-artifacts }} + if: ${{ inputs.upload-artifacts && inputs.build_type != 'release-candidate' }} with: if-no-files-found: 'error' name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} path: ${{ steps.package-name.outputs.CONDA_OUTPUT_DIR }} + + - name: Configure release-candidate store credentials + if: ${{ inputs.build_type == 'release-candidate' }} + uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2 + with: + aws-region: us-east-2 + role-duration-seconds: 43200 # 12h + role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-rapids-release-candidates + - name: Create Conda release catalog companion + if: ${{ inputs.build_type == 'release-candidate' }} + uses: rapidsai/shared-actions/release-catalog-dispatch@d2f44b948645ee187102bf586d2ea44596a637c1 # shared-actions PR 136 candidate-store head + with: + config: >- + { + "release_catalog_key": ${{ toJSON(format('conda:{0}', github.event.repository.name)) }}, + "artifact_directory": ${{ toJSON(steps.package-name.outputs.CONDA_OUTPUT_DIR) }} + } + source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} + candidate-train-sha256: ${{ inputs.candidate-train-sha256 }} + upload-to-s3: 'true' - name: Upload additional artifacts - if: "!cancelled()" + if: ${{ !cancelled() && inputs.build_type != 'release-candidate' }} run: rapids-upload-artifacts-dir "cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)_py${RAPIDS_PY_VERSION//.}" - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main diff --git a/.github/workflows/custom-job.yaml b/.github/workflows/custom-job.yaml index 7bc2560e..e1ebd773 100644 --- a/.github/workflows/custom-job.yaml +++ b/.github/workflows/custom-job.yaml @@ -2,7 +2,7 @@ on: workflow_call: inputs: build_type: - description: "One of: [branch, nightly, pull-request]" + description: "One of: [branch, nightly, pull-request, release-candidate]" required: true type: string branch: @@ -105,6 +105,18 @@ on: default: false type: boolean required: false + release-catalog-config: + description: >- + Optional JSON configuration for the shared-actions release catalog action. In release-candidate mode, + it writes the declared files and evidence to the private candidate store. See + https://github.com/rapidsai/shared-actions/tree/main/release-catalog for configuration details. + default: '' + type: string + required: false + candidate-train-sha256: + description: "Canonical SHA-256 of the release train; required for release-candidate builds." + type: string + default: "" defaults: run: @@ -223,13 +235,29 @@ jobs: MAMBA_USE_SHARDED_REPODATA: false RATTLER_SHARDED: false - name: Upload file to GitHub Artifact + if: ${{ inputs.build_type != 'release-candidate' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ inputs.artifact-name }} path: ${{ inputs.file_to_upload }} if-no-files-found: ignore + - name: Configure release-candidate store credentials + if: ${{ inputs.build_type == 'release-candidate' }} + uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2 + with: + aws-region: us-east-2 + role-duration-seconds: 43200 # 12h + role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-rapids-release-candidates + - name: Create release catalog companion + if: ${{ inputs.build_type == 'release-candidate' && inputs.release-catalog-config != '' }} + uses: rapidsai/shared-actions/release-catalog-dispatch@d2f44b948645ee187102bf586d2ea44596a637c1 # shared-actions PR 136 candidate-store head + with: + config: ${{ inputs.release-catalog-config }} + source-artifact-name: ${{ inputs.artifact-name }} + candidate-train-sha256: ${{ inputs.candidate-train-sha256 }} + upload-to-s3: 'true' - name: Upload additional artifacts - if: "!cancelled()" + if: ${{ !cancelled() && inputs.build_type != 'release-candidate' }} run: rapids-upload-artifacts-dir "custom-job-$(arch)" - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main diff --git a/.github/workflows/wheels-build.yaml b/.github/workflows/wheels-build.yaml index c4050159..fbbcfed2 100644 --- a/.github/workflows/wheels-build.yaml +++ b/.github/workflows/wheels-build.yaml @@ -26,7 +26,7 @@ on: description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string build_type: - description: "One of: [branch, nightly, pull-request]" + description: "One of: [branch, nightly, pull-request, release-candidate]" required: true type: string script: @@ -75,6 +75,10 @@ on: default: true required: false description: "One of [true, false], true if artifacts should be uploaded to GitHub's artifact store" + candidate-train-sha256: + description: "Canonical SHA-256 of the release train; required for release-candidate builds." + type: string + default: "" extra-repo: required: false type: string @@ -251,7 +255,7 @@ jobs: shell: bash -leo pipefail {0} - name: Get package name - if: ${{ inputs.upload-artifacts }} + if: ${{ inputs.upload-artifacts || inputs.build_type == 'release-candidate' }} env: # Pass RAPIDS_PACKAGE_NAME from build-wheel step if available RAPIDS_PACKAGE_NAME: ${{ steps.build-wheel.outputs.rapids-package-name }} @@ -284,7 +288,7 @@ jobs: id: package-name - name: Show files to be uploaded - if: ${{ inputs.upload-artifacts }} + if: ${{ inputs.upload-artifacts || inputs.build_type == 'release-candidate' }} env: WHEEL_OUTPUT_DIR: ${{ steps.package-name.outputs.WHEEL_OUTPUT_DIR }} run: | @@ -292,14 +296,34 @@ jobs: ls -R "$WHEEL_OUTPUT_DIR" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: ${{ inputs.upload-artifacts }} + if: ${{ inputs.upload-artifacts && inputs.build_type != 'release-candidate' }} with: if-no-files-found: 'error' name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} path: ${{ steps.package-name.outputs.WHEEL_OUTPUT_DIR }} + - name: Configure release-candidate store credentials + if: ${{ inputs.build_type == 'release-candidate' }} + uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2 + with: + aws-region: us-east-2 + role-duration-seconds: 43200 # 12h + role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-rapids-release-candidates + - name: Create wheel release catalog companion + if: ${{ inputs.build_type == 'release-candidate' }} + uses: rapidsai/shared-actions/release-catalog-dispatch@d2f44b948645ee187102bf586d2ea44596a637c1 # shared-actions PR 136 candidate-store head + with: + config: >- + { + "release_catalog_key": ${{ toJSON(format('wheel:{0}', github.event.repository.name)) }}, + "artifact_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} + } + source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} + candidate-train-sha256: ${{ inputs.candidate-train-sha256 }} + upload-to-s3: 'true' + - name: Upload additional artifacts - if: "!cancelled()" + if: ${{ !cancelled() && inputs.build_type != 'release-candidate' }} run: rapids-upload-artifacts-dir "cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)_py${RAPIDS_PY_VERSION//.}" - name: Telemetry upload attributes if: ${{ vars.TELEMETRY_ENABLED == 'true' }} diff --git a/README.md b/README.md index 9752c524..c17eb4c0 100644 --- a/README.md +++ b/README.md @@ -96,3 +96,22 @@ wheel-tests: ``` Values passed through `secrets:` are redacted everywhere in the GitHub UI, including in logs, and in most cases are replaced with `***`. + +### Release catalog + +We add additional metadata files to our builds to help track what dependencies +were present at build time (a Software Bill of Materials, SBoM), as well as +keeping track of artifacts as we prepare for releases. The standard Conda and +wheel builders do this automatically and upload an additional +`release-catalog-` GitHub Actions artifact for every package +bundle. + +`custom-job.yaml` can be used to produce artifacts, but the generation of extra +metadata files are opt-in, not automatic. Supplying a non-empty +`release-catalog-config` causes the job to upload the additional +release catalog companion; leaving it empty uploads only the original artifact. + +See the +[`shared-actions` release catalog documentation](https://github.com/rapidsai/shared-actions/tree/main/release-catalog) +for the companion layout, configuration schema, examples, and evidence +semantics.