diff --git a/.github/workflows/build_push_to_idc.yml b/.github/workflows/build_push_to_idc.yml index 83da0949b..24af59aa2 100644 --- a/.github/workflows/build_push_to_idc.yml +++ b/.github/workflows/build_push_to_idc.yml @@ -8,6 +8,14 @@ on: required: true default: main type: string + architecture: + description: Architecture to build and publish + required: true + default: amd64 + type: choice + options: + - amd64 + - all permissions: contents: read @@ -21,10 +29,13 @@ jobs: name: Validate IDC build configuration runs-on: ubuntu-latest outputs: + candidate_image: ${{ steps.resolve.outputs.candidate_image }} + controller_sha: ${{ steps.resolve.outputs.controller_sha }} image_version: ${{ steps.resolve.outputs.image_version }} - source_sha: ${{ steps.resolve.outputs.source_sha }} + matrix: ${{ steps.resolve.outputs.matrix }} source_ref: ${{ steps.resolve.outputs.source_ref }} - controller_sha: ${{ steps.resolve.outputs.controller_sha }} + source_sha: ${{ steps.resolve.outputs.source_sha }} + target_image: ${{ steps.resolve.outputs.target_image }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -35,11 +46,11 @@ jobs: id: resolve shell: bash env: + ARCHITECTURE: ${{ inputs.architecture }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} SOURCE_REF: ${{ inputs.source_ref }} IDC_REGISTRY: ${{ vars.CONTAINER_MIRROR_REGISTRY }} IDC_IMAGE: ${{ vars.CONTAINER_MIRROR_IMAGE }} - IDC_RUNNER: ${{ vars.CONTAINER_MIRROR_RUNNER }} run: | set -euo pipefail if [ "${GITHUB_REF}" != "refs/heads/${DEFAULT_BRANCH}" ]; then @@ -80,7 +91,7 @@ jobs: ;; esac - for key in IDC_REGISTRY IDC_IMAGE IDC_RUNNER; do + for key in IDC_REGISTRY IDC_IMAGE; do if [ -z "${!key}" ]; then echo "::error::Missing required IDC configuration: ${key}" exit 1 @@ -94,37 +105,98 @@ jobs: echo "::error::IDC image must be an untagged repository under the configured IDC registry." exit 1 fi + + case "${ARCHITECTURE}" in + amd64) + matrix='{"include":[{"platform":"linux/amd64","runner":"ubuntu-latest","slug":"linux-amd64"}]}' + architecture_suffix="-amd64" + ;; + all) + matrix='{"include":[{"platform":"linux/amd64","runner":"ubuntu-latest","slug":"linux-amd64"},{"platform":"linux/arm64","runner":"ubuntu-24.04-arm","slug":"linux-arm64"}]}' + architecture_suffix="" + ;; + *) + echo "::error::Unsupported architecture: ${ARCHITECTURE}" + exit 1 + ;; + esac + + image_version="idc-$(date -u +%Y%m%dT%H%M%SZ)-${source_sha}-${GITHUB_RUN_ID}${architecture_suffix}" { + echo "candidate_image=${IDC_IMAGE}-candidates" echo "controller_sha=${controller_sha}" - echo "source_sha=${source_sha}" + echo "image_version=${image_version}" + echo "matrix=${matrix}" echo "source_ref=${source_ref}" - echo "image_version=idc-$(date -u +%Y%m%dT%H%M%SZ)-${source_sha}-${GITHUB_RUN_ID}-amd64" + echo "source_sha=${source_sha}" + echo "target_image=${IDC_IMAGE}" } >> "${GITHUB_OUTPUT}" - publish: - name: Build, verify, and publish IDC image + candidates: + name: Build and verify IDC candidates needs: settings - runs-on: ${{ vars.CONTAINER_MIRROR_RUNNER }} + uses: ./.github/workflows/idc-container-candidates.yml + with: + candidate_image: ${{ needs.settings.outputs.candidate_image }} + controller_sha: ${{ needs.settings.outputs.controller_sha }} + image_version: ${{ needs.settings.outputs.image_version }} + matrix_json: ${{ needs.settings.outputs.matrix }} + source_ref: ${{ needs.settings.outputs.source_ref }} + source_sha: ${{ needs.settings.outputs.source_sha }} + + stage: + name: Assemble verified IDC manifest + needs: + - settings + - candidates + runs-on: ubuntu-latest environment: idc-publication - timeout-minutes: 210 - env: - LOCAL_IMAGE: astra-idc-candidate:${{ needs.settings.outputs.source_sha }} - IMAGE_VERSION: ${{ needs.settings.outputs.image_version }} - RELEASE_SOURCE_SHA: ${{ needs.settings.outputs.source_sha }} - RELEASE_SOURCE_REF: ${{ needs.settings.outputs.source_ref }} - http_proxy: ${{ vars.CONTAINER_MIRROR_HTTP_PROXY }} - https_proxy: ${{ vars.CONTAINER_MIRROR_HTTPS_PROXY }} - no_proxy: ${{ vars.CONTAINER_MIRROR_NO_PROXY }} + timeout-minutes: 20 steps: - - name: Require a self-hosted IDC runner + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + ref: ${{ needs.settings.outputs.controller_sha }} + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: idc-digest-* + path: /tmp/digests + merge-multiple: true + + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ${{ vars.CONTAINER_MIRROR_REGISTRY }} + username: ${{ secrets.IDC_REGISTRY_USERNAME }} + password: ${{ secrets.IDC_REGISTRY_PASSWORD }} + + - name: Assemble or verify the staged manifest shell: bash env: - RUNNER_ENVIRONMENT: ${{ runner.environment }} + CANDIDATE_IMAGE: ${{ needs.settings.outputs.candidate_image }} + IMAGE_VERSION: ${{ needs.settings.outputs.image_version }} + BUILD_MATRIX_JSON: ${{ needs.settings.outputs.matrix }} run: | - if [ "${RUNNER_ENVIRONMENT}" != "self-hosted" ]; then - echo "::error::IDC publication requires a self-hosted runner." - exit 1 - fi + set -euo pipefail + scripts/reconcile-docker-manifest.sh \ + "${CANDIDATE_IMAGE}" "${IMAGE_VERSION}" "${BUILD_MATRIX_JSON}" \ + /tmp/digests verify + + publish: + name: Publish verified IDC image + needs: + - settings + - stage + runs-on: ubuntu-latest + environment: idc-publication + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + ref: ${{ needs.settings.outputs.controller_sha }} - name: Require IDC registry credentials shell: bash @@ -140,124 +212,45 @@ jobs: fi done - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - ref: ${{ needs.settings.outputs.controller_sha }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - ref: ${{ needs.settings.outputs.source_sha }} - path: source - - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - - - name: Build the IDC candidate locally - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: source - file: source/Dockerfile - platforms: linux/amd64 - tags: ${{ env.LOCAL_IMAGE }} - labels: | - org.opencontainers.image.version=${{ env.IMAGE_VERSION }} - org.opencontainers.image.revision=${{ env.RELEASE_SOURCE_SHA }} - org.opencontainers.image.source=https://github.com/matrixorigin/astra - build-args: | - IMAGE_VERSION=${{ env.IMAGE_VERSION }} - IMAGE_REVISION=${{ env.RELEASE_SOURCE_SHA }} - IMAGE_SOURCE_DIRTY=false - IMAGE_BRANCH=${{ env.RELEASE_SOURCE_REF }} - HTTP_PROXY=${{ vars.CONTAINER_MIRROR_HTTP_PROXY }} - HTTPS_PROXY=${{ vars.CONTAINER_MIRROR_HTTPS_PROXY }} - NO_PROXY=${{ vars.CONTAINER_MIRROR_NO_PROXY }} - load: true - push: false - - - name: Verify local candidate identity - shell: bash - run: | - set -euo pipefail - assert_label() { - local label="$1" - local expected="$2" - local actual - actual="$(docker image inspect "${LOCAL_IMAGE}" --format "{{ index .Config.Labels \"${label}\" }}")" - if [ "${actual}" != "${expected}" ]; then - echo "::error::Candidate ${label} is ${actual:-}, expected ${expected}." - exit 1 - fi - } - platform="$(docker image inspect "${LOCAL_IMAGE}" --format '{{.Os}}/{{.Architecture}}')" - if [ "${platform}" != "linux/amd64" ]; then - echo "::error::Candidate platform is ${platform}, expected linux/amd64." - exit 1 - fi - assert_label org.opencontainers.image.version "${IMAGE_VERSION}" - assert_label org.opencontainers.image.revision "${RELEASE_SOURCE_SHA}" - assert_label org.opencontainers.image.ref.name "${RELEASE_SOURCE_REF}" - assert_label org.opencontainers.image.source https://github.com/matrixorigin/astra - - - name: Start local candidate through the all-in-one path + - name: Install pinned crane shell: bash + env: + CRANE_VERSION: v0.20.6 + CRANE_LINUX_X86_64_SHA256: c1d593d01551f2c9a3df5ca0a0be4385a839bd9b86d4a76e18d7b17d16559127 run: | set -euo pipefail - make stack-env - sed -i "s|^ASTRA_IMAGE=.*|ASTRA_IMAGE=${LOCAL_IMAGE}|" deployment/all-in-one/.env - sed -i 's|^MEMORIA_EMBEDDING_PROVIDER=.*|MEMORIA_EMBEDDING_PROVIDER=mock|' deployment/all-in-one/.env - make stack-up - - - name: Verify health and exact memory round trip - run: make stack-verify - - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ${{ vars.CONTAINER_MIRROR_REGISTRY }} - username: ${{ secrets.IDC_REGISTRY_USERNAME }} - password: ${{ secrets.IDC_REGISTRY_PASSWORD }} + mkdir -p "${RUNNER_TEMP}/bin" + curl -fsSL \ + -o "${RUNNER_TEMP}/go-containerregistry.tar.gz" \ + "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" + echo "${CRANE_LINUX_X86_64_SHA256} ${RUNNER_TEMP}/go-containerregistry.tar.gz" \ + | sha256sum --check --strict + tar -xzf "${RUNNER_TEMP}/go-containerregistry.tar.gz" -C "${RUNNER_TEMP}/bin" crane + echo "${RUNNER_TEMP}/bin" >> "${GITHUB_PATH}" + "${RUNNER_TEMP}/bin/crane" version - name: Publish or verify immutable IDC image shell: bash env: - IMAGE_NAME: ${{ vars.CONTAINER_MIRROR_IMAGE }} + CANDIDATE_IMAGE: ${{ needs.settings.outputs.candidate_image }} + IMAGE_NAME: ${{ needs.settings.outputs.target_image }} + IMAGE_VERSION: ${{ needs.settings.outputs.image_version }} + IDC_REGISTRY: ${{ vars.CONTAINER_MIRROR_REGISTRY }} + IDC_REGISTRY_USERNAME: ${{ secrets.IDC_REGISTRY_USERNAME }} + IDC_REGISTRY_PASSWORD: ${{ secrets.IDC_REGISTRY_PASSWORD }} + RELEASE_SOURCE_SHA: ${{ needs.settings.outputs.source_sha }} run: | set -euo pipefail + printf '%s' "${IDC_REGISTRY_PASSWORD}" | crane auth login \ + "${IDC_REGISTRY}" -u "${IDC_REGISTRY_USERNAME}" --password-stdin + source="${CANDIDATE_IMAGE}:${IMAGE_VERSION}" target="${IMAGE_NAME}:${IMAGE_VERSION}" - local_id="$(docker image inspect "${LOCAL_IMAGE}" --format '{{.Id}}')" - target_exists=false - if inspect_output="$(docker buildx imagetools inspect "${target}" 2>&1)"; then - target_exists=true - elif ! grep -Eqi '(: not found|manifest unknown|name unknown|HTTP 404|status[^0-9]*404)' \ - <<< "${inspect_output}"; then - echo "::error::Could not safely determine whether ${target} exists." - printf '%s\n' "${inspect_output}" >&2 - exit 1 - fi - - if [ "${target_exists}" = false ]; then - docker tag "${LOCAL_IMAGE}" "${target}" - docker push "${target}" - fi - docker pull "${target}" - target_id="$(docker image inspect "${target}" --format '{{.Id}}')" - if [ "${target_id}" != "${local_id}" ]; then - echo "::error::${target} does not match the verified local image." - exit 1 - fi + source_digest="$(crane digest "${source}")" + scripts/copy-immutable-container-tag.sh \ + "${source}" "${target}" "https://${IDC_REGISTRY}" { echo "## IDC image published" echo "- Image: \`${target}\`" + echo "- Digest: \`${source_digest}\`" echo "- Source: \`${RELEASE_SOURCE_SHA}\`" } >> "${GITHUB_STEP_SUMMARY}" - - - name: Show service logs after failure - if: ${{ failure() }} - working-directory: deployment/all-in-one - run: docker compose --env-file .env logs --no-color --tail=200 api memoria matrixone || true - - - name: Remove local smoke stack and image - if: ${{ always() }} - run: | - make stack-clean || true - docker image rm "${LOCAL_IMAGE}" "${{ vars.CONTAINER_MIRROR_IMAGE }}:${IMAGE_VERSION}" >/dev/null 2>&1 || true diff --git a/.github/workflows/idc-container-candidates.yml b/.github/workflows/idc-container-candidates.yml new file mode 100644 index 000000000..f774165e8 --- /dev/null +++ b/.github/workflows/idc-container-candidates.yml @@ -0,0 +1,216 @@ +name: Build IDC Container Candidates + +permissions: + contents: read + +on: + workflow_call: + inputs: + candidate_image: + description: IDC staging repository for candidate manifests + required: true + type: string + controller_sha: + description: Trusted workflow and verification-script revision + required: true + type: string + image_version: + description: Candidate image version used for OCI metadata + required: true + type: string + source_sha: + description: Immutable source commit selected by the caller + required: true + type: string + source_ref: + description: Selected source branch or immutable commit for OCI metadata + required: true + type: string + matrix_json: + description: JSON platform build matrix + required: true + type: string + +env: + IMAGE_NAME: ${{ inputs.candidate_image }} + RELEASE_SOURCE_REF: ${{ inputs.source_ref }} + RELEASE_SOURCE_SHA: ${{ inputs.source_sha }} + RELEASE_IMAGE_VERSION: ${{ inputs.image_version }} + +jobs: + build: + name: Build ${{ matrix.platform }} IDC candidate + runs-on: ${{ matrix.runner }} + environment: idc-publication + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.matrix_json) }} + + steps: + - name: Require IDC registry credentials + shell: bash + env: + IDC_REGISTRY_USERNAME: ${{ secrets.IDC_REGISTRY_USERNAME }} + IDC_REGISTRY_PASSWORD: ${{ secrets.IDC_REGISTRY_PASSWORD }} + run: | + set -euo pipefail + for key in IDC_REGISTRY_USERNAME IDC_REGISTRY_PASSWORD; do + if [ -z "${!key}" ]; then + echo "::error::Missing required IDC credential: ${key}" + exit 1 + fi + done + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + ref: ${{ inputs.controller_sha }} + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + ref: ${{ env.RELEASE_SOURCE_SHA }} + path: source + + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ${{ vars.CONTAINER_MIRROR_REGISTRY }} + username: ${{ secrets.IDC_REGISTRY_USERNAME }} + password: ${{ secrets.IDC_REGISTRY_PASSWORD }} + + - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + id: build + with: + context: source + file: source/Dockerfile + platforms: ${{ matrix.platform }} + tags: ${{ env.IMAGE_NAME }} + labels: | + org.opencontainers.image.version=${{ env.RELEASE_IMAGE_VERSION }} + org.opencontainers.image.revision=${{ env.RELEASE_SOURCE_SHA }} + org.opencontainers.image.source=https://github.com/matrixorigin/astra + build-args: | + IMAGE_VERSION=${{ env.RELEASE_IMAGE_VERSION }} + IMAGE_REVISION=${{ env.RELEASE_SOURCE_SHA }} + IMAGE_SOURCE_DIRTY=false + IMAGE_BRANCH=${{ env.RELEASE_SOURCE_REF }} + cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-${{ matrix.slug }} + cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-${{ matrix.slug }},mode=max + outputs: type=image,push-by-digest=true,name-canonical=true,push=true + provenance: mode=max + sbom: true + + - name: Export immutable digest + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + if [[ ! "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "Build returned an invalid image digest: ${digest:-}" >&2 + exit 1 + fi + touch "/tmp/digests/${digest#sha256:}" + + - name: Retain IDC candidate with a run-scoped immutable tag + shell: bash + run: | + set -euo pipefail + digest="${{ steps.build.outputs.digest }}" + scripts/reconcile-docker-candidate-tag.sh \ + "${IMAGE_NAME}" \ + "astra-idc-candidate-${GITHUB_RUN_ID}-${{ matrix.slug }}-${digest#sha256:}" \ + "${digest}" \ + create + + - name: Upload IDC candidate digest + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: idc-digest-${{ matrix.slug }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 30 + + smoke: + name: Verify ${{ matrix.platform }} IDC candidate + needs: build + runs-on: ${{ matrix.runner }} + environment: idc-publication + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.matrix_json) }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + ref: ${{ inputs.controller_sha }} + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: idc-digest-${{ matrix.slug }} + path: /tmp/digest + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ${{ vars.CONTAINER_MIRROR_REGISTRY }} + username: ${{ secrets.IDC_REGISTRY_USERNAME }} + password: ${{ secrets.IDC_REGISTRY_PASSWORD }} + + - name: Start candidate through the released all-in-one path + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + digest_files=(/tmp/digest/*) + if [ "${#digest_files[@]}" -ne 1 ]; then + echo "Expected one candidate digest for ${{ matrix.platform }}, found ${#digest_files[@]}" >&2 + exit 1 + fi + digest="$(basename "${digest_files[0]}")" + if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then + echo "Candidate artifact contains an invalid image digest: ${digest}" >&2 + exit 1 + fi + release_image="${IMAGE_NAME}@sha256:${digest}" + + make stack-env + sed -i "s|^ASTRA_IMAGE=.*|ASTRA_IMAGE=${release_image}|" deployment/all-in-one/.env + sed -i 's|^MEMORIA_EMBEDDING_PROVIDER=.*|MEMORIA_EMBEDDING_PROVIDER=mock|' deployment/all-in-one/.env + make stack-up + + revision="$(docker image inspect "${release_image}" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')" + if [ "${revision}" != "${RELEASE_SOURCE_SHA}" ]; then + echo "Candidate image revision ${revision:-} does not match ${RELEASE_SOURCE_SHA}" >&2 + exit 1 + fi + image_version="$(docker image inspect "${release_image}" --format '{{ index .Config.Labels "org.opencontainers.image.version" }}')" + if [ "${image_version}" != "${RELEASE_IMAGE_VERSION}" ]; then + echo "Candidate image version ${image_version:-} does not match ${RELEASE_IMAGE_VERSION}" >&2 + exit 1 + fi + + - name: Verify health and exact memory round trip + run: make stack-verify + + - name: Write IDC candidate summary + shell: bash + run: | + { + echo "## Verified Astra IDC candidate" + echo "- Platform: \`${{ matrix.platform }}\`" + echo "- Source: \`${RELEASE_SOURCE_SHA}\`" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Show service logs after failure + if: ${{ failure() }} + working-directory: deployment/all-in-one + run: docker compose --env-file .env logs --no-color --tail=200 api memoria matrixone || true + + - name: Remove candidate smoke stack + if: ${{ always() }} + run: make stack-clean || true diff --git a/docs/guides/releasing.md b/docs/guides/releasing.md index a7a04c0f4..dc51c4e57 100644 --- a/docs/guides/releasing.md +++ b/docs/guides/releasing.md @@ -129,35 +129,29 @@ Run **build_push_to_idc** (`build_push_to_idc.yml`) manually from `main`. Its `source_ref` defaults to the latest `main`; set it to `moi-dev` for that branch's latest commit, or to a full commit SHA that is contained in the current `main` or `moi-dev` history. Other branches, tags, abbreviated SHAs, -and commits outside those histories are rejected. The workflow controller and -host-side verification scripts always come from the current protected `main` -revision. This workflow does not create Git tags or GitHub Releases and does -not push to Docker Hub. +and commits outside those histories are rejected. Select `amd64` for a Linux +AMD64 image or `all` for the same native Linux AMD64 and ARM64 matrix used by +the release workflow. This workflow does not create Git tags or GitHub Releases +and does not push images to Docker Hub. Configure repository variables `CONTAINER_MIRROR_REGISTRY` (host and optional -port), `CONTAINER_MIRROR_IMAGE` (full untagged repository), and -`CONTAINER_MIRROR_RUNNER` (a Linux AMD64 Docker-capable self-hosted runner -label). Store `IDC_REGISTRY_USERNAME` and `IDC_REGISTRY_PASSWORD` exclusively +port) and `CONTAINER_MIRROR_IMAGE` (full untagged repository). Store +`IDC_REGISTRY_USERNAME` and `IDC_REGISTRY_PASSWORD` exclusively as secrets in the `idc-publication` Environment; its deployment branch policy -must admit only `main`. Do not keep copies as repository or organization -secrets. This external policy is the trust boundary that prevents a workflow -definition selected from another branch from receiving IDC credentials. Missing -configuration fails before build work, and the admitted ARC runner verifies -`runner.environment` before checkout or registry login. Optional proxy variables are `CONTAINER_MIRROR_HTTP_PROXY`, -`CONTAINER_MIRROR_HTTPS_PROXY`, and `CONTAINER_MIRROR_NO_PROXY`. - -The workflow builds a Linux AMD64 candidate only in the admitted runner's local -Docker store, runs the existing all-in-one smoke test, and authenticates to -Harbor only after verification succeeds. It then publishes -`idc----amd64` directly to IDC. -No candidate manifest or BuildKit cache is pushed to the runtime repository, so -MOI's newest-artifact resolver cannot observe an untagged pre-publication -object. Reruns verify an existing immutable tag against the locally verified -image instead of overwriting it. The final image records the full selected -source commit, its selected branch or SHA, and the canonical -`https://github.com/matrixorigin/astra` OCI source label used by MOI's Astra -revision resolver. The runner must support the existing all-in-one stack, -Python 3, and Docker Buildx. Formal release tags and `latest` are not changed. +normally admits only `main`. Do not keep copies as repository or organization +secrets. + +The workflow copies the established release topology: native GitHub-hosted +runners build each selected platform by digest and run the existing all-in-one +smoke test. Candidates and build caches stay in the IDC-only +`-candidates` repository. After every platform passes, +the workflow assembles the manifest there and copies only that verified, +immutable manifest to the runtime repository as +`idc---[-amd64]`. MOI's +newest-artifact resolver therefore never observes a candidate or cache object. +The final image records the selected commit, branch or SHA, and canonical Astra +OCI source label. Formal release tags and `latest` are not changed. The workflow +controller must be the latest `main` revision admitted by the Environment. ## Prepare a release diff --git a/scripts/README.md b/scripts/README.md index 6ecd77522..631bcfc58 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -263,6 +263,12 @@ reconciliation at the actual publication boundary. `scripts/reconcile-docker-candidate-tag.sh` creates or verifies one immutable, run-scoped staging tag per server platform so registry cleanup cannot discard an otherwise retained recovery candidate. +`scripts/copy-immutable-container-tag.sh` copies a verified manifest between +repositories only after `scripts/inspect-harbor-artifact.py` resolves the exact +target through Harbor's structured API. A 404 from that artifact endpoint +permits first publication for either a new repository or a new tag; +authentication, network, malformed-response, and registry failures fail closed. +An existing tag is accepted only when its digest already matches the source. ### `scripts/verify_github_release_assets.py` diff --git a/scripts/ci/test_release_build_shells.py b/scripts/ci/test_release_build_shells.py index 9ead81940..5d291cf58 100755 --- a/scripts/ci/test_release_build_shells.py +++ b/scripts/ci/test_release_build_shells.py @@ -3,11 +3,14 @@ import os import json +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import re import subprocess import tempfile +import threading import unittest +from urllib.parse import quote ROOT = Path(__file__).resolve().parents[2] @@ -154,11 +157,11 @@ def run_idc_settings(self, **overrides): source_ref = base_sha env = { **os.environ, + "ARCHITECTURE": "amd64", "DEFAULT_BRANCH": "main", "SOURCE_REF": source_ref, "IDC_REGISTRY": "registry.example:5000", "IDC_IMAGE": "registry.example:5000/team/astra", - "IDC_RUNNER": "idc-amd64", "GITHUB_REF": "refs/heads/main", "GITHUB_SHA": main_sha, "GITHUB_RUN_ID": "123", @@ -176,21 +179,200 @@ def test_idc_build_identity(self): self.assertEqual(outputs["controller_sha"], revisions["main"]) self.assertEqual(outputs["source_sha"], revisions["main"]) self.assertEqual(outputs["source_ref"], "main") + self.assertEqual(outputs["candidate_image"], "registry.example:5000/team/astra-candidates") + self.assertEqual(outputs["target_image"], "registry.example:5000/team/astra") self.assertRegex(outputs["image_version"], r"^idc-\d{8}T\d{6}Z-" + revisions["main"] + r"-123-amd64$") - def test_idc_build_stays_local_until_smoke_succeeds(self): + def test_idc_reuses_release_candidate_topology_and_publishes_only_to_idc(self): workflow = (ROOT / ".github/workflows/build_push_to_idc.yml").read_text() - build = workflow.index("Build the IDC candidate locally") - smoke = workflow.index("Verify health and exact memory round trip") - login = workflow.index("docker/login-action") - publish = workflow.index('docker push "${target}"') - self.assertLess(build, smoke) - self.assertLess(smoke, login) - self.assertLess(login, publish) + candidates = (ROOT / ".github/workflows/idc-container-candidates.yml").read_text() + self.assertIn("uses: ./.github/workflows/idc-container-candidates.yml", workflow) + self.assertIn("Assemble verified IDC manifest", workflow) + self.assertIn("scripts/copy-immutable-container-tag.sh", workflow) self.assertIn("environment: idc-publication", workflow) - self.assertIn("load: true", workflow) - self.assertIn("push: false", workflow) - self.assertNotIn("release-container-candidates.yml", workflow) + self.assertIn("push-by-digest=true", candidates) + self.assertIn("make stack-verify", candidates) + self.assertIn("ref: ${{ inputs.controller_sha }}", candidates) + self.assertIn("context: source", candidates) + self.assertIn("file: source/Dockerfile", candidates) + self.assertIn("ubuntu-24.04-arm", workflow) + self.assertNotIn("matrixorigin/astra", candidates.split("org.opencontainers.image.source", 1)[0]) + self.assertNotIn("DOCKERHUB_", workflow + candidates) + + def test_idc_immutable_copy_distinguishes_absence_from_lookup_failures(self): + script = ROOT / "scripts/copy-immutable-container-tag.sh" + source_digest = "sha256:" + "a" * 64 + conflicting_digest = "sha256:" + "b" * 64 + + class HarborHandler(BaseHTTPRequestHandler): + state = "missing" + paths = [] + expected_path = "/api/v2.0/projects/team/repositories/astra/artifacts/release" + + def log_message(self, _format, *_args): + pass + + def do_GET(self): + type(self).paths.append(self.path) + if self.path != type(self).expected_path: + self.send_response(404) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"errors": [{"code": "NOT_FOUND"}]}).encode()) + return + status = { + "missing": 404, + "new_repository": 404, + "unauthorized": 401, + "forbidden": 403, + "server_error": 503, + }.get(type(self).state, 200) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.end_headers() + if type(self).state == "malformed_not_found": + self.wfile.write(b"not a Harbor error envelope") + return + if status == 200: + digest = (source_digest if type(self).state == "same" + else conflicting_digest) + self.wfile.write(json.dumps({"digest": digest}).encode()) + else: + code = "NOT_FOUND" if status == 404 else "TEST" + self.wfile.write(json.dumps({"errors": [{"code": code}]}).encode()) + + with tempfile.TemporaryDirectory() as directory: + fixture = Path(directory) + fake_bin = fixture / "bin" + fake_bin.mkdir() + calls = fixture / "calls" + crane = fake_bin / "crane" + crane.write_text( + '''#!/bin/sh +set -eu +printf '%s\\n' "$*" >> "${ASTRA_TEST_CALLS}" +case "$1 $2" in + "digest source.example/astra:staged") + printf '%s\\n' "${ASTRA_TEST_SOURCE_DIGEST}" + ;; + digest\ *) + if [ -e "${ASTRA_TEST_STATE_DIR}/copied" ]; then + printf '%s\\n' "${ASTRA_TEST_SOURCE_DIGEST}" + else + exit 92 + fi + ;; + "copy --platform=all") + case "${ASTRA_TEST_TARGET_STATE}" in + missing|new_repository) ;; + *) exit 93 ;; + esac + touch "${ASTRA_TEST_STATE_DIR}/copied" + ;; + *) exit 91 ;; +esac +''', + encoding="utf-8", + ) + crane.chmod(0o755) + common_env = { + **os.environ, + "PATH": f"{fake_bin}{os.pathsep}{os.environ['PATH']}", + "ASTRA_TEST_CALLS": str(calls), + "ASTRA_TEST_STATE_DIR": str(fixture), + "ASTRA_TEST_SOURCE_DIGEST": source_digest, + "ASTRA_TEST_CONFLICTING_DIGEST": conflicting_digest, + "IDC_REGISTRY_USERNAME": "release-user", + "IDC_REGISTRY_PASSWORD": "release-password", + "RUNNER_TEMP": str(fixture), + } + + def run(state, repository="team/astra"): + calls.write_text("", encoding="utf-8") + HarborHandler.state = state + HarborHandler.paths = [] + repository_name = repository.partition("/")[2] + encoded_repository_name = quote(quote(repository_name, safe=""), safe="") + HarborHandler.expected_path = ( + "/api/v2.0/projects/team/repositories/" + f"{encoded_repository_name}/artifacts/release" + ) + result = subprocess.run( + [str(script), "source.example/astra:staged", + f"127.0.0.1:{server.server_port}/{repository}:release", + f"http://127.0.0.1:{server.server_port}"], + env={**common_env, "ASTRA_TEST_TARGET_STATE": state}, + capture_output=True, + text=True, + ) + return result, calls.read_text(encoding="utf-8"), HarborHandler.paths + + server = ThreadingHTTPServer(("127.0.0.1", 0), HarborHandler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + + for state in ("missing", "new_repository"): + with self.subTest(state=state): + published, published_calls, paths = run(state) + self.assertEqual(published.returncode, 0, published.stderr) + self.assertIn("copy --platform=all --jobs 2", published_calls) + self.assertEqual( + paths, + ["/api/v2.0/projects/team/repositories/astra/artifacts/release"], + ) + (fixture / "copied").unlink() + + same, same_calls, _ = run("same") + self.assertEqual(same.returncode, 0, same.stderr) + self.assertNotIn("copy ", same_calls) + + conflict, conflict_calls, _ = run("conflict") + self.assertNotEqual(conflict.returncode, 0) + self.assertIn("already exists with digest", conflict.stderr) + self.assertNotIn("copy ", conflict_calls) + + nested_conflict, nested_conflict_calls, nested_paths = run( + "conflict", "team/nested/astra" + ) + self.assertNotEqual(nested_conflict.returncode, 0) + self.assertIn("already exists with digest", nested_conflict.stderr) + self.assertNotIn("copy ", nested_conflict_calls) + self.assertEqual( + nested_paths, + [ + "/api/v2.0/projects/team/repositories/" + "nested%252Fastra/artifacts/release" + ], + ) + + for state in ("unauthorized", "forbidden", "server_error"): + with self.subTest(state=state): + failed, failed_calls, _ = run(state) + self.assertNotEqual(failed.returncode, 0) + self.assertIn("could not safely inspect", failed.stderr) + self.assertNotIn("copy ", failed_calls) + + HarborHandler.state = "malformed_not_found" + malformed, malformed_calls, _ = run("malformed_not_found") + self.assertNotEqual(malformed.returncode, 0) + self.assertNotIn("copy ", malformed_calls) + + unreachable = ThreadingHTTPServer(("127.0.0.1", 0), HarborHandler) + unreachable_port = unreachable.server_port + unreachable.server_close() + calls.write_text("", encoding="utf-8") + network_failure = subprocess.run( + [str(script), "source.example/astra:staged", + f"127.0.0.1:{unreachable_port}/team/astra:release", + f"http://127.0.0.1:{unreachable_port}"], + env={**common_env, "ASTRA_TEST_TARGET_STATE": "network_failure"}, + capture_output=True, + text=True, + ) + self.assertNotEqual(network_failure.returncode, 0) + self.assertNotIn("copy ", calls.read_text(encoding="utf-8")) def test_idc_resolves_moi_dev_and_allowed_historical_commit(self): result, revisions = self.run_idc_settings(SOURCE_REF="moi-dev") @@ -222,16 +404,18 @@ def test_idc_registry_credentials_are_required_before_build(self): self.assertIn("Missing required IDC credential: " + missing, result.stdout) self.assertNotIn("release-password", result.stdout + result.stderr) - def test_idc_rejects_non_main_controller_and_arbitrary_ref(self): + def test_idc_rejects_non_main_controller(self): result, _ = self.run_idc_settings(GITHUB_REF="refs/heads/moi-dev") self.assertNotEqual(result.returncode, 0) self.assertIn("Run this workflow from main", result.stdout) + + def test_idc_rejects_arbitrary_source_ref(self): result, _ = self.run_idc_settings(SOURCE_REF="feature/test") self.assertNotEqual(result.returncode, 0) self.assertIn("source_ref must be main, moi-dev, or a full commit SHA", result.stdout) def test_idc_missing_configuration_stops_before_build(self): - for key in ("IDC_REGISTRY", "IDC_IMAGE", "IDC_RUNNER"): + for key in ("IDC_REGISTRY", "IDC_IMAGE"): with self.subTest(key=key): result, _ = self.run_idc_settings(**{key: ""}) self.assertNotEqual(result.returncode, 0) @@ -249,6 +433,17 @@ def test_idc_rejects_wrong_or_tagged_repository(self): result, _ = self.run_idc_settings(IDC_REGISTRY="https://registry.example") self.assertNotEqual(result.returncode, 0) + def test_idc_architecture_matrix(self): + result, _ = self.run_idc_settings(ARCHITECTURE="all") + self.assertEqual(result.returncode, 0, result.stderr) + outputs = dict(line.split("=", 1) for line in result.stdout.splitlines()) + self.assertIn('"platform":"linux/amd64"', outputs["matrix"]) + self.assertIn('"platform":"linux/arm64"', outputs["matrix"]) + self.assertNotRegex(outputs["image_version"], r"-amd64$") + result, _ = self.run_idc_settings(ARCHITECTURE="s390x") + self.assertNotEqual(result.returncode, 0) + self.assertIn("Unsupported architecture: s390x", result.stdout) + def test_client_arguments_with_and_without_features(self): script = workflow_run_script( ".github/workflows/release-binaries.yml", "Build client candidates" diff --git a/scripts/ci/validate_repository.py b/scripts/ci/validate_repository.py index f63ac0198..b625ddbe5 100755 --- a/scripts/ci/validate_repository.py +++ b/scripts/ci/validate_repository.py @@ -134,6 +134,9 @@ def main() -> None: idc_workflow = Path(".github/workflows/build_push_to_idc.yml").read_text( encoding="utf-8" ) + idc_candidates = Path( + ".github/workflows/idc-container-candidates.yml" + ).read_text(encoding="utf-8") for forbidden in ("push:\n tags:", "on:\n push:"): if forbidden in release_controller: @@ -258,46 +261,78 @@ def main() -> None: for required in ( "source_ref:", - 'GITHUB_REF}" != "refs/heads/${DEFAULT_BRANCH}', + "architecture:", "source_ref commit must belong to main or moi-dev", "environment: idc-publication", - "runs-on: ${{ vars.CONTAINER_MIRROR_RUNNER }}", + "idc-container-candidates.yml", + "Assemble verified IDC manifest", "Require IDC registry credentials", - "IDC publication requires a self-hosted runner", - "Build the IDC candidate locally", - "load: true", - "push: false", - "Verify health and exact memory round trip", - "docker/login-action", - 'docker push "${target}"', - "org.opencontainers.image.source=https://github.com/matrixorigin/astra", - "IMAGE_BRANCH=${{ env.RELEASE_SOURCE_REF }}", + "scripts/copy-immutable-container-tag.sh", + "${IDC_IMAGE}-candidates", + '"runner":"ubuntu-24.04-arm"', ): if required not in idc_workflow: errors.append( - ".github/workflows/build_push_to_idc.yml: missing trusted-controller " - f"or self-hosted admission contract ({required})" + ".github/workflows/build_push_to_idc.yml: missing IDC controller " + f"or publication contract ({required})" ) - idc_build = idc_workflow.find("Build the IDC candidate locally") - idc_smoke = idc_workflow.find("Verify health and exact memory round trip") - idc_login = idc_workflow.find("docker/login-action") - idc_push = idc_workflow.find('docker push "${target}"') - if not 0 <= idc_build < idc_smoke < idc_login < idc_push: - errors.append( - ".github/workflows/build_push_to_idc.yml: the verified local image must " - "pass smoke before Harbor authentication and publication" - ) - for forbidden in ( - "release-container-candidates.yml", + for required in ( + "workflow_call:", + "controller_sha:", "push-by-digest=true", + "context: source", + "file: source/Dockerfile", "buildcache-", - "astra-candidate-${GITHUB_RUN_ID}", + "make stack-up", + "make stack-verify", + "idc-digest-", + "Retain IDC candidate with a run-scoped immutable tag", + "org.opencontainers.image.source=https://github.com/matrixorigin/astra", + "IMAGE_BRANCH=${{ env.RELEASE_SOURCE_REF }}", + ): + if required not in idc_candidates: + errors.append( + ".github/workflows/idc-container-candidates.yml: missing copied " + f"release candidate contract ({required})" + ) + for forbidden in ("DOCKERHUB_", "matrixorigin/astra:"): + if forbidden in idc_workflow + idc_candidates: + errors.append( + ".github/workflows/build_push_to_idc.yml: IDC builds must not publish " + f"to Docker Hub ({forbidden})" + ) + + immutable_copy = Path("scripts/copy-immutable-container-tag.sh").read_text( + encoding="utf-8" + ) + for required in ( + "could not safely inspect", + "inspect-harbor-artifact.py", + "crane copy --platform=all --jobs 2", + "already exists with digest", + "resolves to ${target_digest}, expected ${source_digest}", + ): + if required not in immutable_copy: + errors.append( + "scripts/copy-immutable-container-tag.sh: missing fail-closed " + f"immutable publication contract ({required})" + ) + + harbor_inspector = Path("scripts/inspect-harbor-artifact.py").read_text( + encoding="utf-8" + ) + for required in ( + 'if error.code == 404:', + 'item.get("code") == "NOT_FOUND"', + "return NOT_FOUND", + "Harbor artifact lookup failed with HTTP", + 'document.get("digest")', ): - if forbidden in idc_workflow: + if required not in harbor_inspector: errors.append( - ".github/workflows/build_push_to_idc.yml: IDC runtime repository must " - f"not receive candidate/cache objects ({forbidden})" + "scripts/inspect-harbor-artifact.py: missing structured artifact " + f"lookup contract ({required})" ) manifest_reconciler = Path("scripts/reconcile-docker-manifest.sh").read_text( diff --git a/scripts/copy-immutable-container-tag.sh b/scripts/copy-immutable-container-tag.sh new file mode 100755 index 000000000..b257275f5 --- /dev/null +++ b/scripts/copy-immutable-container-tag.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Copy a verified image to an immutable tag without treating registry failures +# as proof that the target tag is absent. + +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +source_ref="$1" +target_ref="$2" +harbor_api_base_url="$3" + +target_repository="${target_ref%:*}" +target_tag="${target_ref##*:}" +if [[ "${target_repository}" == "${target_ref}" || "${target_tag}" == */* || -z "${target_tag}" ]]; then + echo "target reference must contain an explicit tag: ${target_ref}" >&2 + exit 2 +fi +target_registry="${target_repository%%/*}" +target_repository_path="${target_repository#*/}" +if [[ "${target_registry}" == "${target_repository}" ]]; then + echo "target reference must include a registry and repository: ${target_ref}" >&2 + exit 2 +fi +if [[ "${harbor_api_base_url}" != "https://${target_registry}" && \ + "${harbor_api_base_url}" != "http://${target_registry}" ]]; then + echo "Harbor API base URL must match the target registry" >&2 + exit 2 +fi + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +command -v crane >/dev/null 2>&1 || { + echo "crane is required" >&2 + exit 1 +} + +source_digest="$(crane digest "${source_ref}")" + +target_exists=false +if target_digest="$( + "${script_dir}/inspect-harbor-artifact.py" \ + "${harbor_api_base_url}" "${target_repository_path}" "${target_tag}" +)"; then + target_exists=true +else + lookup_status=$? + if [[ "${lookup_status}" -ne 44 ]]; then + echo "could not safely inspect ${target_ref} through the Harbor API" >&2 + exit 1 + fi +fi + +if [[ "${target_exists}" == true ]]; then + if [[ "${target_digest}" != "${source_digest}" ]]; then + echo "${target_ref} already exists with digest ${target_digest}, expected ${source_digest}" >&2 + exit 1 + fi + echo "verified existing immutable tag ${target_ref} -> ${source_digest}" + exit 0 +fi + +crane copy --platform=all --jobs 2 "${source_ref}" "${target_ref}" +target_digest="$(crane digest "${target_ref}")" +if [[ "${target_digest}" != "${source_digest}" ]]; then + echo "${target_ref} resolves to ${target_digest}, expected ${source_digest}" >&2 + exit 1 +fi + +echo "published immutable tag ${target_ref} -> ${source_digest}" diff --git a/scripts/inspect-harbor-artifact.py b/scripts/inspect-harbor-artifact.py new file mode 100755 index 000000000..bf7d40d40 --- /dev/null +++ b/scripts/inspect-harbor-artifact.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Resolve one Harbor artifact through its structured API response.""" + +from __future__ import annotations + +import base64 +import json +import os +import re +import sys +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlparse +from urllib.request import Request, urlopen + + +NOT_FOUND = 44 + + +def main() -> int: + if len(sys.argv) != 4: + print( + f"usage: {sys.argv[0]} ", + file=sys.stderr, + ) + return 2 + + base_url, repository, reference = sys.argv[1:] + parsed = urlparse(base_url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.path not in {"", "/"}: + print("Harbor API base URL must contain only an HTTP(S) scheme and authority", file=sys.stderr) + return 2 + + project, separator, repository_name = repository.partition("/") + if not separator or not project or not repository_name or not reference: + print("Harbor repository must include a project and repository name", file=sys.stderr) + return 2 + + username = os.environ.get("IDC_REGISTRY_USERNAME", "") + password = os.environ.get("IDC_REGISTRY_PASSWORD", "") + if not username or not password: + print("Harbor registry credentials are required", file=sys.stderr) + return 2 + + credentials = base64.b64encode(f"{username}:{password}".encode()).decode("ascii") + encoded_repository_name = quote(quote(repository_name, safe=""), safe="") + url = ( + f"{base_url.rstrip('/')}/api/v2.0/projects/{quote(project, safe='')}" + f"/repositories/{encoded_repository_name}/artifacts/{quote(reference, safe='')}" + ) + request = Request( + url, + headers={ + "Accept": "application/json", + "Authorization": f"Basic {credentials}", + }, + ) + + try: + with urlopen(request, timeout=30) as response: + document = json.load(response) + except HTTPError as error: + if error.code == 404: + try: + document = json.load(error) + except (UnicodeDecodeError, json.JSONDecodeError): + document = None + errors = document.get("errors") if isinstance(document, dict) else None + if isinstance(errors, list) and any( + isinstance(item, dict) and item.get("code") == "NOT_FOUND" + for item in errors + ): + return NOT_FOUND + print( + f"Harbor artifact lookup failed with HTTP {error.code} {error.reason}", + file=sys.stderr, + ) + return 1 + except (URLError, TimeoutError, OSError) as error: + print(f"Harbor artifact lookup failed: {error}", file=sys.stderr) + return 1 + except (UnicodeDecodeError, json.JSONDecodeError) as error: + print(f"Harbor artifact lookup returned invalid JSON: {error}", file=sys.stderr) + return 1 + + digest = document.get("digest") if isinstance(document, dict) else None + if not isinstance(digest, str) or re.fullmatch(r"sha256:[0-9a-f]{64}", digest) is None: + print("Harbor artifact lookup returned an invalid digest", file=sys.stderr) + return 1 + print(digest) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())