ci(release): reuse verified pipeline for IDC images

.github/workflows/build_push_to_idc.yml

@@ -8,6 +8,14 @@ on:
         required: true
         default: main
         type: string
+      architecture:
+        description: Architecture to build and publish
+        required: true
+        default: amd64
+        type: choice
+        options:
+          - amd64
+          - all
 
 permissions:
   contents: read
@@ -21,10 +29,13 @@ jobs:
     name: Validate IDC build configuration
     runs-on: ubuntu-latest
     outputs:
+      candidate_image: ${{ steps.resolve.outputs.candidate_image }}
+      controller_sha: ${{ steps.resolve.outputs.controller_sha }}
       image_version: ${{ steps.resolve.outputs.image_version }}
-      source_sha: ${{ steps.resolve.outputs.source_sha }}
+      matrix: ${{ steps.resolve.outputs.matrix }}
       source_ref: ${{ steps.resolve.outputs.source_ref }}
-      controller_sha: ${{ steps.resolve.outputs.controller_sha }}
+      source_sha: ${{ steps.resolve.outputs.source_sha }}
+      target_image: ${{ steps.resolve.outputs.target_image }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         with:
@@ -35,11 +46,11 @@ jobs:
         id: resolve
         shell: bash
         env:
+          ARCHITECTURE: ${{ inputs.architecture }}
           DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
           SOURCE_REF: ${{ inputs.source_ref }}
           IDC_REGISTRY: ${{ vars.CONTAINER_MIRROR_REGISTRY }}
           IDC_IMAGE: ${{ vars.CONTAINER_MIRROR_IMAGE }}
-          IDC_RUNNER: ${{ vars.CONTAINER_MIRROR_RUNNER }}
         run: |
           set -euo pipefail
           if [ "${GITHUB_REF}" != "refs/heads/${DEFAULT_BRANCH}" ]; then
@@ -80,7 +91,7 @@ jobs:
               ;;
           esac
 
-          for key in IDC_REGISTRY IDC_IMAGE IDC_RUNNER; do
+          for key in IDC_REGISTRY IDC_IMAGE; do
             if [ -z "${!key}" ]; then
               echo "::error::Missing required IDC configuration: ${key}"
               exit 1
@@ -94,37 +105,98 @@ jobs:
             echo "::error::IDC image must be an untagged repository under the configured IDC registry."
             exit 1
           fi
+
+          case "${ARCHITECTURE}" in
+            amd64)
+              matrix='{"include":[{"platform":"linux/amd64","runner":"ubuntu-latest","slug":"linux-amd64"}]}'
+              architecture_suffix="-amd64"
+              ;;
+            all)
+              matrix='{"include":[{"platform":"linux/amd64","runner":"ubuntu-latest","slug":"linux-amd64"},{"platform":"linux/arm64","runner":"ubuntu-24.04-arm","slug":"linux-arm64"}]}'
+              architecture_suffix=""
+              ;;
+            *)
+              echo "::error::Unsupported architecture: ${ARCHITECTURE}"
+              exit 1
+              ;;
+          esac
+
+          image_version="idc-$(date -u +%Y%m%dT%H%M%SZ)-${source_sha}-${GITHUB_RUN_ID}${architecture_suffix}"
           {
+            echo "candidate_image=${IDC_IMAGE}-candidates"
             echo "controller_sha=${controller_sha}"
-            echo "source_sha=${source_sha}"
+            echo "image_version=${image_version}"
+            echo "matrix=${matrix}"
             echo "source_ref=${source_ref}"
-            echo "image_version=idc-$(date -u +%Y%m%dT%H%M%SZ)-${source_sha}-${GITHUB_RUN_ID}-amd64"
+            echo "source_sha=${source_sha}"
+            echo "target_image=${IDC_IMAGE}"
           } >> "${GITHUB_OUTPUT}"
 
-  publish:
-    name: Build, verify, and publish IDC image
+  candidates:
+    name: Build and verify IDC candidates
     needs: settings
-    runs-on: ${{ vars.CONTAINER_MIRROR_RUNNER }}
+    uses: ./.github/workflows/idc-container-candidates.yml
+    with:
+      candidate_image: ${{ needs.settings.outputs.candidate_image }}
+      controller_sha: ${{ needs.settings.outputs.controller_sha }}
+      image_version: ${{ needs.settings.outputs.image_version }}
+      matrix_json: ${{ needs.settings.outputs.matrix }}
+      source_ref: ${{ needs.settings.outputs.source_ref }}
+      source_sha: ${{ needs.settings.outputs.source_sha }}
+
+  stage:
+    name: Assemble verified IDC manifest
+    needs:
+      - settings
+      - candidates
+    runs-on: ubuntu-latest
     environment: idc-publication
-    timeout-minutes: 210
-    env:
-      LOCAL_IMAGE: astra-idc-candidate:${{ needs.settings.outputs.source_sha }}
-      IMAGE_VERSION: ${{ needs.settings.outputs.image_version }}
-      RELEASE_SOURCE_SHA: ${{ needs.settings.outputs.source_sha }}
-      RELEASE_SOURCE_REF: ${{ needs.settings.outputs.source_ref }}
-      http_proxy: ${{ vars.CONTAINER_MIRROR_HTTP_PROXY }}
-      https_proxy: ${{ vars.CONTAINER_MIRROR_HTTPS_PROXY }}
-      no_proxy: ${{ vars.CONTAINER_MIRROR_NO_PROXY }}
+    timeout-minutes: 20
     steps:
-      - name: Require a self-hosted IDC runner
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+          ref: ${{ needs.settings.outputs.controller_sha }}
+
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          pattern: idc-digest-*
+          path: /tmp/digests
+          merge-multiple: true
+
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+
+      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ${{ vars.CONTAINER_MIRROR_REGISTRY }}
+          username: ${{ secrets.IDC_REGISTRY_USERNAME }}
+          password: ${{ secrets.IDC_REGISTRY_PASSWORD }}
+
+      - name: Assemble or verify the staged manifest
         shell: bash
         env:
-          RUNNER_ENVIRONMENT: ${{ runner.environment }}
+          CANDIDATE_IMAGE: ${{ needs.settings.outputs.candidate_image }}
+          IMAGE_VERSION: ${{ needs.settings.outputs.image_version }}
+          BUILD_MATRIX_JSON: ${{ needs.settings.outputs.matrix }}
         run: |
-          if [ "${RUNNER_ENVIRONMENT}" != "self-hosted" ]; then
-            echo "::error::IDC publication requires a self-hosted runner."
-            exit 1
-          fi
+          set -euo pipefail
+          scripts/reconcile-docker-manifest.sh \
+            "${CANDIDATE_IMAGE}" "${IMAGE_VERSION}" "${BUILD_MATRIX_JSON}" \
+            /tmp/digests verify
+
+  publish:
+    name: Publish verified IDC image
+    needs:
+      - settings
+      - stage
+    runs-on: ubuntu-latest
+    environment: idc-publication
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+          ref: ${{ needs.settings.outputs.controller_sha }}
 
       - name: Require IDC registry credentials
         shell: bash
@@ -140,124 +212,45 @@ jobs:
             fi
           done
 
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-        with:
-          persist-credentials: false
-          ref: ${{ needs.settings.outputs.controller_sha }}
-
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-        with:
-          persist-credentials: false
-          ref: ${{ needs.settings.outputs.source_sha }}
-          path: source
-
-      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
-
-      - name: Build the IDC candidate locally
-        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
-        with:
-          context: source
-          file: source/Dockerfile
-          platforms: linux/amd64
-          tags: ${{ env.LOCAL_IMAGE }}
-          labels: |
-            org.opencontainers.image.version=${{ env.IMAGE_VERSION }}
-            org.opencontainers.image.revision=${{ env.RELEASE_SOURCE_SHA }}
-            org.opencontainers.image.source=https://github.com/matrixorigin/astra
-          build-args: |
-            IMAGE_VERSION=${{ env.IMAGE_VERSION }}
-            IMAGE_REVISION=${{ env.RELEASE_SOURCE_SHA }}
-            IMAGE_SOURCE_DIRTY=false
-            IMAGE_BRANCH=${{ env.RELEASE_SOURCE_REF }}
-            HTTP_PROXY=${{ vars.CONTAINER_MIRROR_HTTP_PROXY }}
-            HTTPS_PROXY=${{ vars.CONTAINER_MIRROR_HTTPS_PROXY }}
-            NO_PROXY=${{ vars.CONTAINER_MIRROR_NO_PROXY }}
-          load: true
-          push: false
-
-      - name: Verify local candidate identity
-        shell: bash
-        run: |
-          set -euo pipefail
-          assert_label() {
-            local label="$1"
-            local expected="$2"
-            local actual
-            actual="$(docker image inspect "${LOCAL_IMAGE}" --format "{{ index .Config.Labels \"${label}\" }}")"
-            if [ "${actual}" != "${expected}" ]; then
-              echo "::error::Candidate ${label} is ${actual:-<missing>}, expected ${expected}."
-              exit 1
-            fi
-          }
-          platform="$(docker image inspect "${LOCAL_IMAGE}" --format '{{.Os}}/{{.Architecture}}')"
-          if [ "${platform}" != "linux/amd64" ]; then
-            echo "::error::Candidate platform is ${platform}, expected linux/amd64."
-            exit 1
-          fi
-          assert_label org.opencontainers.image.version "${IMAGE_VERSION}"
-          assert_label org.opencontainers.image.revision "${RELEASE_SOURCE_SHA}"
-          assert_label org.opencontainers.image.ref.name "${RELEASE_SOURCE_REF}"
-          assert_label org.opencontainers.image.source https://github.com/matrixorigin/astra
-
-      - name: Start local candidate through the all-in-one path
+      - name: Install pinned crane
         shell: bash
+        env:
+          CRANE_VERSION: v0.20.6
+          CRANE_LINUX_X86_64_SHA256: c1d593d01551f2c9a3df5ca0a0be4385a839bd9b86d4a76e18d7b17d16559127
         run: |
           set -euo pipefail
-          make stack-env
-          sed -i "s|^ASTRA_IMAGE=.*|ASTRA_IMAGE=${LOCAL_IMAGE}|" deployment/all-in-one/.env
-          sed -i 's|^MEMORIA_EMBEDDING_PROVIDER=.*|MEMORIA_EMBEDDING_PROVIDER=mock|' deployment/all-in-one/.env
-          make stack-up
-
-      - name: Verify health and exact memory round trip
-        run: make stack-verify
-
-      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
-        with:
-          registry: ${{ vars.CONTAINER_MIRROR_REGISTRY }}
-          username: ${{ secrets.IDC_REGISTRY_USERNAME }}
-          password: ${{ secrets.IDC_REGISTRY_PASSWORD }}
+          mkdir -p "${RUNNER_TEMP}/bin"
+          curl -fsSL \
+            -o "${RUNNER_TEMP}/go-containerregistry.tar.gz" \
+            "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz"
+          echo "${CRANE_LINUX_X86_64_SHA256}  ${RUNNER_TEMP}/go-containerregistry.tar.gz" \
+            | sha256sum --check --strict
+          tar -xzf "${RUNNER_TEMP}/go-containerregistry.tar.gz" -C "${RUNNER_TEMP}/bin" crane
+          echo "${RUNNER_TEMP}/bin" >> "${GITHUB_PATH}"
+          "${RUNNER_TEMP}/bin/crane" version
 
       - name: Publish or verify immutable IDC image
         shell: bash
         env:
-          IMAGE_NAME: ${{ vars.CONTAINER_MIRROR_IMAGE }}
+          CANDIDATE_IMAGE: ${{ needs.settings.outputs.candidate_image }}
+          IMAGE_NAME: ${{ needs.settings.outputs.target_image }}
+          IMAGE_VERSION: ${{ needs.settings.outputs.image_version }}
+          IDC_REGISTRY: ${{ vars.CONTAINER_MIRROR_REGISTRY }}
+          IDC_REGISTRY_USERNAME: ${{ secrets.IDC_REGISTRY_USERNAME }}
+          IDC_REGISTRY_PASSWORD: ${{ secrets.IDC_REGISTRY_PASSWORD }}
+          RELEASE_SOURCE_SHA: ${{ needs.settings.outputs.source_sha }}
         run: |
           set -euo pipefail
+          printf '%s' "${IDC_REGISTRY_PASSWORD}" | crane auth login \
+            "${IDC_REGISTRY}" -u "${IDC_REGISTRY_USERNAME}" --password-stdin
+          source="${CANDIDATE_IMAGE}:${IMAGE_VERSION}"
           target="${IMAGE_NAME}:${IMAGE_VERSION}"
-          local_id="$(docker image inspect "${LOCAL_IMAGE}" --format '{{.Id}}')"
-          target_exists=false
-          if inspect_output="$(docker buildx imagetools inspect "${target}" 2>&1)"; then
-            target_exists=true
-          elif ! grep -Eqi '(: not found|manifest unknown|name unknown|HTTP 404|status[^0-9]*404)' \
-            <<< "${inspect_output}"; then
-            echo "::error::Could not safely determine whether ${target} exists."
-            printf '%s\n' "${inspect_output}" >&2
-            exit 1
-          fi
-
-          if [ "${target_exists}" = false ]; then
-            docker tag "${LOCAL_IMAGE}" "${target}"
-            docker push "${target}"
-          fi
-          docker pull "${target}"
-          target_id="$(docker image inspect "${target}" --format '{{.Id}}')"
-          if [ "${target_id}" != "${local_id}" ]; then
-            echo "::error::${target} does not match the verified local image."
-            exit 1
-          fi
+          source_digest="$(crane digest "${source}")"
+          scripts/copy-immutable-container-tag.sh \
+            "${source}" "${target}" "https://${IDC_REGISTRY}"
           {
             echo "## IDC image published"
             echo "- Image: \`${target}\`"
+            echo "- Digest: \`${source_digest}\`"
             echo "- Source: \`${RELEASE_SOURCE_SHA}\`"
           } >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Show service logs after failure
-        if: ${{ failure() }}
-        working-directory: deployment/all-in-one
-        run: docker compose --env-file .env logs --no-color --tail=200 api memoria matrixone || true
-
-      - name: Remove local smoke stack and image
-        if: ${{ always() }}
-        run: |
-          make stack-clean || true
-          docker image rm "${LOCAL_IMAGE}" "${{ vars.CONTAINER_MIRROR_IMAGE }}:${IMAGE_VERSION}" >/dev/null 2>&1 || true