From c934f78c99beab867c3b2ded3d6c4587aeebbf8f Mon Sep 17 00:00:00 2001 From: Fabrice Brito Date: Sat, 10 Oct 2026 17:03:53 +0200 Subject: [PATCH 1/3] build: support ARM64 and fix isolated tool runtimes --- Dockerfile | 58 ++++++++++++++++++++++++++++++++-------------- ci/test-runtime.sh | 52 +++++++++++++++++++++++++++++++++++++++++ ci/test-yq.sh | 13 +++++++++++ 3 files changed, 105 insertions(+), 18 deletions(-) create mode 100644 ci/test-runtime.sh create mode 100644 ci/test-yq.sh diff --git a/Dockerfile b/Dockerfile index cd4320a..5f23b3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,12 @@ FROM quay.io/jupyter/base-notebook:python-3.12 +# BuildKit supplies amd64 or arm64 for each target platform. +ARG TARGETARCH +RUN case "${TARGETARCH}" in amd64|arm64) ;; *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; esac + ENV DEBIAN_FRONTEND=noninteractive \ USER=jovyan \ - UID=1000 \ + UID=1001 \ GID=100 \ HOME=/workspace @@ -57,7 +61,7 @@ RUN printf '\nexport NVM_DIR="/opt/nvm"\n[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR ARG CODE_RELEASE=4.138.0 RUN mkdir -p /opt/code-server && \ curl -fsSL \ - "https://github.com/coder/code-server/releases/download/v${CODE_RELEASE}/code-server-${CODE_RELEASE}-linux-amd64.tar.gz" \ + "https://github.com/coder/code-server/releases/download/v${CODE_RELEASE}/code-server-${CODE_RELEASE}-linux-${TARGETARCH}.tar.gz" \ | tar -xz --strip-components=1 -C /opt/code-server ENV PATH="/opt/code-server/bin:${PATH}" @@ -67,29 +71,28 @@ ENV PATH="/opt/code-server/bin:${PATH}" # ------------------------------------------------------------------- ARG KUBECTL_VERSION=v1.29.3 RUN curl -fsSL \ - https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl \ + https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl \ -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl ARG TASK_VERSION=v3.41.0 RUN curl -fsSL \ - https://github.com/go-task/task/releases/download/${TASK_VERSION}/task_linux_amd64.tar.gz \ + https://github.com/go-task/task/releases/download/${TASK_VERSION}/task_linux_${TARGETARCH}.tar.gz \ | tar -xz -C /usr/local/bin task && chmod +x /usr/local/bin/task ARG SKAFFOLD_VERSION=2.17.1 RUN curl -fsSL \ - https://storage.googleapis.com/skaffold/releases/v${SKAFFOLD_VERSION}/skaffold-linux-amd64 \ + https://storage.googleapis.com/skaffold/releases/v${SKAFFOLD_VERSION}/skaffold-linux-${TARGETARCH} \ -o /usr/local/bin/skaffold && chmod +x /usr/local/bin/skaffold ARG ORAS_VERSION=1.3.0 RUN curl -fsSL \ - https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz \ + https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_${TARGETARCH}.tar.gz \ | tar -xz -C /usr/local/bin oras && chmod +x /usr/local/bin/oras # ------------------------------------------------------------------- # Python tooling # ------------------------------------------------------------------- -ARG CALRISSIAN_VERSION=0.18.1 -ARG CALRISSIAN_CWL_UTILS_VERSION=0.40 +ARG CALRISSIAN_COMMIT=a95d87dd213d51b11faea7c6def4ced46d466a4a COPY requirements-cwl-uml.txt /tmp/requirements-cwl-uml.txt RUN pip install --no-cache-dir \ @@ -104,10 +107,12 @@ RUN pip install --no-cache-dir \ cwltest && \ python -m bash_kernel.install +# Resolve the commit's cwltool/dependency requirements together, independently +# of the global CWL tools, rather than constraining cwl-utils to the old release. RUN python -m venv /opt/calrissian-venv && \ /opt/calrissian-venv/bin/pip install --no-cache-dir \ - "cwl-utils==${CALRISSIAN_CWL_UTILS_VERSION}" \ - "calrissian==${CALRISSIAN_VERSION}" && \ + "calrissian @ git+https://github.com/duke-gcb/calrissian.git@${CALRISSIAN_COMMIT}" && \ + /opt/calrissian-venv/bin/python -m pip check && \ ln -s /opt/calrissian-venv/bin/calrissian /usr/local/bin/calrissian # ------------------------------------------------------------------- @@ -115,28 +120,37 @@ RUN python -m venv /opt/calrissian-venv && \ # ------------------------------------------------------------------- ARG YQ_VERSION=v4.45.1 RUN curl -fsSL \ - https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 \ + https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${TARGETARCH} \ -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq ARG JQ_VERSION=jq-1.8.1 RUN curl -fsSL \ - https://github.com/jqlang/jq/releases/download/${JQ_VERSION}/jq-linux-amd64 \ + https://github.com/jqlang/jq/releases/download/${JQ_VERSION}/jq-linux-${TARGETARCH} \ -o /usr/local/bin/jq && chmod +x /usr/local/bin/jq # ------------------------------------------------------------------- # hatch # ------------------------------------------------------------------- ARG HATCH_VERSION=1.16.2 -RUN curl -fsSL \ - https://github.com/pypa/hatch/releases/download/hatch-v${HATCH_VERSION}/hatch-x86_64-unknown-linux-gnu.tar.gz \ - | tar -xz -C /usr/local/bin hatch && chmod +x /usr/local/bin/hatch +ENV HATCH_CONFIG=/home/jovyan/.config/hatch/config.toml \ + HATCH_DATA_DIR=/home/jovyan/.local/share/hatch \ + HATCH_CACHE_DIR=/home/jovyan/.cache/hatch +# The release launcher fetches its runtime on first use. Install Hatch and all +# its dependencies now, using the base image's native Python on either platform. +RUN python -m venv /opt/hatch-venv && \ + /opt/hatch-venv/bin/pip install --no-cache-dir "hatch==${HATCH_VERSION}" && \ + /opt/hatch-venv/bin/python -m pip check && \ + ln -s /opt/hatch-venv/bin/hatch /usr/local/bin/hatch && \ + install -d -o ${USER} -g users /home/jovyan/.config/hatch "${HATCH_DATA_DIR}" "${HATCH_CACHE_DIR}" && \ + install -o ${USER} -g users -m 644 /dev/null "${HATCH_CONFIG}" # ------------------------------------------------------------------- # trivy # ------------------------------------------------------------------- ARG TRIVY_VERSION=0.74.0 -RUN curl -fsSL \ - https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb \ +RUN case "${TARGETARCH}" in amd64) TRIVY_ARCH=64bit ;; arm64) TRIVY_ARCH=ARM64 ;; esac && \ + curl -fsSL \ + https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-${TRIVY_ARCH}.deb \ -o /tmp/trivy.deb && \ dpkg -i /tmp/trivy.deb && \ rm /tmp/trivy.deb @@ -165,9 +179,17 @@ RUN apt-get update && apt-get install -y \ cmake --build . -- -j"$(nproc)" && \ cmake --install . && \ ldconfig && \ - rm -rf /tmp/gdal-${GDAL_VER}* && \ + cd / && rm -rf /tmp/gdal-${GDAL_VER}* && \ gdal-config --version +# ------------------------------------------------------------------- +# tomlq requires the Python yq package, whose console launcher shadows the +# pinned Mike Farah binary on Conda's PATH. Keep its modules and tomlq/xq +# launchers; replace only the conflicting command, without changing PATH. +# Prepare the default workspace for non-root tools after runtime installation. +RUN ln -sf /usr/local/bin/yq /opt/conda/bin/yq && \ + install -d -o ${USER} -g users /workspace + # ------------------------------------------------------------------- # Entrypoint # ------------------------------------------------------------------- diff --git a/ci/test-runtime.sh b/ci/test-runtime.sh new file mode 100644 index 0000000..49200aa --- /dev/null +++ b/ci/test-runtime.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Run as jovyan in a disposable container with --network none. +set -euo pipefail +script_dir="$(cd "$(dirname "$0")" && pwd)" +test "$(id -u)" = 1001 +test -w "$HATCH_CONFIG" +test -w "$HATCH_DATA_DIR" +test -w "$HATCH_CACHE_DIR" +test "$(readlink -f "$(command -v hatch)")" = /opt/hatch-venv/bin/hatch +test "$(readlink -f "$(command -v calrissian)")" = /opt/calrissian-venv/bin/calrissian +fixture_dir="$(mktemp -d)" +trap 'rm -rf "$fixture_dir"' EXIT + +[[ "$(hatch --version)" == 'Hatch, version 1.16.2' ]] +cat > "$fixture_dir/pyproject.toml" <<'TOML' +[project] +name = "offline-smoke" +version = "1.2.3" +TOML +cd "$fixture_dir" +[[ "$(hatch version)" == '1.2.3' ]] +/opt/hatch-venv/bin/python -m pip check + +/opt/calrissian-venv/bin/python - <<'PYTHON' +import json +from importlib import metadata +from pathlib import Path + +import calrissian.dask +from packaging.version import Version + +expected = "a95d87dd213d51b11faea7c6def4ced46d466a4a" +distribution = metadata.distribution("calrissian") +provenance = json.loads(distribution.read_text("direct_url.json")) +assert provenance["url"].removesuffix(".git").lower() == "https://github.com/duke-gcb/calrissian" +assert provenance["vcs_info"]["vcs"] == "git" +assert provenance["vcs_info"]["commit_id"] == expected, provenance +assert Version(metadata.version("cwltool")) >= Version("3.1.20260108082145") +package = Path(calrissian.dask.__file__).parent +for resource in ("dask/custom_schema/schema.yaml", "dask/init-dask.py", "dask/dispose-dask.py"): + path = package / resource + assert path.is_file() and path.stat().st_size > 0, path + print(f"Verified installed resource: {path}") +print(f"Verified Calrissian commit {expected}, Dask import, and cwltool {metadata.version('cwltool')}") +PYTHON +/opt/calrissian-venv/bin/python -m pip check +calrissian --help > "$fixture_dir/calrissian-help.txt" +for flag in --dask-gateway-url --dask-script-configmap; do + grep -Fq -- "$flag" "$fixture_dir/calrissian-help.txt" +done +bash "$script_dir/test-yq.sh" +echo 'Offline Hatch, Calrissian package, and YAML/TOML checks passed (no Kubernetes Dask execution tested).' diff --git a/ci/test-yq.sh b/ci/test-yq.sh new file mode 100644 index 0000000..5d0a860 --- /dev/null +++ b/ci/test-yq.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Run inside the image as its default user, exercising commands through PATH. +set -euo pipefail +fixture_dir="$(mktemp -d)" +trap 'rm -rf "$fixture_dir"' EXIT +version="$(yq --version)" +[[ "$version" == 'yq (https://github.com/mikefarah/yq/) version v4.45.1' ]] +printf 'image_version: "1.4.0"\n' > "$fixture_dir/image.yaml" +[[ "$(yq eval '.image_version' "$fixture_dir/image.yaml")" == '1.4.0' ]] +printf '[image]\nversion = "1.4.0"\n' > "$fixture_dir/image.toml" +[[ "$(tomlq -r '.image.version' "$fixture_dir/image.toml")" == '1.4.0' ]] +python -m pip check +printf 'Default yq, TOML processing, and Python dependencies passed: %s\n' "$version" From c4b346c831b3622079152b0fb215591f0ce69f6b Mon Sep 17 00:00:00 2001 From: Fabrice Brito Date: Sat, 10 Oct 2026 17:03:53 +0200 Subject: [PATCH 2/3] ci: test and publish multi-platform container images --- .github/workflows/build-image.yml | 293 +++++++++++++++--------------- Taskfile.yaml | 20 +- ci/test-image.sh | 95 ++++++++++ 3 files changed, 256 insertions(+), 152 deletions(-) create mode 100644 ci/test-image.sh diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 37a52db..080bc7b 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -3,179 +3,172 @@ name: build on: push: branches: [develop, main] - tags: - - 'v*' + tags: ['v*'] + pull_request: + branches: [develop, main] workflow_dispatch: inputs: GDAL_VER: - description: "GDAL version to use for Docker build" + description: GDAL version to use for Docker build required: false - default: "3.12.1" + default: '3.12.1' + +permissions: + contents: read + jobs: - # ------------------------------------------------------------ - # Resolve release tag from ref - # ------------------------------------------------------------ release-meta: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: tag: ${{ steps.meta.outputs.tag }} + image: ${{ steps.meta.outputs.image }} steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install PyYAML - id: meta + env: + EVENT_NAME: ${{ github.event_name }} run: | set -euo pipefail - if [[ "${GITHUB_REF}" == "refs/heads/develop" ]]; then - TAG="latest-dev" - elif [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then - TAG="latest" - elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then - TAG="${GITHUB_REF#refs/tags/v}" - - # -------------------------------------------------- - # Validate against release.yaml - # -------------------------------------------------- - RELEASE_VERSION=$(yq -r '.image_version' release.yaml) - - if [[ -z "${RELEASE_VERSION}" || "${RELEASE_VERSION}" == "null" ]]; then - echo "image_version not found in release.yaml" - exit 1 - fi - - if [[ "${TAG}" != "${RELEASE_VERSION}" ]]; then - echo "Git tag v${TAG} does not match release.yaml version ${RELEASE_VERSION}" - exit 1 - fi - else - echo "Unsupported ref ${GITHUB_REF}" - exit 1 - fi - echo "tag=${TAG}" >> "$GITHUB_OUTPUT" - - # ------------------------------------------------------------ - # Build, scan, SBOM, push - # ------------------------------------------------------------ - build-scan-push: - needs: release-meta - runs-on: ubuntu-latest - + python - <<'PY' + import os, yaml + with open('release.yaml') as stream: + release = yaml.safe_load(stream) + ref = os.environ['GITHUB_REF'] + if os.environ['EVENT_NAME'] == 'pull_request': + tag = 'pr-test' + elif ref == 'refs/heads/develop': + tag = 'latest-dev' + elif ref == 'refs/heads/main': + tag = 'latest' + elif ref.startswith('refs/tags/v'): + tag = ref.removeprefix('refs/tags/v') + if tag != str(release['image_version']): + raise SystemExit('Git tag does not match release.yaml image_version') + else: + raise SystemExit(f'Unsupported release ref: {ref}') + image = '/'.join(release[key] for key in ('image_registry', 'image_prefix', 'image_name')) + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write(f'tag={tag}\nimage={image}\n') + PY + + extension-releases: + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - - name: Install tooling - run: | - sudo apt-get update - sudo apt-get install -y jq skopeo - curl -sL https://github.com/oras-project/oras/releases/download/v1.2.2/oras_1.2.2_linux_amd64.tar.gz | tar -xz oras - sudo mv oras /usr/local/bin/oras - - - name: Configure Docker Hub auth for Kaniko - run: | - if [[ -n "${{ secrets.dockerhub-username }}" ]]; then - mkdir -p $HOME/.docker - cat > $HOME/.docker/config.json <> $GITHUB_ENV - - name: Build image with Kaniko (to tar) - uses: docker://gcr.io/kaniko-project/executor:debug - env: - DOCKER_CONFIG: /github/home/.docker - with: - args: > - --context . - --dockerfile Dockerfile - --build-arg GDAL_VER=${{ env.GDAL_VER }} - --no-push - --tar-path image.tar - - - name: Download latest released extension VSIXs + - name: Download the same released VSIXs for both architectures env: GITHUB_TOKEN: ${{ github.token }} run: python3 ci/test-extensions.py download extension-vsix - - - name: Test extension installation and runtime requirements + - uses: actions/upload-artifact@v4 + with: + name: extension-vsix + path: extension-vsix/ + + build-test: + needs: [release-meta, extension-releases] + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: actions/download-artifact@v4 + with: + name: extension-vsix + path: extension-vsix + - name: Build and load architecture image + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/${{ matrix.arch }} + load: true + tags: pde-extension-test:ci + build-args: GDAL_VER=${{ inputs.GDAL_VER || '3.12.1' }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + provenance: false + - name: Test tools, identity, extensions, and entrypoint + env: + PLATFORM: linux/${{ matrix.arch }} + run: bash ci/test-image.sh pde-extension-test:ci "$PLATFORM" "$PWD/extension-vsix" + - name: Save tested image + run: docker save pde-extension-test:ci -o image.tar + - name: Install pinned scan tooling + env: + ARCH: ${{ matrix.arch }} run: | set -euo pipefail - skopeo copy docker-archive:image.tar docker-daemon:pde-extension-test:ci - docker run --rm --platform linux/amd64 \ - --entrypoint python3 \ - -v "$PWD/ci:/opt/extension-ci:ro" \ - -v "$PWD/extension-vsix:/opt/extension-vsix:ro" \ - pde-extension-test:ci \ - /opt/extension-ci/test-extensions.py test /opt/extension-vsix - - - name: Record tested extension releases - if: always() - uses: actions/upload-artifact@v4 - with: - name: tested-extension-releases - path: extension-vsix/*.release.json - if-no-files-found: ignore - - # -------------------------------------------------------- - # Trivy scan (relaxed for now) - # -------------------------------------------------------- - - name: Trivy scan (debug mode) - uses: docker://aquasec/trivy:0.50.2 + case "$ARCH" in amd64) TRIVY_ARCH=64bit ;; arm64) TRIVY_ARCH=ARM64 ;; esac + curl -fsSL "https://github.com/aquasecurity/trivy/releases/download/v0.74.0/trivy_0.74.0_Linux-${TRIVY_ARCH}.tar.gz" | tar -xz trivy + sudo install trivy /usr/local/bin/trivy + - name: Trivy scan (relaxed for now) env: TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db + run: trivy image --input image.tar --severity HIGH,CRITICAL --exit-code 0 --no-progress + - name: Generate architecture SBOM + run: trivy image --input image.tar --format spdx-json --output sbom.spdx.json + - name: Upload tested image and its SBOM + uses: actions/upload-artifact@v4 with: - args: > - image - --input image.tar - --severity HIGH,CRITICAL - --exit-code 0 - --no-progress - - - name: Generate SBOM (SPDX JSON) - uses: docker://aquasec/trivy:0.50.2 - env: - TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db + name: image-${{ matrix.arch }} + path: | + image.tar + sbom.spdx.json + retention-days: 1 + + # Neither architecture is published until both native build/test jobs pass. + publish: + if: github.event_name != 'pull_request' + needs: [release-meta, build-test] + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + steps: + - uses: actions/download-artifact@v4 with: - args: > - image - --input image.tar - --format spdx-json - --output sbom.spdx.json - - # -------------------------------------------------------- - # Push image and attach SBOM - # -------------------------------------------------------- - - name: Push image with Skopeo - id: push + pattern: image-* + path: images + - uses: docker/setup-buildx-action@v3 + - name: Install publishing tools run: | set -euo pipefail - - TAG=${{ needs.release-meta.outputs.tag }} - REGISTRY=$(yq -r '.image_registry' release.yaml) - IMAGE_PREFIX=$(yq -r '.image_prefix' release.yaml) - IMAGE=$(yq -r '.image_name' release.yaml) - - IMAGE_REF="${REGISTRY}/${IMAGE_PREFIX}/${IMAGE}:${TAG}" - - skopeo copy \ - --dest-creds="${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - docker-archive:image.tar \ - docker://${IMAGE_REF} - - DIGEST=$(skopeo inspect \ - --creds="${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - docker://${IMAGE_REF} \ - | jq -r .Digest) - - echo "${REGISTRY}/${IMAGE_PREFIX}/${IMAGE}@${DIGEST}" > image.digest + sudo apt-get update + sudo apt-get install -y skopeo jq + ARCH=$(dpkg --print-architecture) + curl -fsSL "https://github.com/oras-project/oras/releases/download/v1.2.2/oras_1.2.2_linux_${ARCH}.tar.gz" | tar -xz oras + sudo install oras /usr/local/bin/oras + - name: Publish architecture images, attach SBOMs, and create combined manifest + env: + IMAGE: ${{ needs.release-meta.outputs.image }} + TAG: ${{ needs.release-meta.outputs.tag }} + REGISTRY_TOKEN: ${{ github.token }} + REGISTRY_USER: ${{ github.actor }} + run: | + set -euo pipefail + REGISTRY="${IMAGE%%/*}" + printf '%s' "$REGISTRY_TOKEN" | docker login "$REGISTRY" --username "$REGISTRY_USER" --password-stdin + printf '%s' "$REGISTRY_TOKEN" | oras login "$REGISTRY" --username "$REGISTRY_USER" --password-stdin + refs=() + for arch in amd64 arm64; do + ARCH_REF="${IMAGE}:${TAG}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${arch}" + skopeo copy --dest-creds "${REGISTRY_USER}:${REGISTRY_TOKEN}" \ + "docker-archive:images/image-${arch}/image.tar" "docker://${ARCH_REF}" + DIGEST=$(skopeo inspect --creds "${REGISTRY_USER}:${REGISTRY_TOKEN}" "docker://${ARCH_REF}" | jq -r .Digest) + # Attach each SBOM to the exact image digest it describes, not the index. + (cd "images/image-${arch}" && oras attach --artifact-type application/spdx+json \ + "${IMAGE}@${DIGEST}" sbom.spdx.json:application/spdx+json) + refs+=("${IMAGE}@${DIGEST}") + done + docker buildx imagetools create --tag "${IMAGE}:${TAG}" "${refs[@]}" + docker buildx imagetools inspect "${IMAGE}:${TAG}" diff --git a/Taskfile.yaml b/Taskfile.yaml index 2c48d5e..8a63f8c 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -2,7 +2,10 @@ version: '3' vars: IMAGE: eoepca/pde-code-server:local + PLATFORM: linux/amd64 GDAL_VER: '3.12.1' + OUTPUT: /tmp/pde-code-server-multi.oci.tar + VSIX_DIR: /tmp/pde-extension-vsix tasks: default: @@ -11,9 +14,22 @@ tasks: - task: build build: - desc: Build the AMD64 container image (override IMAGE or GDAL_VER as needed) + desc: Build and load one platform (PLATFORM=linux/amd64 or linux/arm64) preconditions: - sh: command -v docker msg: Docker must be installed to build the image. + - sh: 'test {{.PLATFORM | shellQuote}} = linux/amd64 || test {{.PLATFORM | shellQuote}} = linux/arm64' + msg: PLATFORM must be linux/amd64 or linux/arm64. cmds: - - docker build --platform=linux/amd64 --build-arg GDAL_VER={{.GDAL_VER | shellQuote}} -t {{.IMAGE | shellQuote}} . + - docker buildx build --platform={{.PLATFORM | shellQuote}} --build-arg GDAL_VER={{.GDAL_VER | shellQuote}} --load -t {{.IMAGE | shellQuote}} . + + build:multi: + desc: Build both platforms into a local OCI archive (requires a docker-container Buildx builder) + cmds: + - docker buildx build --platform=linux/amd64,linux/arm64 --build-arg GDAL_VER={{.GDAL_VER | shellQuote}} --output type=oci,dest={{.OUTPUT | shellQuote}} -t {{.IMAGE | shellQuote}} . + + test: + desc: Check tools, identity, startup, and released extensions in the local image + cmds: + - python3 ci/test-extensions.py download {{.VSIX_DIR | shellQuote}} + - bash ci/test-image.sh {{.IMAGE | shellQuote}} {{.PLATFORM | shellQuote}} {{.VSIX_DIR | shellQuote}} diff --git a/ci/test-image.sh b/ci/test-image.sh new file mode 100644 index 0000000..75271e7 --- /dev/null +++ b/ci/test-image.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Run against a loaded image; uses native execution or registered binfmt/QEMU. +set -euo pipefail +image="${1:?image required}" +platform="${2:?platform required}" +vsix_dir="${3:?VSIX directory required}" +repo_dir="$(cd "$(dirname "$0")/.." && pwd)" +arch="${platform#linux/}" +case "$platform" in linux/amd64|linux/arm64) ;; *) exit 1 ;; esac +actual="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" +[[ "$actual" == "$platform" ]] || { echo "Expected $platform, got $actual" >&2; exit 1; } +# Exercise cold-start runtime availability before any network-enabled tool check. +docker run --rm --network none --platform "$platform" --entrypoint bash \ + -v "$repo_dir/ci:/opt/image-ci:ro" "$image" /opt/image-ci/test-runtime.sh +docker run --rm --platform "$platform" --entrypoint bash "$image" -ec ' + test "$(id -u)" = 1001 + test "$UID" = 1001 + test "$(id -un)" = jovyan + test "$(id -g)" = 100 + test "$(dpkg --print-architecture)" = "$1" + code-server --version + node --version + kubectl version --client=true + task --version + skaffold version + /usr/local/bin/oras version + yq --version + jq --version + hatch --version + trivy --version + java -version + dot -V + gdalinfo --version + calrissian --version + cwltool --version + cwltest --help >/dev/null + podman --version + skopeo --version + uv --version + QT_QPA_PLATFORM=offscreen nextcloudcmd --version + python3 -m pip check + /opt/calrissian-venv/bin/python -m pip check + python3 -c "import cwl_loader, cwl2puml" +' bash "$arch" +# Check the ENV value directly: Bash replaces UID with the current numeric UID. +docker run --rm -i --platform "$platform" --entrypoint python3 "$image" - "$arch" <<'PYTHON' +import os +from pathlib import Path +import shutil +import struct +import sys + +assert os.environ["UID"] == "1001" +assert os.getuid() == 1001 +machine = {"amd64": 62, "arm64": 183}[sys.argv[1]] +executables = ["/opt/code-server/lib/node", shutil.which("node"), shutil.which("trivy"), + "/opt/hatch-venv/bin/python", "/opt/calrissian-venv/bin/python"] +executables += [f"/usr/local/bin/{name}" for name in + ("kubectl", "task", "skaffold", "oras", "yq", "jq")] +for executable in executables: + with Path(executable).open("rb") as stream: + header = stream.read(20) + assert header[:4] == b"\x7fELF" and header[5] == 1, executable + assert struct.unpack("/dev/null + fi +} +trap cleanup EXIT +container="$(docker run -d --platform "$platform" \ + -e JHSINGLE_NATIVE_PROXY_AUTHTYPE=none "$image")" +deadline=$((SECONDS + 180)) +while ((SECONDS < deadline)); do + if docker exec "$container" curl -fsS --max-time 5 http://127.0.0.1:8888/ >/dev/null; then + docker exec "$container" test -L /workspace/drive + docker exec "$container" test "$(docker exec "$container" readlink /workspace/drive)" = /home/jovyan/drive + echo "Image, extensions, and entrypoint checks passed on $platform" + exit 0 + fi + [[ "$(docker inspect --format '{{.State.Running}}' "$container")" == true ]] || exit 1 + sleep 2 +done +echo "Entrypoint did not become ready within 180 seconds" >&2 +exit 1 From 591a6d8f8e7dd7e1b8bdd1d196f826e2cbf824cf Mon Sep 17 00:00:00 2001 From: Fabrice Brito Date: Sat, 10 Oct 2026 17:03:53 +0200 Subject: [PATCH 3/3] docs: document multi-platform builds and runtime fixes --- CHANGELOG.md | 25 ++++++++++++ README.md | 108 ++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 114 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cda76e..33a025b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,31 @@ Notable project changes are recorded here for users and contributors. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [Unreleased] + +### Added + +- AMD64 and ARM64 binary downloads and native CI build/test jobs, with a combined + platform manifest and architecture-specific SBOM attachments. +- Local Buildx tasks for either platform and a multi-platform OCI archive, plus + checks for tool execution, user identity, extensions, and entrypoint startup. + +### Changed + +- Align the `UID` environment variable with the actual `jovyan` UID of 1001. +- Use Trivy 0.74.0 for CI scans, matching the pinned image scanner; both platform + assets are available, while the previous 0.50.2 binary asset URLs returned 404. + +### Fixed + +- Install Calrissian from commit `a95d87dd213d51b11faea7c6def4ced46d466a4a` + with compatible isolated dependencies and bundled Dask resources. +- Preinstall Hatch 1.16.2 and its runtime dependencies in an isolated environment, + avoiding a first-use download; add offline Hatch and Calrissian smoke checks. +- Ensure the default workspace is writable by jovyan for tool configuration. +- Make Mike Farah's pinned `yq` the default command while retaining the Python + `yq` dependency and `tomlq`/`xq` launchers; smoke-test YAML and TOML processing. + ## [1.4.0] - 2026-10-10 ### Added diff --git a/README.md b/README.md index 4aba2c0..d8eac1b 100644 --- a/README.md +++ b/README.md @@ -13,21 +13,42 @@ This repository contains a Dockerfile to build a container that exposes [Code Se ### Building the Docker Image -Build the local AMD64 image with: +Build a local image for AMD64 or ARM64 with: ```bash -docker build --platform=linux/amd64 -t eoepca/pde-code-server:local . +task build PLATFORM=linux/amd64 IMAGE=eoepca/pde-code-server:amd64 +task build PLATFORM=linux/arm64 IMAGE=eoepca/pde-code-server:arm64 ``` -With [Task](https://taskfile.dev/) installed, run `task build` (or `task`). +Install [Task](https://taskfile.dev/) and Docker with Buildx. `task build` (or +`task`) defaults to `linux/amd64` and `eoepca/pde-code-server:local`. Override the image tag or GDAL version when needed: ```bash task build IMAGE=eoepca/pde-code-server:dev GDAL_VER=3.12.1 ``` -The image includes the required OpenJDK 17 JDK and is built for AMD64 because -the standalone binaries used by the image are AMD64 builds. +Both `linux/amd64` and `linux/arm64` are supported. BuildKit supplies `TARGETARCH` +for binary downloads; Node.js, Python packages, GDAL compilation, and apt packages +use the target architecture. All image tool versions remain pinned at their +existing versions, including Nextcloud 3.11.0-1.1build4 and OpenJDK 17. + +Use a native Docker host for the selected platform when possible. Cross-platform +builds and tests require QEMU/binfmt registration (for example, +`docker run --privileged --rm tonistiigi/binfmt --install arm64` on an AMD64 host). +GDAL source compilation under emulation can take substantially longer. + +To build both platforms into a local OCI archive without publishing: + +```bash +docker buildx create --name pde-multi --driver docker-container --use +docker buildx inspect --bootstrap +task build:multi OUTPUT=/tmp/pde-code-server-multi.oci.tar +``` + +The OCI archive contains both platforms. For local runtime testing, use the +single-platform `task build` commands above, which load an image into Docker. +Choose distinct tags to keep both images available. ### Running Locally @@ -36,13 +57,15 @@ local use, disable proxy authentication and bind the service to localhost: ```bash docker run --rm -it \ - --platform=linux/amd64 \ + --platform="${PLATFORM:-linux/amd64}" \ -e JHSINGLE_NATIVE_PROXY_AUTHTYPE=none \ -p 127.0.0.1:8888:8888 \ -v "$PWD:/workspace" \ eoepca/pde-code-server:local ``` +Set `PLATFORM=linux/arm64` for an ARM64 local image. + Open . If port 8888 is already in use, change the host port (the first port in `-p`) and use that port in the URL. The container image must be rebuilt after changes to the entrypoint. At startup, `nc-sync` @@ -55,7 +78,8 @@ state and should not be committed. ### Extension compatibility checks -Before publishing an image, CI downloads the latest stable release VSIXs from +Before publishing an image, CI builds and tests AMD64 and ARM64 on native +GitHub runners (`ubuntu-24.04` and `ubuntu-24.04-arm`). It downloads the latest stable release VSIXs from `eoap/eoap-validator-vscode`, `eoap/cwl-metadata-editor`, and `eoap/cwl-uml-viewer`. It installs them with `code-server` as the image's default user and checks the installed IDs and versions. It also checks Python dependency @@ -63,6 +87,10 @@ consistency, installs the validator's bundled wheel in a fresh virtual environme loads its CLI, and renders the UML viewer's sample through its installed Python bridge and bundled Java renderer for every available diagram type. +Both architectures use the same downloaded VSIXs. Tool execution, ELF binary architecture, image platform, +user identity (UID 1001 / GID 100), Calrissian isolation, Nextcloud symlink, and +standalone entrypoint startup are checked on each platform. + Failures block image publication. The tested release tags and download URLs are saved as a CI artifact. These checks use the image's default runtime settings; they do not exercise editor UI interactions or extension activation. Latest @@ -70,17 +98,43 @@ upstream releases are intentionally used, so a new incompatible release can fail CI without changes here. Extensions are installed only in the disposable test container. -To run the same check against a locally built image: +To run the full checks against each locally built image: + +```bash +task test PLATFORM=linux/amd64 IMAGE=eoepca/pde-code-server:amd64 +task test PLATFORM=linux/arm64 IMAGE=eoepca/pde-code-server:arm64 +``` + +`task test` downloads the latest VSIXs, then checks tools, extensions, and startup. +It also verifies that the default `yq` is Mike Farah's pinned v4.45.1, reads a +YAML fixture with `yq eval`, reads a TOML fixture with `tomlq`, and runs +`python -m pip check`. The Python `yq` package remains installed for `tomlq`; +only its conflicting `yq` launcher is replaced, leaving Conda's PATH intact. +The focused runtime check also runs Hatch version and local project metadata +commands without networking, verifies Calrissian's Git commit provenance, Dask +import, CLI flags, and bundled schema/init/dispose scripts, and checks both +isolated environments' dependencies. It does not execute Dask on Kubernetes. +To run these checks against a loaded image on either platform: + +```bash +docker run --rm --network none --platform linux/amd64 --entrypoint bash \ + -v "$PWD/ci:/opt/image-ci:ro" eoepca/pde-code-server:amd64 /opt/image-ci/test-runtime.sh +docker run --rm --network none --platform linux/arm64 --entrypoint bash \ + -v "$PWD/ci:/opt/image-ci:ro" eoepca/pde-code-server:arm64 /opt/image-ci/test-runtime.sh +``` + +To reuse the same downloaded release assets for both platforms: ```bash python3 ci/test-extensions.py download /tmp/pde-extension-vsix -docker run --rm --platform linux/amd64 --entrypoint python3 \ - -v "$PWD/ci:/opt/extension-ci:ro" \ - -v /tmp/pde-extension-vsix:/opt/extension-vsix:ro \ - eoepca/pde-code-server:local \ - /opt/extension-ci/test-extensions.py test /opt/extension-vsix +bash ci/test-image.sh eoepca/pde-code-server:amd64 linux/amd64 /tmp/pde-extension-vsix +bash ci/test-image.sh eoepca/pde-code-server:arm64 linux/arm64 /tmp/pde-extension-vsix ``` +The UML viewer's bundled dependencies are tested as shipped, including its native +Python modules; CI does not patch the VSIX or skip rendering on ARM64. A future +upstream incompatibility will fail the corresponding platform check. + This image is based on the Jupyter Python 3.12 base notebook image (currently Ubuntu 24.04) and provides a curated set of development, Kubernetes, and Earth-Observation workflow tools. @@ -166,10 +220,13 @@ Installed via pip (Python 3.12): Reference implementation of the Common Workflow Language. -* calrissian: 0.18.1 +* calrissian: commit `a95d87dd213d51b11faea7c6def4ced46d466a4a` - CWL runner for Kubernetes, installed in `/opt/calrissian-venv` with - `cwl-utils==0.40` to isolate its dependencies from the image's CWL tooling. + CWL runner for Kubernetes with Dask support, installed from the pinned upstream + Git commit in `/opt/calrissian-venv`. Its dependencies are resolved together, + including `cwltool>=3.1.20260108082145`, separately from the image's CWL tooling. + `CALRISSIAN_COMMIT` is the build argument; changing it also requires updating + the expected commit in `ci/test-runtime.sh`. * jhsingle-native-proxy (>= 0.0.9) @@ -191,12 +248,19 @@ Installed via pip (Python 3.12): - hatch: 1.16.2 - Python project manager and build tool, installed as a standalone binary. + Python project manager and build tool, installed with all runtime dependencies + in `/opt/hatch-venv`, using the base image's native Python on AMD64 and ARM64. + `/usr/local/bin/hatch` links to its console script. Jovyan can use it offline + without downloading Hatch itself. `HATCH_CONFIG`, `HATCH_DATA_DIR`, and + `HATCH_CACHE_DIR` point to writable locations under `/home/jovyan`, independent + of the `/workspace` project mount. Project environment creation, project + dependencies, plugins, or additional Python versions may still need network + access; the offline smoke check only reads local project metadata. ### User & Runtime Environment - User: jovyan -- UID / GID: 1001 +- UID / GID: 1001 / 100 - Home / Workdir: /workspace - Passwordless sudo enabled for the user. @@ -218,7 +282,13 @@ ghcr.io//pde-code-server The registry owner corresponds to the GitHub repository owner (user or organization). -Images are built using Kaniko and pushed using OCI-compliant tooling. +Images are built with Buildx. Each platform is tested, scanned with Trivy 0.74.0, +and given its own SPDX JSON SBOM. After both platform jobs succeed, CI publishes +the tested images and attaches each SBOM with ORAS to that architecture image's +exact registry digest. It then publishes a combined multi-platform manifest under +the branch or release tag. The SBOMs describe individual platform images, not the +combined index. Vulnerability findings remain informational, as in the previous +workflow. Pull requests run builds and checks without publishing. ### Tagging Strategy