From 66787bcbae30dd0862932a244ab73ce9290e15ef Mon Sep 17 00:00:00 2001 From: Simone Vaccari Date: Fri, 9 Oct 2026 16:34:15 +0200 Subject: [PATCH 01/14] updating resources for ESAAPEX-136 --- Dockerfile | 46 ++++++++++++++++++++++++++++------------ README.md | 34 +++++++++++++++++++++++------ entrypoint.sh | 11 +++++++++- release.yaml | 2 +- requirements-cwl-uml.txt | 12 +++++++++++ 5 files changed, 83 insertions(+), 22 deletions(-) create mode 100644 requirements-cwl-uml.txt diff --git a/Dockerfile b/Dockerfile index 72b3b4c..cd4320a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,6 @@ RUN apt-get update && apt-get install -y \ ca-certificates \ curl \ git \ - nodejs \ - npm \ nano \ net-tools \ sudo \ @@ -26,20 +24,33 @@ RUN apt-get update && apt-get install -y \ tree \ podman \ skopeo \ + openjdk-17-jdk-headless \ nextcloud-desktop-cmd=3.11.0-1.1build4 \ && rm -rf /var/lib/apt/lists/* -# Optional Java runtime (enable with --build-arg INSTALL_JRE=true) -ARG INSTALL_JRE=true -RUN if [ "${INSTALL_JRE}" = "true" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends openjdk-17-jre-headless && \ - rm -rf /var/lib/apt/lists/*; \ - fi - RUN usermod -u 1001 ${USER} && \ echo "${USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/${USER} +# ------------------------------------------------------------------- +# NVM and Node.js +# ------------------------------------------------------------------- +ARG NVM_VERSION=v0.40.3 +ARG NODE_VERSION=22.15.0 + +ENV NVM_DIR=/opt/nvm + +RUN mkdir -p "${NVM_DIR}" && \ + curl -fsSL \ + "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" \ + | bash && \ + bash -c '. "${NVM_DIR}/nvm.sh" && nvm install "${NODE_VERSION}" && nvm alias default "${NODE_VERSION}"' && \ + chown -R jovyan:users "${NVM_DIR}" + +ENV PATH="${NVM_DIR}/versions/node/v${NODE_VERSION}/bin:${PATH}" + +RUN printf '\nexport NVM_DIR="/opt/nvm"\n[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"\n' \ + >> /etc/bash.bashrc + # ------------------------------------------------------------------- # code-server # ------------------------------------------------------------------- @@ -78,7 +89,11 @@ RUN curl -fsSL \ # Python tooling # ------------------------------------------------------------------- ARG CALRISSIAN_VERSION=0.18.1 +ARG CALRISSIAN_CWL_UTILS_VERSION=0.40 +COPY requirements-cwl-uml.txt /tmp/requirements-cwl-uml.txt + RUN pip install --no-cache-dir \ + -r /tmp/requirements-cwl-uml.txt \ awscli \ awscli-plugin-endpoint \ "jhsingle-native-proxy>=0.0.9" \ @@ -86,10 +101,15 @@ RUN pip install --no-cache-dir \ tomlq \ uv \ cwltool \ - cwltest \ - "calrissian==${CALRISSIAN_VERSION}" && \ + cwltest && \ python -m bash_kernel.install +RUN python -m venv /opt/calrissian-venv && \ + /opt/calrissian-venv/bin/pip install --no-cache-dir \ + "cwl-utils==${CALRISSIAN_CWL_UTILS_VERSION}" \ + "calrissian==${CALRISSIAN_VERSION}" && \ + ln -s /opt/calrissian-venv/bin/calrissian /usr/local/bin/calrissian + # ------------------------------------------------------------------- # yq / jq # ------------------------------------------------------------------- @@ -126,7 +146,7 @@ RUN curl -fsSL \ # ------------------------------------------------------------------- ARG GDAL_VER=3.12.1 RUN apt-get update && apt-get install -y \ - cmake ninja-build libproj-dev proj-data proj-bin && \ + cmake g++ ninja-build xz-utils libproj-dev proj-data proj-bin && \ rm -rf /var/lib/apt/lists/* && \ set -e && \ cd /tmp && \ diff --git a/README.md b/README.md index 33ea518..bfef563 100644 --- a/README.md +++ b/README.md @@ -13,19 +13,37 @@ This repository contains a Dockerfile to build a container that exposes [Code Se ### Building the Docker Image -To build the Docker image, run the following command: +Build the local AMD64 image with: ```bash -docker build -t eoepca/pde-code-server . +docker build --platform=linux/amd64 -t eoepca/pde-code-server:local . ``` -To include OpenJDK 17 JRE (headless), enable the optional build argument -(disabled by default): +The image includes the required OpenJDK 17 JDK and is built for AMD64 because +the standalone binaries used by the image are AMD64 builds. + +### Running Locally + +The proxy uses OAuth by default for ApplicationHub deployments. For standalone +local use, disable proxy authentication and bind the service to localhost: ```bash -docker build --build-arg INSTALL_JRE=true -t eoepca/pde-code-server . +docker run --rm -it \ + --platform=linux/amd64 \ + -e JHSINGLE_NATIVE_PROXY_AUTHTYPE=none \ + -p 127.0.0.1:8888:8888 \ + -v "$PWD:/workspace" \ + eoepca/pde-code-server:local ``` +Open . If port 8888 is already in use, change the host +port (the first port in `-p`) and use that port in the URL. The container +image must be rebuilt after changes to the entrypoint. At startup, `nc-sync` +creates `/workspace/drive` as a symlink to `/home/jovyan/drive` inside the +container. With the project directory mounted at `/workspace`, this symlink +also appears in the project directory on the host; it is generated runtime +state and should not be committed. + ## Installed Tooling This image is based on Debian bookworm and Python 3.12, and provides a curated set of development, Kubernetes, and Earth-Observation workflow tools. @@ -36,8 +54,10 @@ All non-distro binaries are pinned to explicit versions to ensure reproducibilit - OS: Debian GNU/Linux 12 (bookworm) - Python: 3.12.11 -- Node.js: 18.x (Debian package) -- npm: bundled with Node.js +- NVM: v0.40.3, installed at `/opt/nvm` +- Node.js: v22.15.0, installed via NVM and available on `PATH` +- npm: bundled with Node.js 22 +- Java: OpenJDK 17 JDK (headless), required and installed from Debian packages Installed system utilities: diff --git a/entrypoint.sh b/entrypoint.sh index 6d87921..74c2059 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -34,8 +34,17 @@ destport=$((port + 1)) echo "Using internal port $destport" +auth_type="${JHSINGLE_NATIVE_PROXY_AUTHTYPE:-oauth}" +case "$auth_type" in + oauth|none) ;; + *) + echo "Unsupported JHSINGLE_NATIVE_PROXY_AUTHTYPE: $auth_type (expected oauth or none)" >&2 + exit 1 + ;; +esac + if [ -z "$CODE_SERVER_WS" ]; then CODE_SERVER_WS="/workspace" fi -jhsingle-native-proxy --port $port --destport $destport code-server {--}auth none {--}bind-addr 0.0.0.0:$destport {--}user-data-dir /workspace $CODE_SERVER_WS +jhsingle-native-proxy --authtype "$auth_type" --port "$port" --destport "$destport" code-server {--}auth none {--}bind-addr "0.0.0.0:$destport" {--}user-data-dir /workspace "$CODE_SERVER_WS" diff --git a/release.yaml b/release.yaml index 25e4f74..d48539b 100644 --- a/release.yaml +++ b/release.yaml @@ -1,4 +1,4 @@ image_name: pde-code-server image_prefix: eoepca -image_version: 1.3.0 +image_version: 1.3.1 image_registry: ghcr.io \ No newline at end of file diff --git a/requirements-cwl-uml.txt b/requirements-cwl-uml.txt new file mode 100644 index 0000000..56fa9b4 --- /dev/null +++ b/requirements-cwl-uml.txt @@ -0,0 +1,12 @@ +transpiler-mate-runtime==1.3.0 +cwl-loader==0.27.0 +transpiler-mate-api==1.0.1 +cwl2click==0.10.1 +cwl2markdown==0.2.1 +cwl2puml==0.49.1 +cwl2inputs==0.1.1 +cwl2ogc==0.22.2 +cwl2oci==0.1.1 +cwl2sbom==0.1.1 +cwl-baseline-plugin==0.1.1 +cwl2codemeta==0.1.1 \ No newline at end of file From 6e98e44f7609eacde47edfd48017c64b97d07b62 Mon Sep 17 00:00:00 2001 From: Fabrice Brito Date: Sat, 10 Oct 2026 15:05:06 +0200 Subject: [PATCH 02/14] ci: verify released VSIX installation and runtime requirements --- .github/workflows/build-image.yml | 24 ++++++ README.md | 28 +++++++ ci/test-extensions.py | 133 ++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 ci/test-extensions.py diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 6d18df5..37a52db 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -103,6 +103,30 @@ jobs: --no-push --tar-path image.tar + - name: Download latest released extension VSIXs + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 ci/test-extensions.py download extension-vsix + + - name: Test extension installation and runtime requirements + run: | + set -euo pipefail + skopeo copy docker-archive:image.tar docker-daemon:pde-extension-test:ci + docker run --rm --platform linux/amd64 \ + --entrypoint python3 \ + -v "$PWD/ci:/opt/extension-ci:ro" \ + -v "$PWD/extension-vsix:/opt/extension-vsix:ro" \ + pde-extension-test:ci \ + /opt/extension-ci/test-extensions.py test /opt/extension-vsix + + - name: Record tested extension releases + if: always() + uses: actions/upload-artifact@v4 + with: + name: tested-extension-releases + path: extension-vsix/*.release.json + if-no-files-found: ignore + # -------------------------------------------------------- # Trivy scan (relaxed for now) # -------------------------------------------------------- diff --git a/README.md b/README.md index bfef563..0a6dc56 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,34 @@ state and should not be committed. ## Installed Tooling +### Extension compatibility checks + +Before publishing an image, CI downloads the latest stable release VSIXs from +`eoap/eoap-validator-vscode`, `eoap/cwl-metadata-editor`, and +`eoap/cwl-uml-viewer`. It installs them with `code-server` as the image's default +user and checks the installed IDs and versions. It also checks Python dependency +consistency, installs the validator's bundled wheel in a fresh virtual environment, +loads its CLI, and renders the UML viewer's sample through its installed Python +bridge and bundled Java renderer for every available diagram type. + +Failures block image publication. The tested release tags and download URLs are +saved as a CI artifact. These checks use the image's default runtime settings; +they do not exercise editor UI interactions or extension activation. Latest +upstream releases are intentionally used, so a new incompatible release can fail +CI without changes here. Extensions are installed only in the disposable test +container. + +To run the same check against a locally built image: + +```bash +python3 ci/test-extensions.py download /tmp/pde-extension-vsix +docker run --rm --platform linux/amd64 --entrypoint python3 \ + -v "$PWD/ci:/opt/extension-ci:ro" \ + -v /tmp/pde-extension-vsix:/opt/extension-vsix:ro \ + eoepca/pde-code-server:local \ + /opt/extension-ci/test-extensions.py test /opt/extension-vsix +``` + This image is based on Debian bookworm and Python 3.12, and provides a curated set of development, Kubernetes, and Earth-Observation workflow tools. All non-distro binaries are pinned to explicit versions to ensure reproducibility. diff --git a/ci/test-extensions.py b/ci/test-extensions.py new file mode 100644 index 0000000..e1a49ea --- /dev/null +++ b/ci/test-extensions.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Download released VSIXs on the runner, or smoke-test them in the built image.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import tempfile +import urllib.request +import zipfile + +REPOSITORIES = ("eoap-validator-vscode", "cwl-metadata-editor", "cwl-uml-viewer") + + +def run(*args, **kwargs): + print("+", " ".join(map(str, args)), flush=True) + try: + return subprocess.run(list(map(str, args)), check=True, timeout=600, **kwargs) + except subprocess.CalledProcessError as error: + if error.stdout: + print(error.stdout, flush=True) + if error.stderr: + print(error.stderr, flush=True) + raise + + +def download(destination): + destination.mkdir(parents=True, exist_ok=True) + for repo in REPOSITORIES: + headers = {"Accept": "application/vnd.github+json"} + if os.environ.get("GITHUB_TOKEN"): + headers["Authorization"] = "Bearer " + os.environ["GITHUB_TOKEN"] + request = urllib.request.Request( + f"https://api.github.com/repos/eoap/{repo}/releases/latest", headers=headers + ) + with urllib.request.urlopen(request, timeout=60) as response: + release = json.load(response) + assets = [a for a in release["assets"] if a["name"].endswith(".vsix")] + if len(assets) != 1: + raise RuntimeError(f"{repo}: expected one VSIX, found {len(assets)}") + print(f"{repo}: {release['tag_name']} ({assets[0]['name']})", flush=True) + # Do not forward the API token to release asset download hosts. + with urllib.request.urlopen(assets[0]["browser_download_url"], timeout=120) as response: + (destination / f"{repo}.vsix").write_bytes(response.read()) + (destination / f"{repo}.release.json").write_text(json.dumps({ + "tag": release["tag_name"], "url": assets[0]["browser_download_url"] + }, indent=2) + "\n") + + +def test(source): + with tempfile.TemporaryDirectory(prefix="extension-smoke-") as temporary: + root = Path(temporary) + extensions = root / "extensions" + user_data = root / "user-data" + cli = ("code-server", "--extensions-dir", extensions, "--user-data-dir", user_data) + installed = {} + for repo in REPOSITORIES: + vsix = source / f"{repo}.vsix" + with zipfile.ZipFile(vsix) as archive: + manifest = json.loads(archive.read("extension/package.json")) + identifier = f"{manifest['publisher']}.{manifest['name']}" + print(f"Installing {identifier}@{manifest['version']}; VS Code {manifest['engines']['vscode']}", flush=True) + # No --force: incompatible VS Code engines must fail installation. + run(*cli, "--install-extension", vsix) + installed[repo] = (identifier, manifest["version"]) + listed = run(*cli, "--list-extensions", "--show-versions", capture_output=True, text=True).stdout + print(listed, flush=True) + for identifier, version in installed.values(): + if f"{identifier}@{version}".lower() not in listed.lower().splitlines(): + raise RuntimeError(f"Extension missing after installation: {identifier}@{version}") + + def extension_path(repo): + identifier, version = installed[repo] + matches = list(extensions.glob(f"{identifier}-{version}*")) + if len(matches) != 1: + raise RuntimeError(f"Cannot locate installed {identifier}: {matches}") + return matches[0] + + run("python3", "-m", "pip", "check") + # Exercise the validator's default first-use venv and bundled wheel. + validator = extension_path("eoap-validator-vscode") + wheels = list((validator / "vendor").glob("eoap_validator-*.whl")) + if len(wheels) != 1: + raise RuntimeError(f"Expected one bundled validator wheel: {wheels}") + venv = root / "validator-venv" + run("python3", "-m", "venv", venv) + python = venv / "bin/python" + run(python, "-m", "pip", "install", "--disable-pip-version-check", wheels[0]) + run(python, "-m", "pip", "check") + run(python, "-m", "eoap_validator", "--help") + + # Use the installed bridge exactly as the extension does, including its + # vendored sys.path and isolated Python mode. Global imports are insufficient. + uml = extension_path("cwl-uml-viewer") + fixture = uml / "examples/echo.cwl" + request = {"path": str(fixture), "text": fixture.read_text()} + + def bridge(extra): + result = run("python3", "-I", uml / "python/bridge.py", + input=json.dumps(request | extra), capture_output=True, text=True) + payload = json.loads(result.stdout) + if "error" in payload: + raise RuntimeError(payload["error"]) + return payload + + info = bridge({"action": "inspect"}) + if not info["workflows"] or not info["diagrams"]: + raise RuntimeError(f"No UML workflows or diagrams: {info}") + jars = list((uml / "vendor").glob("plantuml-asl-*.jar")) + if len(jars) != 1: + raise RuntimeError(f"Expected one bundled PlantUML renderer: {jars}") + for diagram in info["diagrams"]: + puml = bridge({"action": "render", "diagram": diagram, + "workflow": info["workflows"][0]})["puml"] + svg = run("java", "-Djava.awt.headless=true", + "-DPLANTUML_SECURITY_PROFILE=SANDBOX", "-jar", jars[0], + "-pipe", "-charset", "UTF-8", "-tsvg", "-failfast2", + input=puml, capture_output=True, text=True).stdout + if " Date: Sat, 10 Oct 2026 15:05:12 +0200 Subject: [PATCH 03/14] build: add Taskfile for local AMD64 image builds --- README.md | 7 +++++++ Taskfile.yaml | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 Taskfile.yaml diff --git a/README.md b/README.md index 0a6dc56..9bfb7b9 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,13 @@ Build the local AMD64 image with: docker build --platform=linux/amd64 -t eoepca/pde-code-server:local . ``` +With [Task](https://taskfile.dev/) installed, run `task build` (or `task`). +Override the image tag or GDAL version when needed: + +```bash +task build IMAGE=eoepca/pde-code-server:dev GDAL_VER=3.12.1 +``` + The image includes the required OpenJDK 17 JDK and is built for AMD64 because the standalone binaries used by the image are AMD64 builds. diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 0000000..2c48d5e --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,19 @@ +version: '3' + +vars: + IMAGE: eoepca/pde-code-server:local + GDAL_VER: '3.12.1' + +tasks: + default: + desc: Build the local container image + cmds: + - task: build + + build: + desc: Build the AMD64 container image (override IMAGE or GDAL_VER as needed) + preconditions: + - sh: command -v docker + msg: Docker must be installed to build the image. + cmds: + - docker build --platform=linux/amd64 --build-arg GDAL_VER={{.GDAL_VER | shellQuote}} -t {{.IMAGE | shellQuote}} . From 7de234879a9bf196ec77cb9d895c73de61cc9f41 Mon Sep 17 00:00:00 2001 From: Fabrice Brito Date: Sat, 10 Oct 2026 15:14:04 +0200 Subject: [PATCH 04/14] release: prepare 1.4.0 --- CHANGELOG.md | 31 ++++++++++++++++++++++++++++++- README.md | 11 +++++++---- release.yaml | 4 ++-- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a18fb8b..1cda76e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,33 @@ Notable project changes are recorded here for users and contributors. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [1.4.0] - 2026-10-10 + +### Added + +- Pinned CWL and transpiler tooling for EOAP extension runtime requirements. +- CI installation checks for the latest released EOAP Validator, CWL Metadata + Editor, and CWL UML Viewer VSIXs, with Python dependency checks, validator + environment setup, and rendering checks for all available UML diagram types. + Failures block image publication; tested release metadata is saved as an artifact. +- A Taskfile for local AMD64 image builds with image tag and GDAL version overrides. +- Standalone local operation through `JHSINGLE_NATIVE_PROXY_AUTHTYPE=none`, + with OAuth retained as the default. + +### Changed + +- Install Node.js 22.15.0 through NVM 0.40.3 instead of distro Node.js packages. +- Require the OpenJDK 17 headless JDK, replacing the optional Java runtime and + removing the `INSTALL_JRE` build argument. +- Install Calrissian 0.18.1 and its pinned `cwl-utils` 0.40 dependency in an + isolated virtual environment to avoid conflicts with the image's CWL tooling. +- Document local image builds, standalone usage, and extension compatibility checks. + +### Fixed + +- Include the C++ compiler and XZ utilities required for the GDAL source build. +- Validate proxy authentication modes and quote entrypoint arguments. + ## [1.3.0] - 2026-10-06 ### Added @@ -65,6 +92,8 @@ at the earliest Git tag. tooling versions, and install code-server 4.108.1 from its release tarball. - Refactor the CI container build and publishing workflow. -[Unreleased]: https://github.com/EOEPCA/pde-code-server/compare/v1.2.0...HEAD +[Unreleased]: https://github.com/EOEPCA/pde-code-server/compare/v1.4.0...HEAD +[1.4.0]: https://github.com/EOEPCA/pde-code-server/compare/v1.3.0...v1.4.0 +[1.3.0]: https://github.com/EOEPCA/pde-code-server/compare/v1.2.0...v1.3.0 [1.2.0]: https://github.com/EOEPCA/pde-code-server/compare/v1.1.0...v1.2.0 [1.1.0]: https://github.com/EOEPCA/pde-code-server/releases/tag/v1.1.0 diff --git a/README.md b/README.md index 9bfb7b9..4aba2c0 100644 --- a/README.md +++ b/README.md @@ -81,18 +81,20 @@ docker run --rm --platform linux/amd64 --entrypoint python3 \ /opt/extension-ci/test-extensions.py test /opt/extension-vsix ``` -This image is based on Debian bookworm and Python 3.12, and provides a curated set of development, Kubernetes, and Earth-Observation workflow tools. +This image is based on the Jupyter Python 3.12 base notebook image (currently +Ubuntu 24.04) and provides a curated set of development, Kubernetes, and +Earth-Observation workflow tools. All non-distro binaries are pinned to explicit versions to ensure reproducibility. ### Base System -- OS: Debian GNU/Linux 12 (bookworm) +- OS: Ubuntu 24.04 (from the current Jupyter base image) - Python: 3.12.11 - NVM: v0.40.3, installed at `/opt/nvm` - Node.js: v22.15.0, installed via NVM and available on `PATH` - npm: bundled with Node.js 22 -- Java: OpenJDK 17 JDK (headless), required and installed from Debian packages +- Java: OpenJDK 17 JDK (headless), required and installed from distro packages Installed system utilities: @@ -166,7 +168,8 @@ Installed via pip (Python 3.12): * calrissian: 0.18.1 - CWL runner for Kubernetes. + CWL runner for Kubernetes, installed in `/opt/calrissian-venv` with + `cwl-utils==0.40` to isolate its dependencies from the image's CWL tooling. * jhsingle-native-proxy (>= 0.0.9) diff --git a/release.yaml b/release.yaml index d48539b..66849b1 100644 --- a/release.yaml +++ b/release.yaml @@ -1,4 +1,4 @@ image_name: pde-code-server image_prefix: eoepca -image_version: 1.3.1 -image_registry: ghcr.io \ No newline at end of file +image_version: 1.4.0 +image_registry: ghcr.io From c934f78c99beab867c3b2ded3d6c4587aeebbf8f Mon Sep 17 00:00:00 2001 From: Fabrice Brito Date: Sat, 10 Oct 2026 17:03:53 +0200 Subject: [PATCH 05/14] build: support ARM64 and fix isolated tool runtimes --- Dockerfile | 58 ++++++++++++++++++++++++++++++++-------------- ci/test-runtime.sh | 52 +++++++++++++++++++++++++++++++++++++++++ ci/test-yq.sh | 13 +++++++++++ 3 files changed, 105 insertions(+), 18 deletions(-) create mode 100644 ci/test-runtime.sh create mode 100644 ci/test-yq.sh diff --git a/Dockerfile b/Dockerfile index cd4320a..5f23b3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,12 @@ FROM quay.io/jupyter/base-notebook:python-3.12 +# BuildKit supplies amd64 or arm64 for each target platform. +ARG TARGETARCH +RUN case "${TARGETARCH}" in amd64|arm64) ;; *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; esac + ENV DEBIAN_FRONTEND=noninteractive \ USER=jovyan \ - UID=1000 \ + UID=1001 \ GID=100 \ HOME=/workspace @@ -57,7 +61,7 @@ RUN printf '\nexport NVM_DIR="/opt/nvm"\n[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR ARG CODE_RELEASE=4.138.0 RUN mkdir -p /opt/code-server && \ curl -fsSL \ - "https://github.com/coder/code-server/releases/download/v${CODE_RELEASE}/code-server-${CODE_RELEASE}-linux-amd64.tar.gz" \ + "https://github.com/coder/code-server/releases/download/v${CODE_RELEASE}/code-server-${CODE_RELEASE}-linux-${TARGETARCH}.tar.gz" \ | tar -xz --strip-components=1 -C /opt/code-server ENV PATH="/opt/code-server/bin:${PATH}" @@ -67,29 +71,28 @@ ENV PATH="/opt/code-server/bin:${PATH}" # ------------------------------------------------------------------- ARG KUBECTL_VERSION=v1.29.3 RUN curl -fsSL \ - https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl \ + https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl \ -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl ARG TASK_VERSION=v3.41.0 RUN curl -fsSL \ - https://github.com/go-task/task/releases/download/${TASK_VERSION}/task_linux_amd64.tar.gz \ + https://github.com/go-task/task/releases/download/${TASK_VERSION}/task_linux_${TARGETARCH}.tar.gz \ | tar -xz -C /usr/local/bin task && chmod +x /usr/local/bin/task ARG SKAFFOLD_VERSION=2.17.1 RUN curl -fsSL \ - https://storage.googleapis.com/skaffold/releases/v${SKAFFOLD_VERSION}/skaffold-linux-amd64 \ + https://storage.googleapis.com/skaffold/releases/v${SKAFFOLD_VERSION}/skaffold-linux-${TARGETARCH} \ -o /usr/local/bin/skaffold && chmod +x /usr/local/bin/skaffold ARG ORAS_VERSION=1.3.0 RUN curl -fsSL \ - https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz \ + https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_${TARGETARCH}.tar.gz \ | tar -xz -C /usr/local/bin oras && chmod +x /usr/local/bin/oras # ------------------------------------------------------------------- # Python tooling # ------------------------------------------------------------------- -ARG CALRISSIAN_VERSION=0.18.1 -ARG CALRISSIAN_CWL_UTILS_VERSION=0.40 +ARG CALRISSIAN_COMMIT=a95d87dd213d51b11faea7c6def4ced46d466a4a COPY requirements-cwl-uml.txt /tmp/requirements-cwl-uml.txt RUN pip install --no-cache-dir \ @@ -104,10 +107,12 @@ RUN pip install --no-cache-dir \ cwltest && \ python -m bash_kernel.install +# Resolve the commit's cwltool/dependency requirements together, independently +# of the global CWL tools, rather than constraining cwl-utils to the old release. RUN python -m venv /opt/calrissian-venv && \ /opt/calrissian-venv/bin/pip install --no-cache-dir \ - "cwl-utils==${CALRISSIAN_CWL_UTILS_VERSION}" \ - "calrissian==${CALRISSIAN_VERSION}" && \ + "calrissian @ git+https://github.com/duke-gcb/calrissian.git@${CALRISSIAN_COMMIT}" && \ + /opt/calrissian-venv/bin/python -m pip check && \ ln -s /opt/calrissian-venv/bin/calrissian /usr/local/bin/calrissian # ------------------------------------------------------------------- @@ -115,28 +120,37 @@ RUN python -m venv /opt/calrissian-venv && \ # ------------------------------------------------------------------- ARG YQ_VERSION=v4.45.1 RUN curl -fsSL \ - https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 \ + https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${TARGETARCH} \ -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq ARG JQ_VERSION=jq-1.8.1 RUN curl -fsSL \ - https://github.com/jqlang/jq/releases/download/${JQ_VERSION}/jq-linux-amd64 \ + https://github.com/jqlang/jq/releases/download/${JQ_VERSION}/jq-linux-${TARGETARCH} \ -o /usr/local/bin/jq && chmod +x /usr/local/bin/jq # ------------------------------------------------------------------- # hatch # ------------------------------------------------------------------- ARG HATCH_VERSION=1.16.2 -RUN curl -fsSL \ - https://github.com/pypa/hatch/releases/download/hatch-v${HATCH_VERSION}/hatch-x86_64-unknown-linux-gnu.tar.gz \ - | tar -xz -C /usr/local/bin hatch && chmod +x /usr/local/bin/hatch +ENV HATCH_CONFIG=/home/jovyan/.config/hatch/config.toml \ + HATCH_DATA_DIR=/home/jovyan/.local/share/hatch \ + HATCH_CACHE_DIR=/home/jovyan/.cache/hatch +# The release launcher fetches its runtime on first use. Install Hatch and all +# its dependencies now, using the base image's native Python on either platform. +RUN python -m venv /opt/hatch-venv && \ + /opt/hatch-venv/bin/pip install --no-cache-dir "hatch==${HATCH_VERSION}" && \ + /opt/hatch-venv/bin/python -m pip check && \ + ln -s /opt/hatch-venv/bin/hatch /usr/local/bin/hatch && \ + install -d -o ${USER} -g users /home/jovyan/.config/hatch "${HATCH_DATA_DIR}" "${HATCH_CACHE_DIR}" && \ + install -o ${USER} -g users -m 644 /dev/null "${HATCH_CONFIG}" # ------------------------------------------------------------------- # trivy # ------------------------------------------------------------------- ARG TRIVY_VERSION=0.74.0 -RUN curl -fsSL \ - https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb \ +RUN case "${TARGETARCH}" in amd64) TRIVY_ARCH=64bit ;; arm64) TRIVY_ARCH=ARM64 ;; esac && \ + curl -fsSL \ + https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-${TRIVY_ARCH}.deb \ -o /tmp/trivy.deb && \ dpkg -i /tmp/trivy.deb && \ rm /tmp/trivy.deb @@ -165,9 +179,17 @@ RUN apt-get update && apt-get install -y \ cmake --build . -- -j"$(nproc)" && \ cmake --install . && \ ldconfig && \ - rm -rf /tmp/gdal-${GDAL_VER}* && \ + cd / && rm -rf /tmp/gdal-${GDAL_VER}* && \ gdal-config --version +# ------------------------------------------------------------------- +# tomlq requires the Python yq package, whose console launcher shadows the +# pinned Mike Farah binary on Conda's PATH. Keep its modules and tomlq/xq +# launchers; replace only the conflicting command, without changing PATH. +# Prepare the default workspace for non-root tools after runtime installation. +RUN ln -sf /usr/local/bin/yq /opt/conda/bin/yq && \ + install -d -o ${USER} -g users /workspace + # ------------------------------------------------------------------- # Entrypoint # ------------------------------------------------------------------- diff --git a/ci/test-runtime.sh b/ci/test-runtime.sh new file mode 100644 index 0000000..49200aa --- /dev/null +++ b/ci/test-runtime.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Run as jovyan in a disposable container with --network none. +set -euo pipefail +script_dir="$(cd "$(dirname "$0")" && pwd)" +test "$(id -u)" = 1001 +test -w "$HATCH_CONFIG" +test -w "$HATCH_DATA_DIR" +test -w "$HATCH_CACHE_DIR" +test "$(readlink -f "$(command -v hatch)")" = /opt/hatch-venv/bin/hatch +test "$(readlink -f "$(command -v calrissian)")" = /opt/calrissian-venv/bin/calrissian +fixture_dir="$(mktemp -d)" +trap 'rm -rf "$fixture_dir"' EXIT + +[[ "$(hatch --version)" == 'Hatch, version 1.16.2' ]] +cat > "$fixture_dir/pyproject.toml" <<'TOML' +[project] +name = "offline-smoke" +version = "1.2.3" +TOML +cd "$fixture_dir" +[[ "$(hatch version)" == '1.2.3' ]] +/opt/hatch-venv/bin/python -m pip check + +/opt/calrissian-venv/bin/python - <<'PYTHON' +import json +from importlib import metadata +from pathlib import Path + +import calrissian.dask +from packaging.version import Version + +expected = "a95d87dd213d51b11faea7c6def4ced46d466a4a" +distribution = metadata.distribution("calrissian") +provenance = json.loads(distribution.read_text("direct_url.json")) +assert provenance["url"].removesuffix(".git").lower() == "https://github.com/duke-gcb/calrissian" +assert provenance["vcs_info"]["vcs"] == "git" +assert provenance["vcs_info"]["commit_id"] == expected, provenance +assert Version(metadata.version("cwltool")) >= Version("3.1.20260108082145") +package = Path(calrissian.dask.__file__).parent +for resource in ("dask/custom_schema/schema.yaml", "dask/init-dask.py", "dask/dispose-dask.py"): + path = package / resource + assert path.is_file() and path.stat().st_size > 0, path + print(f"Verified installed resource: {path}") +print(f"Verified Calrissian commit {expected}, Dask import, and cwltool {metadata.version('cwltool')}") +PYTHON +/opt/calrissian-venv/bin/python -m pip check +calrissian --help > "$fixture_dir/calrissian-help.txt" +for flag in --dask-gateway-url --dask-script-configmap; do + grep -Fq -- "$flag" "$fixture_dir/calrissian-help.txt" +done +bash "$script_dir/test-yq.sh" +echo 'Offline Hatch, Calrissian package, and YAML/TOML checks passed (no Kubernetes Dask execution tested).' diff --git a/ci/test-yq.sh b/ci/test-yq.sh new file mode 100644 index 0000000..5d0a860 --- /dev/null +++ b/ci/test-yq.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Run inside the image as its default user, exercising commands through PATH. +set -euo pipefail +fixture_dir="$(mktemp -d)" +trap 'rm -rf "$fixture_dir"' EXIT +version="$(yq --version)" +[[ "$version" == 'yq (https://github.com/mikefarah/yq/) version v4.45.1' ]] +printf 'image_version: "1.4.0"\n' > "$fixture_dir/image.yaml" +[[ "$(yq eval '.image_version' "$fixture_dir/image.yaml")" == '1.4.0' ]] +printf '[image]\nversion = "1.4.0"\n' > "$fixture_dir/image.toml" +[[ "$(tomlq -r '.image.version' "$fixture_dir/image.toml")" == '1.4.0' ]] +python -m pip check +printf 'Default yq, TOML processing, and Python dependencies passed: %s\n' "$version" From c4b346c831b3622079152b0fb215591f0ce69f6b Mon Sep 17 00:00:00 2001 From: Fabrice Brito Date: Sat, 10 Oct 2026 17:03:53 +0200 Subject: [PATCH 06/14] ci: test and publish multi-platform container images --- .github/workflows/build-image.yml | 293 +++++++++++++++--------------- Taskfile.yaml | 20 +- ci/test-image.sh | 95 ++++++++++ 3 files changed, 256 insertions(+), 152 deletions(-) create mode 100644 ci/test-image.sh diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 37a52db..080bc7b 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -3,179 +3,172 @@ name: build on: push: branches: [develop, main] - tags: - - 'v*' + tags: ['v*'] + pull_request: + branches: [develop, main] workflow_dispatch: inputs: GDAL_VER: - description: "GDAL version to use for Docker build" + description: GDAL version to use for Docker build required: false - default: "3.12.1" + default: '3.12.1' + +permissions: + contents: read + jobs: - # ------------------------------------------------------------ - # Resolve release tag from ref - # ------------------------------------------------------------ release-meta: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: tag: ${{ steps.meta.outputs.tag }} + image: ${{ steps.meta.outputs.image }} steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install PyYAML - id: meta + env: + EVENT_NAME: ${{ github.event_name }} run: | set -euo pipefail - if [[ "${GITHUB_REF}" == "refs/heads/develop" ]]; then - TAG="latest-dev" - elif [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then - TAG="latest" - elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then - TAG="${GITHUB_REF#refs/tags/v}" - - # -------------------------------------------------- - # Validate against release.yaml - # -------------------------------------------------- - RELEASE_VERSION=$(yq -r '.image_version' release.yaml) - - if [[ -z "${RELEASE_VERSION}" || "${RELEASE_VERSION}" == "null" ]]; then - echo "image_version not found in release.yaml" - exit 1 - fi - - if [[ "${TAG}" != "${RELEASE_VERSION}" ]]; then - echo "Git tag v${TAG} does not match release.yaml version ${RELEASE_VERSION}" - exit 1 - fi - else - echo "Unsupported ref ${GITHUB_REF}" - exit 1 - fi - echo "tag=${TAG}" >> "$GITHUB_OUTPUT" - - # ------------------------------------------------------------ - # Build, scan, SBOM, push - # ------------------------------------------------------------ - build-scan-push: - needs: release-meta - runs-on: ubuntu-latest - + python - <<'PY' + import os, yaml + with open('release.yaml') as stream: + release = yaml.safe_load(stream) + ref = os.environ['GITHUB_REF'] + if os.environ['EVENT_NAME'] == 'pull_request': + tag = 'pr-test' + elif ref == 'refs/heads/develop': + tag = 'latest-dev' + elif ref == 'refs/heads/main': + tag = 'latest' + elif ref.startswith('refs/tags/v'): + tag = ref.removeprefix('refs/tags/v') + if tag != str(release['image_version']): + raise SystemExit('Git tag does not match release.yaml image_version') + else: + raise SystemExit(f'Unsupported release ref: {ref}') + image = '/'.join(release[key] for key in ('image_registry', 'image_prefix', 'image_name')) + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write(f'tag={tag}\nimage={image}\n') + PY + + extension-releases: + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - - name: Install tooling - run: | - sudo apt-get update - sudo apt-get install -y jq skopeo - curl -sL https://github.com/oras-project/oras/releases/download/v1.2.2/oras_1.2.2_linux_amd64.tar.gz | tar -xz oras - sudo mv oras /usr/local/bin/oras - - - name: Configure Docker Hub auth for Kaniko - run: | - if [[ -n "${{ secrets.dockerhub-username }}" ]]; then - mkdir -p $HOME/.docker - cat > $HOME/.docker/config.json <> $GITHUB_ENV - - name: Build image with Kaniko (to tar) - uses: docker://gcr.io/kaniko-project/executor:debug - env: - DOCKER_CONFIG: /github/home/.docker - with: - args: > - --context . - --dockerfile Dockerfile - --build-arg GDAL_VER=${{ env.GDAL_VER }} - --no-push - --tar-path image.tar - - - name: Download latest released extension VSIXs + - name: Download the same released VSIXs for both architectures env: GITHUB_TOKEN: ${{ github.token }} run: python3 ci/test-extensions.py download extension-vsix - - - name: Test extension installation and runtime requirements + - uses: actions/upload-artifact@v4 + with: + name: extension-vsix + path: extension-vsix/ + + build-test: + needs: [release-meta, extension-releases] + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: actions/download-artifact@v4 + with: + name: extension-vsix + path: extension-vsix + - name: Build and load architecture image + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/${{ matrix.arch }} + load: true + tags: pde-extension-test:ci + build-args: GDAL_VER=${{ inputs.GDAL_VER || '3.12.1' }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + provenance: false + - name: Test tools, identity, extensions, and entrypoint + env: + PLATFORM: linux/${{ matrix.arch }} + run: bash ci/test-image.sh pde-extension-test:ci "$PLATFORM" "$PWD/extension-vsix" + - name: Save tested image + run: docker save pde-extension-test:ci -o image.tar + - name: Install pinned scan tooling + env: + ARCH: ${{ matrix.arch }} run: | set -euo pipefail - skopeo copy docker-archive:image.tar docker-daemon:pde-extension-test:ci - docker run --rm --platform linux/amd64 \ - --entrypoint python3 \ - -v "$PWD/ci:/opt/extension-ci:ro" \ - -v "$PWD/extension-vsix:/opt/extension-vsix:ro" \ - pde-extension-test:ci \ - /opt/extension-ci/test-extensions.py test /opt/extension-vsix - - - name: Record tested extension releases - if: always() - uses: actions/upload-artifact@v4 - with: - name: tested-extension-releases - path: extension-vsix/*.release.json - if-no-files-found: ignore - - # -------------------------------------------------------- - # Trivy scan (relaxed for now) - # -------------------------------------------------------- - - name: Trivy scan (debug mode) - uses: docker://aquasec/trivy:0.50.2 + case "$ARCH" in amd64) TRIVY_ARCH=64bit ;; arm64) TRIVY_ARCH=ARM64 ;; esac + curl -fsSL "https://github.com/aquasecurity/trivy/releases/download/v0.74.0/trivy_0.74.0_Linux-${TRIVY_ARCH}.tar.gz" | tar -xz trivy + sudo install trivy /usr/local/bin/trivy + - name: Trivy scan (relaxed for now) env: TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db + run: trivy image --input image.tar --severity HIGH,CRITICAL --exit-code 0 --no-progress + - name: Generate architecture SBOM + run: trivy image --input image.tar --format spdx-json --output sbom.spdx.json + - name: Upload tested image and its SBOM + uses: actions/upload-artifact@v4 with: - args: > - image - --input image.tar - --severity HIGH,CRITICAL - --exit-code 0 - --no-progress - - - name: Generate SBOM (SPDX JSON) - uses: docker://aquasec/trivy:0.50.2 - env: - TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db + name: image-${{ matrix.arch }} + path: | + image.tar + sbom.spdx.json + retention-days: 1 + + # Neither architecture is published until both native build/test jobs pass. + publish: + if: github.event_name != 'pull_request' + needs: [release-meta, build-test] + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + steps: + - uses: actions/download-artifact@v4 with: - args: > - image - --input image.tar - --format spdx-json - --output sbom.spdx.json - - # -------------------------------------------------------- - # Push image and attach SBOM - # -------------------------------------------------------- - - name: Push image with Skopeo - id: push + pattern: image-* + path: images + - uses: docker/setup-buildx-action@v3 + - name: Install publishing tools run: | set -euo pipefail - - TAG=${{ needs.release-meta.outputs.tag }} - REGISTRY=$(yq -r '.image_registry' release.yaml) - IMAGE_PREFIX=$(yq -r '.image_prefix' release.yaml) - IMAGE=$(yq -r '.image_name' release.yaml) - - IMAGE_REF="${REGISTRY}/${IMAGE_PREFIX}/${IMAGE}:${TAG}" - - skopeo copy \ - --dest-creds="${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - docker-archive:image.tar \ - docker://${IMAGE_REF} - - DIGEST=$(skopeo inspect \ - --creds="${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - docker://${IMAGE_REF} \ - | jq -r .Digest) - - echo "${REGISTRY}/${IMAGE_PREFIX}/${IMAGE}@${DIGEST}" > image.digest + sudo apt-get update + sudo apt-get install -y skopeo jq + ARCH=$(dpkg --print-architecture) + curl -fsSL "https://github.com/oras-project/oras/releases/download/v1.2.2/oras_1.2.2_linux_${ARCH}.tar.gz" | tar -xz oras + sudo install oras /usr/local/bin/oras + - name: Publish architecture images, attach SBOMs, and create combined manifest + env: + IMAGE: ${{ needs.release-meta.outputs.image }} + TAG: ${{ needs.release-meta.outputs.tag }} + REGISTRY_TOKEN: ${{ github.token }} + REGISTRY_USER: ${{ github.actor }} + run: | + set -euo pipefail + REGISTRY="${IMAGE%%/*}" + printf '%s' "$REGISTRY_TOKEN" | docker login "$REGISTRY" --username "$REGISTRY_USER" --password-stdin + printf '%s' "$REGISTRY_TOKEN" | oras login "$REGISTRY" --username "$REGISTRY_USER" --password-stdin + refs=() + for arch in amd64 arm64; do + ARCH_REF="${IMAGE}:${TAG}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${arch}" + skopeo copy --dest-creds "${REGISTRY_USER}:${REGISTRY_TOKEN}" \ + "docker-archive:images/image-${arch}/image.tar" "docker://${ARCH_REF}" + DIGEST=$(skopeo inspect --creds "${REGISTRY_USER}:${REGISTRY_TOKEN}" "docker://${ARCH_REF}" | jq -r .Digest) + # Attach each SBOM to the exact image digest it describes, not the index. + (cd "images/image-${arch}" && oras attach --artifact-type application/spdx+json \ + "${IMAGE}@${DIGEST}" sbom.spdx.json:application/spdx+json) + refs+=("${IMAGE}@${DIGEST}") + done + docker buildx imagetools create --tag "${IMAGE}:${TAG}" "${refs[@]}" + docker buildx imagetools inspect "${IMAGE}:${TAG}" diff --git a/Taskfile.yaml b/Taskfile.yaml index 2c48d5e..8a63f8c 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -2,7 +2,10 @@ version: '3' vars: IMAGE: eoepca/pde-code-server:local + PLATFORM: linux/amd64 GDAL_VER: '3.12.1' + OUTPUT: /tmp/pde-code-server-multi.oci.tar + VSIX_DIR: /tmp/pde-extension-vsix tasks: default: @@ -11,9 +14,22 @@ tasks: - task: build build: - desc: Build the AMD64 container image (override IMAGE or GDAL_VER as needed) + desc: Build and load one platform (PLATFORM=linux/amd64 or linux/arm64) preconditions: - sh: command -v docker msg: Docker must be installed to build the image. + - sh: 'test {{.PLATFORM | shellQuote}} = linux/amd64 || test {{.PLATFORM | shellQuote}} = linux/arm64' + msg: PLATFORM must be linux/amd64 or linux/arm64. cmds: - - docker build --platform=linux/amd64 --build-arg GDAL_VER={{.GDAL_VER | shellQuote}} -t {{.IMAGE | shellQuote}} . + - docker buildx build --platform={{.PLATFORM | shellQuote}} --build-arg GDAL_VER={{.GDAL_VER | shellQuote}} --load -t {{.IMAGE | shellQuote}} . + + build:multi: + desc: Build both platforms into a local OCI archive (requires a docker-container Buildx builder) + cmds: + - docker buildx build --platform=linux/amd64,linux/arm64 --build-arg GDAL_VER={{.GDAL_VER | shellQuote}} --output type=oci,dest={{.OUTPUT | shellQuote}} -t {{.IMAGE | shellQuote}} . + + test: + desc: Check tools, identity, startup, and released extensions in the local image + cmds: + - python3 ci/test-extensions.py download {{.VSIX_DIR | shellQuote}} + - bash ci/test-image.sh {{.IMAGE | shellQuote}} {{.PLATFORM | shellQuote}} {{.VSIX_DIR | shellQuote}} diff --git a/ci/test-image.sh b/ci/test-image.sh new file mode 100644 index 0000000..75271e7 --- /dev/null +++ b/ci/test-image.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Run against a loaded image; uses native execution or registered binfmt/QEMU. +set -euo pipefail +image="${1:?image required}" +platform="${2:?platform required}" +vsix_dir="${3:?VSIX directory required}" +repo_dir="$(cd "$(dirname "$0")/.." && pwd)" +arch="${platform#linux/}" +case "$platform" in linux/amd64|linux/arm64) ;; *) exit 1 ;; esac +actual="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" +[[ "$actual" == "$platform" ]] || { echo "Expected $platform, got $actual" >&2; exit 1; } +# Exercise cold-start runtime availability before any network-enabled tool check. +docker run --rm --network none --platform "$platform" --entrypoint bash \ + -v "$repo_dir/ci:/opt/image-ci:ro" "$image" /opt/image-ci/test-runtime.sh +docker run --rm --platform "$platform" --entrypoint bash "$image" -ec ' + test "$(id -u)" = 1001 + test "$UID" = 1001 + test "$(id -un)" = jovyan + test "$(id -g)" = 100 + test "$(dpkg --print-architecture)" = "$1" + code-server --version + node --version + kubectl version --client=true + task --version + skaffold version + /usr/local/bin/oras version + yq --version + jq --version + hatch --version + trivy --version + java -version + dot -V + gdalinfo --version + calrissian --version + cwltool --version + cwltest --help >/dev/null + podman --version + skopeo --version + uv --version + QT_QPA_PLATFORM=offscreen nextcloudcmd --version + python3 -m pip check + /opt/calrissian-venv/bin/python -m pip check + python3 -c "import cwl_loader, cwl2puml" +' bash "$arch" +# Check the ENV value directly: Bash replaces UID with the current numeric UID. +docker run --rm -i --platform "$platform" --entrypoint python3 "$image" - "$arch" <<'PYTHON' +import os +from pathlib import Path +import shutil +import struct +import sys + +assert os.environ["UID"] == "1001" +assert os.getuid() == 1001 +machine = {"amd64": 62, "arm64": 183}[sys.argv[1]] +executables = ["/opt/code-server/lib/node", shutil.which("node"), shutil.which("trivy"), + "/opt/hatch-venv/bin/python", "/opt/calrissian-venv/bin/python"] +executables += [f"/usr/local/bin/{name}" for name in + ("kubectl", "task", "skaffold", "oras", "yq", "jq")] +for executable in executables: + with Path(executable).open("rb") as stream: + header = stream.read(20) + assert header[:4] == b"\x7fELF" and header[5] == 1, executable + assert struct.unpack("/dev/null + fi +} +trap cleanup EXIT +container="$(docker run -d --platform "$platform" \ + -e JHSINGLE_NATIVE_PROXY_AUTHTYPE=none "$image")" +deadline=$((SECONDS + 180)) +while ((SECONDS < deadline)); do + if docker exec "$container" curl -fsS --max-time 5 http://127.0.0.1:8888/ >/dev/null; then + docker exec "$container" test -L /workspace/drive + docker exec "$container" test "$(docker exec "$container" readlink /workspace/drive)" = /home/jovyan/drive + echo "Image, extensions, and entrypoint checks passed on $platform" + exit 0 + fi + [[ "$(docker inspect --format '{{.State.Running}}' "$container")" == true ]] || exit 1 + sleep 2 +done +echo "Entrypoint did not become ready within 180 seconds" >&2 +exit 1 From 591a6d8f8e7dd7e1b8bdd1d196f826e2cbf824cf Mon Sep 17 00:00:00 2001 From: Fabrice Brito Date: Sat, 10 Oct 2026 17:03:53 +0200 Subject: [PATCH 07/14] docs: document multi-platform builds and runtime fixes --- CHANGELOG.md | 25 ++++++++++++ README.md | 108 ++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 114 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cda76e..33a025b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,31 @@ Notable project changes are recorded here for users and contributors. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [Unreleased] + +### Added + +- AMD64 and ARM64 binary downloads and native CI build/test jobs, with a combined + platform manifest and architecture-specific SBOM attachments. +- Local Buildx tasks for either platform and a multi-platform OCI archive, plus + checks for tool execution, user identity, extensions, and entrypoint startup. + +### Changed + +- Align the `UID` environment variable with the actual `jovyan` UID of 1001. +- Use Trivy 0.74.0 for CI scans, matching the pinned image scanner; both platform + assets are available, while the previous 0.50.2 binary asset URLs returned 404. + +### Fixed + +- Install Calrissian from commit `a95d87dd213d51b11faea7c6def4ced46d466a4a` + with compatible isolated dependencies and bundled Dask resources. +- Preinstall Hatch 1.16.2 and its runtime dependencies in an isolated environment, + avoiding a first-use download; add offline Hatch and Calrissian smoke checks. +- Ensure the default workspace is writable by jovyan for tool configuration. +- Make Mike Farah's pinned `yq` the default command while retaining the Python + `yq` dependency and `tomlq`/`xq` launchers; smoke-test YAML and TOML processing. + ## [1.4.0] - 2026-10-10 ### Added diff --git a/README.md b/README.md index 4aba2c0..d8eac1b 100644 --- a/README.md +++ b/README.md @@ -13,21 +13,42 @@ This repository contains a Dockerfile to build a container that exposes [Code Se ### Building the Docker Image -Build the local AMD64 image with: +Build a local image for AMD64 or ARM64 with: ```bash -docker build --platform=linux/amd64 -t eoepca/pde-code-server:local . +task build PLATFORM=linux/amd64 IMAGE=eoepca/pde-code-server:amd64 +task build PLATFORM=linux/arm64 IMAGE=eoepca/pde-code-server:arm64 ``` -With [Task](https://taskfile.dev/) installed, run `task build` (or `task`). +Install [Task](https://taskfile.dev/) and Docker with Buildx. `task build` (or +`task`) defaults to `linux/amd64` and `eoepca/pde-code-server:local`. Override the image tag or GDAL version when needed: ```bash task build IMAGE=eoepca/pde-code-server:dev GDAL_VER=3.12.1 ``` -The image includes the required OpenJDK 17 JDK and is built for AMD64 because -the standalone binaries used by the image are AMD64 builds. +Both `linux/amd64` and `linux/arm64` are supported. BuildKit supplies `TARGETARCH` +for binary downloads; Node.js, Python packages, GDAL compilation, and apt packages +use the target architecture. All image tool versions remain pinned at their +existing versions, including Nextcloud 3.11.0-1.1build4 and OpenJDK 17. + +Use a native Docker host for the selected platform when possible. Cross-platform +builds and tests require QEMU/binfmt registration (for example, +`docker run --privileged --rm tonistiigi/binfmt --install arm64` on an AMD64 host). +GDAL source compilation under emulation can take substantially longer. + +To build both platforms into a local OCI archive without publishing: + +```bash +docker buildx create --name pde-multi --driver docker-container --use +docker buildx inspect --bootstrap +task build:multi OUTPUT=/tmp/pde-code-server-multi.oci.tar +``` + +The OCI archive contains both platforms. For local runtime testing, use the +single-platform `task build` commands above, which load an image into Docker. +Choose distinct tags to keep both images available. ### Running Locally @@ -36,13 +57,15 @@ local use, disable proxy authentication and bind the service to localhost: ```bash docker run --rm -it \ - --platform=linux/amd64 \ + --platform="${PLATFORM:-linux/amd64}" \ -e JHSINGLE_NATIVE_PROXY_AUTHTYPE=none \ -p 127.0.0.1:8888:8888 \ -v "$PWD:/workspace" \ eoepca/pde-code-server:local ``` +Set `PLATFORM=linux/arm64` for an ARM64 local image. + Open . If port 8888 is already in use, change the host port (the first port in `-p`) and use that port in the URL. The container image must be rebuilt after changes to the entrypoint. At startup, `nc-sync` @@ -55,7 +78,8 @@ state and should not be committed. ### Extension compatibility checks -Before publishing an image, CI downloads the latest stable release VSIXs from +Before publishing an image, CI builds and tests AMD64 and ARM64 on native +GitHub runners (`ubuntu-24.04` and `ubuntu-24.04-arm`). It downloads the latest stable release VSIXs from `eoap/eoap-validator-vscode`, `eoap/cwl-metadata-editor`, and `eoap/cwl-uml-viewer`. It installs them with `code-server` as the image's default user and checks the installed IDs and versions. It also checks Python dependency @@ -63,6 +87,10 @@ consistency, installs the validator's bundled wheel in a fresh virtual environme loads its CLI, and renders the UML viewer's sample through its installed Python bridge and bundled Java renderer for every available diagram type. +Both architectures use the same downloaded VSIXs. Tool execution, ELF binary architecture, image platform, +user identity (UID 1001 / GID 100), Calrissian isolation, Nextcloud symlink, and +standalone entrypoint startup are checked on each platform. + Failures block image publication. The tested release tags and download URLs are saved as a CI artifact. These checks use the image's default runtime settings; they do not exercise editor UI interactions or extension activation. Latest @@ -70,17 +98,43 @@ upstream releases are intentionally used, so a new incompatible release can fail CI without changes here. Extensions are installed only in the disposable test container. -To run the same check against a locally built image: +To run the full checks against each locally built image: + +```bash +task test PLATFORM=linux/amd64 IMAGE=eoepca/pde-code-server:amd64 +task test PLATFORM=linux/arm64 IMAGE=eoepca/pde-code-server:arm64 +``` + +`task test` downloads the latest VSIXs, then checks tools, extensions, and startup. +It also verifies that the default `yq` is Mike Farah's pinned v4.45.1, reads a +YAML fixture with `yq eval`, reads a TOML fixture with `tomlq`, and runs +`python -m pip check`. The Python `yq` package remains installed for `tomlq`; +only its conflicting `yq` launcher is replaced, leaving Conda's PATH intact. +The focused runtime check also runs Hatch version and local project metadata +commands without networking, verifies Calrissian's Git commit provenance, Dask +import, CLI flags, and bundled schema/init/dispose scripts, and checks both +isolated environments' dependencies. It does not execute Dask on Kubernetes. +To run these checks against a loaded image on either platform: + +```bash +docker run --rm --network none --platform linux/amd64 --entrypoint bash \ + -v "$PWD/ci:/opt/image-ci:ro" eoepca/pde-code-server:amd64 /opt/image-ci/test-runtime.sh +docker run --rm --network none --platform linux/arm64 --entrypoint bash \ + -v "$PWD/ci:/opt/image-ci:ro" eoepca/pde-code-server:arm64 /opt/image-ci/test-runtime.sh +``` + +To reuse the same downloaded release assets for both platforms: ```bash python3 ci/test-extensions.py download /tmp/pde-extension-vsix -docker run --rm --platform linux/amd64 --entrypoint python3 \ - -v "$PWD/ci:/opt/extension-ci:ro" \ - -v /tmp/pde-extension-vsix:/opt/extension-vsix:ro \ - eoepca/pde-code-server:local \ - /opt/extension-ci/test-extensions.py test /opt/extension-vsix +bash ci/test-image.sh eoepca/pde-code-server:amd64 linux/amd64 /tmp/pde-extension-vsix +bash ci/test-image.sh eoepca/pde-code-server:arm64 linux/arm64 /tmp/pde-extension-vsix ``` +The UML viewer's bundled dependencies are tested as shipped, including its native +Python modules; CI does not patch the VSIX or skip rendering on ARM64. A future +upstream incompatibility will fail the corresponding platform check. + This image is based on the Jupyter Python 3.12 base notebook image (currently Ubuntu 24.04) and provides a curated set of development, Kubernetes, and Earth-Observation workflow tools. @@ -166,10 +220,13 @@ Installed via pip (Python 3.12): Reference implementation of the Common Workflow Language. -* calrissian: 0.18.1 +* calrissian: commit `a95d87dd213d51b11faea7c6def4ced46d466a4a` - CWL runner for Kubernetes, installed in `/opt/calrissian-venv` with - `cwl-utils==0.40` to isolate its dependencies from the image's CWL tooling. + CWL runner for Kubernetes with Dask support, installed from the pinned upstream + Git commit in `/opt/calrissian-venv`. Its dependencies are resolved together, + including `cwltool>=3.1.20260108082145`, separately from the image's CWL tooling. + `CALRISSIAN_COMMIT` is the build argument; changing it also requires updating + the expected commit in `ci/test-runtime.sh`. * jhsingle-native-proxy (>= 0.0.9) @@ -191,12 +248,19 @@ Installed via pip (Python 3.12): - hatch: 1.16.2 - Python project manager and build tool, installed as a standalone binary. + Python project manager and build tool, installed with all runtime dependencies + in `/opt/hatch-venv`, using the base image's native Python on AMD64 and ARM64. + `/usr/local/bin/hatch` links to its console script. Jovyan can use it offline + without downloading Hatch itself. `HATCH_CONFIG`, `HATCH_DATA_DIR`, and + `HATCH_CACHE_DIR` point to writable locations under `/home/jovyan`, independent + of the `/workspace` project mount. Project environment creation, project + dependencies, plugins, or additional Python versions may still need network + access; the offline smoke check only reads local project metadata. ### User & Runtime Environment - User: jovyan -- UID / GID: 1001 +- UID / GID: 1001 / 100 - Home / Workdir: /workspace - Passwordless sudo enabled for the user. @@ -218,7 +282,13 @@ ghcr.io//pde-code-server The registry owner corresponds to the GitHub repository owner (user or organization). -Images are built using Kaniko and pushed using OCI-compliant tooling. +Images are built with Buildx. Each platform is tested, scanned with Trivy 0.74.0, +and given its own SPDX JSON SBOM. After both platform jobs succeed, CI publishes +the tested images and attaches each SBOM with ORAS to that architecture image's +exact registry digest. It then publishes a combined multi-platform manifest under +the branch or release tag. The SBOMs describe individual platform images, not the +combined index. Vulnerability findings remain informational, as in the previous +workflow. Pull requests run builds and checks without publishing. ### Tagging Strategy From d46f22e8c739edfa449c427807fde435c0a71a6b Mon Sep 17 00:00:00 2001 From: fbrito Date: Mon, 12 Oct 2026 12:42:25 +0200 Subject: [PATCH 08/14] fix(hatch): upgrade to 1.16.5 for virtualenv compatibility Exercise default environment creation and activation in the offline runtime checks and update the documented Hatch version. Fixes #18 --- CHANGELOG.md | 5 +++-- Dockerfile | 2 +- README.md | 5 +++-- ci/test-runtime.sh | 16 +++++++++++++++- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33a025b..8154769 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,8 +23,9 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - Install Calrissian from commit `a95d87dd213d51b11faea7c6def4ced46d466a4a` with compatible isolated dependencies and bundled Dask resources. -- Preinstall Hatch 1.16.2 and its runtime dependencies in an isolated environment, - avoiding a first-use download; add offline Hatch and Calrissian smoke checks. +- Preinstall Hatch 1.16.5 and its runtime dependencies in an isolated environment, + avoiding a first-use download and fixing interpreter discovery with virtualenv + 21; add offline Hatch environment creation/activation and Calrissian smoke checks. - Ensure the default workspace is writable by jovyan for tool configuration. - Make Mike Farah's pinned `yq` the default command while retaining the Python `yq` dependency and `tomlq`/`xq` launchers; smoke-test YAML and TOML processing. diff --git a/Dockerfile b/Dockerfile index 5f23b3d..3c3db4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -131,7 +131,7 @@ RUN curl -fsSL \ # ------------------------------------------------------------------- # hatch # ------------------------------------------------------------------- -ARG HATCH_VERSION=1.16.2 +ARG HATCH_VERSION=1.16.5 ENV HATCH_CONFIG=/home/jovyan/.config/hatch/config.toml \ HATCH_DATA_DIR=/home/jovyan/.local/share/hatch \ HATCH_CACHE_DIR=/home/jovyan/.cache/hatch diff --git a/README.md b/README.md index d8eac1b..7874e71 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,8 @@ YAML fixture with `yq eval`, reads a TOML fixture with `tomlq`, and runs `python -m pip check`. The Python `yq` package remains installed for `tomlq`; only its conflicting `yq` launcher is replaced, leaving Conda's PATH intact. The focused runtime check also runs Hatch version and local project metadata -commands without networking, verifies Calrissian's Git commit provenance, Dask +commands, creates and activates a default environment without networking (skipping +project installation), verifies Calrissian's Git commit provenance, Dask import, CLI flags, and bundled schema/init/dispose scripts, and checks both isolated environments' dependencies. It does not execute Dask on Kubernetes. To run these checks against a loaded image on either platform: @@ -246,7 +247,7 @@ Installed via pip (Python 3.12): ### Python Build & Packaging -- hatch: 1.16.2 +- hatch: 1.16.5 Python project manager and build tool, installed with all runtime dependencies in `/opt/hatch-venv`, using the base image's native Python on AMD64 and ARM64. diff --git a/ci/test-runtime.sh b/ci/test-runtime.sh index 49200aa..5caa7c1 100644 --- a/ci/test-runtime.sh +++ b/ci/test-runtime.sh @@ -11,14 +11,28 @@ test "$(readlink -f "$(command -v calrissian)")" = /opt/calrissian-venv/bin/calr fixture_dir="$(mktemp -d)" trap 'rm -rf "$fixture_dir"' EXIT -[[ "$(hatch --version)" == 'Hatch, version 1.16.2' ]] +[[ "$(hatch --version)" == 'Hatch, version 1.16.5' ]] cat > "$fixture_dir/pyproject.toml" <<'TOML' [project] name = "offline-smoke" version = "1.2.3" + +[tool.hatch.envs.default] +skip-install = true TOML cd "$fixture_dir" [[ "$(hatch version)" == '1.2.3' ]] +# Exercise interpreter discovery, which failed with older Hatch and virtualenv 21. +# Skip project installation so this regression check can run without networking. +hatch env create default +environment_dir="$(hatch env find default)" +test -x "$environment_dir/bin/python" +( + source "$environment_dir/bin/activate" + test "$(command -v python)" = "$environment_dir/bin/python" + python -c 'import os, sys; assert sys.prefix == os.environ["VIRTUAL_ENV"]' +) +hatch env remove default /opt/hatch-venv/bin/python -m pip check /opt/calrissian-venv/bin/python - <<'PYTHON' From 9637086e48ffd09f192d7981422c67bfd8fae552 Mon Sep 17 00:00:00 2001 From: ldonnini Date: Mon, 12 Oct 2026 23:09:07 +0200 Subject: [PATCH 09/14] fix(image): unblock bash kernel venv activation and workspace cache --- CHANGELOG.md | 6 +++++- Dockerfile | 9 ++++++++- ci/test-runtime.sh | 4 ++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8154769..1179f45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,11 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - Preinstall Hatch 1.16.5 and its runtime dependencies in an isolated environment, avoiding a first-use download and fixing interpreter discovery with virtualenv 21; add offline Hatch environment creation/activation and Calrissian smoke checks. -- Ensure the default workspace is writable by jovyan for tool configuration. +- Set `VIRTUAL_ENV_DISABLE_PROMPT=1` for the Bash kernel only: when `~/.bashrc` activates a + virtual environment, activating another one in a notebook cell restored the pre-kernel + prompt and the cell never finished. Terminals keep the `(venv)` prompt. +- Ensure the default workspace is writable by jovyan for tool configuration, and + remove the root-owned build cache that blocked `uv` and pip caches there. - Make Mike Farah's pinned `yq` the default command while retaining the Python `yq` dependency and `tomlq`/`xq` launchers; smoke-test YAML and TOML processing. diff --git a/Dockerfile b/Dockerfile index 3c3db4d..b3993ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -107,6 +107,11 @@ RUN pip install --no-cache-dir \ cwltest && \ python -m bash_kernel.install +# Bash kernels detect a finished cell by their prompt. When ~/.bashrc activates a virtual +# environment, activating another one in a cell restores the pre-kernel prompt and the cell +# never finishes; keep venv activation away from the kernel prompt (terminals keep theirs). +RUN python -c "import json, pathlib; p = pathlib.Path('/usr/local/share/jupyter/kernels/bash/kernel.json'); spec = json.loads(p.read_text()); spec.setdefault('env', {})['VIRTUAL_ENV_DISABLE_PROMPT'] = '1'; p.write_text(json.dumps(spec))" + # Resolve the commit's cwltool/dependency requirements together, independently # of the global CWL tools, rather than constraining cwl-utils to the old release. RUN python -m venv /opt/calrissian-venv && \ @@ -186,8 +191,10 @@ RUN apt-get update && apt-get install -y \ # tomlq requires the Python yq package, whose console launcher shadows the # pinned Mike Farah binary on Conda's PATH. Keep its modules and tomlq/xq # launchers; replace only the conflicting command, without changing PATH. -# Prepare the default workspace for non-root tools after runtime installation. +# Prepare the default workspace for non-root tools after runtime installation, +# dropping the root-owned cache that build-isolated pip runs leave in HOME. RUN ln -sf /usr/local/bin/yq /opt/conda/bin/yq && \ + rm -rf /workspace/.cache && \ install -d -o ${USER} -g users /workspace # ------------------------------------------------------------------- diff --git a/ci/test-runtime.sh b/ci/test-runtime.sh index 5caa7c1..9b263f1 100644 --- a/ci/test-runtime.sh +++ b/ci/test-runtime.sh @@ -6,6 +6,8 @@ test "$(id -u)" = 1001 test -w "$HATCH_CONFIG" test -w "$HATCH_DATA_DIR" test -w "$HATCH_CACHE_DIR" +# Tools such as uv and pip create caches under the default /workspace HOME. +test -z "$(find /workspace ! -user jovyan -print -quit)" test "$(readlink -f "$(command -v hatch)")" = /opt/hatch-venv/bin/hatch test "$(readlink -f "$(command -v calrissian)")" = /opt/calrissian-venv/bin/calrissian fixture_dir="$(mktemp -d)" @@ -33,6 +35,8 @@ test -x "$environment_dir/bin/python" python -c 'import os, sys; assert sys.prefix == os.environ["VIRTUAL_ENV"]' ) hatch env remove default +# Activating a venv in a Bash kernel must not touch the prompt the kernel waits for. +jupyter kernelspec list --json | python -c 'import json, sys; spec = json.load(sys.stdin)["kernelspecs"]["bash"]["spec"]; assert spec["env"]["VIRTUAL_ENV_DISABLE_PROMPT"] == "1", spec' /opt/hatch-venv/bin/python -m pip check /opt/calrissian-venv/bin/python - <<'PYTHON' From af17d0ff50a5479cbcf15786682891f1dd2729c4 Mon Sep 17 00:00:00 2001 From: mlongobardo-gituname Date: Fri, 30 Oct 2026 11:09:36 +0100 Subject: [PATCH 10/14] release(*,d): update changelog --- CHANGELOG.md | 52 ++++++++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1179f45..3188587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,40 +4,15 @@ Notable project changes are recorded here for users and contributors. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). -## [Unreleased] +## [1.4.0] - 2026-10-30 ### Added - - AMD64 and ARM64 binary downloads and native CI build/test jobs, with a combined platform manifest and architecture-specific SBOM attachments. - Local Buildx tasks for either platform and a multi-platform OCI archive, plus checks for tool execution, user identity, extensions, and entrypoint startup. - -### Changed - -- Align the `UID` environment variable with the actual `jovyan` UID of 1001. -- Use Trivy 0.74.0 for CI scans, matching the pinned image scanner; both platform - assets are available, while the previous 0.50.2 binary asset URLs returned 404. - -### Fixed - -- Install Calrissian from commit `a95d87dd213d51b11faea7c6def4ced46d466a4a` - with compatible isolated dependencies and bundled Dask resources. -- Preinstall Hatch 1.16.5 and its runtime dependencies in an isolated environment, - avoiding a first-use download and fixing interpreter discovery with virtualenv - 21; add offline Hatch environment creation/activation and Calrissian smoke checks. -- Set `VIRTUAL_ENV_DISABLE_PROMPT=1` for the Bash kernel only: when `~/.bashrc` activates a - virtual environment, activating another one in a notebook cell restored the pre-kernel - prompt and the cell never finished. Terminals keep the `(venv)` prompt. -- Ensure the default workspace is writable by jovyan for tool configuration, and - remove the root-owned build cache that blocked `uv` and pip caches there. -- Make Mike Farah's pinned `yq` the default command while retaining the Python - `yq` dependency and `tomlq`/`xq` launchers; smoke-test YAML and TOML processing. - -## [1.4.0] - 2026-10-10 - -### Added - +- Pull-request CI validation on both architectures, using the same downloaded + extension releases for AMD64 and ARM64. - Pinned CWL and transpiler tooling for EOAP extension runtime requirements. - CI installation checks for the latest released EOAP Validator, CWL Metadata Editor, and CWL UML Viewer VSIXs, with Python dependency checks, validator @@ -55,12 +30,29 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - Install Calrissian 0.18.1 and its pinned `cwl-utils` 0.40 dependency in an isolated virtual environment to avoid conflicts with the image's CWL tooling. - Document local image builds, standalone usage, and extension compatibility checks. - +- Replace Kaniko builds with Docker Buildx and publish the tested images only + after both native architecture build/test jobs pass. +- Align the `UID` environment variable with the actual `jovyan` UID of 1001. +- Use Trivy 0.74.0 for CI scans, matching the pinned image scanner; both platform + assets are available, while the previous 0.50.2 binary asset URLs returned 404. ### Fixed - Include the C++ compiler and XZ utilities required for the GDAL source build. - Validate proxy authentication modes and quote entrypoint arguments. - +- Install Calrissian from commit `a95d87dd213d51b11faea7c6def4ced46d466a4a` + with compatible isolated dependencies and bundled Dask resources. +- Preinstall Hatch 1.16.5 and its runtime dependencies in an isolated environment, + avoiding a first-use download and fixing interpreter discovery with virtualenv + 21; add offline Hatch environment creation/activation and Calrissian smoke checks. +- Set `VIRTUAL_ENV_DISABLE_PROMPT=1` for the Bash kernel only: when `~/.bashrc` activates a + virtual environment, activating another one in a notebook cell restored the pre-kernel + prompt and the cell never finished. Terminals keep the `(venv)` prompt. +- Ensure the default workspace is writable by jovyan for tool configuration, and + remove the root-owned build cache that blocked `uv` and pip caches there. +- Store Hatch configuration, data, and caches in writable directories under + `/home/jovyan`, including when a volume is mounted at `/workspace`. +- Make Mike Farah's pinned `yq` the default command while retaining the Python + `yq` dependency and `tomlq`/`xq` launchers; smoke-test YAML and TOML processing. ## [1.3.0] - 2026-10-06 ### Added From 836e895b0bb44b65453492442c17ab34cb529b90 Mon Sep 17 00:00:00 2001 From: mlongobardo-gituname Date: Mon, 2 Nov 2026 10:59:26 +0100 Subject: [PATCH 11/14] feat(Docker): Fedora transition --- .github/workflows/build-image.yml | 2 + CHANGELOG.md | 8 +++ Dockerfile | 80 +++++++++++++---------- README.md | 57 ++++++++++++---- Taskfile.yaml | 5 ++ ci/fixtures/mastering-app-package/bash-rc | 5 ++ ci/fixtures/mastering-app-package/init.sh | 42 ++++++++++++ ci/test-image.sh | 9 ++- ci/test-mastering.sh | 63 ++++++++++++++++++ ci/test-runtime.sh | 3 + 10 files changed, 226 insertions(+), 48 deletions(-) create mode 100644 ci/fixtures/mastering-app-package/bash-rc create mode 100644 ci/fixtures/mastering-app-package/init.sh create mode 100644 ci/test-mastering.sh diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 080bc7b..b5d9d79 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -101,6 +101,8 @@ jobs: env: PLATFORM: linux/${{ matrix.arch }} run: bash ci/test-image.sh pde-extension-test:ci "$PLATFORM" "$PWD/extension-vsix" + - name: Test mastering-app-package initialization and chart startup + run: bash ci/test-mastering.sh pde-extension-test:ci linux/${{ matrix.arch }} - name: Save tested image run: docker save pde-extension-test:ci -o image.tar - name: Install pinned scan tooling diff --git a/CHANGELOG.md b/CHANGELOG.md index 3188587..29849c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ Notable project changes are recorded here for users and contributors. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## Unreleased + +- Migrate the runtime OS to Fedora 44 and code-server to 4.144.0, retaining + the Python 3.12 Conda/Jupyter environment and PDE tools, identity and paths. +- Retain Java 17 through Temurin and use Fedora's Nextcloud client packages. +- Add an integration gate on both architectures for the mastering-app-package + init script, extensions, Python environment, S3 setup and direct editor startup. + ## [1.4.0] - 2026-10-30 ### Added diff --git a/Dockerfile b/Dockerfile index b3993ee..b3b4fbc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,28 @@ -FROM quay.io/jupyter/base-notebook:python-3.12 +# Retain the complete Python/Jupyter/Conda toolchain while moving the OS to Fedora. +FROM quay.io/jupyter/base-notebook:python-3.12 AS python-runtime +FROM eclipse-temurin:17-jdk AS java-runtime +FROM fedora:44 + +COPY --from=java-runtime /opt/java/openjdk /opt/java/openjdk +COPY --from=python-runtime /opt/conda /opt/conda +COPY --from=python-runtime /usr/local/bin/start*.py /usr/local/bin/ +COPY --from=python-runtime /usr/local/bin/start*.sh /usr/local/bin/ +COPY --from=python-runtime /usr/local/bin/fix-permissions /usr/local/bin/ +ENV JAVA_HOME=/opt/java/openjdk \ + CONDA_DIR=/opt/conda \ + NB_USER=jovyan \ + NB_UID=1001 \ + NB_GID=100 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + SHELL=/bin/bash \ + PATH=/opt/java/openjdk/bin:/opt/conda/bin:${PATH} # BuildKit supplies amd64 or arm64 for each target platform. ARG TARGETARCH RUN case "${TARGETARCH}" in amd64|arm64) ;; *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; esac -ENV DEBIAN_FRONTEND=noninteractive \ - USER=jovyan \ +ENV USER=jovyan \ UID=1001 \ GID=100 \ HOME=/workspace @@ -15,25 +32,18 @@ USER root # ------------------------------------------------------------------- # Base system packages (runtime only) # ------------------------------------------------------------------- -RUN apt-get update && apt-get install -y \ - ca-certificates \ - curl \ - git \ - nano \ - net-tools \ - sudo \ - wget \ - graphviz \ - file \ - tree \ - podman \ - skopeo \ - openjdk-17-jdk-headless \ - nextcloud-desktop-cmd=3.11.0-1.1build4 \ - && rm -rf /var/lib/apt/lists/* - -RUN usermod -u 1001 ${USER} && \ - echo "${USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/${USER} +RUN dnf upgrade -y && dnf install -y \ + ca-certificates curl git git-lfs nano net-tools sudo wget graphviz file tree \ + podman skopeo fontconfig freetype nextcloud-client \ + htop openssh-clients procps-ng zsh dumb-init glibc-langpack-en \ + findutils which tar gzip bzip2 unzip hostname shadow-utils \ + && dnf clean all && rm -rf /var/cache/dnf + +RUN groupadd -g 100 users 2>/dev/null || getent group 100 +RUN useradd -m -u 1001 -g 100 -s /bin/bash jovyan && \ + echo "jovyan ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/jovyan && \ + chmod 440 /etc/sudoers.d/jovyan && \ + git lfs install --system # ------------------------------------------------------------------- # NVM and Node.js @@ -53,12 +63,12 @@ RUN mkdir -p "${NVM_DIR}" && \ ENV PATH="${NVM_DIR}/versions/node/v${NODE_VERSION}/bin:${PATH}" RUN printf '\nexport NVM_DIR="/opt/nvm"\n[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"\n' \ - >> /etc/bash.bashrc + >> /etc/bashrc # ------------------------------------------------------------------- # code-server # ------------------------------------------------------------------- -ARG CODE_RELEASE=4.138.0 +ARG CODE_RELEASE=4.144.0 RUN mkdir -p /opt/code-server && \ curl -fsSL \ "https://github.com/coder/code-server/releases/download/v${CODE_RELEASE}/code-server-${CODE_RELEASE}-linux-${TARGETARCH}.tar.gz" \ @@ -155,22 +165,19 @@ RUN python -m venv /opt/hatch-venv && \ ARG TRIVY_VERSION=0.74.0 RUN case "${TARGETARCH}" in amd64) TRIVY_ARCH=64bit ;; arm64) TRIVY_ARCH=ARM64 ;; esac && \ curl -fsSL \ - https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-${TRIVY_ARCH}.deb \ - -o /tmp/trivy.deb && \ - dpkg -i /tmp/trivy.deb && \ - rm /tmp/trivy.deb + https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-${TRIVY_ARCH}.tar.gz \ + | tar -xz -C /usr/local/bin trivy # ------------------------------------------------------------------- # GDAL # ------------------------------------------------------------------- ARG GDAL_VER=3.12.1 -RUN apt-get update && apt-get install -y \ - cmake g++ ninja-build xz-utils libproj-dev proj-data proj-bin && \ - rm -rf /var/lib/apt/lists/* && \ - set -e && \ +RUN dnf install -y \ + cmake gcc-c++ ninja-build xz proj-devel proj-data proj && \ + dnf clean all && rm -rf /var/cache/dnf && \ cd /tmp && \ - curl -fsSL -o gdal-${GDAL_VER}.tar.xz https://download.osgeo.org/gdal/${GDAL_VER}/gdal-${GDAL_VER}.tar.xz \ - || curl -fsSL -o gdal-${GDAL_VER}.tar.gz https://download.osgeo.org/gdal/${GDAL_VER}/gdal-${GDAL_VER}.tar.gz && \ + (curl -fsSL -o gdal-${GDAL_VER}.tar.xz https://download.osgeo.org/gdal/${GDAL_VER}/gdal-${GDAL_VER}.tar.xz \ + || { rm -f gdal-${GDAL_VER}.tar.xz; curl -fsSL -o gdal-${GDAL_VER}.tar.gz https://download.osgeo.org/gdal/${GDAL_VER}/gdal-${GDAL_VER}.tar.gz; }) && \ if [ -f gdal-${GDAL_VER}.tar.xz ]; then \ tar -xJf gdal-${GDAL_VER}.tar.xz; \ else \ @@ -180,9 +187,12 @@ RUN apt-get update && apt-get install -y \ mkdir build && cd build && \ cmake -G Ninja ../ \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/usr/local && \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_INSTALL_LIBDIR=lib && \ cmake --build . -- -j"$(nproc)" && \ cmake --install . && \ + echo /usr/local/lib > /etc/ld.so.conf.d/gdal.conf && \ + echo /usr/local/lib64 >> /etc/ld.so.conf.d/gdal.conf && \ ldconfig && \ cd / && rm -rf /tmp/gdal-${GDAL_VER}* && \ gdal-config --version diff --git a/README.md b/README.md index 7874e71..b5d6c07 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,10 @@ task build IMAGE=eoepca/pde-code-server:dev GDAL_VER=3.12.1 ``` Both `linux/amd64` and `linux/arm64` are supported. BuildKit supplies `TARGETARCH` -for binary downloads; Node.js, Python packages, GDAL compilation, and apt packages -use the target architecture. All image tool versions remain pinned at their -existing versions, including Nextcloud 3.11.0-1.1build4 and OpenJDK 17. +for binary downloads; Node.js, Python packages, GDAL compilation, and Fedora RPM packages +use the target architecture. code-server is upgraded to 4.144.0. Nextcloud follows Fedora 44 packages; +Java remains OpenJDK 17, supplied by Eclipse Temurin. Other explicitly pinned +tool versions are retained. Use a native Docker host for the selected platform when possible. Cross-platform builds and tests require QEMU/binfmt registration (for example, @@ -74,6 +75,35 @@ container. With the project directory mounted at `/workspace`, this symlink also appears in the project directory on the host; it is generated runtime state and should not be committed. +## Mastering Application Package compatibility + +The migration preserves `jovyan` (1001:100), `HOME=/workspace`, passwordless sudo, +Python 3.12, Conda, Bash/Python notebook kernels, NVM, all existing PDE tools, +and the ApplicationHub proxy/Nextcloud entrypoint. The workshop Helm chart can +continue running its init container and direct code-server command unchanged. +Workspace volumes must be writable by UID 1001 or GID 100 (`fsGroup: 100`). + +Run the integration check with disposable Docker volumes and LocalStack: + +```bash +bash ci/test-mastering.sh eoepca/pde-code-server:fedora-arm64 linux/arm64 +# Test the current local workshop files instead of the checked-in snapshot: +bash ci/test-mastering.sh eoepca/pde-code-server:fedora-arm64 linux/arm64 \ + /Users/mlongobardo/projects/dev-platform-eoap/mastering-app-package/files +``` + +This executes the unmodified `init.sh`, verifies all four extensions, geospatial +Python imports, pip dependencies, the kernel interpreter, interactive venv +activation, downloaded kubectl, and S3 bucket creation, then starts code-server +using the chart's command and XDG settings. It needs network access for GitHub, +Open VSX, PyPI, Kubernetes downloads, and the LocalStack image. It does not test +Kubernetes scheduling, a real Nextcloud service, or browser extension activation. +CI runs this check on both architectures before publishing. + +The fixture in `ci/fixtures/mastering-app-package` was copied from the local +workshop checkout at commit `49bb077b8753bddfb215827a04a3cf61d265811b`; +refresh it when the workshop initialization changes. + ## Installed Tooling ### Extension compatibility checks @@ -136,25 +166,28 @@ The UML viewer's bundled dependencies are tested as shipped, including its nativ Python modules; CI does not patch the VSIX or skip rendering on ARM64. A future upstream incompatibility will fail the corresponding platform check. -This image is based on the Jupyter Python 3.12 base notebook image (currently -Ubuntu 24.04) and provides a curated set of development, Kubernetes, and -Earth-Observation workflow tools. +This image uses Fedora 44, following the Fedora recipe in coder/code-server +with the PDE runtime contract preserved. The complete Python 3.12 Conda/Jupyter +environment is copied from the Jupyter base notebook image, so notebook tools +and `/opt/conda` paths remain available. Fedora supplies the OS and system tools. -All non-distro binaries are pinned to explicit versions to ensure reproducibility. +Standalone tool releases are pinned. Fedora packages and the Python 3.12 Jupyter +and Java 17 Temurin source images receive updates; builds are not fully locked. ### Base System -- OS: Ubuntu 24.04 (from the current Jupyter base image) -- Python: 3.12.11 +- OS: Fedora 44 +- Python: 3.12 (Jupyter/Conda toolchain) - NVM: v0.40.3, installed at `/opt/nvm` - Node.js: v22.15.0, installed via NVM and available on `PATH` - npm: bundled with Node.js 22 -- Java: OpenJDK 17 JDK (headless), required and installed from distro packages +- Java: Eclipse Temurin OpenJDK 17 JDK, at `/opt/java/openjdk` Installed system utilities: - curl, wget -- git +- git, git-lfs +- htop, openssh-clients, zsh, dumb-init - sudo - nano - net-tools @@ -165,7 +198,7 @@ Installed system utilities: ### code-server -- code-server: 4.138.0 +- code-server: 4.144.0 Installed from official release tarball and available in PATH: diff --git a/Taskfile.yaml b/Taskfile.yaml index 8a63f8c..08fe992 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -33,3 +33,8 @@ tasks: cmds: - python3 ci/test-extensions.py download {{.VSIX_DIR | shellQuote}} - bash ci/test-image.sh {{.IMAGE | shellQuote}} {{.PLATFORM | shellQuote}} {{.VSIX_DIR | shellQuote}} + + test:mastering: + desc: Run the workshop init script and chart startup against disposable LocalStack and storage + cmds: + - bash ci/test-mastering.sh {{.IMAGE | shellQuote}} {{.PLATFORM | shellQuote}} diff --git a/ci/fixtures/mastering-app-package/bash-rc b/ci/fixtures/mastering-app-package/bash-rc new file mode 100644 index 0000000..00dd390 --- /dev/null +++ b/ci/fixtures/mastering-app-package/bash-rc @@ -0,0 +1,5 @@ +alias ll="ls -l" + +alias aws="aws --endpoint-url=http://eoap-mastering-app-package-localstack:4566" + +source /workspace/.venv/bin/activate \ No newline at end of file diff --git a/ci/fixtures/mastering-app-package/init.sh b/ci/fixtures/mastering-app-package/init.sh new file mode 100644 index 0000000..e2bb1af --- /dev/null +++ b/ci/fixtures/mastering-app-package/init.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -x + +cd /workspace + +if [ ! -d mastering-app-package/.git ]; then + git clone 'https://github.com/eoap/mastering-app-package.git' || exit $? +fi + +code-server --install-extension ms-python.python --install-extension redhat.vscode-yaml --install-extension sbg-rabix.benten-cwl --install-extension ms-toolsai.jupyter + +ln -sfn /workspace/.local/share/code-server/extensions /workspace/extensions + +mkdir -p /workspace/User/ + +echo '{"workbench.colorTheme": "Visual Studio Dark"}' > /workspace/User/settings.json + +python -m venv /workspace/.venv +source /workspace/.venv/bin/activate +/workspace/.venv/bin/python -m pip install --upgrade "pip @ https://files.pythonhosted.org/packages/3f/06/3e4420302363fbb71be3c7f35ce042f54bfca330e3e66b3a2e3c4cca579f/pip-26.2.2-py3-none-any.whl#sha256=fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70" || exit $? +/workspace/.venv/bin/python -m pip install --upgrade "certifi>=2026.10.19" || exit $? +/workspace/.venv/bin/python -m pip install --no-cache-dir rasterio click pystac loguru pyproj shapely scikit-image pystac rio_stac ipykernel stactools[validate] calrissian matplotlib pandas nose2 +/workspace/.venv/bin/python -m ipykernel install --user --name mastering_env --display-name "Python (Mastering Application Package)" + +echo "**** install kubectl ****" +case "$(uname -m)" in + aarch64|arm64) kubectl_arch=arm64 ;; + x86_64) kubectl_arch=amd64 ;; + *) echo "Unsupported architecture: $(uname -m)" >&2; exit 1 ;; +esac +curl -fsSL -o kubectl "https://dl.k8s.io/release/$(curl -fsSL https://dl.k8s.io/release/stable.txt)/bin/linux/${kubectl_arch}/kubectl" || exit $? +chmod +x kubectl +mv ./kubectl /workspace/.venv/bin/kubectl + +export AWS_DEFAULT_REGION="us-east-1" + +export AWS_ACCESS_KEY_ID="test" + +export AWS_SECRET_ACCESS_KEY="test" + +aws s3 mb s3://results --endpoint-url=http://eoap-mastering-app-package-localstack:4566 diff --git a/ci/test-image.sh b/ci/test-image.sh index 75271e7..a7dd5d8 100644 --- a/ci/test-image.sh +++ b/ci/test-image.sh @@ -17,7 +17,9 @@ docker run --rm --platform "$platform" --entrypoint bash "$image" -ec ' test "$UID" = 1001 test "$(id -un)" = jovyan test "$(id -g)" = 100 - test "$(dpkg --print-architecture)" = "$1" + . /etc/os-release + test "$ID" = fedora + test "$VERSION_ID" = 44 code-server --version node --version kubectl version --client=true @@ -29,6 +31,7 @@ docker run --rm --platform "$platform" --entrypoint bash "$image" -ec ' hatch --version trivy --version java -version + java -XshowSettings:properties -version 2>&1 | grep -Eq "java.specification.version = 17$" dot -V gdalinfo --version calrissian --version @@ -38,6 +41,10 @@ docker run --rm --platform "$platform" --entrypoint bash "$image" -ec ' skopeo --version uv --version QT_QPA_PLATFORM=offscreen nextcloudcmd --version + QT_QPA_PLATFORM=offscreen nextcloudcmd --help > /tmp/nextcloud-help + for option in --user --password --exclude --path; do + grep -Fq -- "$option" /tmp/nextcloud-help + done python3 -m pip check /opt/calrissian-venv/bin/python -m pip check python3 -c "import cwl_loader, cwl2puml" diff --git a/ci/test-mastering.sh b/ci/test-mastering.sh new file mode 100644 index 0000000..f2c75e8 --- /dev/null +++ b/ci/test-mastering.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Execute the unmodified workshop init script against disposable storage and S3. +set -euo pipefail +image="${1:?image required}" +platform="${2:?platform required}" +repo_dir="$(cd "$(dirname "$0")/.." && pwd)" +fixtures="${3:-$repo_dir/ci/fixtures/mastering-app-package}" +fixtures="$(cd "$fixtures" && pwd)" +name="pde-mastering-$$" +cleanup() { + docker rm -f "$name-init" "$name-server" "$name-s3" >/dev/null 2>&1 || true + docker volume rm "$name-workspace" >/dev/null 2>&1 || true + docker network rm "$name" >/dev/null 2>&1 || true +} +trap cleanup EXIT +docker network create "$name" >/dev/null +docker volume create "$name-workspace" >/dev/null +docker run -d --platform "$platform" --name "$name-s3" --network "$name" \ + --network-alias eoap-mastering-app-package-localstack \ + -e SERVICES=s3 localstack/localstack:4.0.3 >/dev/null +ready=false +for ((i=0; i<90; i++)); do + if docker exec "$name-s3" awslocal s3 ls >/dev/null 2>&1; then ready=true; break; fi + sleep 2 +done +[[ "$ready" == true ]] || { docker logs "$name-s3"; exit 1; } +# Match a PVC made writable through the chart's fsGroup=100. +docker run --rm --platform "$platform" --user root --entrypoint bash \ + -v "$name-workspace:/workspace" "$image" -ec 'chown 1001:100 /workspace' +docker run --name "$name-init" --platform "$platform" --network "$name" \ + --entrypoint bash -v "$name-workspace:/workspace" \ + -v "$fixtures:/opt/init:ro" "$image" -ec ' + bash /opt/init/init.sh + test -d /workspace/mastering-app-package/.git + test "$(readlink /workspace/extensions)" = /workspace/.local/share/code-server/extensions + code-server --list-extensions > /tmp/extensions + for extension in ms-python.python redhat.vscode-yaml sbg-rabix.benten-cwl ms-toolsai.jupyter; do + grep -Fxi "$extension" /tmp/extensions + done + /workspace/.venv/bin/python -m pip check + /workspace/.venv/bin/python -c "import rasterio, click, pystac, loguru, pyproj, shapely, skimage, rio_stac, ipykernel, stactools, calrissian, matplotlib, pandas, nose2" + /workspace/.venv/bin/kubectl version --client + jupyter kernelspec list --json | python -c '\''import json,sys; s=json.load(sys.stdin)["kernelspecs"]["mastering_env"]["spec"]; assert s["argv"][0] == "/workspace/.venv/bin/python"'\'' + cp /opt/init/bash-rc /workspace/.bashrc + bash -ic '\''test "$VIRTUAL_ENV" = /workspace/.venv; python -c "import rasterio"'\'' + ' +docker exec "$name-s3" awslocal s3api head-bucket --bucket results +# Match the chart: it invokes code-server directly, bypassing the image entrypoint. +docker run -d --name "$name-server" --platform "$platform" --network "$name" \ + -v "$name-workspace:/workspace" --entrypoint code-server \ + -e XDG_CONFIG_HOME=/workspace/.local -e XDG_DATA_HOME=/workspace/.local/share/ \ + -e XDG_RUNTIME_DIR=/workspace/.local "$image" \ + --auth none --bind-addr 0.0.0.0:8080 --user-data-dir /workspace \ + /workspace/mastering-app-package >/dev/null +for ((i=0; i<90; i++)); do + if docker exec "$name-server" curl -fsS http://127.0.0.1:8080/ >/dev/null; then + echo "Mastering init, Python imports, extensions, S3, kernel, shell and chart startup passed ($platform)" + exit 0 + fi + sleep 2 +done +docker logs "$name-server" +exit 1 diff --git a/ci/test-runtime.sh b/ci/test-runtime.sh index 9b263f1..b8fa53b 100644 --- a/ci/test-runtime.sh +++ b/ci/test-runtime.sh @@ -3,6 +3,9 @@ set -euo pipefail script_dir="$(cd "$(dirname "$0")" && pwd)" test "$(id -u)" = 1001 +test -w /opt/conda +test -w /opt/conda/bin +python -c 'import sys; assert sys.version_info[:2] == (3, 12)' test -w "$HATCH_CONFIG" test -w "$HATCH_DATA_DIR" test -w "$HATCH_CACHE_DIR" From 9190c35149afb7248900a8b7da47c5a8b7d6e155 Mon Sep 17 00:00:00 2001 From: mlongobardo-gituname Date: Mon, 2 Nov 2026 11:23:34 +0100 Subject: [PATCH 12/14] test(*sh): going on with tests --- CHANGELOG.md | 2 ++ README.md | 15 ++++++++++++++- ci/test-image.sh | 26 +------------------------- ci/test-runtime.sh | 1 - ci/test-startup.sh | 29 +++++++++++++++++++++++++++++ entrypoint.sh | 3 ++- 6 files changed, 48 insertions(+), 28 deletions(-) create mode 100644 ci/test-startup.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 29849c9..8a165fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - Migrate the runtime OS to Fedora 44 and code-server to 4.144.0, retaining the Python 3.12 Conda/Jupyter environment and PDE tools, identity and paths. +- Probe code-server `/healthz` with a 120-second readiness window so cold + editor startup does not stall the native proxy. - Retain Java 17 through Temurin and use Fedora's Nextcloud client packages. - Add an integration gate on both architectures for the mastering-app-package init script, extensions, Python environment, S3 setup and direct editor startup. diff --git a/README.md b/README.md index b5d6c07..8fe22f2 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,8 @@ Choose distinct tags to keep both images available. ### Running Locally -The proxy uses OAuth by default for ApplicationHub deployments. For standalone +The proxy uses `/healthz` for code-server readiness with a 120-second cold-start +window. It uses OAuth by default for ApplicationHub deployments. For standalone local use, disable proxy authentication and bind the service to localhost: ```bash @@ -104,6 +105,18 @@ The fixture in `ci/fixtures/mastering-app-package` was copied from the local workshop checkout at commit `49bb077b8753bddfb215827a04a3cf61d265811b`; refresh it when the workshop initialization changes. +Local validation on 2026-11-02 passed on ARM64: tool and dependency checks, +all three released EOAP extensions (including UML rendering), the current local +workshop `init.sh`, and both direct and proxied startup. AMD64 is covered by the +native CI job but was not built or executed locally during this migration. +The tested local image is `eoepca/pde-code-server:fedora-arm64`. + +To rerun only the proxy/Nextcloud-symlink startup check: + +```bash +bash ci/test-startup.sh eoepca/pde-code-server:fedora-arm64 linux/arm64 +``` + ## Installed Tooling ### Extension compatibility checks diff --git a/ci/test-image.sh b/ci/test-image.sh index a7dd5d8..90528eb 100644 --- a/ci/test-image.sh +++ b/ci/test-image.sh @@ -75,28 +75,4 @@ docker run --rm --platform "$platform" --entrypoint python3 \ -v "$repo_dir/ci:/opt/extension-ci:ro" -v "$vsix_dir:/opt/extension-vsix:ro" \ "$image" /opt/extension-ci/test-extensions.py test /opt/extension-vsix -container="" -# Invoked by the EXIT trap. -# shellcheck disable=SC2329 -cleanup() { - if [[ -n "$container" ]]; then - docker logs "$container" || true - docker rm -f "$container" >/dev/null - fi -} -trap cleanup EXIT -container="$(docker run -d --platform "$platform" \ - -e JHSINGLE_NATIVE_PROXY_AUTHTYPE=none "$image")" -deadline=$((SECONDS + 180)) -while ((SECONDS < deadline)); do - if docker exec "$container" curl -fsS --max-time 5 http://127.0.0.1:8888/ >/dev/null; then - docker exec "$container" test -L /workspace/drive - docker exec "$container" test "$(docker exec "$container" readlink /workspace/drive)" = /home/jovyan/drive - echo "Image, extensions, and entrypoint checks passed on $platform" - exit 0 - fi - [[ "$(docker inspect --format '{{.State.Running}}' "$container")" == true ]] || exit 1 - sleep 2 -done -echo "Entrypoint did not become ready within 180 seconds" >&2 -exit 1 +bash "$repo_dir/ci/test-startup.sh" "$image" "$platform" diff --git a/ci/test-runtime.sh b/ci/test-runtime.sh index b8fa53b..2c4d1ad 100644 --- a/ci/test-runtime.sh +++ b/ci/test-runtime.sh @@ -3,7 +3,6 @@ set -euo pipefail script_dir="$(cd "$(dirname "$0")" && pwd)" test "$(id -u)" = 1001 -test -w /opt/conda test -w /opt/conda/bin python -c 'import sys; assert sys.version_info[:2] == (3, 12)' test -w "$HATCH_CONFIG" diff --git a/ci/test-startup.sh b/ci/test-startup.sh new file mode 100644 index 0000000..73bdd91 --- /dev/null +++ b/ci/test-startup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail +image="${1:?image required}" +platform="${2:?platform required}" +container="" +# Invoked by the EXIT trap. +# shellcheck disable=SC2329 +cleanup() { + if [[ -n "$container" ]]; then + docker logs "$container" || true + docker rm -f "$container" >/dev/null + fi +} +trap cleanup EXIT +container="$(docker run -d --platform "$platform" \ + -e JHSINGLE_NATIVE_PROXY_AUTHTYPE=none "$image")" +deadline=$((SECONDS + 180)) +while ((SECONDS < deadline)); do + if docker exec "$container" curl -fsS --max-time 5 http://127.0.0.1:8888/ >/dev/null; then + docker exec "$container" test -L /workspace/drive + docker exec "$container" test "$(docker exec "$container" readlink /workspace/drive)" = /home/jovyan/drive + echo "Entrypoint checks passed on $platform" + exit 0 + fi + [[ "$(docker inspect --format '{{.State.Running}}' "$container")" == true ]] || exit 1 + sleep 2 +done +echo "Entrypoint did not become ready within 180 seconds" >&2 +exit 1 diff --git a/entrypoint.sh b/entrypoint.sh index 74c2059..4a6c2e4 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -47,4 +47,5 @@ if [ -z "$CODE_SERVER_WS" ]; then CODE_SERVER_WS="/workspace" fi -jhsingle-native-proxy --authtype "$auth_type" --port "$port" --destport "$destport" code-server {--}auth none {--}bind-addr "0.0.0.0:$destport" {--}user-data-dir /workspace "$CODE_SERVER_WS" +# Probe the lightweight endpoint: loading the editor page can outlast cold-start readiness. +jhsingle-native-proxy --ready-check-path /healthz --ready-timeout 120 --authtype "$auth_type" --port "$port" --destport "$destport" code-server {--}auth none {--}bind-addr "0.0.0.0:$destport" {--}user-data-dir /workspace "$CODE_SERVER_WS" From ce04b133536611b04c94a061016a20c7ff4ee8d3 Mon Sep 17 00:00:00 2001 From: mlongobardo-gituname Date: Mon, 2 Nov 2026 11:39:40 +0100 Subject: [PATCH 13/14] feat(Docker): trivy to 0.76.1 --- CHANGELOG.md | 20 +++++++++----------- Dockerfile | 2 +- README.md | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a165fb..486559d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,17 +4,8 @@ Notable project changes are recorded here for users and contributors. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). -## Unreleased -- Migrate the runtime OS to Fedora 44 and code-server to 4.144.0, retaining - the Python 3.12 Conda/Jupyter environment and PDE tools, identity and paths. -- Probe code-server `/healthz` with a 120-second readiness window so cold - editor startup does not stall the native proxy. -- Retain Java 17 through Temurin and use Fedora's Nextcloud client packages. -- Add an integration gate on both architectures for the mastering-app-package - init script, extensions, Python environment, S3 setup and direct editor startup. - -## [1.4.0] - 2026-10-30 +## [1.4.0] - 2026-11-02 ### Added - AMD64 and ARM64 binary downloads and native CI build/test jobs, with a combined @@ -46,7 +37,14 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - Use Trivy 0.74.0 for CI scans, matching the pinned image scanner; both platform assets are available, while the previous 0.50.2 binary asset URLs returned 404. ### Fixed - +- Migrate the runtime OS to Fedora 44 and code-server to 4.144.0, retaining + the Python 3.12 Conda/Jupyter environment and PDE tools, identity and paths. +- Probe code-server `/healthz` with a 120-second readiness window so cold + editor startup does not stall the native proxy. +- Retain Java 17 through Temurin and use Fedora's Nextcloud client packages. +- Add an integration gate on both architectures for the mastering-app-package + init script, extensions, Python environment, S3 setup and direct editor startup. +- Trivy Transition to 0.76.1 - Include the C++ compiler and XZ utilities required for the GDAL source build. - Validate proxy authentication modes and quote entrypoint arguments. - Install Calrissian from commit `a95d87dd213d51b11faea7c6def4ced46d466a4a` diff --git a/Dockerfile b/Dockerfile index b3b4fbc..c1218ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -162,7 +162,7 @@ RUN python -m venv /opt/hatch-venv && \ # ------------------------------------------------------------------- # trivy # ------------------------------------------------------------------- -ARG TRIVY_VERSION=0.74.0 +ARG TRIVY_VERSION=0.76.1 RUN case "${TARGETARCH}" in amd64) TRIVY_ARCH=64bit ;; arm64) TRIVY_ARCH=ARM64 ;; esac && \ curl -fsSL \ https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-${TRIVY_ARCH}.tar.gz \ diff --git a/README.md b/README.md index 8fe22f2..02669a7 100644 --- a/README.md +++ b/README.md @@ -329,7 +329,7 @@ ghcr.io//pde-code-server The registry owner corresponds to the GitHub repository owner (user or organization). -Images are built with Buildx. Each platform is tested, scanned with Trivy 0.74.0, +Images are built with Buildx. Each platform is tested, scanned with Trivy 0.76.1, and given its own SPDX JSON SBOM. After both platform jobs succeed, CI publishes the tested images and attaches each SBOM with ORAS to that architecture image's exact registry digest. It then publishes a combined multi-platform manifest under From 69206b5dbc0a74cfb2e539bd5180dfc0a6aa820f Mon Sep 17 00:00:00 2001 From: mlongobardo-gituname Date: Wed, 4 Nov 2026 09:35:24 +0100 Subject: [PATCH 14/14] feat(Docker): podman must work inside code-server --- .github/workflows/build-image.yml | 2 ++ CHANGELOG.md | 11 ++++++ Dockerfile | 22 ++++++++++-- README.md | 56 +++++++++++++++++++++++++++++++ Taskfile.yaml | 5 +++ ci/test-gdal.sh | 20 +++++++++++ ci/test-podman.sh | 29 ++++++++++++++++ ci/test-runtime.sh | 1 + 8 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 ci/test-gdal.sh create mode 100644 ci/test-podman.sh diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index b5d9d79..7d2c929 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -103,6 +103,8 @@ jobs: run: bash ci/test-image.sh pde-extension-test:ci "$PLATFORM" "$PWD/extension-vsix" - name: Test mastering-app-package initialization and chart startup run: bash ci/test-mastering.sh pde-extension-test:ci linux/${{ matrix.arch }} + - name: Test nested rootless Podman pull, build, and execution + run: bash ci/test-podman.sh pde-extension-test:ci linux/${{ matrix.arch }} - name: Save tested image run: docker save pde-extension-test:ci -o image.tar - name: Install pinned scan tooling diff --git a/CHANGELOG.md b/CHANGELOG.md index 486559d..43f2e05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ Notable project changes are recorded here for users and contributors. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## Unreleased + +- Remove GDAL's temporary build toolchain and unused development dependencies + in the build layer, retaining PROJ/data and linked runtime libraries. Add a + compressed-raster and reprojection regression check. + +- Give rootless Podman's UID/GID mapping helpers the setuid permissions needed + to unpack images containing additional user/group IDs, including Python slim. +- Test actual nested rootless Podman builds and non-root execution on both CI + architectures, using a disposable workspace volume. + ## [1.4.0] - 2026-11-02 ### Added diff --git a/Dockerfile b/Dockerfile index c1218ff..120977f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -172,7 +172,10 @@ RUN case "${TARGETARCH}" in amd64) TRIVY_ARCH=64bit ;; arm64) TRIVY_ARCH=ARM64 ; # GDAL # ------------------------------------------------------------------- ARG GDAL_VER=3.12.1 -RUN dnf install -y \ +# Keep runtime RPMs explicitly: the source-installed GDAL has no RPM dependency +# record, and removing proj-devel would otherwise also autoremove PROJ/data. +# Install and remove the build toolchain in this same layer to reclaim its size. +RUN set -eu && set -o pipefail && dnf install -y \ cmake gcc-c++ ninja-build xz proj-devel proj-data proj && \ dnf clean all && rm -rf /var/cache/dnf && \ cd /tmp && \ @@ -195,7 +198,15 @@ RUN dnf install -y \ echo /usr/local/lib64 >> /etc/ld.so.conf.d/gdal.conf && \ ldconfig && \ cd / && rm -rf /tmp/gdal-${GDAL_VER}* && \ - gdal-config --version + gdal-config --version && \ + ldd /usr/local/lib/libgdal.so > /tmp/gdal-runtime-libs && \ + ! grep -q 'not found' /tmp/gdal-runtime-libs && \ + dnf -y mark user proj proj-data xz && \ + awk '/=> \// {print $3}' /tmp/gdal-runtime-libs \ + | xargs -r rpm -qf --qf '%{NAME}\n' | sort -u | xargs -r dnf -y mark user && \ + dnf remove -y cmake gcc-c++ ninja-build proj-devel && \ + dnf clean all && rm -rf /var/cache/dnf /tmp/gdal-runtime-libs && \ + gdalinfo --version # ------------------------------------------------------------------- # tomlq requires the Python yq package, whose console launcher shadows the @@ -216,6 +227,13 @@ RUN chmod 755 /usr/local/bin/nc-sync COPY entrypoint.sh /opt/entrypoint.sh RUN chmod +x /opt/entrypoint.sh +# Fedora's shadow-utils helpers are installed without setuid/file capabilities. +# Rootless Podman needs these narrowly scoped helpers to apply jovyan's ranges. +RUN test -x /usr/bin/newuidmap && test -x /usr/bin/newgidmap && \ + grep -q '^jovyan:' /etc/subuid && grep -q '^jovyan:' /etc/subgid && \ + chown root:root /usr/bin/newuidmap /usr/bin/newgidmap && \ + chmod 4755 /usr/bin/newuidmap /usr/bin/newgidmap + USER ${USER} ENV GDAL_CONFIG=/usr/local/bin/gdal-config diff --git a/README.md b/README.md index 02669a7..a000ded 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,49 @@ container. With the project directory mounted at `/workspace`, this symlink also appears in the project directory on the host; it is generated runtime state and should not be committed. +## Building containers with rootless Podman + +The image runs Podman as `jovyan` (UID 1001, GID 100). It provides subordinate +UID/GID ranges in `/etc/subuid` and `/etc/subgid`, and root-owned setuid +`newuidmap`/`newgidmap` helpers. The outer container must permit nested user +namespaces and helper privilege escalation. The APEx privileged pod setup can +provide this; overriding the user to UID 1000 does not match the image account. +Keep the main container and init-container identity aligned with UID 1001. + +For an existing Fedora image affected by missing helper privileges, run these +commands in the code-server terminal **as jovyan (UID 1001)**: + +```bash +sudo chown root:root /usr/bin/newuidmap /usr/bin/newgidmap +sudo chmod 4755 /usr/bin/newuidmap /usr/bin/newgidmap +podman system migrate +podman unshare cat /proc/self/uid_map /proc/self/gid_map +podman pull docker.io/library/python:3.12-slim +``` + +`podman system migrate` stops that user's running Podman containers and replaces +the cached user namespace. Each mapping should include the user's own ID plus +at least 65,536 subordinate IDs. If the ranges are absent or the actual UID is +1000, correct the deployment identity/ranges first. If helpers still report +`Operation not permitted`, inspect the outer pod's privilege-escalation and +user-namespace restrictions. Do not use `podman system reset` for this repair; +it deletes container storage. The terminal repair lasts only until the outer +container is replaced; rebuild the image for the persistent fix. + +Keep Podman storage on the mounted workspace volume. Nested overlay storage in +Docker's writable container layer produced an `exec ... Invalid argument` error +in local testing; an independent workspace volume worked with the default driver. +The regression test uses an anonymous workspace volume and a privileged outer +container, then pulls Python slim, builds a layer with UID 1234/GID 42 ownership, +and runs it as that non-root user. CI runs it on both architectures: + +```bash +task test:podman IMAGE=eoepca/pde-code-server:podman-fix-arm64 PLATFORM=linux/arm64 +``` + +This test passed locally on ARM64 on 2026-11-04 using the prior validated image +plus the helper-permissions fix. AMD64 execution remains covered by CI. + ## Mastering Application Package compatibility The migration preserves `jovyan` (1001:100), `HOME=/workspace`, passwordless sudo, @@ -304,6 +347,19 @@ Installed via pip (Python 3.12): dependencies, plugins, or additional Python versions may still need network access; the offline smoke check only reads local project metadata. +### GDAL build cleanup + +GDAL is compiled from source, then CMake, GCC/G++, Ninja, PROJ development +headers, and their unused dependencies are removed in the same image layer. +The ARM64 cleanup test removed 53 RPMs (about 348 MiB of installed files). +Compressed image savings vary. PROJ tools and coordinate grids, `xz`, GDAL's +linked runtime libraries, and GDAL's own headers/`gdal-config` remain installed. +CI checks compressed GeoTIFF creation and EPSG reprojection after cleanup. + +Compilers are no longer preinstalled for source builds inside the development +environment. If a project needs them, install the required development packages +with `sudo dnf install gcc-c++ cmake ninja-build proj-devel`. + ### User & Runtime Environment - User: jovyan diff --git a/Taskfile.yaml b/Taskfile.yaml index 08fe992..8df4bed 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -38,3 +38,8 @@ tasks: desc: Run the workshop init script and chart startup against disposable LocalStack and storage cmds: - bash ci/test-mastering.sh {{.IMAGE | shellQuote}} {{.PLATFORM | shellQuote}} + + test:podman: + desc: Test rootless Podman builds inside a disposable privileged container and workspace volume + cmds: + - bash ci/test-podman.sh {{.IMAGE | shellQuote}} {{.PLATFORM | shellQuote}} diff --git a/ci/test-gdal.sh b/ci/test-gdal.sh new file mode 100644 index 0000000..132341d --- /dev/null +++ b/ci/test-gdal.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Exercise GDAL's shared libraries, raster codecs, and PROJ data after cleanup. +set -euo pipefail +fixture_dir="$(mktemp -d)" +trap 'rm -rf "$fixture_dir"' EXIT +gdal-config --version +gdal_create -q -of GTiff -outsize 8 8 -bands 1 -burn 7 \ + -co COMPRESS=DEFLATE -a_srs EPSG:4326 -a_ullr 10 45 11 44 \ + "$fixture_dir/source.tif" +gdalwarp -q -t_srs EPSG:3857 -co COMPRESS=LZW \ + "$fixture_dir/source.tif" "$fixture_dir/projected.tif" +gdalinfo -json -stats "$fixture_dir/projected.tif" | python -c ' +import json, sys +info = json.load(sys.stdin) +assert info["driverShortName"] == "GTiff", info +assert "3857" in info["coordinateSystem"]["wkt"], info +band = info["bands"][0] +assert band["minimum"] == 7 and band["maximum"] == 7, band +print("GDAL compressed GeoTIFF and EPSG reprojection checks passed") +' diff --git a/ci/test-podman.sh b/ci/test-podman.sh new file mode 100644 index 0000000..df08789 --- /dev/null +++ b/ci/test-podman.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Exercise rootless Podman inside a disposable privileged outer container. +# An anonymous workspace volume mirrors a PVC and avoids overlay-on-overlay. +# Docker removes that volume with the container; no host Docker socket is mounted. +set -euo pipefail +image="${1:?image required}" +platform="${2:?platform required}" +docker run --rm -i --privileged --platform "$platform" -v /workspace --entrypoint bash "$image" -se <<'SCRIPT' +set -euo pipefail +test "$(id -u)" = 1001 +test -u /usr/bin/newuidmap +test -u /usr/bin/newgidmap +# Require subordinate ranges, not just a single-ID fallback namespace. +podman unshare cat /proc/self/uid_map /proc/self/gid_map | python -c ' +import sys +rows = [list(map(int, line.split())) for line in sys.stdin] +assert len(rows) == 4 and rows[1][2] >= 65536 and rows[3][2] >= 65536, rows +' +context="$(mktemp -d)" +trap 'rm -rf "$context"' EXIT +cat > "$context/Containerfile" <<'CONTAINERFILE' +FROM docker.io/library/python:3.12-slim +RUN python -c 'import os; assert os.stat("/etc/gshadow").st_gid == 42; open("/mapped-owner", "w").close(); os.chown("/mapped-owner", 1234, 42)' +USER 1234:42 +CMD ["python", "-c", "import os; s=os.stat('/mapped-owner'); assert (os.getuid(),os.getgid(),s.st_uid,s.st_gid)==(1234,42,1234,42); print('Nested build and non-root execution passed')"] +CONTAINERFILE +podman build --tag localhost/pde-podman-test "$context" +podman run --rm localhost/pde-podman-test +SCRIPT diff --git a/ci/test-runtime.sh b/ci/test-runtime.sh index 2c4d1ad..41d6905 100644 --- a/ci/test-runtime.sh +++ b/ci/test-runtime.sh @@ -69,4 +69,5 @@ for flag in --dask-gateway-url --dask-script-configmap; do grep -Fq -- "$flag" "$fixture_dir/calrissian-help.txt" done bash "$script_dir/test-yq.sh" +bash "$script_dir/test-gdal.sh" echo 'Offline Hatch, Calrissian package, and YAML/TOML checks passed (no Kubernetes Dask execution tested).'