diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 6d18df5..7d2c929 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -3,155 +3,176 @@ name: build on: push: branches: [develop, main] - tags: - - 'v*' + tags: ['v*'] + pull_request: + branches: [develop, main] workflow_dispatch: inputs: GDAL_VER: - description: "GDAL version to use for Docker build" + description: GDAL version to use for Docker build required: false - default: "3.12.1" + default: '3.12.1' + +permissions: + contents: read + jobs: - # ------------------------------------------------------------ - # Resolve release tag from ref - # ------------------------------------------------------------ release-meta: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: tag: ${{ steps.meta.outputs.tag }} + image: ${{ steps.meta.outputs.image }} steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install PyYAML - id: meta + env: + EVENT_NAME: ${{ github.event_name }} run: | set -euo pipefail - if [[ "${GITHUB_REF}" == "refs/heads/develop" ]]; then - TAG="latest-dev" - elif [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then - TAG="latest" - elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then - TAG="${GITHUB_REF#refs/tags/v}" - - # -------------------------------------------------- - # Validate against release.yaml - # -------------------------------------------------- - RELEASE_VERSION=$(yq -r '.image_version' release.yaml) - - if [[ -z "${RELEASE_VERSION}" || "${RELEASE_VERSION}" == "null" ]]; then - echo "image_version not found in release.yaml" - exit 1 - fi - - if [[ "${TAG}" != "${RELEASE_VERSION}" ]]; then - echo "Git tag v${TAG} does not match release.yaml version ${RELEASE_VERSION}" - exit 1 - fi - else - echo "Unsupported ref ${GITHUB_REF}" - exit 1 - fi - echo "tag=${TAG}" >> "$GITHUB_OUTPUT" - - # ------------------------------------------------------------ - # Build, scan, SBOM, push - # ------------------------------------------------------------ - build-scan-push: - needs: release-meta - runs-on: ubuntu-latest - + python - <<'PY' + import os, yaml + with open('release.yaml') as stream: + release = yaml.safe_load(stream) + ref = os.environ['GITHUB_REF'] + if os.environ['EVENT_NAME'] == 'pull_request': + tag = 'pr-test' + elif ref == 'refs/heads/develop': + tag = 'latest-dev' + elif ref == 'refs/heads/main': + tag = 'latest' + elif ref.startswith('refs/tags/v'): + tag = ref.removeprefix('refs/tags/v') + if tag != str(release['image_version']): + raise SystemExit('Git tag does not match release.yaml image_version') + else: + raise SystemExit(f'Unsupported release ref: {ref}') + image = '/'.join(release[key] for key in ('image_registry', 'image_prefix', 'image_name')) + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write(f'tag={tag}\nimage={image}\n') + PY + + extension-releases: + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - - name: Install tooling - run: | - sudo apt-get update - sudo apt-get install -y jq skopeo - curl -sL https://github.com/oras-project/oras/releases/download/v1.2.2/oras_1.2.2_linux_amd64.tar.gz | tar -xz oras - sudo mv oras /usr/local/bin/oras - - - name: Configure Docker Hub auth for Kaniko - run: | - if [[ -n "${{ secrets.dockerhub-username }}" ]]; then - mkdir -p $HOME/.docker - cat > $HOME/.docker/config.json <> $GITHUB_ENV - - name: Build image with Kaniko (to tar) - uses: docker://gcr.io/kaniko-project/executor:debug + - name: Download the same released VSIXs for both architectures env: - DOCKER_CONFIG: /github/home/.docker + GITHUB_TOKEN: ${{ github.token }} + run: python3 ci/test-extensions.py download extension-vsix + - uses: actions/upload-artifact@v4 with: - args: > - --context . - --dockerfile Dockerfile - --build-arg GDAL_VER=${{ env.GDAL_VER }} - --no-push - --tar-path image.tar - - # -------------------------------------------------------- - # Trivy scan (relaxed for now) - # -------------------------------------------------------- - - name: Trivy scan (debug mode) - uses: docker://aquasec/trivy:0.50.2 - env: - TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db + name: extension-vsix + path: extension-vsix/ + + build-test: + needs: [release-meta, extension-releases] + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: actions/download-artifact@v4 with: - args: > - image - --input image.tar - --severity HIGH,CRITICAL - --exit-code 0 - --no-progress - - - name: Generate SBOM (SPDX JSON) - uses: docker://aquasec/trivy:0.50.2 + name: extension-vsix + path: extension-vsix + - name: Build and load architecture image + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/${{ matrix.arch }} + load: true + tags: pde-extension-test:ci + build-args: GDAL_VER=${{ inputs.GDAL_VER || '3.12.1' }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + provenance: false + - name: Test tools, identity, extensions, and entrypoint + env: + PLATFORM: linux/${{ matrix.arch }} + run: bash ci/test-image.sh pde-extension-test:ci "$PLATFORM" "$PWD/extension-vsix" + - name: Test mastering-app-package initialization and chart startup + run: bash ci/test-mastering.sh pde-extension-test:ci linux/${{ matrix.arch }} + - name: Test nested rootless Podman pull, build, and execution + run: bash ci/test-podman.sh pde-extension-test:ci linux/${{ matrix.arch }} + - name: Save tested image + run: docker save pde-extension-test:ci -o image.tar + - name: Install pinned scan tooling + env: + ARCH: ${{ matrix.arch }} + run: | + set -euo pipefail + case "$ARCH" in amd64) TRIVY_ARCH=64bit ;; arm64) TRIVY_ARCH=ARM64 ;; esac + curl -fsSL "https://github.com/aquasecurity/trivy/releases/download/v0.74.0/trivy_0.74.0_Linux-${TRIVY_ARCH}.tar.gz" | tar -xz trivy + sudo install trivy /usr/local/bin/trivy + - name: Trivy scan (relaxed for now) env: TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db + run: trivy image --input image.tar --severity HIGH,CRITICAL --exit-code 0 --no-progress + - name: Generate architecture SBOM + run: trivy image --input image.tar --format spdx-json --output sbom.spdx.json + - name: Upload tested image and its SBOM + uses: actions/upload-artifact@v4 with: - args: > - image - --input image.tar - --format spdx-json - --output sbom.spdx.json - - # -------------------------------------------------------- - # Push image and attach SBOM - # -------------------------------------------------------- - - name: Push image with Skopeo - id: push + name: image-${{ matrix.arch }} + path: | + image.tar + sbom.spdx.json + retention-days: 1 + + # Neither architecture is published until both native build/test jobs pass. + publish: + if: github.event_name != 'pull_request' + needs: [release-meta, build-test] + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: image-* + path: images + - uses: docker/setup-buildx-action@v3 + - name: Install publishing tools run: | set -euo pipefail - - TAG=${{ needs.release-meta.outputs.tag }} - REGISTRY=$(yq -r '.image_registry' release.yaml) - IMAGE_PREFIX=$(yq -r '.image_prefix' release.yaml) - IMAGE=$(yq -r '.image_name' release.yaml) - - IMAGE_REF="${REGISTRY}/${IMAGE_PREFIX}/${IMAGE}:${TAG}" - - skopeo copy \ - --dest-creds="${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - docker-archive:image.tar \ - docker://${IMAGE_REF} - - DIGEST=$(skopeo inspect \ - --creds="${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - docker://${IMAGE_REF} \ - | jq -r .Digest) - - echo "${REGISTRY}/${IMAGE_PREFIX}/${IMAGE}@${DIGEST}" > image.digest + sudo apt-get update + sudo apt-get install -y skopeo jq + ARCH=$(dpkg --print-architecture) + curl -fsSL "https://github.com/oras-project/oras/releases/download/v1.2.2/oras_1.2.2_linux_${ARCH}.tar.gz" | tar -xz oras + sudo install oras /usr/local/bin/oras + - name: Publish architecture images, attach SBOMs, and create combined manifest + env: + IMAGE: ${{ needs.release-meta.outputs.image }} + TAG: ${{ needs.release-meta.outputs.tag }} + REGISTRY_TOKEN: ${{ github.token }} + REGISTRY_USER: ${{ github.actor }} + run: | + set -euo pipefail + REGISTRY="${IMAGE%%/*}" + printf '%s' "$REGISTRY_TOKEN" | docker login "$REGISTRY" --username "$REGISTRY_USER" --password-stdin + printf '%s' "$REGISTRY_TOKEN" | oras login "$REGISTRY" --username "$REGISTRY_USER" --password-stdin + refs=() + for arch in amd64 arm64; do + ARCH_REF="${IMAGE}:${TAG}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${arch}" + skopeo copy --dest-creds "${REGISTRY_USER}:${REGISTRY_TOKEN}" \ + "docker-archive:images/image-${arch}/image.tar" "docker://${ARCH_REF}" + DIGEST=$(skopeo inspect --creds "${REGISTRY_USER}:${REGISTRY_TOKEN}" "docker://${ARCH_REF}" | jq -r .Digest) + # Attach each SBOM to the exact image digest it describes, not the index. + (cd "images/image-${arch}" && oras attach --artifact-type application/spdx+json \ + "${IMAGE}@${DIGEST}" sbom.spdx.json:application/spdx+json) + refs+=("${IMAGE}@${DIGEST}") + done + docker buildx imagetools create --tag "${IMAGE}:${TAG}" "${refs[@]}" + docker buildx imagetools inspect "${IMAGE}:${TAG}" diff --git a/CHANGELOG.md b/CHANGELOG.md index a18fb8b..43f2e05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,74 @@ Notable project changes are recorded here for users and contributors. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## Unreleased + +- Remove GDAL's temporary build toolchain and unused development dependencies + in the build layer, retaining PROJ/data and linked runtime libraries. Add a + compressed-raster and reprojection regression check. + +- Give rootless Podman's UID/GID mapping helpers the setuid permissions needed + to unpack images containing additional user/group IDs, including Python slim. +- Test actual nested rootless Podman builds and non-root execution on both CI + architectures, using a disposable workspace volume. + +## [1.4.0] - 2026-11-02 + +### Added +- AMD64 and ARM64 binary downloads and native CI build/test jobs, with a combined + platform manifest and architecture-specific SBOM attachments. +- Local Buildx tasks for either platform and a multi-platform OCI archive, plus + checks for tool execution, user identity, extensions, and entrypoint startup. +- Pull-request CI validation on both architectures, using the same downloaded + extension releases for AMD64 and ARM64. +- Pinned CWL and transpiler tooling for EOAP extension runtime requirements. +- CI installation checks for the latest released EOAP Validator, CWL Metadata + Editor, and CWL UML Viewer VSIXs, with Python dependency checks, validator + environment setup, and rendering checks for all available UML diagram types. + Failures block image publication; tested release metadata is saved as an artifact. +- A Taskfile for local AMD64 image builds with image tag and GDAL version overrides. +- Standalone local operation through `JHSINGLE_NATIVE_PROXY_AUTHTYPE=none`, + with OAuth retained as the default. + +### Changed + +- Install Node.js 22.15.0 through NVM 0.40.3 instead of distro Node.js packages. +- Require the OpenJDK 17 headless JDK, replacing the optional Java runtime and + removing the `INSTALL_JRE` build argument. +- Install Calrissian 0.18.1 and its pinned `cwl-utils` 0.40 dependency in an + isolated virtual environment to avoid conflicts with the image's CWL tooling. +- Document local image builds, standalone usage, and extension compatibility checks. +- Replace Kaniko builds with Docker Buildx and publish the tested images only + after both native architecture build/test jobs pass. +- Align the `UID` environment variable with the actual `jovyan` UID of 1001. +- Use Trivy 0.74.0 for CI scans, matching the pinned image scanner; both platform + assets are available, while the previous 0.50.2 binary asset URLs returned 404. +### Fixed +- Migrate the runtime OS to Fedora 44 and code-server to 4.144.0, retaining + the Python 3.12 Conda/Jupyter environment and PDE tools, identity and paths. +- Probe code-server `/healthz` with a 120-second readiness window so cold + editor startup does not stall the native proxy. +- Retain Java 17 through Temurin and use Fedora's Nextcloud client packages. +- Add an integration gate on both architectures for the mastering-app-package + init script, extensions, Python environment, S3 setup and direct editor startup. +- Trivy Transition to 0.76.1 +- Include the C++ compiler and XZ utilities required for the GDAL source build. +- Validate proxy authentication modes and quote entrypoint arguments. +- Install Calrissian from commit `a95d87dd213d51b11faea7c6def4ced46d466a4a` + with compatible isolated dependencies and bundled Dask resources. +- Preinstall Hatch 1.16.5 and its runtime dependencies in an isolated environment, + avoiding a first-use download and fixing interpreter discovery with virtualenv + 21; add offline Hatch environment creation/activation and Calrissian smoke checks. +- Set `VIRTUAL_ENV_DISABLE_PROMPT=1` for the Bash kernel only: when `~/.bashrc` activates a + virtual environment, activating another one in a notebook cell restored the pre-kernel + prompt and the cell never finished. Terminals keep the `(venv)` prompt. +- Ensure the default workspace is writable by jovyan for tool configuration, and + remove the root-owned build cache that blocked `uv` and pip caches there. +- Store Hatch configuration, data, and caches in writable directories under + `/home/jovyan`, including when a volume is mounted at `/workspace`. +- Make Mike Farah's pinned `yq` the default command while retaining the Python + `yq` dependency and `tomlq`/`xq` launchers; smoke-test YAML and TOML processing. ## [1.3.0] - 2026-10-06 ### Added @@ -65,6 +133,8 @@ at the earliest Git tag. tooling versions, and install code-server 4.108.1 from its release tarball. - Refactor the CI container build and publishing workflow. -[Unreleased]: https://github.com/EOEPCA/pde-code-server/compare/v1.2.0...HEAD +[Unreleased]: https://github.com/EOEPCA/pde-code-server/compare/v1.4.0...HEAD +[1.4.0]: https://github.com/EOEPCA/pde-code-server/compare/v1.3.0...v1.4.0 +[1.3.0]: https://github.com/EOEPCA/pde-code-server/compare/v1.2.0...v1.3.0 [1.2.0]: https://github.com/EOEPCA/pde-code-server/compare/v1.1.0...v1.2.0 [1.1.0]: https://github.com/EOEPCA/pde-code-server/releases/tag/v1.1.0 diff --git a/Dockerfile b/Dockerfile index 72b3b4c..120977f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,29 @@ -FROM quay.io/jupyter/base-notebook:python-3.12 - -ENV DEBIAN_FRONTEND=noninteractive \ - USER=jovyan \ - UID=1000 \ +# Retain the complete Python/Jupyter/Conda toolchain while moving the OS to Fedora. +FROM quay.io/jupyter/base-notebook:python-3.12 AS python-runtime +FROM eclipse-temurin:17-jdk AS java-runtime +FROM fedora:44 + +COPY --from=java-runtime /opt/java/openjdk /opt/java/openjdk +COPY --from=python-runtime /opt/conda /opt/conda +COPY --from=python-runtime /usr/local/bin/start*.py /usr/local/bin/ +COPY --from=python-runtime /usr/local/bin/start*.sh /usr/local/bin/ +COPY --from=python-runtime /usr/local/bin/fix-permissions /usr/local/bin/ +ENV JAVA_HOME=/opt/java/openjdk \ + CONDA_DIR=/opt/conda \ + NB_USER=jovyan \ + NB_UID=1001 \ + NB_GID=100 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + SHELL=/bin/bash \ + PATH=/opt/java/openjdk/bin:/opt/conda/bin:${PATH} + +# BuildKit supplies amd64 or arm64 for each target platform. +ARG TARGETARCH +RUN case "${TARGETARCH}" in amd64|arm64) ;; *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; esac + +ENV USER=jovyan \ + UID=1001 \ GID=100 \ HOME=/workspace @@ -11,42 +32,46 @@ USER root # ------------------------------------------------------------------- # Base system packages (runtime only) # ------------------------------------------------------------------- -RUN apt-get update && apt-get install -y \ - ca-certificates \ - curl \ - git \ - nodejs \ - npm \ - nano \ - net-tools \ - sudo \ - wget \ - graphviz \ - file \ - tree \ - podman \ - skopeo \ - nextcloud-desktop-cmd=3.11.0-1.1build4 \ - && rm -rf /var/lib/apt/lists/* - -# Optional Java runtime (enable with --build-arg INSTALL_JRE=true) -ARG INSTALL_JRE=true -RUN if [ "${INSTALL_JRE}" = "true" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends openjdk-17-jre-headless && \ - rm -rf /var/lib/apt/lists/*; \ - fi - -RUN usermod -u 1001 ${USER} && \ - echo "${USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/${USER} +RUN dnf upgrade -y && dnf install -y \ + ca-certificates curl git git-lfs nano net-tools sudo wget graphviz file tree \ + podman skopeo fontconfig freetype nextcloud-client \ + htop openssh-clients procps-ng zsh dumb-init glibc-langpack-en \ + findutils which tar gzip bzip2 unzip hostname shadow-utils \ + && dnf clean all && rm -rf /var/cache/dnf + +RUN groupadd -g 100 users 2>/dev/null || getent group 100 +RUN useradd -m -u 1001 -g 100 -s /bin/bash jovyan && \ + echo "jovyan ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/jovyan && \ + chmod 440 /etc/sudoers.d/jovyan && \ + git lfs install --system + +# ------------------------------------------------------------------- +# NVM and Node.js +# ------------------------------------------------------------------- +ARG NVM_VERSION=v0.40.3 +ARG NODE_VERSION=22.15.0 + +ENV NVM_DIR=/opt/nvm + +RUN mkdir -p "${NVM_DIR}" && \ + curl -fsSL \ + "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" \ + | bash && \ + bash -c '. "${NVM_DIR}/nvm.sh" && nvm install "${NODE_VERSION}" && nvm alias default "${NODE_VERSION}"' && \ + chown -R jovyan:users "${NVM_DIR}" + +ENV PATH="${NVM_DIR}/versions/node/v${NODE_VERSION}/bin:${PATH}" + +RUN printf '\nexport NVM_DIR="/opt/nvm"\n[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"\n' \ + >> /etc/bashrc # ------------------------------------------------------------------- # code-server # ------------------------------------------------------------------- -ARG CODE_RELEASE=4.138.0 +ARG CODE_RELEASE=4.144.0 RUN mkdir -p /opt/code-server && \ curl -fsSL \ - "https://github.com/coder/code-server/releases/download/v${CODE_RELEASE}/code-server-${CODE_RELEASE}-linux-amd64.tar.gz" \ + "https://github.com/coder/code-server/releases/download/v${CODE_RELEASE}/code-server-${CODE_RELEASE}-linux-${TARGETARCH}.tar.gz" \ | tar -xz --strip-components=1 -C /opt/code-server ENV PATH="/opt/code-server/bin:${PATH}" @@ -56,29 +81,32 @@ ENV PATH="/opt/code-server/bin:${PATH}" # ------------------------------------------------------------------- ARG KUBECTL_VERSION=v1.29.3 RUN curl -fsSL \ - https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl \ + https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl \ -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl ARG TASK_VERSION=v3.41.0 RUN curl -fsSL \ - https://github.com/go-task/task/releases/download/${TASK_VERSION}/task_linux_amd64.tar.gz \ + https://github.com/go-task/task/releases/download/${TASK_VERSION}/task_linux_${TARGETARCH}.tar.gz \ | tar -xz -C /usr/local/bin task && chmod +x /usr/local/bin/task ARG SKAFFOLD_VERSION=2.17.1 RUN curl -fsSL \ - https://storage.googleapis.com/skaffold/releases/v${SKAFFOLD_VERSION}/skaffold-linux-amd64 \ + https://storage.googleapis.com/skaffold/releases/v${SKAFFOLD_VERSION}/skaffold-linux-${TARGETARCH} \ -o /usr/local/bin/skaffold && chmod +x /usr/local/bin/skaffold ARG ORAS_VERSION=1.3.0 RUN curl -fsSL \ - https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz \ + https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_${TARGETARCH}.tar.gz \ | tar -xz -C /usr/local/bin oras && chmod +x /usr/local/bin/oras # ------------------------------------------------------------------- # Python tooling # ------------------------------------------------------------------- -ARG CALRISSIAN_VERSION=0.18.1 +ARG CALRISSIAN_COMMIT=a95d87dd213d51b11faea7c6def4ced46d466a4a +COPY requirements-cwl-uml.txt /tmp/requirements-cwl-uml.txt + RUN pip install --no-cache-dir \ + -r /tmp/requirements-cwl-uml.txt \ awscli \ awscli-plugin-endpoint \ "jhsingle-native-proxy>=0.0.9" \ @@ -86,52 +114,73 @@ RUN pip install --no-cache-dir \ tomlq \ uv \ cwltool \ - cwltest \ - "calrissian==${CALRISSIAN_VERSION}" && \ + cwltest && \ python -m bash_kernel.install +# Bash kernels detect a finished cell by their prompt. When ~/.bashrc activates a virtual +# environment, activating another one in a cell restores the pre-kernel prompt and the cell +# never finishes; keep venv activation away from the kernel prompt (terminals keep theirs). +RUN python -c "import json, pathlib; p = pathlib.Path('/usr/local/share/jupyter/kernels/bash/kernel.json'); spec = json.loads(p.read_text()); spec.setdefault('env', {})['VIRTUAL_ENV_DISABLE_PROMPT'] = '1'; p.write_text(json.dumps(spec))" + +# Resolve the commit's cwltool/dependency requirements together, independently +# of the global CWL tools, rather than constraining cwl-utils to the old release. +RUN python -m venv /opt/calrissian-venv && \ + /opt/calrissian-venv/bin/pip install --no-cache-dir \ + "calrissian @ git+https://github.com/duke-gcb/calrissian.git@${CALRISSIAN_COMMIT}" && \ + /opt/calrissian-venv/bin/python -m pip check && \ + ln -s /opt/calrissian-venv/bin/calrissian /usr/local/bin/calrissian + # ------------------------------------------------------------------- # yq / jq # ------------------------------------------------------------------- ARG YQ_VERSION=v4.45.1 RUN curl -fsSL \ - https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 \ + https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${TARGETARCH} \ -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq ARG JQ_VERSION=jq-1.8.1 RUN curl -fsSL \ - https://github.com/jqlang/jq/releases/download/${JQ_VERSION}/jq-linux-amd64 \ + https://github.com/jqlang/jq/releases/download/${JQ_VERSION}/jq-linux-${TARGETARCH} \ -o /usr/local/bin/jq && chmod +x /usr/local/bin/jq # ------------------------------------------------------------------- # hatch # ------------------------------------------------------------------- -ARG HATCH_VERSION=1.16.2 -RUN curl -fsSL \ - https://github.com/pypa/hatch/releases/download/hatch-v${HATCH_VERSION}/hatch-x86_64-unknown-linux-gnu.tar.gz \ - | tar -xz -C /usr/local/bin hatch && chmod +x /usr/local/bin/hatch +ARG HATCH_VERSION=1.16.5 +ENV HATCH_CONFIG=/home/jovyan/.config/hatch/config.toml \ + HATCH_DATA_DIR=/home/jovyan/.local/share/hatch \ + HATCH_CACHE_DIR=/home/jovyan/.cache/hatch +# The release launcher fetches its runtime on first use. Install Hatch and all +# its dependencies now, using the base image's native Python on either platform. +RUN python -m venv /opt/hatch-venv && \ + /opt/hatch-venv/bin/pip install --no-cache-dir "hatch==${HATCH_VERSION}" && \ + /opt/hatch-venv/bin/python -m pip check && \ + ln -s /opt/hatch-venv/bin/hatch /usr/local/bin/hatch && \ + install -d -o ${USER} -g users /home/jovyan/.config/hatch "${HATCH_DATA_DIR}" "${HATCH_CACHE_DIR}" && \ + install -o ${USER} -g users -m 644 /dev/null "${HATCH_CONFIG}" # ------------------------------------------------------------------- # trivy # ------------------------------------------------------------------- -ARG TRIVY_VERSION=0.74.0 -RUN curl -fsSL \ - https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb \ - -o /tmp/trivy.deb && \ - dpkg -i /tmp/trivy.deb && \ - rm /tmp/trivy.deb +ARG TRIVY_VERSION=0.76.1 +RUN case "${TARGETARCH}" in amd64) TRIVY_ARCH=64bit ;; arm64) TRIVY_ARCH=ARM64 ;; esac && \ + curl -fsSL \ + https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-${TRIVY_ARCH}.tar.gz \ + | tar -xz -C /usr/local/bin trivy # ------------------------------------------------------------------- # GDAL # ------------------------------------------------------------------- ARG GDAL_VER=3.12.1 -RUN apt-get update && apt-get install -y \ - cmake ninja-build libproj-dev proj-data proj-bin && \ - rm -rf /var/lib/apt/lists/* && \ - set -e && \ +# Keep runtime RPMs explicitly: the source-installed GDAL has no RPM dependency +# record, and removing proj-devel would otherwise also autoremove PROJ/data. +# Install and remove the build toolchain in this same layer to reclaim its size. +RUN set -eu && set -o pipefail && dnf install -y \ + cmake gcc-c++ ninja-build xz proj-devel proj-data proj && \ + dnf clean all && rm -rf /var/cache/dnf && \ cd /tmp && \ - curl -fsSL -o gdal-${GDAL_VER}.tar.xz https://download.osgeo.org/gdal/${GDAL_VER}/gdal-${GDAL_VER}.tar.xz \ - || curl -fsSL -o gdal-${GDAL_VER}.tar.gz https://download.osgeo.org/gdal/${GDAL_VER}/gdal-${GDAL_VER}.tar.gz && \ + (curl -fsSL -o gdal-${GDAL_VER}.tar.xz https://download.osgeo.org/gdal/${GDAL_VER}/gdal-${GDAL_VER}.tar.xz \ + || { rm -f gdal-${GDAL_VER}.tar.xz; curl -fsSL -o gdal-${GDAL_VER}.tar.gz https://download.osgeo.org/gdal/${GDAL_VER}/gdal-${GDAL_VER}.tar.gz; }) && \ if [ -f gdal-${GDAL_VER}.tar.xz ]; then \ tar -xJf gdal-${GDAL_VER}.tar.xz; \ else \ @@ -141,12 +190,33 @@ RUN apt-get update && apt-get install -y \ mkdir build && cd build && \ cmake -G Ninja ../ \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/usr/local && \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_INSTALL_LIBDIR=lib && \ cmake --build . -- -j"$(nproc)" && \ cmake --install . && \ + echo /usr/local/lib > /etc/ld.so.conf.d/gdal.conf && \ + echo /usr/local/lib64 >> /etc/ld.so.conf.d/gdal.conf && \ ldconfig && \ - rm -rf /tmp/gdal-${GDAL_VER}* && \ - gdal-config --version + cd / && rm -rf /tmp/gdal-${GDAL_VER}* && \ + gdal-config --version && \ + ldd /usr/local/lib/libgdal.so > /tmp/gdal-runtime-libs && \ + ! grep -q 'not found' /tmp/gdal-runtime-libs && \ + dnf -y mark user proj proj-data xz && \ + awk '/=> \// {print $3}' /tmp/gdal-runtime-libs \ + | xargs -r rpm -qf --qf '%{NAME}\n' | sort -u | xargs -r dnf -y mark user && \ + dnf remove -y cmake gcc-c++ ninja-build proj-devel && \ + dnf clean all && rm -rf /var/cache/dnf /tmp/gdal-runtime-libs && \ + gdalinfo --version + +# ------------------------------------------------------------------- +# tomlq requires the Python yq package, whose console launcher shadows the +# pinned Mike Farah binary on Conda's PATH. Keep its modules and tomlq/xq +# launchers; replace only the conflicting command, without changing PATH. +# Prepare the default workspace for non-root tools after runtime installation, +# dropping the root-owned cache that build-isolated pip runs leave in HOME. +RUN ln -sf /usr/local/bin/yq /opt/conda/bin/yq && \ + rm -rf /workspace/.cache && \ + install -d -o ${USER} -g users /workspace # ------------------------------------------------------------------- # Entrypoint @@ -157,6 +227,13 @@ RUN chmod 755 /usr/local/bin/nc-sync COPY entrypoint.sh /opt/entrypoint.sh RUN chmod +x /opt/entrypoint.sh +# Fedora's shadow-utils helpers are installed without setuid/file capabilities. +# Rootless Podman needs these narrowly scoped helpers to apply jovyan's ranges. +RUN test -x /usr/bin/newuidmap && test -x /usr/bin/newgidmap && \ + grep -q '^jovyan:' /etc/subuid && grep -q '^jovyan:' /etc/subgid && \ + chown root:root /usr/bin/newuidmap /usr/bin/newgidmap && \ + chmod 4755 /usr/bin/newuidmap /usr/bin/newgidmap + USER ${USER} ENV GDAL_CONFIG=/usr/local/bin/gdal-config diff --git a/README.md b/README.md index 33ea518..a000ded 100644 --- a/README.md +++ b/README.md @@ -13,36 +13,237 @@ This repository contains a Dockerfile to build a container that exposes [Code Se ### Building the Docker Image -To build the Docker image, run the following command: +Build a local image for AMD64 or ARM64 with: ```bash -docker build -t eoepca/pde-code-server . +task build PLATFORM=linux/amd64 IMAGE=eoepca/pde-code-server:amd64 +task build PLATFORM=linux/arm64 IMAGE=eoepca/pde-code-server:arm64 ``` -To include OpenJDK 17 JRE (headless), enable the optional build argument -(disabled by default): +Install [Task](https://taskfile.dev/) and Docker with Buildx. `task build` (or +`task`) defaults to `linux/amd64` and `eoepca/pde-code-server:local`. +Override the image tag or GDAL version when needed: ```bash -docker build --build-arg INSTALL_JRE=true -t eoepca/pde-code-server . +task build IMAGE=eoepca/pde-code-server:dev GDAL_VER=3.12.1 +``` + +Both `linux/amd64` and `linux/arm64` are supported. BuildKit supplies `TARGETARCH` +for binary downloads; Node.js, Python packages, GDAL compilation, and Fedora RPM packages +use the target architecture. code-server is upgraded to 4.144.0. Nextcloud follows Fedora 44 packages; +Java remains OpenJDK 17, supplied by Eclipse Temurin. Other explicitly pinned +tool versions are retained. + +Use a native Docker host for the selected platform when possible. Cross-platform +builds and tests require QEMU/binfmt registration (for example, +`docker run --privileged --rm tonistiigi/binfmt --install arm64` on an AMD64 host). +GDAL source compilation under emulation can take substantially longer. + +To build both platforms into a local OCI archive without publishing: + +```bash +docker buildx create --name pde-multi --driver docker-container --use +docker buildx inspect --bootstrap +task build:multi OUTPUT=/tmp/pde-code-server-multi.oci.tar +``` + +The OCI archive contains both platforms. For local runtime testing, use the +single-platform `task build` commands above, which load an image into Docker. +Choose distinct tags to keep both images available. + +### Running Locally + +The proxy uses `/healthz` for code-server readiness with a 120-second cold-start +window. It uses OAuth by default for ApplicationHub deployments. For standalone +local use, disable proxy authentication and bind the service to localhost: + +```bash +docker run --rm -it \ + --platform="${PLATFORM:-linux/amd64}" \ + -e JHSINGLE_NATIVE_PROXY_AUTHTYPE=none \ + -p 127.0.0.1:8888:8888 \ + -v "$PWD:/workspace" \ + eoepca/pde-code-server:local +``` + +Set `PLATFORM=linux/arm64` for an ARM64 local image. + +Open . If port 8888 is already in use, change the host +port (the first port in `-p`) and use that port in the URL. The container +image must be rebuilt after changes to the entrypoint. At startup, `nc-sync` +creates `/workspace/drive` as a symlink to `/home/jovyan/drive` inside the +container. With the project directory mounted at `/workspace`, this symlink +also appears in the project directory on the host; it is generated runtime +state and should not be committed. + +## Building containers with rootless Podman + +The image runs Podman as `jovyan` (UID 1001, GID 100). It provides subordinate +UID/GID ranges in `/etc/subuid` and `/etc/subgid`, and root-owned setuid +`newuidmap`/`newgidmap` helpers. The outer container must permit nested user +namespaces and helper privilege escalation. The APEx privileged pod setup can +provide this; overriding the user to UID 1000 does not match the image account. +Keep the main container and init-container identity aligned with UID 1001. + +For an existing Fedora image affected by missing helper privileges, run these +commands in the code-server terminal **as jovyan (UID 1001)**: + +```bash +sudo chown root:root /usr/bin/newuidmap /usr/bin/newgidmap +sudo chmod 4755 /usr/bin/newuidmap /usr/bin/newgidmap +podman system migrate +podman unshare cat /proc/self/uid_map /proc/self/gid_map +podman pull docker.io/library/python:3.12-slim +``` + +`podman system migrate` stops that user's running Podman containers and replaces +the cached user namespace. Each mapping should include the user's own ID plus +at least 65,536 subordinate IDs. If the ranges are absent or the actual UID is +1000, correct the deployment identity/ranges first. If helpers still report +`Operation not permitted`, inspect the outer pod's privilege-escalation and +user-namespace restrictions. Do not use `podman system reset` for this repair; +it deletes container storage. The terminal repair lasts only until the outer +container is replaced; rebuild the image for the persistent fix. + +Keep Podman storage on the mounted workspace volume. Nested overlay storage in +Docker's writable container layer produced an `exec ... Invalid argument` error +in local testing; an independent workspace volume worked with the default driver. +The regression test uses an anonymous workspace volume and a privileged outer +container, then pulls Python slim, builds a layer with UID 1234/GID 42 ownership, +and runs it as that non-root user. CI runs it on both architectures: + +```bash +task test:podman IMAGE=eoepca/pde-code-server:podman-fix-arm64 PLATFORM=linux/arm64 +``` + +This test passed locally on ARM64 on 2026-11-04 using the prior validated image +plus the helper-permissions fix. AMD64 execution remains covered by CI. + +## Mastering Application Package compatibility + +The migration preserves `jovyan` (1001:100), `HOME=/workspace`, passwordless sudo, +Python 3.12, Conda, Bash/Python notebook kernels, NVM, all existing PDE tools, +and the ApplicationHub proxy/Nextcloud entrypoint. The workshop Helm chart can +continue running its init container and direct code-server command unchanged. +Workspace volumes must be writable by UID 1001 or GID 100 (`fsGroup: 100`). + +Run the integration check with disposable Docker volumes and LocalStack: + +```bash +bash ci/test-mastering.sh eoepca/pde-code-server:fedora-arm64 linux/arm64 +# Test the current local workshop files instead of the checked-in snapshot: +bash ci/test-mastering.sh eoepca/pde-code-server:fedora-arm64 linux/arm64 \ + /Users/mlongobardo/projects/dev-platform-eoap/mastering-app-package/files +``` + +This executes the unmodified `init.sh`, verifies all four extensions, geospatial +Python imports, pip dependencies, the kernel interpreter, interactive venv +activation, downloaded kubectl, and S3 bucket creation, then starts code-server +using the chart's command and XDG settings. It needs network access for GitHub, +Open VSX, PyPI, Kubernetes downloads, and the LocalStack image. It does not test +Kubernetes scheduling, a real Nextcloud service, or browser extension activation. +CI runs this check on both architectures before publishing. + +The fixture in `ci/fixtures/mastering-app-package` was copied from the local +workshop checkout at commit `49bb077b8753bddfb215827a04a3cf61d265811b`; +refresh it when the workshop initialization changes. + +Local validation on 2026-11-02 passed on ARM64: tool and dependency checks, +all three released EOAP extensions (including UML rendering), the current local +workshop `init.sh`, and both direct and proxied startup. AMD64 is covered by the +native CI job but was not built or executed locally during this migration. +The tested local image is `eoepca/pde-code-server:fedora-arm64`. + +To rerun only the proxy/Nextcloud-symlink startup check: + +```bash +bash ci/test-startup.sh eoepca/pde-code-server:fedora-arm64 linux/arm64 ``` ## Installed Tooling -This image is based on Debian bookworm and Python 3.12, and provides a curated set of development, Kubernetes, and Earth-Observation workflow tools. +### Extension compatibility checks + +Before publishing an image, CI builds and tests AMD64 and ARM64 on native +GitHub runners (`ubuntu-24.04` and `ubuntu-24.04-arm`). It downloads the latest stable release VSIXs from +`eoap/eoap-validator-vscode`, `eoap/cwl-metadata-editor`, and +`eoap/cwl-uml-viewer`. It installs them with `code-server` as the image's default +user and checks the installed IDs and versions. It also checks Python dependency +consistency, installs the validator's bundled wheel in a fresh virtual environment, +loads its CLI, and renders the UML viewer's sample through its installed Python +bridge and bundled Java renderer for every available diagram type. + +Both architectures use the same downloaded VSIXs. Tool execution, ELF binary architecture, image platform, +user identity (UID 1001 / GID 100), Calrissian isolation, Nextcloud symlink, and +standalone entrypoint startup are checked on each platform. + +Failures block image publication. The tested release tags and download URLs are +saved as a CI artifact. These checks use the image's default runtime settings; +they do not exercise editor UI interactions or extension activation. Latest +upstream releases are intentionally used, so a new incompatible release can fail +CI without changes here. Extensions are installed only in the disposable test +container. -All non-distro binaries are pinned to explicit versions to ensure reproducibility. +To run the full checks against each locally built image: + +```bash +task test PLATFORM=linux/amd64 IMAGE=eoepca/pde-code-server:amd64 +task test PLATFORM=linux/arm64 IMAGE=eoepca/pde-code-server:arm64 +``` + +`task test` downloads the latest VSIXs, then checks tools, extensions, and startup. +It also verifies that the default `yq` is Mike Farah's pinned v4.45.1, reads a +YAML fixture with `yq eval`, reads a TOML fixture with `tomlq`, and runs +`python -m pip check`. The Python `yq` package remains installed for `tomlq`; +only its conflicting `yq` launcher is replaced, leaving Conda's PATH intact. +The focused runtime check also runs Hatch version and local project metadata +commands, creates and activates a default environment without networking (skipping +project installation), verifies Calrissian's Git commit provenance, Dask +import, CLI flags, and bundled schema/init/dispose scripts, and checks both +isolated environments' dependencies. It does not execute Dask on Kubernetes. +To run these checks against a loaded image on either platform: + +```bash +docker run --rm --network none --platform linux/amd64 --entrypoint bash \ + -v "$PWD/ci:/opt/image-ci:ro" eoepca/pde-code-server:amd64 /opt/image-ci/test-runtime.sh +docker run --rm --network none --platform linux/arm64 --entrypoint bash \ + -v "$PWD/ci:/opt/image-ci:ro" eoepca/pde-code-server:arm64 /opt/image-ci/test-runtime.sh +``` + +To reuse the same downloaded release assets for both platforms: + +```bash +python3 ci/test-extensions.py download /tmp/pde-extension-vsix +bash ci/test-image.sh eoepca/pde-code-server:amd64 linux/amd64 /tmp/pde-extension-vsix +bash ci/test-image.sh eoepca/pde-code-server:arm64 linux/arm64 /tmp/pde-extension-vsix +``` + +The UML viewer's bundled dependencies are tested as shipped, including its native +Python modules; CI does not patch the VSIX or skip rendering on ARM64. A future +upstream incompatibility will fail the corresponding platform check. + +This image uses Fedora 44, following the Fedora recipe in coder/code-server +with the PDE runtime contract preserved. The complete Python 3.12 Conda/Jupyter +environment is copied from the Jupyter base notebook image, so notebook tools +and `/opt/conda` paths remain available. Fedora supplies the OS and system tools. + +Standalone tool releases are pinned. Fedora packages and the Python 3.12 Jupyter +and Java 17 Temurin source images receive updates; builds are not fully locked. ### Base System -- OS: Debian GNU/Linux 12 (bookworm) -- Python: 3.12.11 -- Node.js: 18.x (Debian package) -- npm: bundled with Node.js +- OS: Fedora 44 +- Python: 3.12 (Jupyter/Conda toolchain) +- NVM: v0.40.3, installed at `/opt/nvm` +- Node.js: v22.15.0, installed via NVM and available on `PATH` +- npm: bundled with Node.js 22 +- Java: Eclipse Temurin OpenJDK 17 JDK, at `/opt/java/openjdk` Installed system utilities: - curl, wget -- git +- git, git-lfs +- htop, openssh-clients, zsh, dumb-init - sudo - nano - net-tools @@ -53,7 +254,7 @@ Installed system utilities: ### code-server -- code-server: 4.138.0 +- code-server: 4.144.0 Installed from official release tarball and available in PATH: @@ -109,9 +310,13 @@ Installed via pip (Python 3.12): Reference implementation of the Common Workflow Language. -* calrissian: 0.18.1 +* calrissian: commit `a95d87dd213d51b11faea7c6def4ced46d466a4a` - CWL runner for Kubernetes. + CWL runner for Kubernetes with Dask support, installed from the pinned upstream + Git commit in `/opt/calrissian-venv`. Its dependencies are resolved together, + including `cwltool>=3.1.20260108082145`, separately from the image's CWL tooling. + `CALRISSIAN_COMMIT` is the build argument; changing it also requires updating + the expected commit in `ci/test-runtime.sh`. * jhsingle-native-proxy (>= 0.0.9) @@ -131,14 +336,34 @@ Installed via pip (Python 3.12): ### Python Build & Packaging -- hatch: 1.16.2 +- hatch: 1.16.5 + + Python project manager and build tool, installed with all runtime dependencies + in `/opt/hatch-venv`, using the base image's native Python on AMD64 and ARM64. + `/usr/local/bin/hatch` links to its console script. Jovyan can use it offline + without downloading Hatch itself. `HATCH_CONFIG`, `HATCH_DATA_DIR`, and + `HATCH_CACHE_DIR` point to writable locations under `/home/jovyan`, independent + of the `/workspace` project mount. Project environment creation, project + dependencies, plugins, or additional Python versions may still need network + access; the offline smoke check only reads local project metadata. + +### GDAL build cleanup + +GDAL is compiled from source, then CMake, GCC/G++, Ninja, PROJ development +headers, and their unused dependencies are removed in the same image layer. +The ARM64 cleanup test removed 53 RPMs (about 348 MiB of installed files). +Compressed image savings vary. PROJ tools and coordinate grids, `xz`, GDAL's +linked runtime libraries, and GDAL's own headers/`gdal-config` remain installed. +CI checks compressed GeoTIFF creation and EPSG reprojection after cleanup. - Python project manager and build tool, installed as a standalone binary. +Compilers are no longer preinstalled for source builds inside the development +environment. If a project needs them, install the required development packages +with `sudo dnf install gcc-c++ cmake ninja-build proj-devel`. ### User & Runtime Environment - User: jovyan -- UID / GID: 1001 +- UID / GID: 1001 / 100 - Home / Workdir: /workspace - Passwordless sudo enabled for the user. @@ -160,7 +385,13 @@ ghcr.io//pde-code-server The registry owner corresponds to the GitHub repository owner (user or organization). -Images are built using Kaniko and pushed using OCI-compliant tooling. +Images are built with Buildx. Each platform is tested, scanned with Trivy 0.76.1, +and given its own SPDX JSON SBOM. After both platform jobs succeed, CI publishes +the tested images and attaches each SBOM with ORAS to that architecture image's +exact registry digest. It then publishes a combined multi-platform manifest under +the branch or release tag. The SBOMs describe individual platform images, not the +combined index. Vulnerability findings remain informational, as in the previous +workflow. Pull requests run builds and checks without publishing. ### Tagging Strategy diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 0000000..8df4bed --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,45 @@ +version: '3' + +vars: + IMAGE: eoepca/pde-code-server:local + PLATFORM: linux/amd64 + GDAL_VER: '3.12.1' + OUTPUT: /tmp/pde-code-server-multi.oci.tar + VSIX_DIR: /tmp/pde-extension-vsix + +tasks: + default: + desc: Build the local container image + cmds: + - task: build + + build: + desc: Build and load one platform (PLATFORM=linux/amd64 or linux/arm64) + preconditions: + - sh: command -v docker + msg: Docker must be installed to build the image. + - sh: 'test {{.PLATFORM | shellQuote}} = linux/amd64 || test {{.PLATFORM | shellQuote}} = linux/arm64' + msg: PLATFORM must be linux/amd64 or linux/arm64. + cmds: + - docker buildx build --platform={{.PLATFORM | shellQuote}} --build-arg GDAL_VER={{.GDAL_VER | shellQuote}} --load -t {{.IMAGE | shellQuote}} . + + build:multi: + desc: Build both platforms into a local OCI archive (requires a docker-container Buildx builder) + cmds: + - docker buildx build --platform=linux/amd64,linux/arm64 --build-arg GDAL_VER={{.GDAL_VER | shellQuote}} --output type=oci,dest={{.OUTPUT | shellQuote}} -t {{.IMAGE | shellQuote}} . + + test: + desc: Check tools, identity, startup, and released extensions in the local image + cmds: + - python3 ci/test-extensions.py download {{.VSIX_DIR | shellQuote}} + - bash ci/test-image.sh {{.IMAGE | shellQuote}} {{.PLATFORM | shellQuote}} {{.VSIX_DIR | shellQuote}} + + test:mastering: + desc: Run the workshop init script and chart startup against disposable LocalStack and storage + cmds: + - bash ci/test-mastering.sh {{.IMAGE | shellQuote}} {{.PLATFORM | shellQuote}} + + test:podman: + desc: Test rootless Podman builds inside a disposable privileged container and workspace volume + cmds: + - bash ci/test-podman.sh {{.IMAGE | shellQuote}} {{.PLATFORM | shellQuote}} diff --git a/ci/fixtures/mastering-app-package/bash-rc b/ci/fixtures/mastering-app-package/bash-rc new file mode 100644 index 0000000..00dd390 --- /dev/null +++ b/ci/fixtures/mastering-app-package/bash-rc @@ -0,0 +1,5 @@ +alias ll="ls -l" + +alias aws="aws --endpoint-url=http://eoap-mastering-app-package-localstack:4566" + +source /workspace/.venv/bin/activate \ No newline at end of file diff --git a/ci/fixtures/mastering-app-package/init.sh b/ci/fixtures/mastering-app-package/init.sh new file mode 100644 index 0000000..e2bb1af --- /dev/null +++ b/ci/fixtures/mastering-app-package/init.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -x + +cd /workspace + +if [ ! -d mastering-app-package/.git ]; then + git clone 'https://github.com/eoap/mastering-app-package.git' || exit $? +fi + +code-server --install-extension ms-python.python --install-extension redhat.vscode-yaml --install-extension sbg-rabix.benten-cwl --install-extension ms-toolsai.jupyter + +ln -sfn /workspace/.local/share/code-server/extensions /workspace/extensions + +mkdir -p /workspace/User/ + +echo '{"workbench.colorTheme": "Visual Studio Dark"}' > /workspace/User/settings.json + +python -m venv /workspace/.venv +source /workspace/.venv/bin/activate +/workspace/.venv/bin/python -m pip install --upgrade "pip @ https://files.pythonhosted.org/packages/3f/06/3e4420302363fbb71be3c7f35ce042f54bfca330e3e66b3a2e3c4cca579f/pip-26.2.2-py3-none-any.whl#sha256=fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70" || exit $? +/workspace/.venv/bin/python -m pip install --upgrade "certifi>=2026.10.19" || exit $? +/workspace/.venv/bin/python -m pip install --no-cache-dir rasterio click pystac loguru pyproj shapely scikit-image pystac rio_stac ipykernel stactools[validate] calrissian matplotlib pandas nose2 +/workspace/.venv/bin/python -m ipykernel install --user --name mastering_env --display-name "Python (Mastering Application Package)" + +echo "**** install kubectl ****" +case "$(uname -m)" in + aarch64|arm64) kubectl_arch=arm64 ;; + x86_64) kubectl_arch=amd64 ;; + *) echo "Unsupported architecture: $(uname -m)" >&2; exit 1 ;; +esac +curl -fsSL -o kubectl "https://dl.k8s.io/release/$(curl -fsSL https://dl.k8s.io/release/stable.txt)/bin/linux/${kubectl_arch}/kubectl" || exit $? +chmod +x kubectl +mv ./kubectl /workspace/.venv/bin/kubectl + +export AWS_DEFAULT_REGION="us-east-1" + +export AWS_ACCESS_KEY_ID="test" + +export AWS_SECRET_ACCESS_KEY="test" + +aws s3 mb s3://results --endpoint-url=http://eoap-mastering-app-package-localstack:4566 diff --git a/ci/test-extensions.py b/ci/test-extensions.py new file mode 100644 index 0000000..e1a49ea --- /dev/null +++ b/ci/test-extensions.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Download released VSIXs on the runner, or smoke-test them in the built image.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import tempfile +import urllib.request +import zipfile + +REPOSITORIES = ("eoap-validator-vscode", "cwl-metadata-editor", "cwl-uml-viewer") + + +def run(*args, **kwargs): + print("+", " ".join(map(str, args)), flush=True) + try: + return subprocess.run(list(map(str, args)), check=True, timeout=600, **kwargs) + except subprocess.CalledProcessError as error: + if error.stdout: + print(error.stdout, flush=True) + if error.stderr: + print(error.stderr, flush=True) + raise + + +def download(destination): + destination.mkdir(parents=True, exist_ok=True) + for repo in REPOSITORIES: + headers = {"Accept": "application/vnd.github+json"} + if os.environ.get("GITHUB_TOKEN"): + headers["Authorization"] = "Bearer " + os.environ["GITHUB_TOKEN"] + request = urllib.request.Request( + f"https://api.github.com/repos/eoap/{repo}/releases/latest", headers=headers + ) + with urllib.request.urlopen(request, timeout=60) as response: + release = json.load(response) + assets = [a for a in release["assets"] if a["name"].endswith(".vsix")] + if len(assets) != 1: + raise RuntimeError(f"{repo}: expected one VSIX, found {len(assets)}") + print(f"{repo}: {release['tag_name']} ({assets[0]['name']})", flush=True) + # Do not forward the API token to release asset download hosts. + with urllib.request.urlopen(assets[0]["browser_download_url"], timeout=120) as response: + (destination / f"{repo}.vsix").write_bytes(response.read()) + (destination / f"{repo}.release.json").write_text(json.dumps({ + "tag": release["tag_name"], "url": assets[0]["browser_download_url"] + }, indent=2) + "\n") + + +def test(source): + with tempfile.TemporaryDirectory(prefix="extension-smoke-") as temporary: + root = Path(temporary) + extensions = root / "extensions" + user_data = root / "user-data" + cli = ("code-server", "--extensions-dir", extensions, "--user-data-dir", user_data) + installed = {} + for repo in REPOSITORIES: + vsix = source / f"{repo}.vsix" + with zipfile.ZipFile(vsix) as archive: + manifest = json.loads(archive.read("extension/package.json")) + identifier = f"{manifest['publisher']}.{manifest['name']}" + print(f"Installing {identifier}@{manifest['version']}; VS Code {manifest['engines']['vscode']}", flush=True) + # No --force: incompatible VS Code engines must fail installation. + run(*cli, "--install-extension", vsix) + installed[repo] = (identifier, manifest["version"]) + listed = run(*cli, "--list-extensions", "--show-versions", capture_output=True, text=True).stdout + print(listed, flush=True) + for identifier, version in installed.values(): + if f"{identifier}@{version}".lower() not in listed.lower().splitlines(): + raise RuntimeError(f"Extension missing after installation: {identifier}@{version}") + + def extension_path(repo): + identifier, version = installed[repo] + matches = list(extensions.glob(f"{identifier}-{version}*")) + if len(matches) != 1: + raise RuntimeError(f"Cannot locate installed {identifier}: {matches}") + return matches[0] + + run("python3", "-m", "pip", "check") + # Exercise the validator's default first-use venv and bundled wheel. + validator = extension_path("eoap-validator-vscode") + wheels = list((validator / "vendor").glob("eoap_validator-*.whl")) + if len(wheels) != 1: + raise RuntimeError(f"Expected one bundled validator wheel: {wheels}") + venv = root / "validator-venv" + run("python3", "-m", "venv", venv) + python = venv / "bin/python" + run(python, "-m", "pip", "install", "--disable-pip-version-check", wheels[0]) + run(python, "-m", "pip", "check") + run(python, "-m", "eoap_validator", "--help") + + # Use the installed bridge exactly as the extension does, including its + # vendored sys.path and isolated Python mode. Global imports are insufficient. + uml = extension_path("cwl-uml-viewer") + fixture = uml / "examples/echo.cwl" + request = {"path": str(fixture), "text": fixture.read_text()} + + def bridge(extra): + result = run("python3", "-I", uml / "python/bridge.py", + input=json.dumps(request | extra), capture_output=True, text=True) + payload = json.loads(result.stdout) + if "error" in payload: + raise RuntimeError(payload["error"]) + return payload + + info = bridge({"action": "inspect"}) + if not info["workflows"] or not info["diagrams"]: + raise RuntimeError(f"No UML workflows or diagrams: {info}") + jars = list((uml / "vendor").glob("plantuml-asl-*.jar")) + if len(jars) != 1: + raise RuntimeError(f"Expected one bundled PlantUML renderer: {jars}") + for diagram in info["diagrams"]: + puml = bridge({"action": "render", "diagram": diagram, + "workflow": info["workflows"][0]})["puml"] + svg = run("java", "-Djava.awt.headless=true", + "-DPLANTUML_SECURITY_PROFILE=SANDBOX", "-jar", jars[0], + "-pipe", "-charset", "UTF-8", "-tsvg", "-failfast2", + input=puml, capture_output=True, text=True).stdout + if "&2; exit 1; } +# Exercise cold-start runtime availability before any network-enabled tool check. +docker run --rm --network none --platform "$platform" --entrypoint bash \ + -v "$repo_dir/ci:/opt/image-ci:ro" "$image" /opt/image-ci/test-runtime.sh +docker run --rm --platform "$platform" --entrypoint bash "$image" -ec ' + test "$(id -u)" = 1001 + test "$UID" = 1001 + test "$(id -un)" = jovyan + test "$(id -g)" = 100 + . /etc/os-release + test "$ID" = fedora + test "$VERSION_ID" = 44 + code-server --version + node --version + kubectl version --client=true + task --version + skaffold version + /usr/local/bin/oras version + yq --version + jq --version + hatch --version + trivy --version + java -version + java -XshowSettings:properties -version 2>&1 | grep -Eq "java.specification.version = 17$" + dot -V + gdalinfo --version + calrissian --version + cwltool --version + cwltest --help >/dev/null + podman --version + skopeo --version + uv --version + QT_QPA_PLATFORM=offscreen nextcloudcmd --version + QT_QPA_PLATFORM=offscreen nextcloudcmd --help > /tmp/nextcloud-help + for option in --user --password --exclude --path; do + grep -Fq -- "$option" /tmp/nextcloud-help + done + python3 -m pip check + /opt/calrissian-venv/bin/python -m pip check + python3 -c "import cwl_loader, cwl2puml" +' bash "$arch" +# Check the ENV value directly: Bash replaces UID with the current numeric UID. +docker run --rm -i --platform "$platform" --entrypoint python3 "$image" - "$arch" <<'PYTHON' +import os +from pathlib import Path +import shutil +import struct +import sys + +assert os.environ["UID"] == "1001" +assert os.getuid() == 1001 +machine = {"amd64": 62, "arm64": 183}[sys.argv[1]] +executables = ["/opt/code-server/lib/node", shutil.which("node"), shutil.which("trivy"), + "/opt/hatch-venv/bin/python", "/opt/calrissian-venv/bin/python"] +executables += [f"/usr/local/bin/{name}" for name in + ("kubectl", "task", "skaffold", "oras", "yq", "jq")] +for executable in executables: + with Path(executable).open("rb") as stream: + header = stream.read(20) + assert header[:4] == b"\x7fELF" and header[5] == 1, executable + assert struct.unpack("/dev/null 2>&1 || true + docker volume rm "$name-workspace" >/dev/null 2>&1 || true + docker network rm "$name" >/dev/null 2>&1 || true +} +trap cleanup EXIT +docker network create "$name" >/dev/null +docker volume create "$name-workspace" >/dev/null +docker run -d --platform "$platform" --name "$name-s3" --network "$name" \ + --network-alias eoap-mastering-app-package-localstack \ + -e SERVICES=s3 localstack/localstack:4.0.3 >/dev/null +ready=false +for ((i=0; i<90; i++)); do + if docker exec "$name-s3" awslocal s3 ls >/dev/null 2>&1; then ready=true; break; fi + sleep 2 +done +[[ "$ready" == true ]] || { docker logs "$name-s3"; exit 1; } +# Match a PVC made writable through the chart's fsGroup=100. +docker run --rm --platform "$platform" --user root --entrypoint bash \ + -v "$name-workspace:/workspace" "$image" -ec 'chown 1001:100 /workspace' +docker run --name "$name-init" --platform "$platform" --network "$name" \ + --entrypoint bash -v "$name-workspace:/workspace" \ + -v "$fixtures:/opt/init:ro" "$image" -ec ' + bash /opt/init/init.sh + test -d /workspace/mastering-app-package/.git + test "$(readlink /workspace/extensions)" = /workspace/.local/share/code-server/extensions + code-server --list-extensions > /tmp/extensions + for extension in ms-python.python redhat.vscode-yaml sbg-rabix.benten-cwl ms-toolsai.jupyter; do + grep -Fxi "$extension" /tmp/extensions + done + /workspace/.venv/bin/python -m pip check + /workspace/.venv/bin/python -c "import rasterio, click, pystac, loguru, pyproj, shapely, skimage, rio_stac, ipykernel, stactools, calrissian, matplotlib, pandas, nose2" + /workspace/.venv/bin/kubectl version --client + jupyter kernelspec list --json | python -c '\''import json,sys; s=json.load(sys.stdin)["kernelspecs"]["mastering_env"]["spec"]; assert s["argv"][0] == "/workspace/.venv/bin/python"'\'' + cp /opt/init/bash-rc /workspace/.bashrc + bash -ic '\''test "$VIRTUAL_ENV" = /workspace/.venv; python -c "import rasterio"'\'' + ' +docker exec "$name-s3" awslocal s3api head-bucket --bucket results +# Match the chart: it invokes code-server directly, bypassing the image entrypoint. +docker run -d --name "$name-server" --platform "$platform" --network "$name" \ + -v "$name-workspace:/workspace" --entrypoint code-server \ + -e XDG_CONFIG_HOME=/workspace/.local -e XDG_DATA_HOME=/workspace/.local/share/ \ + -e XDG_RUNTIME_DIR=/workspace/.local "$image" \ + --auth none --bind-addr 0.0.0.0:8080 --user-data-dir /workspace \ + /workspace/mastering-app-package >/dev/null +for ((i=0; i<90; i++)); do + if docker exec "$name-server" curl -fsS http://127.0.0.1:8080/ >/dev/null; then + echo "Mastering init, Python imports, extensions, S3, kernel, shell and chart startup passed ($platform)" + exit 0 + fi + sleep 2 +done +docker logs "$name-server" +exit 1 diff --git a/ci/test-podman.sh b/ci/test-podman.sh new file mode 100644 index 0000000..df08789 --- /dev/null +++ b/ci/test-podman.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Exercise rootless Podman inside a disposable privileged outer container. +# An anonymous workspace volume mirrors a PVC and avoids overlay-on-overlay. +# Docker removes that volume with the container; no host Docker socket is mounted. +set -euo pipefail +image="${1:?image required}" +platform="${2:?platform required}" +docker run --rm -i --privileged --platform "$platform" -v /workspace --entrypoint bash "$image" -se <<'SCRIPT' +set -euo pipefail +test "$(id -u)" = 1001 +test -u /usr/bin/newuidmap +test -u /usr/bin/newgidmap +# Require subordinate ranges, not just a single-ID fallback namespace. +podman unshare cat /proc/self/uid_map /proc/self/gid_map | python -c ' +import sys +rows = [list(map(int, line.split())) for line in sys.stdin] +assert len(rows) == 4 and rows[1][2] >= 65536 and rows[3][2] >= 65536, rows +' +context="$(mktemp -d)" +trap 'rm -rf "$context"' EXIT +cat > "$context/Containerfile" <<'CONTAINERFILE' +FROM docker.io/library/python:3.12-slim +RUN python -c 'import os; assert os.stat("/etc/gshadow").st_gid == 42; open("/mapped-owner", "w").close(); os.chown("/mapped-owner", 1234, 42)' +USER 1234:42 +CMD ["python", "-c", "import os; s=os.stat('/mapped-owner'); assert (os.getuid(),os.getgid(),s.st_uid,s.st_gid)==(1234,42,1234,42); print('Nested build and non-root execution passed')"] +CONTAINERFILE +podman build --tag localhost/pde-podman-test "$context" +podman run --rm localhost/pde-podman-test +SCRIPT diff --git a/ci/test-runtime.sh b/ci/test-runtime.sh new file mode 100644 index 0000000..41d6905 --- /dev/null +++ b/ci/test-runtime.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Run as jovyan in a disposable container with --network none. +set -euo pipefail +script_dir="$(cd "$(dirname "$0")" && pwd)" +test "$(id -u)" = 1001 +test -w /opt/conda/bin +python -c 'import sys; assert sys.version_info[:2] == (3, 12)' +test -w "$HATCH_CONFIG" +test -w "$HATCH_DATA_DIR" +test -w "$HATCH_CACHE_DIR" +# Tools such as uv and pip create caches under the default /workspace HOME. +test -z "$(find /workspace ! -user jovyan -print -quit)" +test "$(readlink -f "$(command -v hatch)")" = /opt/hatch-venv/bin/hatch +test "$(readlink -f "$(command -v calrissian)")" = /opt/calrissian-venv/bin/calrissian +fixture_dir="$(mktemp -d)" +trap 'rm -rf "$fixture_dir"' EXIT + +[[ "$(hatch --version)" == 'Hatch, version 1.16.5' ]] +cat > "$fixture_dir/pyproject.toml" <<'TOML' +[project] +name = "offline-smoke" +version = "1.2.3" + +[tool.hatch.envs.default] +skip-install = true +TOML +cd "$fixture_dir" +[[ "$(hatch version)" == '1.2.3' ]] +# Exercise interpreter discovery, which failed with older Hatch and virtualenv 21. +# Skip project installation so this regression check can run without networking. +hatch env create default +environment_dir="$(hatch env find default)" +test -x "$environment_dir/bin/python" +( + source "$environment_dir/bin/activate" + test "$(command -v python)" = "$environment_dir/bin/python" + python -c 'import os, sys; assert sys.prefix == os.environ["VIRTUAL_ENV"]' +) +hatch env remove default +# Activating a venv in a Bash kernel must not touch the prompt the kernel waits for. +jupyter kernelspec list --json | python -c 'import json, sys; spec = json.load(sys.stdin)["kernelspecs"]["bash"]["spec"]; assert spec["env"]["VIRTUAL_ENV_DISABLE_PROMPT"] == "1", spec' +/opt/hatch-venv/bin/python -m pip check + +/opt/calrissian-venv/bin/python - <<'PYTHON' +import json +from importlib import metadata +from pathlib import Path + +import calrissian.dask +from packaging.version import Version + +expected = "a95d87dd213d51b11faea7c6def4ced46d466a4a" +distribution = metadata.distribution("calrissian") +provenance = json.loads(distribution.read_text("direct_url.json")) +assert provenance["url"].removesuffix(".git").lower() == "https://github.com/duke-gcb/calrissian" +assert provenance["vcs_info"]["vcs"] == "git" +assert provenance["vcs_info"]["commit_id"] == expected, provenance +assert Version(metadata.version("cwltool")) >= Version("3.1.20260108082145") +package = Path(calrissian.dask.__file__).parent +for resource in ("dask/custom_schema/schema.yaml", "dask/init-dask.py", "dask/dispose-dask.py"): + path = package / resource + assert path.is_file() and path.stat().st_size > 0, path + print(f"Verified installed resource: {path}") +print(f"Verified Calrissian commit {expected}, Dask import, and cwltool {metadata.version('cwltool')}") +PYTHON +/opt/calrissian-venv/bin/python -m pip check +calrissian --help > "$fixture_dir/calrissian-help.txt" +for flag in --dask-gateway-url --dask-script-configmap; do + grep -Fq -- "$flag" "$fixture_dir/calrissian-help.txt" +done +bash "$script_dir/test-yq.sh" +bash "$script_dir/test-gdal.sh" +echo 'Offline Hatch, Calrissian package, and YAML/TOML checks passed (no Kubernetes Dask execution tested).' diff --git a/ci/test-startup.sh b/ci/test-startup.sh new file mode 100644 index 0000000..73bdd91 --- /dev/null +++ b/ci/test-startup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail +image="${1:?image required}" +platform="${2:?platform required}" +container="" +# Invoked by the EXIT trap. +# shellcheck disable=SC2329 +cleanup() { + if [[ -n "$container" ]]; then + docker logs "$container" || true + docker rm -f "$container" >/dev/null + fi +} +trap cleanup EXIT +container="$(docker run -d --platform "$platform" \ + -e JHSINGLE_NATIVE_PROXY_AUTHTYPE=none "$image")" +deadline=$((SECONDS + 180)) +while ((SECONDS < deadline)); do + if docker exec "$container" curl -fsS --max-time 5 http://127.0.0.1:8888/ >/dev/null; then + docker exec "$container" test -L /workspace/drive + docker exec "$container" test "$(docker exec "$container" readlink /workspace/drive)" = /home/jovyan/drive + echo "Entrypoint checks passed on $platform" + exit 0 + fi + [[ "$(docker inspect --format '{{.State.Running}}' "$container")" == true ]] || exit 1 + sleep 2 +done +echo "Entrypoint did not become ready within 180 seconds" >&2 +exit 1 diff --git a/ci/test-yq.sh b/ci/test-yq.sh new file mode 100644 index 0000000..5d0a860 --- /dev/null +++ b/ci/test-yq.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Run inside the image as its default user, exercising commands through PATH. +set -euo pipefail +fixture_dir="$(mktemp -d)" +trap 'rm -rf "$fixture_dir"' EXIT +version="$(yq --version)" +[[ "$version" == 'yq (https://github.com/mikefarah/yq/) version v4.45.1' ]] +printf 'image_version: "1.4.0"\n' > "$fixture_dir/image.yaml" +[[ "$(yq eval '.image_version' "$fixture_dir/image.yaml")" == '1.4.0' ]] +printf '[image]\nversion = "1.4.0"\n' > "$fixture_dir/image.toml" +[[ "$(tomlq -r '.image.version' "$fixture_dir/image.toml")" == '1.4.0' ]] +python -m pip check +printf 'Default yq, TOML processing, and Python dependencies passed: %s\n' "$version" diff --git a/entrypoint.sh b/entrypoint.sh index 6d87921..4a6c2e4 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -34,8 +34,18 @@ destport=$((port + 1)) echo "Using internal port $destport" +auth_type="${JHSINGLE_NATIVE_PROXY_AUTHTYPE:-oauth}" +case "$auth_type" in + oauth|none) ;; + *) + echo "Unsupported JHSINGLE_NATIVE_PROXY_AUTHTYPE: $auth_type (expected oauth or none)" >&2 + exit 1 + ;; +esac + if [ -z "$CODE_SERVER_WS" ]; then CODE_SERVER_WS="/workspace" fi -jhsingle-native-proxy --port $port --destport $destport code-server {--}auth none {--}bind-addr 0.0.0.0:$destport {--}user-data-dir /workspace $CODE_SERVER_WS +# Probe the lightweight endpoint: loading the editor page can outlast cold-start readiness. +jhsingle-native-proxy --ready-check-path /healthz --ready-timeout 120 --authtype "$auth_type" --port "$port" --destport "$destport" code-server {--}auth none {--}bind-addr "0.0.0.0:$destport" {--}user-data-dir /workspace "$CODE_SERVER_WS" diff --git a/release.yaml b/release.yaml index 25e4f74..66849b1 100644 --- a/release.yaml +++ b/release.yaml @@ -1,4 +1,4 @@ image_name: pde-code-server image_prefix: eoepca -image_version: 1.3.0 -image_registry: ghcr.io \ No newline at end of file +image_version: 1.4.0 +image_registry: ghcr.io diff --git a/requirements-cwl-uml.txt b/requirements-cwl-uml.txt new file mode 100644 index 0000000..56fa9b4 --- /dev/null +++ b/requirements-cwl-uml.txt @@ -0,0 +1,12 @@ +transpiler-mate-runtime==1.3.0 +cwl-loader==0.27.0 +transpiler-mate-api==1.0.1 +cwl2click==0.10.1 +cwl2markdown==0.2.1 +cwl2puml==0.49.1 +cwl2inputs==0.1.1 +cwl2ogc==0.22.2 +cwl2oci==0.1.1 +cwl2sbom==0.1.1 +cwl-baseline-plugin==0.1.1 +cwl2codemeta==0.1.1 \ No newline at end of file