diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 71f70f7..f4488c3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -103,7 +103,7 @@ jobs: UBUNTU_VERSION: ${{ matrix.ubuntu }} OPENPBS_VERSION: ${{ env.OPENPBS_VERSION }} SPACK_STACK_VERSION: ${{ env.SPACK_STACK_VERSION }} - run: docker compose -f docker-compose-test.yml up --pull never -d + run: docker compose -f docker-compose-test.yml up --pull never -d --wait - name: Check cluster logs run: docker compose -f docker-compose-test.yml logs - name: Check status of the cluster containers @@ -306,7 +306,7 @@ jobs: UBUNTU_VERSION: ${{ matrix.ubuntu }} OPENPBS_VERSION: ${{ env.OPENPBS_VERSION }} SPACK_STACK_VERSION: ${{ env.SPACK_STACK_VERSION }} - run: docker compose -f docker-compose-test.yml up --pull never -d + run: docker compose -f docker-compose-test.yml up --pull never -d --wait - name: Check cluster logs run: docker compose -f docker-compose-test.yml logs - name: Check status of the cluster containers diff --git a/.github/workflows/package-cleanup.yaml b/.github/workflows/package-cleanup.yaml index b6f74ae..eeb655e 100644 --- a/.github/workflows/package-cleanup.yaml +++ b/.github/workflows/package-cleanup.yaml @@ -1,6 +1,10 @@ name: PackageCleanup on: + push: + branches: [ main ] + pull_request: + branches: [ main ] workflow_dispatch: inputs: buildcache_cutoff: @@ -12,94 +16,85 @@ on: required: false default: 'true' +env: + # Must stay in sync with docker.yml. Used to construct per-(ubuntu, spack-stack) + # buildcache repo names for the cleanup-stale-buildcache job. + SPACK_STACK_VERSION: 2.1.0 + jobs: - cleanup-untagged: + + # Remove untagged versions of our published images. New publishes leave the + # previous tagged versions as "untagged" if the tag was moved (e.g. when + # `latest` is re-pointed). Runs continuously to keep the registry tidy. + cleanup-packages: runs-on: ubuntu-latest - if: github.event_name == 'workflow_dispatch' permissions: packages: write contents: read + strategy: + fail-fast: false + matrix: + container: [frontend, server, mom] steps: - - name: Remove untagged versions of dockerspackstackpbscluster/pbs-spack-stack-frontend - uses: actions/delete-package-versions@v5 - with: - package-name: 'dockerspackstackpbscluster/pbs-spack-stack-frontend' - package-type: 'container' - min-versions-to-keep: 0 - delete-only-untagged-versions: 'true' - - name: Remove untagged versions of dockerspackstackpbscluster/pbs-spack-stack-server - uses: actions/delete-package-versions@v5 - with: - package-name: 'dockerspackstackpbscluster/pbs-spack-stack-server' - package-type: 'container' - min-versions-to-keep: 0 - delete-only-untagged-versions: 'true' - - name: Remove untagged versions of dockerspackstackpbscluster/pbs-spack-stack-mom - uses: actions/delete-package-versions@v5 - with: - package-name: 'dockerspackstackpbscluster/pbs-spack-stack-mom' - package-type: 'container' - min-versions-to-keep: 0 - delete-only-untagged-versions: 'true' - - name: Remove untagged versions of dockerspackstackpbscluster/frontend-cache-amd64 - uses: actions/delete-package-versions@v5 - with: - package-name: 'dockerspackstackpbscluster/frontend-cache-amd64' - package-type: 'container' - min-versions-to-keep: 0 - delete-only-untagged-versions: 'true' - - name: Remove untagged versions of dockerspackstackpbscluster/frontend-cache-arm64 - uses: actions/delete-package-versions@v5 - with: - package-name: 'dockerspackstackpbscluster/frontend-cache-arm64' - package-type: 'container' - min-versions-to-keep: 0 - delete-only-untagged-versions: 'true' - - name: Remove untagged versions of dockerspackstackpbscluster/server-cache-amd64 - uses: actions/delete-package-versions@v5 - with: - package-name: 'dockerspackstackpbscluster/server-cache-amd64' - package-type: 'container' - min-versions-to-keep: 0 - delete-only-untagged-versions: 'true' - - name: Remove untagged versions of dockerspackstackpbscluster/server-cache-arm64 + - + name: Remove untagged versions of dockerspackstackpbscluster/pbs-spack-stack-${{ matrix.container }} uses: actions/delete-package-versions@v5 - with: - package-name: 'dockerspackstackpbscluster/server-cache-arm64' - package-type: 'container' - min-versions-to-keep: 0 - delete-only-untagged-versions: 'true' - - name: Remove untagged versions of dockerspackstackpbscluster/mom-cache-amd64 - uses: actions/delete-package-versions@v5 - with: - package-name: 'dockerspackstackpbscluster/mom-cache-amd64' + with: + package-name: 'dockerspackstackpbscluster/pbs-spack-stack-${{ matrix.container }}' package-type: 'container' min-versions-to-keep: 0 delete-only-untagged-versions: 'true' - - name: Remove untagged versions of dockerspackstackpbscluster/mom-cache-arm64 + + # Remove untagged versions of the docker layer caches written by CI. Each + # (container, ubuntu, arch) combination has its own cache package -- when CI + # pushes a new :cache tag, the previous version becomes untagged. + cleanup-caches: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + strategy: + fail-fast: false + matrix: + container: [frontend, server, mom] + ubuntu: ['24.04', '26.04'] + arch: [amd64, arm64] + steps: + - + name: Remove untagged versions of dockerspackstackpbscluster/${{ matrix.container }}-cache-ubuntu-${{ matrix.ubuntu }}-${{ matrix.arch }} uses: actions/delete-package-versions@v5 - with: - package-name: 'dockerspackstackpbscluster/mom-cache-arm64' + with: + package-name: 'dockerspackstackpbscluster/${{ matrix.container }}-cache-ubuntu-${{ matrix.ubuntu }}-${{ matrix.arch }}' package-type: 'container' min-versions-to-keep: 0 delete-only-untagged-versions: 'true' + # Age-based cleanup of stale spack OCI buildcache entries. Manual only (cutoff + # input is required) and parameterized so the operator can do dry runs first. + # One job per (ubuntu, spack-stack) cache repo, since each gets its own + # per-spec-hash blob/tag population from autopush. cleanup-stale-buildcache: if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.buildcache_cutoff != '' }} runs-on: ubuntu-latest permissions: packages: write contents: read + strategy: + fail-fast: false + matrix: + ubuntu: ['24.04', '26.04'] steps: - - name: Clean stale buildcache entries + - + name: Clean stale buildcache entries for ubuntu-${{ matrix.ubuntu }}-spack-stack-${{ env.SPACK_STACK_VERSION }} uses: actions/github-script@v7 with: script: | const cutoff = new Date('${{ github.event.inputs.buildcache_cutoff }}'); const dryRun = '${{ github.event.inputs.dry_run }}' === 'true'; const org = 'noaa-gsl'; - const packageName = 'dockerspackstackpbscluster/buildcache'; + const packageName = 'dockerspackstackpbscluster/buildcache-ubuntu-${{ matrix.ubuntu }}-spack-stack-${{ env.SPACK_STACK_VERSION }}'; + console.log(`Package: ${packageName}`); console.log(`Cutoff date: ${cutoff.toISOString()}`); console.log(`Dry run: ${dryRun}`); @@ -123,6 +118,7 @@ jobs: const createdAt = new Date(version.created_at); const tags = version.metadata?.container?.tags || []; + // Never delete index entries - they are updated in place by spack buildcache update-index const isIndex = tags.some(t => t.includes('index') || t.startsWith('_')); if (isIndex) { console.log(`Preserving index: ${version.id} (tags: ${tags.join(', ')})`); @@ -153,4 +149,4 @@ jobs: page++; } - console.log(`\nSummary: ${deleted} ${dryRun ? 'would be ' : ''}deleted, ${kept} kept`); + console.log(`\nSummary for ${packageName}: ${deleted} ${dryRun ? 'would be ' : ''}deleted, ${kept} kept`); diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4f6b7af --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Artifacts generated by running test/test_hello.sh +test/hello.exe +test/hello.out +test/hello.raw +test/hello.pbs +test/hello.pbs.log +test/*.OU +test/*.ER + +# macOS +.DS_Store diff --git a/README.md b/README.md index c642da1..0b16bb4 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ updated for PBS and OpenPBS. There are three containers: * A frontend container that acts as a PBS cluster login node. - Spack-stack is installed on the frontend in /opt which is mounted + Spack-stack is installed on the frontend in /opt/spack-stack which is mounted across the cluster as a shared volume using docker compose * A server container that acts as a PBS server/controller node * A mom container that acts as a PBS compute node @@ -20,10 +20,14 @@ These containers are launched using Docker Compose to build a fully functioning PBS cluster. A `docker-compose.yml` file defines the cluster, specifying ports and volumes to be shared. Multiple instances of the mom container can be -added to `docker-compose.yml` to create clusters of different -sizes. The cluster behaves as if it were running on multiple -nodes even if the containers are all running on the same host -machine. +added to `docker-compose.yml` to grow the cluster: the PBS server +pre-declares nodes `pbsnode1` through `pbsnode10` (raise the cap with +the `PBS_MAX_NODES` environment variable on the server container), so +adding a `pbsnodeN` service with a matching `hostname: pbsnodeN` is all +it takes for that node to join. Declared nodes without a running +container simply appear `down` in `pbsnodes -a`. The cluster behaves +as if it were running on multiple nodes even if the containers are all +running on the same host machine. # Image tags and base selection @@ -66,7 +70,7 @@ UBUNTU_VERSION=24.04 docker compose up -d --pull never ## Quickest path: docker compose `docker compose build` reads `.env` and constructs the full set of build args -automatically. To build all three containers (frontend, master, node) for the +automatically. To build all three containers (frontend, server, mom) for the default Ubuntu version: ```bash @@ -135,7 +139,7 @@ docker buildx build \ frontend/ ``` -The frontend build compiles ~355 scientific software packages and can take +The frontend build compiles ~356 scientific software packages and can take many hours on first build from an empty buildcache. Subsequent builds reuse cached packages from GHCR and finish much faster. @@ -194,7 +198,7 @@ docker compose down -v # the -v flag deletes the named volumes UBUNTU_VERSION=24.04 docker compose up -d --pull never ``` -Without `-v`, the new container will mount the previous run's `/opt`, which +Without `-v`, the new container will mount the previous run's `/opt/spack-stack`, which contains spack-built binaries linked against the *previous* OS's glibc. The cluster will appear to start fine but `qsub` or other PBS job submission of any spack-built executable will fail with `GLIBC_X.YZ not found`. @@ -215,7 +219,7 @@ docker compose -f docker-compose.yml ps ``` To check status of PBS: ``` -docker exec spack-stack-frontend qstat +docker exec spack-stack-frontend bash -lc "qstat" ``` To submit a simple PBS job: ``` diff --git a/docker-compose-test.yml b/docker-compose-test.yml index f87ad9c..61d25f8 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -76,36 +76,26 @@ services: BASE_IMAGE_TAG: ubuntu-${UBUNTU_VERSION:-26.04}-openpbs-${OPENPBS_VERSION} container_name: spack-stack-node1 hostname: pbsnode1 - environment: - - PBS_NODENAME=pbsnode1 pbsnode2: <<: *node-common container_name: spack-stack-node2 hostname: pbsnode2 - environment: - - PBS_NODENAME=pbsnode2 pbsnode3: <<: *node-common container_name: spack-stack-node3 hostname: pbsnode3 - environment: - - PBS_NODENAME=pbsnode3 pbsnode4: <<: *node-common container_name: spack-stack-node4 hostname: pbsnode4 - environment: - - PBS_NODENAME=pbsnode4 pbsnode5: <<: *node-common container_name: spack-stack-node5 hostname: pbsnode5 - environment: - - PBS_NODENAME=pbsnode5 volumes: home-vol: diff --git a/docker-compose.yml b/docker-compose.yml index 18ee8e2..8bf0870 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,36 +73,26 @@ services: BASE_IMAGE_TAG: ubuntu-${UBUNTU_VERSION:-26.04}-openpbs-${OPENPBS_VERSION} container_name: spack-stack-node1 hostname: pbsnode1 - environment: - - PBS_NODENAME=pbsnode1 pbsnode2: <<: *node-common container_name: spack-stack-node2 hostname: pbsnode2 - environment: - - PBS_NODENAME=pbsnode2 pbsnode3: <<: *node-common container_name: spack-stack-node3 hostname: pbsnode3 - environment: - - PBS_NODENAME=pbsnode3 pbsnode4: <<: *node-common container_name: spack-stack-node4 hostname: pbsnode4 - environment: - - PBS_NODENAME=pbsnode4 pbsnode5: <<: *node-common container_name: spack-stack-node5 hostname: pbsnode5 - environment: - - PBS_NODENAME=pbsnode5 volumes: home-vol: diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 785c9d8..46990ff 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -114,6 +114,15 @@ RUN cd /opt/spack-stack \ ' prefix: /opt/pbs' \ > /tmp/openpbs-tm.yaml \ && spack -e . config add -f /tmp/openpbs-tm.yaml \ + # Pin met to a single graphics variant. A from-scratch concretization resolves all + # consumers (metplus -> global-workflow/neptune/ufs-srw envs) to met~graphics, but with + # reuse:true + unify:when_possible the solver will greedily reuse a stale met+graphics + # blob if one lingers in the buildcache -- producing two met@12.1.1 specs that collide on + # the unhashed module name (met/12.1.1.lua). Requiring ~graphics forbids the +graphics + # variant so reuse can never reintroduce the duplicate, on any arch or cache state. The + # graphics extras are unused here (metplus plotting variants are off), so this is the + # lighter build and changes nothing functionally. \ + && spack -e . config add 'packages:met:require:"~graphics"' \ # Use the spack-stack site default target (x86_64_v3 on x86_64 hosts). \ # Earlier this block added `target:[x86_64]` for buildcache portability, but spack merged \ # that with the site default to `[x86_64, x86_64_v3]`, and the concretizer's choice between \ diff --git a/test/test_hello.sh b/test/test_hello.sh index 188afb3..e02bd89 100755 --- a/test/test_hello.sh +++ b/test/test_hello.sh @@ -38,7 +38,7 @@ mpiexec ./hello.exe > hello.raw EOF # -W block=true makes qsub wait for the job to finish and exit with the job's -# status, so a job failure trips `set -e` just like the old synchronous srun did. +# status, so a job failure trips `set -e` instead of being silently masked. qsub -W block=true hello.pbs sort hello.raw > hello.out