Updates to cleanup leftovers from the port from Slurm

.github/workflows/docker.yml

@@ -103,7 +103,7 @@ jobs:
           UBUNTU_VERSION: ${{ matrix.ubuntu }}
           OPENPBS_VERSION: ${{ env.OPENPBS_VERSION }}
           SPACK_STACK_VERSION: ${{ env.SPACK_STACK_VERSION }}
-        run: docker compose -f docker-compose-test.yml up --pull never -d
+        run: docker compose -f docker-compose-test.yml up --pull never -d --wait
       - name: Check cluster logs
         run: docker compose -f docker-compose-test.yml logs
       - name: Check status of the cluster containers
@@ -306,7 +306,7 @@ jobs:
           UBUNTU_VERSION: ${{ matrix.ubuntu }}
           OPENPBS_VERSION: ${{ env.OPENPBS_VERSION }}
           SPACK_STACK_VERSION: ${{ env.SPACK_STACK_VERSION }}
-        run: docker compose -f docker-compose-test.yml up --pull never -d
+        run: docker compose -f docker-compose-test.yml up --pull never -d --wait
       - name: Check cluster logs
         run: docker compose -f docker-compose-test.yml logs
       - name: Check status of the cluster containers

.github/workflows/package-cleanup.yaml

@@ -1,6 +1,10 @@
 name: PackageCleanup
 
 on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
   workflow_dispatch:
     inputs:
       buildcache_cutoff:
@@ -12,94 +16,85 @@ on:
         required: false
         default: 'true'
 
+env:
+  # Must stay in sync with docker.yml. Used to construct per-(ubuntu, spack-stack)
+  # buildcache repo names for the cleanup-stale-buildcache job.
+  SPACK_STACK_VERSION: 2.1.0
+
 jobs:
-  cleanup-untagged:
+
+  # Remove untagged versions of our published images. New publishes leave the
+  # previous tagged versions as "untagged" if the tag was moved (e.g. when
+  # `latest` is re-pointed). Runs continuously to keep the registry tidy.
+  cleanup-packages:
     runs-on: ubuntu-latest
-    if: github.event_name == 'workflow_dispatch'
     permissions:
       packages: write
       contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        container: [frontend, server, mom]
     steps:
-      - name: Remove untagged versions of dockerspackstackpbscluster/pbs-spack-stack-frontend
-        uses: actions/delete-package-versions@v5
-        with: 
-          package-name: 'dockerspackstackpbscluster/pbs-spack-stack-frontend'
-          package-type: 'container'
-          min-versions-to-keep: 0
-          delete-only-untagged-versions: 'true'
-      - name: Remove untagged versions of dockerspackstackpbscluster/pbs-spack-stack-server
-        uses: actions/delete-package-versions@v5
-        with: 
-          package-name: 'dockerspackstackpbscluster/pbs-spack-stack-server'
-          package-type: 'container'
-          min-versions-to-keep: 0
-          delete-only-untagged-versions: 'true'
-      - name: Remove untagged versions of dockerspackstackpbscluster/pbs-spack-stack-mom
-        uses: actions/delete-package-versions@v5
-        with: 
-          package-name: 'dockerspackstackpbscluster/pbs-spack-stack-mom'
-          package-type: 'container'
-          min-versions-to-keep: 0
-          delete-only-untagged-versions: 'true'
-      - name: Remove untagged versions of dockerspackstackpbscluster/frontend-cache-amd64
-        uses: actions/delete-package-versions@v5
-        with: 
-          package-name: 'dockerspackstackpbscluster/frontend-cache-amd64'
-          package-type: 'container'
-          min-versions-to-keep: 0
-          delete-only-untagged-versions: 'true'
-      - name: Remove untagged versions of dockerspackstackpbscluster/frontend-cache-arm64
-        uses: actions/delete-package-versions@v5
-        with: 
-          package-name: 'dockerspackstackpbscluster/frontend-cache-arm64'
-          package-type: 'container'
-          min-versions-to-keep: 0
-          delete-only-untagged-versions: 'true'
-      - name: Remove untagged versions of dockerspackstackpbscluster/server-cache-amd64
-        uses: actions/delete-package-versions@v5
-        with: 
-          package-name: 'dockerspackstackpbscluster/server-cache-amd64'
-          package-type: 'container'
-          min-versions-to-keep: 0
-          delete-only-untagged-versions: 'true'
-      - name: Remove untagged versions of dockerspackstackpbscluster/server-cache-arm64
+      -
+        name: Remove untagged versions of dockerspackstackpbscluster/pbs-spack-stack-${{ matrix.container }}
         uses: actions/delete-package-versions@v5
-        with: 
-          package-name: 'dockerspackstackpbscluster/server-cache-arm64'
-          package-type: 'container'
-          min-versions-to-keep: 0
-          delete-only-untagged-versions: 'true'
-      - name: Remove untagged versions of dockerspackstackpbscluster/mom-cache-amd64
-        uses: actions/delete-package-versions@v5
-        with: 
-          package-name: 'dockerspackstackpbscluster/mom-cache-amd64'
+        with:
+          package-name: 'dockerspackstackpbscluster/pbs-spack-stack-${{ matrix.container }}'
           package-type: 'container'
           min-versions-to-keep: 0
           delete-only-untagged-versions: 'true'
-      - name: Remove untagged versions of dockerspackstackpbscluster/mom-cache-arm64
+
+  # Remove untagged versions of the docker layer caches written by CI. Each
+  # (container, ubuntu, arch) combination has its own cache package -- when CI
+  # pushes a new :cache tag, the previous version becomes untagged.
+  cleanup-caches:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        container: [frontend, server, mom]
+        ubuntu: ['24.04', '26.04']
+        arch: [amd64, arm64]
+    steps:
+      -
+        name: Remove untagged versions of dockerspackstackpbscluster/${{ matrix.container }}-cache-ubuntu-${{ matrix.ubuntu }}-${{ matrix.arch }}
         uses: actions/delete-package-versions@v5
-        with: 
-          package-name: 'dockerspackstackpbscluster/mom-cache-arm64'
+        with:
+          package-name: 'dockerspackstackpbscluster/${{ matrix.container }}-cache-ubuntu-${{ matrix.ubuntu }}-${{ matrix.arch }}'
           package-type: 'container'
           min-versions-to-keep: 0
           delete-only-untagged-versions: 'true'
 
+  # Age-based cleanup of stale spack OCI buildcache entries. Manual only (cutoff
+  # input is required) and parameterized so the operator can do dry runs first.
+  # One job per (ubuntu, spack-stack) cache repo, since each gets its own
+  # per-spec-hash blob/tag population from autopush.
   cleanup-stale-buildcache:
     if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.buildcache_cutoff != '' }}
     runs-on: ubuntu-latest
     permissions:
       packages: write
       contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        ubuntu: ['24.04', '26.04']
     steps:
-      - name: Clean stale buildcache entries
+      -
+        name: Clean stale buildcache entries for ubuntu-${{ matrix.ubuntu }}-spack-stack-${{ env.SPACK_STACK_VERSION }}
         uses: actions/github-script@v7
         with:
           script: |
             const cutoff = new Date('${{ github.event.inputs.buildcache_cutoff }}');
             const dryRun = '${{ github.event.inputs.dry_run }}' === 'true';
             const org = 'noaa-gsl';
-            const packageName = 'dockerspackstackpbscluster/buildcache';
+            const packageName = 'dockerspackstackpbscluster/buildcache-ubuntu-${{ matrix.ubuntu }}-spack-stack-${{ env.SPACK_STACK_VERSION }}';
 
+            console.log(`Package: ${packageName}`);
             console.log(`Cutoff date: ${cutoff.toISOString()}`);
             console.log(`Dry run: ${dryRun}`);
 
@@ -123,6 +118,7 @@ jobs:
                 const createdAt = new Date(version.created_at);
                 const tags = version.metadata?.container?.tags || [];
 
+                // Never delete index entries - they are updated in place by spack buildcache update-index
                 const isIndex = tags.some(t => t.includes('index') || t.startsWith('_'));
                 if (isIndex) {
                   console.log(`Preserving index: ${version.id} (tags: ${tags.join(', ')})`);
@@ -153,4 +149,4 @@ jobs:
               page++;
             }
 
-            console.log(`\nSummary: ${deleted} ${dryRun ? 'would be ' : ''}deleted, ${kept} kept`);
+            console.log(`\nSummary for ${packageName}: ${deleted} ${dryRun ? 'would be ' : ''}deleted, ${kept} kept`);

.gitignore

@@ -0,0 +1,11 @@
+# Artifacts generated by running test/test_hello.sh
+test/hello.exe
+test/hello.out
+test/hello.raw
+test/hello.pbs
+test/hello.pbs.log
+test/*.OU
+test/*.ER
+
+# macOS
+.DS_Store

README.md

@@ -11,7 +11,7 @@ updated for PBS and OpenPBS.
 There are three containers:
 
 * A frontend container that acts as a PBS cluster login node.
-  Spack-stack is installed on the frontend in /opt which is mounted
+  Spack-stack is installed on the frontend in /opt/spack-stack which is mounted
   across the cluster as a shared volume using docker compose
 * A server container that acts as a PBS server/controller node
 * A mom container that acts as a PBS compute node
@@ -20,10 +20,14 @@ These containers are launched using Docker Compose to build
 a fully functioning PBS cluster.  A `docker-compose.yml`
 file defines the cluster, specifying ports and volumes to
 be shared.  Multiple instances of the mom container can be
-added to `docker-compose.yml` to create clusters of different
-sizes.  The cluster behaves as if it were running on multiple
-nodes even if the containers are all running on the same host
-machine.
+added to `docker-compose.yml` to grow the cluster: the PBS server
+pre-declares nodes `pbsnode1` through `pbsnode10` (raise the cap with
+the `PBS_MAX_NODES` environment variable on the server container), so
+adding a `pbsnodeN` service with a matching `hostname: pbsnodeN` is all
+it takes for that node to join. Declared nodes without a running
+container simply appear `down` in `pbsnodes -a`.  The cluster behaves
+as if it were running on multiple nodes even if the containers are all
+running on the same host machine.
 
 # Image tags and base selection
 
@@ -66,7 +70,7 @@ UBUNTU_VERSION=24.04 docker compose up -d --pull never
 ## Quickest path: docker compose
 
 `docker compose build` reads `.env` and constructs the full set of build args
-automatically. To build all three containers (frontend, master, node) for the
+automatically. To build all three containers (frontend, server, mom) for the
 default Ubuntu version:
 
 ```bash
@@ -135,7 +139,7 @@ docker buildx build \
   frontend/
 ```
 
-The frontend build compiles ~355 scientific software packages and can take
+The frontend build compiles ~356 scientific software packages and can take
 many hours on first build from an empty buildcache. Subsequent builds reuse
 cached packages from GHCR and finish much faster.
 
@@ -194,7 +198,7 @@ docker compose down -v   # the -v flag deletes the named volumes
 UBUNTU_VERSION=24.04 docker compose up -d --pull never
 ```
 
-Without `-v`, the new container will mount the previous run's `/opt`, which
+Without `-v`, the new container will mount the previous run's `/opt/spack-stack`, which
 contains spack-built binaries linked against the *previous* OS's glibc. The
 cluster will appear to start fine but `qsub` or other PBS job submission of any
 spack-built executable will fail with `GLIBC_X.YZ not found`.
@@ -215,7 +219,7 @@ docker compose -f docker-compose.yml ps
 ```
 To check status of PBS:
 ```
-docker exec spack-stack-frontend qstat
+docker exec spack-stack-frontend bash -lc "qstat"
 ```
 To submit a simple PBS job:
 ```

docker-compose-test.yml

@@ -76,36 +76,26 @@ services:
         BASE_IMAGE_TAG: ubuntu-${UBUNTU_VERSION:-26.04}-openpbs-${OPENPBS_VERSION}
     container_name: spack-stack-node1
     hostname: pbsnode1
-    environment:
-      - PBS_NODENAME=pbsnode1
 
   pbsnode2:
     <<: *node-common
     container_name: spack-stack-node2
     hostname: pbsnode2
-    environment:
-      - PBS_NODENAME=pbsnode2
 
   pbsnode3:
     <<: *node-common
     container_name: spack-stack-node3
     hostname: pbsnode3
-    environment:
-      - PBS_NODENAME=pbsnode3
 
   pbsnode4:
     <<: *node-common
     container_name: spack-stack-node4
     hostname: pbsnode4
-    environment:
-      - PBS_NODENAME=pbsnode4
 
   pbsnode5:
     <<: *node-common
     container_name: spack-stack-node5
     hostname: pbsnode5
-    environment:
-      - PBS_NODENAME=pbsnode5
 
 volumes:
   home-vol:

docker-compose.yml

@@ -73,36 +73,26 @@ services:
         BASE_IMAGE_TAG: ubuntu-${UBUNTU_VERSION:-26.04}-openpbs-${OPENPBS_VERSION}
     container_name: spack-stack-node1
     hostname: pbsnode1
-    environment:
-      - PBS_NODENAME=pbsnode1
 
   pbsnode2:
     <<: *node-common
     container_name: spack-stack-node2
     hostname: pbsnode2
-    environment:
-      - PBS_NODENAME=pbsnode2
 
   pbsnode3:
     <<: *node-common
     container_name: spack-stack-node3
     hostname: pbsnode3
-    environment:
-      - PBS_NODENAME=pbsnode3
 
   pbsnode4:
     <<: *node-common
     container_name: spack-stack-node4
     hostname: pbsnode4
-    environment:
-      - PBS_NODENAME=pbsnode4
 
   pbsnode5:
     <<: *node-common
     container_name: spack-stack-node5
     hostname: pbsnode5
-    environment:
-      - PBS_NODENAME=pbsnode5
 
 volumes:
   home-vol:

frontend/Dockerfile

@@ -114,6 +114,15 @@ RUN cd /opt/spack-stack \
    '      prefix: /opt/pbs' \
    > /tmp/openpbs-tm.yaml \
  && spack -e . config add -f /tmp/openpbs-tm.yaml \
+ # Pin met to a single graphics variant. A from-scratch concretization resolves all
+ # consumers (metplus -> global-workflow/neptune/ufs-srw envs) to met~graphics, but with
+ # reuse:true + unify:when_possible the solver will greedily reuse a stale met+graphics
+ # blob if one lingers in the buildcache -- producing two met@12.1.1 specs that collide on
+ # the unhashed module name (met/12.1.1.lua). Requiring ~graphics forbids the +graphics
+ # variant so reuse can never reintroduce the duplicate, on any arch or cache state. The
+ # graphics extras are unused here (metplus plotting variants are off), so this is the
+ # lighter build and changes nothing functionally. \
+ && spack -e . config add 'packages:met:require:"~graphics"' \
  # Use the spack-stack site default target (x86_64_v3 on x86_64 hosts). \
  # Earlier this block added `target:[x86_64]` for buildcache portability, but spack merged \
  # that with the site default to `[x86_64, x86_64_v3]`, and the concretizer's choice between \

test/test_hello.sh

@@ -38,7 +38,7 @@ mpiexec ./hello.exe > hello.raw
 EOF
 
 # -W block=true makes qsub wait for the job to finish and exit with the job's
-# status, so a job failure trips `set -e` just like the old synchronous srun did.
+# status, so a job failure trips `set -e` instead of being silently masked.
 qsub -W block=true hello.pbs
 
 sort hello.raw > hello.out