From 4e7f12db78f6e2632d62e874109fc94fc61fc4fa Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 2 Sep 2026 10:48:46 -0500 Subject: [PATCH 1/2] CI: bump NVHPC to 26.3, pin generic x86-64 codegen The MFC NVHPC cpu job fails intermittently with an nvfortran backend crash while compiling m_model.fpp: LLVM ERROR: Cannot select: v4f32 = X86ISD::RSQRT14S in function '@m_model_s_read_stl_binary_' nvfortran-Fatal-.../24.5/compilers/share/llvm/bin/llc TERMINATED by signal 6 nvfortran defaults to -tp=native, so it tunes for whichever host the job lands on. GitHub's ubuntu-22.04 pool mixes AVX-512-capable and non-capable CPUs, so on an AVX-512 draw 24.5 emits an RSQRT14S its own LLVM backend cannot lower. That is why the job passes on some runs and aborts on others with no change to ffmt. Pin -tp=px so codegen no longer depends on the runner draw, matching the flags MFC sets in its own NVHPC CI, and move the container to 26.3, the newest release MFC exercises upstream. --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ecaad6..cb1b2c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,13 +116,22 @@ jobs: mfc-nvhpc: name: MFC NVHPC ${{ matrix.target }} runs-on: ubuntu-22.04 - container: nvcr.io/nvidia/nvhpc:24.5-devel-cuda_multi-ubuntu22.04 + container: nvcr.io/nvidia/nvhpc:26.3-devel-cuda_multi-ubuntu22.04 env: CC: nvc CXX: nvc++ FC: nvfortran OMPI_ALLOW_RUN_AS_ROOT: 1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + # Target generic x86-64 rather than the runner's own CPU. GitHub's + # ubuntu-22.04 pool mixes AVX-512-capable and non-capable hosts, and + # nvfortran's default -tp=native turns that into nondeterministic + # codegen: on an AVX-512 draw it emitted an RSQRT14S that its own LLVM + # backend could not select, aborting llc while compiling m_model.fpp. + # Mirrors the flags MFC uses in its own NVHPC CI. + FFLAGS: -tp=px -Kieee -noswitcherror + CFLAGS: -tp=px + CXXFLAGS: -tp=px strategy: fail-fast: false matrix: From 322ca831eafd326e69682e70a54c449f2cf664fa Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 2 Sep 2026 10:53:23 -0500 Subject: [PATCH 2/2] CI: drive the NVHPC container by hand so the image pull can retry The container: directive pulls the ~25-30 GB cuda_multi image with no retry, and nvcr.io intermittently times out under load ("context deadline exceeded"), turning a registry hiccup into a red build. Replace it with an explicit docker pull loop (5 attempts, backing off 30s per attempt; pulls resume completed layers, so retries are cheap) plus a long-lived container driven by docker exec, which preserves installed packages and env across steps the way container: did. Same approach MFC uses in its own NVHPC CI. The workspace is bind-mounted at /workspace, so git is told to trust it to avoid dubious-ownership noise. Compiler env moves from the job-level env: block onto docker run -e. --- .github/workflows/ci.yml | 113 +++++++++++++++++++++++++++------------ 1 file changed, 78 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb1b2c4..d4816c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,27 +116,14 @@ jobs: mfc-nvhpc: name: MFC NVHPC ${{ matrix.target }} runs-on: ubuntu-22.04 - container: nvcr.io/nvidia/nvhpc:26.3-devel-cuda_multi-ubuntu22.04 - env: - CC: nvc - CXX: nvc++ - FC: nvfortran - OMPI_ALLOW_RUN_AS_ROOT: 1 - OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 - # Target generic x86-64 rather than the runner's own CPU. GitHub's - # ubuntu-22.04 pool mixes AVX-512-capable and non-capable hosts, and - # nvfortran's default -tp=native turns that into nondeterministic - # codegen: on an AVX-512 draw it emitted an RSQRT14S that its own LLVM - # backend could not select, aborting llc while compiling m_model.fpp. - # Mirrors the flags MFC uses in its own NVHPC CI. - FFLAGS: -tp=px -Kieee -noswitcherror - CFLAGS: -tp=px - CXXFLAGS: -tp=px strategy: fail-fast: false matrix: target: [cpu, gpu-omp] + env: + NVHPC_IMAGE: nvcr.io/nvidia/nvhpc:26.3-devel-cuda_multi-ubuntu22.04 steps: + # Free disk on the host before pulling the ~25-30 GB cuda_multi image. - name: Free disk space uses: jlumbroso/free-disk-space@main with: @@ -148,41 +135,97 @@ jobs: - uses: actions/checkout@v4 + # Retry the pull: nvcr.io intermittently times out ("context deadline + # exceeded") under load. Pulls resume completed layers, so retries are + # cheap. This is why the job drives a container by hand instead of using + # the container: directive, which offers no way to retry. + - name: Pull NVHPC container + run: | + for attempt in 1 2 3 4 5; do + docker pull "$NVHPC_IMAGE" && exit 0 + echo "docker pull failed (attempt $attempt/5); retrying in $((attempt * 30))s..." + sleep $((attempt * 30)) + done + echo "::error::Failed to pull $NVHPC_IMAGE after 5 attempts" + exit 1 + + # A long-lived container preserves installed packages and env across + # steps, the way container: did. + # + # -tp=px targets generic x86-64 rather than the runner's own CPU. + # GitHub's ubuntu-22.04 pool mixes AVX-512-capable and non-capable + # hosts, and nvfortran's default -tp=native turns that into + # nondeterministic codegen: on an AVX-512 draw it emitted an RSQRT14S + # that its own LLVM backend could not select, aborting llc while + # compiling m_model.fpp. Mirrors the flags MFC uses in its own NVHPC CI. + - name: Start NVHPC container + run: | + docker run -d --name nvhpc \ + --security-opt seccomp=unconfined \ + -v "${{ github.workspace }}:/workspace" \ + -w /workspace \ + -e CC=nvc \ + -e CXX=nvc++ \ + -e FC=nvfortran \ + -e OMPI_ALLOW_RUN_AS_ROOT=1 \ + -e OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \ + -e "FFLAGS=-tp=px -Kieee -noswitcherror" \ + -e CFLAGS=-tp=px \ + -e CXXFLAGS=-tp=px \ + "$NVHPC_IMAGE" sleep infinity + - name: Install dependencies run: | - apt-get update -y - apt-get install -y cmake python3 python3-venv python3-pip \ - libfftw3-dev libhdf5-dev hdf5-tools \ - openmpi-bin libopenmpi-dev git curl + docker exec nvhpc bash -c ' + set -e + apt-get update -y + apt-get install -y cmake python3 python3-venv python3-pip \ + libfftw3-dev libhdf5-dev hdf5-tools \ + openmpi-bin libopenmpi-dev git curl + # The repo is bind-mounted from the host, so git sees a different + # owner. Mark it safe to suppress "dubious ownership" errors. + git config --global --add safe.directory /workspace + ' - name: Install Rust run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - echo "$HOME/.cargo/bin" >> $GITHUB_PATH + docker exec nvhpc bash -c ' + set -e + curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + ' - name: Build ffmt run: | - . "$HOME/.cargo/env" - cargo build --release + docker exec nvhpc bash -c ' + set -e + . "$HOME/.cargo/env" + cargo build --release + ' - name: Clone MFC - run: git clone --depth 1 https://github.com/MFlowCode/MFC.git /tmp/MFC + run: | + docker exec nvhpc git clone --depth 1 \ + https://github.com/MFlowCode/MFC.git /tmp/MFC - name: Format MFC source with ffmt run: | - . "$HOME/.cargo/env" - ./target/release/ffmt -j $(nproc) /tmp/MFC/src/ 2>/dev/null || true - ./target/release/ffmt -j $(nproc) /tmp/MFC/src/ 2>/dev/null || true + docker exec nvhpc bash -c ' + ./target/release/ffmt -j $(nproc) /tmp/MFC/src/ 2>/dev/null || true + ./target/release/ffmt -j $(nproc) /tmp/MFC/src/ 2>/dev/null || true + ' - name: Build MFC run: | - cd /tmp/MFC - pip install -e toolchain/ - GPU_FLAG="" - if [ "${{ matrix.target }}" = "gpu-acc" ]; then GPU_FLAG="--gpu acc"; fi - if [ "${{ matrix.target }}" = "gpu-omp" ]; then GPU_FLAG="--gpu mp"; fi - JOBS=${{ matrix.target == 'cpu' && '$(nproc)' || '2' }} - ./mfc.sh build -j $JOBS $GPU_FLAG + docker exec nvhpc bash -c ' + set -e + cd /tmp/MFC + pip install -e toolchain/ + GPU_FLAG="" + if [ "${{ matrix.target }}" = "gpu-acc" ]; then GPU_FLAG="--gpu acc"; fi + if [ "${{ matrix.target }}" = "gpu-omp" ]; then GPU_FLAG="--gpu mp"; fi + JOBS=${{ matrix.target == 'cpu' && '$(nproc)' || '2' }} + ./mfc.sh build -j $JOBS $GPU_FLAG + ' docs: name: Documentation