(4.1) ci: demonstrate ci:+cross/full fail message

.cargo/config.toml

@@ -12,13 +12,26 @@ CARGO_LLVM_COV_TARGET_DIR = { value = "target/llvm-cov/build", relative = true,
 CARGO_LLVM_COV_BUILD_DIR = { value = "target/llvm-cov/target", relative = true, force = false }
 
 [build]
-rustflags = ["--cfg=tokio_unstable"]
+# Register `emulated` so cfg_attr sites don't trip unexpected_cfgs natively.
+rustflags = ["--cfg=tokio_unstable", "--check-cfg=cfg(emulated)"]
 
 [target.wasm32-wasip1]
 # Trailing `--` separates wasmtime's CLI from the module + module args
 # that cargo appends, so e.g. `--nocapture` from libtest reaches the
 # wasm module instead of being parsed as a wasmtime option.
 runner = "wasmtime run --dir . --"
 
+[target.x86_64-unknown-linux-gnu]
+runner = "scripts/test-runner.sh x86_64"
+
+[target.x86_64-unknown-linux-musl]
+runner = "scripts/test-runner.sh x86_64"
+
+[target.aarch64-unknown-linux-gnu]
+runner = "scripts/test-runner.sh aarch64"
+
+[target.aarch64-unknown-linux-musl]
+runner = "scripts/test-runner.sh aarch64"
+
 [alias]
 nt = "nextest"

.config/nextest.toml

@@ -146,3 +146,8 @@ vm = { max-threads = 1 }
 
 [profile.miri]
 slow-timeout = { period = "500s" }
+
+[profile.cross-qemu]
+slow-timeout = { period = "60s", terminate-after = 5 }
+fail-fast = false
+failure-output = "immediate-final"

.github/actions/sticky-pr-comment/action.yml

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright Open Network Fabric Authors
+
+# Create-or-update a sticky PR comment keyed by an HTML-comment marker.
+# Caller needs `pull-requests: write`.
+
+name: "Sticky PR comment"
+description: "Create-or-update a sticky comment on the current pull request, keyed by marker"
+
+inputs:
+  marker:
+    description: "HTML-comment marker (e.g. `<!-- ci:dev:cross-advisory -->`) uniquely identifying this comment.  Prepended to the body automatically."
+    required: true
+  body:
+    description: "Markdown body of the comment (without the marker)."
+    required: true
+  pr-number:
+    description: "Pull request number to comment on."
+    default: "${{ github.event.pull_request.number }}"
+  token:
+    description: "GitHub token with `pull-requests: write`."
+    default: "${{ github.token }}"
+
+runs:
+  using: "composite"
+  steps:
+    - name: "Create or update sticky comment"
+      uses: "actions/github-script@v8"
+      env:
+        STICKY_MARKER: "${{ inputs.marker }}"
+        STICKY_BODY: "${{ inputs.body }}"
+        STICKY_PR_NUMBER: "${{ inputs.pr-number }}"
+      with:
+        github-token: "${{ inputs.token }}"
+        script: |
+          const marker = process.env.STICKY_MARKER;
+          const userBody = process.env.STICKY_BODY;
+          const prNumber = Number.parseInt(process.env.STICKY_PR_NUMBER, 10);
+          if (!marker || !marker.startsWith('<!--') || !marker.endsWith('-->')) {
+            core.setFailed(
+              `sticky-pr-comment: 'marker' must be an HTML comment ` +
+              `(\`<!-- ... -->\`); got: ${JSON.stringify(marker)}`,
+            );
+            return;
+          }
+          if (!Number.isInteger(prNumber) || prNumber <= 0) {
+            core.setFailed(
+              `sticky-pr-comment: 'pr-number' must be a positive integer; ` +
+              `got: ${JSON.stringify(process.env.STICKY_PR_NUMBER)}.  ` +
+              `(Are you invoking this on a non-pull_request event without ` +
+              `passing pr-number explicitly?)`,
+            );
+            return;
+          }
+          const body = `${marker}\n${userBody}`;
+          const comments = await github.paginate(
+            github.rest.issues.listComments,
+            {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              per_page: 100,
+            },
+          );
+          const existing = comments.find(
+            c => typeof c.body === 'string' && c.body.startsWith(marker),
+          );
+          if (existing) {
+            await github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: existing.id,
+              body,
+            });
+            core.info(`sticky-pr-comment: updated comment ${existing.id}`);
+          } else {
+            const { data: created } = await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              body,
+            });
+            core.info(`sticky-pr-comment: created comment ${created.id}`);
+          }

.github/workflows/bump.yml

@@ -149,9 +149,12 @@ jobs:
           token: "${{ steps.app-token.outputs.token }}"
           branch: "bump/cargo-upgrades"
           title: "bump(cargo)!: :rocket: upgrades available"
+          # ci:+cross/full opts the bump PR into the full cross matrix
+          # (build + qemu-user test on musl) to catch upstream regressions.
           labels: |
             automated
             dependencies
+            ci:+cross/full
           signoff: "true"
           sign-commits: "true"
           body: |

.github/workflows/dev.yml

@@ -447,6 +447,7 @@ jobs:
         github.event_name == 'pull_request'
         && (
           contains(github.event.pull_request.labels.*.name, 'ci:+cross')
+          || contains(github.event.pull_request.labels.*.name, 'ci:+cross/full')
         )
         || (github.event_name == 'push' || github.event_name == 'merge_group')
       }}
@@ -489,6 +490,25 @@ jobs:
     steps:
       - *checkout
       - *nix-setup
+      - name: "test"
+        if: >-
+          ${{
+            matrix.recipe.args == 'dataplane'
+            && matrix.libc == 'musl'
+            && github.event_name == 'pull_request'
+            && (
+              contains(github.event.pull_request.labels.*.name, 'ci:+cross')
+              || contains(github.event.pull_request.labels.*.name, 'ci:+cross/full')
+            )
+          }}
+        uses: *just
+        env:
+          JUST_VARS: >-
+            platform=${{ matrix.platform }}
+            profile=${{ matrix.profile }}
+            libc=${{ matrix.libc }}
+        with:
+          recipe: "test"
       - name: "${{ matrix.platform }}/${{ matrix.libc }}/${{ matrix.profile }}/${{ matrix.recipe.args }}"
         uses: *just
         env:
@@ -535,25 +555,6 @@ jobs:
         uses: *just
         with:
           recipe: "test"
-      - name: "shuttle_pct"
-        env:
-          JUST_VARS: >-
-            docker_sock=/run/docker/docker.sock
-            debug_justfile=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_justfile || false }}
-            profile=release
-            features=shuttle_pct
-            oci_repo=ghcr.io
-        uses: *just
-        with:
-          recipe: "test"
-      # The `loom` feature flips `concurrency::sync` to loom's
-      # primitives workspace-wide, which breaks crates that rely on
-      # `Weak`, `Arc::downgrade`, etc. (those aren't in
-      # `loom::sync`).  Scope the loom build to only the concurrency
-      # package (which hosts the core concurrency tests) so workspace
-      # feature unification doesn't poison unrelated crates.
-      #
-      # TODO: gate tests which can't be used with loom
       - name: "loom"
         env:
           JUST_VARS: >-
@@ -565,7 +566,6 @@ jobs:
         uses: *just
         with:
           recipe: "test"
-          recipe_args: "concurrency"
       - *tmate
 
   vlab:
@@ -655,6 +655,13 @@ jobs:
   summary:
     name: "Summary"
     runs-on: "ubuntu-latest"
+    # `pull-requests: write` is for the "Surface cross failures" step.
+    # `actions: read` lets that same step query the cross matrix's real
+    # job outcomes via the REST API.
+    permissions:
+      actions: "read"
+      contents: "read"
+      pull-requests: "write"
     needs:
       - check
       - sanitize
@@ -713,6 +720,76 @@ jobs:
         run: |
           echo '::error:: Some vlab job(s) failed'
           exit 1
+      # Cross is advisory (continue-on-error: true at the job level),
+      # which makes `needs.cross.result` always report `success` to
+      # dependents -- the very property that keeps leg flakes from
+      # blocking merge also hides genuine failures from this gate.
+      # Read the underlying matrix job outcomes from the REST API
+      # instead, then exit 1 only via the sticky comment path.
+      - name: "Detect cross matrix failures"
+        id: "cross_status"
+        if: >-
+          ${{
+            github.event_name == 'pull_request'
+            && needs.cross.result != 'skipped'
+          }}
+        uses: "actions/github-script@v8"
+        with:
+          script: |
+            // Cross matrix jobs are named
+            // `${recipe.name}/${recipe.args}/${platform}/${libc}`.
+            // `recipe.name` is always "build-container" in the `cross`
+            // job's matrix, so the prefix uniquely identifies them.
+            const jobs = await github.paginate(
+              github.rest.actions.listJobsForWorkflowRun,
+              {
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.runId,
+                per_page: 100,
+              },
+            );
+            const cross = jobs.filter(j => j.name.startsWith('build-container/'));
+            const failed = cross.filter(j => j.conclusion === 'failure');
+            core.setOutput('failed_count', String(failed.length));
+            core.setOutput('failed_names', failed.map(j => j.name).join(', '));
+            const heading = failed.length === 0
+              ? `All ${cross.length} cross matrix entries passed.`
+              : `${failed.length} of ${cross.length} cross matrix entries failed.`;
+            await core.summary.addHeading('Cross build advisory').addRaw(heading).write();
+      # The sticky-comment composite action is a local action; it has
+      # to be on the runner's disk for `uses:` to resolve.  The summary
+      # job has no other reason to check the repo out, so do a sparse
+      # checkout of just the action directory, gated on the same
+      # failure condition as the comment step itself.
+      - name: "Checkout sticky-pr-comment action"
+        if: >-
+          ${{
+            github.event_name == 'pull_request'
+            && steps.cross_status.outputs.failed_count != ''
+            && steps.cross_status.outputs.failed_count != '0'
+          }}
+        uses: "actions/checkout@v6"
+        with:
+          persist-credentials: "false"
+          sparse-checkout: ".github/actions/sticky-pr-comment"
+          sparse-checkout-cone-mode: "false"
+      - name: "Surface cross failures via sticky PR comment"
+        if: >-
+          ${{
+            github.event_name == 'pull_request'
+            && steps.cross_status.outputs.failed_count != ''
+            && steps.cross_status.outputs.failed_count != '0'
+          }}
+        uses: "./.github/actions/sticky-pr-comment"
+        with:
+          marker: "<!-- ci:dev:cross-advisory -->"
+          body: |
+            ## :warning: Cross build advisory: failure detected :warning:
+
+            [Please investigate before merging](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+
+            Failed legs: ${{ steps.cross_status.outputs.failed_names }}
 
   publish:
     runs-on: lab

.github/workflows/lint-opengrep.yml

@@ -40,4 +40,6 @@ jobs:
               shell: "bash"
               run: |
                   set -euo pipefail
-                  opengrep scan --experimental --verbose --error
+                  opengrep scan --experimental --verbose --error \
+                      --config auto \
+                      --config .semgrep/rules

.gitignore

@@ -1,5 +1,7 @@
 **.profraw
 **/__fuzz__/**
+# qemu-user core dumps from SIGABRT under emulated tests.
+**/qemu_*.core
 result*
 result*/**
 **/target/**

.semgrep/rules/no-std-sync-direct.yaml

@@ -0,0 +1,80 @@
+# Path-aware enforcement of the `concurrency::sync` facade.
+#
+# `clippy::disallowed_types` resolves through re-exports: it sees
+# `concurrency::sync::Arc` as `std::sync::Arc` (same canonical path),
+# so it cannot distinguish "imported via the workspace facade" from
+# "imported directly from std::sync". opengrep's source-level pattern
+# matching has no such limit — it sees what was written, not what was
+# resolved.
+#
+# These rules forbid both `use std::sync::X;` imports and bare
+# fully-qualified path expressions (`std::sync::X::new(...)`,
+# `std::sync::X<T>`) for any type the `concurrency::sync` facade
+# exposes. Switching to `concurrency::sync` is the migration that lets
+# future loom/shuttle test builds hook these types via the backend
+# cfg. The lock types (Mutex/RwLock and guards) are also caught by
+# `clippy::disallowed_types` because parking_lot provides a distinct
+# production type; this rule is the only enforcement for Arc / Weak /
+# atomic / LazyLock / OnceLock / Once / Barrier / Condvar.
+#
+# The `concurrency` crate itself is exempt (it IS the facade) and so are
+# its tests (which use the facade by design but are flagged by clippy's
+# alias resolution; that allow is per-file). Anywhere else, a hit means
+# "switch to `use concurrency::sync::X;` and drop the FQN".
+rules:
+  - id: rust-no-direct-std-sync-import
+    languages: [rust]
+    severity: ERROR
+    message: |
+      Import synchronization primitives via `concurrency::sync::*`, not
+      `std::sync::*`. The workspace's `concurrency::sync` facade hands
+      you parking_lot-backed locks in production and loom/shuttle
+      equivalents under the model-checker features; importing directly
+      from `std::sync` bypasses that backend swap.
+    paths:
+      exclude:
+        # The wrapper layer is allowed to use std::sync directly.
+        - concurrency/src/sync/
+        # quiescent/slot are concurrency-internal users of the facade
+        # that still parse as std::sync because the lint matches paths
+        # textually here too.
+        - concurrency/src/quiescent.rs
+        - concurrency/src/slot.rs
+        # Tests for the concurrency crate use the wrapped types
+        # deliberately; the allow is at the file head.
+        - concurrency/tests/
+    pattern-either:
+      # Direct imports: `use std::sync::X;`.
+      - pattern: use std::sync::Arc;
+      - pattern: use std::sync::Weak;
+      - pattern: use std::sync::Mutex;
+      - pattern: use std::sync::MutexGuard;
+      - pattern: use std::sync::RwLock;
+      - pattern: use std::sync::RwLockReadGuard;
+      - pattern: use std::sync::RwLockWriteGuard;
+      - pattern: use std::sync::OnceLock;
+      - pattern: use std::sync::LazyLock;
+      - pattern: use std::sync::Once;
+      - pattern: use std::sync::Barrier;
+      - pattern: use std::sync::Condvar;
+      - pattern: use std::sync::atomic;
+      - pattern: use std::sync::atomic::$X;
+      - pattern: use std::sync::PoisonError;
+      # Grouped imports across one or many lines:
+      # `use std::sync::{Arc, Mutex};` and the multi-line variant.
+      # AST-level matching for `use ... :: { ... }` is hit-and-miss
+      # across tree-sitter versions; a regex with `(?s)` (dotall)
+      # spans newlines without anchoring on the import keyword's
+      # column.
+      - pattern-regex: '(?s)use\s+std::sync::\{[^}]*\b(?:Arc|Weak|Mutex|MutexGuard|RwLock|RwLockReadGuard|RwLockWriteGuard|OnceLock|LazyLock|Once|Barrier|Condvar|PoisonError|atomic)\b'
+      # Fully-qualified path expressions / type references that skip
+      # the `use` line entirely (e.g. `static X: std::sync::LazyLock
+      # <T> = std::sync::LazyLock::new(...)`, or inline turbofish
+      # like `std::sync::Arc::<T>::new`).  Bare `pattern: std::sync
+      # ::X` does not fire reliably in opengrep's Rust frontend
+      # across path-vs-type contexts; a regex is the dependable
+      # backstop.  The leading `(?m)^(?!\s*(?://|\*))` skips lines
+      # whose first non-whitespace is a `//` line/doc comment or a
+      # block-comment continuation `*`, so rustdoc intra-doc links
+      # like `/// [std::sync::Arc]` don't false-positive.
+      - pattern-regex: '(?m)^(?!\s*(?://|\*)).*\bstd::sync::(?:Arc|Weak|Mutex|MutexGuard|RwLock|RwLockReadGuard|RwLockWriteGuard|OnceLock|LazyLock|Once|Barrier|Condvar|PoisonError|atomic::[A-Za-z_][A-Za-z0-9_]*)\b'