perf(ci): parallelize complete collector ShellCheck gate

.github/scripts/check-collector-shells.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Preserve source companions while bounding independent ShellCheck groups.
+set -euo pipefail
+root=$(cd -- "$(dirname -- "$0")/../.." && pwd)
+if [[ ${1:-} == --root && $# == 2 ]]; then
+  root=$(cd -- "$2" && pwd)
+elif (( $# != 0 )); then
+  printf 'usage: %s [--root directory]\n' "$0" >&2
+  exit 2
+fi
+groups="$root/.github/scripts/collector-shellcheck-groups.json"
+cd -- "$root/rust_hft"
+scripts=(../deployment/aliyun/*.sh)
+[[ -f ${scripts[0]} ]] || { echo 'collector shell input set is empty' >&2; exit 1; }
+work=$(mktemp -d)
+trap 'rm -rf -- "$work"' EXIT
+jq -e 'type=="array" and length>0 and all(.[]; type=="array" and length>0
+  and all(.[]; type=="string" and startswith("../deployment/aliyun/") and endswith(".sh")))' "$groups" >/dev/null
+printf '%s\0' "${scripts[@]}" | sort -z >"$work/expected"
+jq -j '.[][] | . + "\u0000"' "$groups" | sort -zu >"$work/actual"
+cmp -s "$work/expected" "$work/actual" || {
+  echo 'ShellCheck groups must cover the exact collector script set' >&2; exit 1;
+}
+began=$SECONDS
+for script in "${scripts[@]}"; do bash -n "$script"; done
+printf 'collector_shell_phase name=syntax files=%s elapsed_seconds=%s\n' \
+  "${#scripts[@]}" "$((SECONDS - began))"
+began=$SECONDS
+# Keep the original cwd, source context and flags. No -x or warning suppression.
+jq -c '.[]' "$groups" | while IFS= read -r group; do printf '%s\0' "$group"; done \
+  | xargs -0 -P 4 -n 1 bash -c '
+      mapfile -d "" -t scripts < <(jq -j ".[] | . + \"\\u0000\"" <<<"$1")
+      shellcheck -- "${scripts[@]}"
+    ' _
+printf 'collector_shell_phase name=shellcheck workers=4 groups=%s files=%s elapsed_seconds=%s\n' \
+  "$(jq length "$groups")" "${#scripts[@]}" "$((SECONDS - began))"

.github/scripts/collector-shellcheck-groups.json

@@ -0,0 +1,84 @@
+[
+  [
+    "../deployment/aliyun/test-monday-collector-health.sh",
+    "../deployment/aliyun/test-recovery-retention-fixture.sh",
+    "../deployment/aliyun/host-rust-lob-recovery-queue.sh",
+    "../deployment/aliyun/rust-lob-control-plane-lib.sh"
+  ],
+  [
+    "../deployment/aliyun/test-rust-lob-retention.sh",
+    "../deployment/aliyun/test-recovery-retention-fixture.sh",
+    "../deployment/aliyun/host-rust-lob-recovery-queue.sh",
+    "../deployment/aliyun/rust-lob-control-plane-lib.sh",
+    "../deployment/aliyun/test-rust-lob-recovery-queue.sh"
+  ],
+  [
+    "../deployment/aliyun/test-rust-lob-control-plane.sh",
+    "../deployment/aliyun/host-rust-lob-controller-release.sh",
+    "../deployment/aliyun/test-rust-lob-controller-release.sh",
+    "../deployment/aliyun/rust-lob-control-plane-lib.sh"
+  ],
+  [
+    "../deployment/aliyun/test-polymarket-raw-ops-control-plane.sh"
+  ],
+  [
+    "../deployment/aliyun/host-rust-lob-shadow-gate.sh",
+    "../deployment/aliyun/host-rust-lob-cutover.sh",
+    "../deployment/aliyun/rust-lob-control-plane-lib.sh"
+  ],
+  [
+    "../deployment/aliyun/host-rust-lob-restore.sh",
+    "../deployment/aliyun/host-rust-lob-readback.sh",
+    "../deployment/aliyun/publish-rust-lob-pair-release.sh",
+    "../deployment/aliyun/test-host-rust-lob-cutover.sh",
+    "../deployment/aliyun/test-rust-lob-rollback-unit-bindings.sh",
+    "../deployment/aliyun/rust-lob-control-plane-lib.sh"
+  ],
+  [
+    "../deployment/aliyun/binance-lob-slice-materialize.sh",
+    "../deployment/aliyun/binance-usdm-reference-shadow-gate.sh",
+    "../deployment/aliyun/bybit-options-control-plane-lib.sh",
+    "../deployment/aliyun/collector-monitor-retained.sh",
+    "../deployment/aliyun/host-bybit-options-cutover.sh",
+    "../deployment/aliyun/host-collector-health-unit-release.sh",
+    "../deployment/aliyun/monday-collector-health.sh",
+    "../deployment/aliyun/polymarket-market-tape-canary-monitor.sh",
+    "../deployment/aliyun/polymarket-raw-ops-cutover.sh",
+    "../deployment/aliyun/polymarket-raw-ops-shadow-gate.sh",
+    "../deployment/aliyun/test-binance-fee-cutover.sh",
+    "../deployment/aliyun/test-binance-lob-slice-materialize.sh",
+    "../deployment/aliyun/test-binance-usdm-reference-host-gate.sh",
+    "../deployment/aliyun/test-binance-usdm-reference-shadow-gate.sh",
+    "../deployment/aliyun/test-bybit-options-release-contract.sh",
+    "../deployment/aliyun/test-collector-health-unit-release.sh",
+    "../deployment/aliyun/test-data-completeness-check.sh",
+    "../deployment/aliyun/test-polymarket-market-recorder-release.sh",
+    "../deployment/aliyun/test-polymarket-market-tape-upload-contract.sh",
+    "../deployment/aliyun/test-polymarket-reference-upload-contract.sh",
+    "../deployment/aliyun/test-trading-ecs-host-contract.sh",
+    "../deployment/aliyun/trading-ecs-hostctl.sh",
+    "../deployment/aliyun/binance-fee-cutover.sh",
+    "../deployment/aliyun/binance-usdm-reference-cutover.sh",
+    "../deployment/aliyun/bybit-options-archiver-deploy.sh",
+    "../deployment/aliyun/collector-monitor-release.sh",
+    "../deployment/aliyun/data-completeness-check.sh",
+    "../deployment/aliyun/host-bybit-options-shadow-gate.sh",
+    "../deployment/aliyun/host-collector-monitor-install.sh",
+    "../deployment/aliyun/polymarket-market-recorder-deploy.sh",
+    "../deployment/aliyun/polymarket-market-tape-upload-watchdog.sh",
+    "../deployment/aliyun/polymarket-raw-ops-gate-control.sh",
+    "../deployment/aliyun/rust-lob-control-plane.sh",
+    "../deployment/aliyun/test-binance-fee-release-contract.sh",
+    "../deployment/aliyun/test-binance-usdm-account-release-contract.sh",
+    "../deployment/aliyun/test-binance-usdm-reference-release-contract.sh",
+    "../deployment/aliyun/test-bybit-options-cutover.sh",
+    "../deployment/aliyun/test-bybit-options-shadow-gate.sh",
+    "../deployment/aliyun/test-collector-monitor-release.sh",
+    "../deployment/aliyun/test-host-bybit-options-cutover.sh",
+    "../deployment/aliyun/test-polymarket-market-tape-canary-monitor.sh",
+    "../deployment/aliyun/test-polymarket-raw-ops-stage.sh",
+    "../deployment/aliyun/test-rust-lob-restore.sh",
+    "../deployment/aliyun/test-upload-cpu-limits.sh",
+    "../deployment/aliyun/trading-ecs-runtime.sh"
+  ]
+]

.github/scripts/select-rust-ci-scope.sh

@@ -746,7 +746,7 @@ for path in "${paths[@]}"; do
       [[ $event == pull_request ]] && select_job ploy/commit-hygiene
       continue
       ;;
-    .github/scripts/run-collector-control-contracts.sh|.github/scripts/test-collector-control-scheduling.sh)
+    .github/scripts/run-collector-control-contracts.sh|.github/scripts/test-collector-control-scheduling.sh|.github/scripts/check-collector-shells.sh|.github/scripts/collector-shellcheck-groups.json)
       control=true
       select_job ci/ci-contracts
       select_job ploy/workflow-lint

.github/scripts/test-collector-control-scheduling.sh

@@ -109,4 +109,99 @@ grep -q 'register the fee cutover contract' "$root/error"
 sed 's/^  test-binance-fee-cutover.sh$/  # test-binance-fee-cutover.sh/' "$runner" >"$root/.github/scripts/run-collector-control-contracts.sh"
 if check_fee 2>"$root/error"; then exit 1; fi
 grep -q 'register the fee cutover contract' "$root/error"
-printf 'collector scheduling: coverage, isolation, failures, cancellation and fee admission passed\n'
+
+# Preserve the full source-aware input union and retain a failing static gate.
+lint_runner="$source_root/.github/scripts/check-collector-shells.sh"
+fixture shell-lint
+mkdir -p "$root/bin" "$root/.github/scripts"
+lint_groups="$root/.github/scripts/collector-shellcheck-groups.json"
+cp "$source_root/.github/scripts/collector-shellcheck-groups.json" "$lint_groups"
+while IFS= read -r -d '' file; do
+  printf '#!/usr/bin/env bash\n:\n' >"$root/rust_hft/$file"
+done < <(jq -j '.[][] | . + "\u0000"' "$lint_groups" | sort -zu)
+printf '#!/usr/bin/env bash\n:\n' >"$root/deployment/aliyun/space name.sh"
+jq '.[-1] += ["../deployment/aliyun/space name.sh"]' "$lint_groups" >"$root/groups.new"
+mv "$root/groups.new" "$lint_groups"
+cp "$lint_groups" "$root/groups.saved"
+cat >"$root/bin/shellcheck" <<'LINT'
+#!/usr/bin/env bash
+set -euo pipefail
+[[ $1 == -- && $# -gt 1 ]]
+shift
+health=false companion=false
+for file in "$@"; do
+  [[ $file != ../deployment/aliyun/test-monday-collector-health.sh ]] || health=true
+  [[ $file != ../deployment/aliyun/test-recovery-retention-fixture.sh ]] || companion=true
+done
+if [[ $health == true && $companion != true ]]; then
+  echo 'missing retention source companion' >&2; exit 38
+fi
+exec 9>>"$LINT_ROOT/lint.lock"
+flock 9
+active=$(cat "$LINT_ROOT/lint.active")
+active=$((active + 1))
+printf '%s\n' "$active" >"$LINT_ROOT/lint.active"
+peak=$(cat "$LINT_ROOT/lint.peak")
+if (( active > peak )); then printf '%s\n' "$active" >"$LINT_ROOT/lint.peak"; fi
+printf '%s\0' "$@" >>"$LINT_ROOT/lint.calls"
+printf '%s\n' "$BASHPID" >>"$LINT_ROOT/lint.workers"
+flock -u 9
+sleep 0.1
+flock 9
+active=$(cat "$LINT_ROOT/lint.active")
+printf '%s\n' "$((active - 1))" >"$LINT_ROOT/lint.active"
+flock -u 9
+for file in "$@"; do
+  if [[ ${file##*/} == ${LINT_FAIL:-} ]]; then exit 37; fi
+done
+LINT
+chmod +x "$root/bin/shellcheck"
+reset_lint() {
+  : >"$root/lint.calls"; : >"$root/lint.workers"
+  printf '0\n' >"$root/lint.active"
+  printf '0\n' >"$root/lint.peak"
+}
+run_lint() {
+  LINT_ROOT="$root" LINT_FAIL="${1:-}" PATH="$root/bin:$PATH" \
+    bash "$lint_runner" --root "$root"
+}
+check_lint_coverage() {
+  (cd "$root/rust_hft" || exit; printf '%s\0' ../deployment/aliyun/*.sh) | sort -z >"$root/lint.expected"
+  sort -zu "$root/lint.calls" >"$root/lint.actual"
+  cmp "$root/lint.expected" "$root/lint.actual"
+  [[ $(cat "$root/lint.active") == 0 ]]
+  [[ $(wc -l <"$root/lint.workers") -eq $(jq length "$lint_groups") ]]
+}
+reset_lint
+run_lint >"$root/lint.output"
+check_lint_coverage
+peak=$(cat "$root/lint.peak")
+(( peak > 1 && peak <= 4 ))
+reset_lint
+if run_lint test-trading-ecs-host-contract.sh >"$root/lint.output" 2>&1; then
+  echo 'parallel lint failure did not fail the static gate' >&2; exit 1
+fi
+check_lint_coverage
+reset_lint
+jq '.[0] -= ["../deployment/aliyun/test-recovery-retention-fixture.sh"]' "$lint_groups" >"$root/groups.new"
+mv "$root/groups.new" "$lint_groups"
+if run_lint >"$root/lint.output" 2>&1; then
+  echo 'split source companions passed the static gate' >&2; exit 1
+fi
+grep -q 'missing retention source companion' "$root/lint.output"
+cp "$root/groups.saved" "$lint_groups"
+reset_lint
+printf '#!/usr/bin/env bash\n:\n' >"$root/deployment/aliyun/unregistered.sh"
+if run_lint >"$root/lint.output" 2>&1; then
+  echo 'unregistered collector script passed the static gate' >&2; exit 1
+fi
+grep -q 'exact collector script set' "$root/lint.output"
+[[ ! -s $root/lint.calls ]]
+rm "$root/deployment/aliyun/unregistered.sh"
+reset_lint
+printf 'if\n' >"$root/deployment/aliyun/test-trading-ecs-host-contract.sh"
+if run_lint >"$root/lint.output" 2>&1; then
+  echo 'invalid shell syntax passed the static gate' >&2; exit 1
+fi
+[[ ! -s $root/lint.calls ]]
+printf 'collector scheduling: coverage, isolation, failures, cancellation, fee admission and source-aware lint passed\n'

.github/workflows/ci.yml

@@ -173,10 +173,7 @@ jobs:
       - name: Verify collector release shell contracts
         working-directory: rust_hft
         run: |
-          for script in ../deployment/aliyun/*.sh; do
-            bash -n "$script"
-          done
-          shellcheck ../deployment/aliyun/*.sh
+          bash ../.github/scripts/check-collector-shells.sh
           bash ../.github/scripts/run-collector-control-contracts.sh
 
   monitor_contracts: