diff --git a/.github/scripts/check-collector-shells.sh b/.github/scripts/check-collector-shells.sh new file mode 100755 index 000000000..eb7630932 --- /dev/null +++ b/.github/scripts/check-collector-shells.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Preserve source companions while bounding independent ShellCheck groups. +set -euo pipefail +root=$(cd -- "$(dirname -- "$0")/../.." && pwd) +if [[ ${1:-} == --root && $# == 2 ]]; then + root=$(cd -- "$2" && pwd) +elif (( $# != 0 )); then + printf 'usage: %s [--root directory]\n' "$0" >&2 + exit 2 +fi +groups="$root/.github/scripts/collector-shellcheck-groups.json" +cd -- "$root/rust_hft" +scripts=(../deployment/aliyun/*.sh) +[[ -f ${scripts[0]} ]] || { echo 'collector shell input set is empty' >&2; exit 1; } +work=$(mktemp -d) +trap 'rm -rf -- "$work"' EXIT +jq -e 'type=="array" and length>0 and all(.[]; type=="array" and length>0 + and all(.[]; type=="string" and startswith("../deployment/aliyun/") and endswith(".sh")))' "$groups" >/dev/null +printf '%s\0' "${scripts[@]}" | sort -z >"$work/expected" +jq -j '.[][] | . + "\u0000"' "$groups" | sort -zu >"$work/actual" +cmp -s "$work/expected" "$work/actual" || { + echo 'ShellCheck groups must cover the exact collector script set' >&2; exit 1; +} +began=$SECONDS +for script in "${scripts[@]}"; do bash -n "$script"; done +printf 'collector_shell_phase name=syntax files=%s elapsed_seconds=%s\n' \ + "${#scripts[@]}" "$((SECONDS - began))" +began=$SECONDS +# Keep the original cwd, source context and flags. No -x or warning suppression. +jq -c '.[]' "$groups" | while IFS= read -r group; do printf '%s\0' "$group"; done \ + | xargs -0 -P 4 -n 1 bash -c ' + mapfile -d "" -t scripts < <(jq -j ".[] | . + \"\\u0000\"" <<<"$1") + shellcheck -- "${scripts[@]}" + ' _ +printf 'collector_shell_phase name=shellcheck workers=4 groups=%s files=%s elapsed_seconds=%s\n' \ + "$(jq length "$groups")" "${#scripts[@]}" "$((SECONDS - began))" diff --git a/.github/scripts/collector-shellcheck-groups.json b/.github/scripts/collector-shellcheck-groups.json new file mode 100644 index 000000000..0f776ace4 --- /dev/null +++ b/.github/scripts/collector-shellcheck-groups.json @@ -0,0 +1,84 @@ +[ + [ + "../deployment/aliyun/test-monday-collector-health.sh", + "../deployment/aliyun/test-recovery-retention-fixture.sh", + "../deployment/aliyun/host-rust-lob-recovery-queue.sh", + "../deployment/aliyun/rust-lob-control-plane-lib.sh" + ], + [ + "../deployment/aliyun/test-rust-lob-retention.sh", + "../deployment/aliyun/test-recovery-retention-fixture.sh", + "../deployment/aliyun/host-rust-lob-recovery-queue.sh", + "../deployment/aliyun/rust-lob-control-plane-lib.sh", + "../deployment/aliyun/test-rust-lob-recovery-queue.sh" + ], + [ + "../deployment/aliyun/test-rust-lob-control-plane.sh", + "../deployment/aliyun/host-rust-lob-controller-release.sh", + "../deployment/aliyun/test-rust-lob-controller-release.sh", + "../deployment/aliyun/rust-lob-control-plane-lib.sh" + ], + [ + "../deployment/aliyun/test-polymarket-raw-ops-control-plane.sh" + ], + [ + "../deployment/aliyun/host-rust-lob-shadow-gate.sh", + "../deployment/aliyun/host-rust-lob-cutover.sh", + "../deployment/aliyun/rust-lob-control-plane-lib.sh" + ], + [ + "../deployment/aliyun/host-rust-lob-restore.sh", + "../deployment/aliyun/host-rust-lob-readback.sh", + "../deployment/aliyun/publish-rust-lob-pair-release.sh", + "../deployment/aliyun/test-host-rust-lob-cutover.sh", + "../deployment/aliyun/test-rust-lob-rollback-unit-bindings.sh", + "../deployment/aliyun/rust-lob-control-plane-lib.sh" + ], + [ + "../deployment/aliyun/binance-lob-slice-materialize.sh", + "../deployment/aliyun/binance-usdm-reference-shadow-gate.sh", + "../deployment/aliyun/bybit-options-control-plane-lib.sh", + "../deployment/aliyun/collector-monitor-retained.sh", + "../deployment/aliyun/host-bybit-options-cutover.sh", + "../deployment/aliyun/host-collector-health-unit-release.sh", + "../deployment/aliyun/monday-collector-health.sh", + "../deployment/aliyun/polymarket-market-tape-canary-monitor.sh", + "../deployment/aliyun/polymarket-raw-ops-cutover.sh", + "../deployment/aliyun/polymarket-raw-ops-shadow-gate.sh", + "../deployment/aliyun/test-binance-fee-cutover.sh", + "../deployment/aliyun/test-binance-lob-slice-materialize.sh", + "../deployment/aliyun/test-binance-usdm-reference-host-gate.sh", + "../deployment/aliyun/test-binance-usdm-reference-shadow-gate.sh", + "../deployment/aliyun/test-bybit-options-release-contract.sh", + "../deployment/aliyun/test-collector-health-unit-release.sh", + "../deployment/aliyun/test-data-completeness-check.sh", + "../deployment/aliyun/test-polymarket-market-recorder-release.sh", + "../deployment/aliyun/test-polymarket-market-tape-upload-contract.sh", + "../deployment/aliyun/test-polymarket-reference-upload-contract.sh", + "../deployment/aliyun/test-trading-ecs-host-contract.sh", + "../deployment/aliyun/trading-ecs-hostctl.sh", + "../deployment/aliyun/binance-fee-cutover.sh", + "../deployment/aliyun/binance-usdm-reference-cutover.sh", + "../deployment/aliyun/bybit-options-archiver-deploy.sh", + "../deployment/aliyun/collector-monitor-release.sh", + "../deployment/aliyun/data-completeness-check.sh", + "../deployment/aliyun/host-bybit-options-shadow-gate.sh", + "../deployment/aliyun/host-collector-monitor-install.sh", + "../deployment/aliyun/polymarket-market-recorder-deploy.sh", + "../deployment/aliyun/polymarket-market-tape-upload-watchdog.sh", + "../deployment/aliyun/polymarket-raw-ops-gate-control.sh", + "../deployment/aliyun/rust-lob-control-plane.sh", + "../deployment/aliyun/test-binance-fee-release-contract.sh", + "../deployment/aliyun/test-binance-usdm-account-release-contract.sh", + "../deployment/aliyun/test-binance-usdm-reference-release-contract.sh", + "../deployment/aliyun/test-bybit-options-cutover.sh", + "../deployment/aliyun/test-bybit-options-shadow-gate.sh", + "../deployment/aliyun/test-collector-monitor-release.sh", + "../deployment/aliyun/test-host-bybit-options-cutover.sh", + "../deployment/aliyun/test-polymarket-market-tape-canary-monitor.sh", + "../deployment/aliyun/test-polymarket-raw-ops-stage.sh", + "../deployment/aliyun/test-rust-lob-restore.sh", + "../deployment/aliyun/test-upload-cpu-limits.sh", + "../deployment/aliyun/trading-ecs-runtime.sh" + ] +] diff --git a/.github/scripts/select-rust-ci-scope.sh b/.github/scripts/select-rust-ci-scope.sh index 222800062..e10e7462e 100755 --- a/.github/scripts/select-rust-ci-scope.sh +++ b/.github/scripts/select-rust-ci-scope.sh @@ -746,7 +746,7 @@ for path in "${paths[@]}"; do [[ $event == pull_request ]] && select_job ploy/commit-hygiene continue ;; - .github/scripts/run-collector-control-contracts.sh|.github/scripts/test-collector-control-scheduling.sh) + .github/scripts/run-collector-control-contracts.sh|.github/scripts/test-collector-control-scheduling.sh|.github/scripts/check-collector-shells.sh|.github/scripts/collector-shellcheck-groups.json) control=true select_job ci/ci-contracts select_job ploy/workflow-lint diff --git a/.github/scripts/test-collector-control-scheduling.sh b/.github/scripts/test-collector-control-scheduling.sh index 6ff44eda6..415861942 100755 --- a/.github/scripts/test-collector-control-scheduling.sh +++ b/.github/scripts/test-collector-control-scheduling.sh @@ -109,4 +109,99 @@ grep -q 'register the fee cutover contract' "$root/error" sed 's/^ test-binance-fee-cutover.sh$/ # test-binance-fee-cutover.sh/' "$runner" >"$root/.github/scripts/run-collector-control-contracts.sh" if check_fee 2>"$root/error"; then exit 1; fi grep -q 'register the fee cutover contract' "$root/error" -printf 'collector scheduling: coverage, isolation, failures, cancellation and fee admission passed\n' + +# Preserve the full source-aware input union and retain a failing static gate. +lint_runner="$source_root/.github/scripts/check-collector-shells.sh" +fixture shell-lint +mkdir -p "$root/bin" "$root/.github/scripts" +lint_groups="$root/.github/scripts/collector-shellcheck-groups.json" +cp "$source_root/.github/scripts/collector-shellcheck-groups.json" "$lint_groups" +while IFS= read -r -d '' file; do + printf '#!/usr/bin/env bash\n:\n' >"$root/rust_hft/$file" +done < <(jq -j '.[][] | . + "\u0000"' "$lint_groups" | sort -zu) +printf '#!/usr/bin/env bash\n:\n' >"$root/deployment/aliyun/space name.sh" +jq '.[-1] += ["../deployment/aliyun/space name.sh"]' "$lint_groups" >"$root/groups.new" +mv "$root/groups.new" "$lint_groups" +cp "$lint_groups" "$root/groups.saved" +cat >"$root/bin/shellcheck" <<'LINT' +#!/usr/bin/env bash +set -euo pipefail +[[ $1 == -- && $# -gt 1 ]] +shift +health=false companion=false +for file in "$@"; do + [[ $file != ../deployment/aliyun/test-monday-collector-health.sh ]] || health=true + [[ $file != ../deployment/aliyun/test-recovery-retention-fixture.sh ]] || companion=true +done +if [[ $health == true && $companion != true ]]; then + echo 'missing retention source companion' >&2; exit 38 +fi +exec 9>>"$LINT_ROOT/lint.lock" +flock 9 +active=$(cat "$LINT_ROOT/lint.active") +active=$((active + 1)) +printf '%s\n' "$active" >"$LINT_ROOT/lint.active" +peak=$(cat "$LINT_ROOT/lint.peak") +if (( active > peak )); then printf '%s\n' "$active" >"$LINT_ROOT/lint.peak"; fi +printf '%s\0' "$@" >>"$LINT_ROOT/lint.calls" +printf '%s\n' "$BASHPID" >>"$LINT_ROOT/lint.workers" +flock -u 9 +sleep 0.1 +flock 9 +active=$(cat "$LINT_ROOT/lint.active") +printf '%s\n' "$((active - 1))" >"$LINT_ROOT/lint.active" +flock -u 9 +for file in "$@"; do + if [[ ${file##*/} == ${LINT_FAIL:-} ]]; then exit 37; fi +done +LINT +chmod +x "$root/bin/shellcheck" +reset_lint() { + : >"$root/lint.calls"; : >"$root/lint.workers" + printf '0\n' >"$root/lint.active" + printf '0\n' >"$root/lint.peak" +} +run_lint() { + LINT_ROOT="$root" LINT_FAIL="${1:-}" PATH="$root/bin:$PATH" \ + bash "$lint_runner" --root "$root" +} +check_lint_coverage() { + (cd "$root/rust_hft" || exit; printf '%s\0' ../deployment/aliyun/*.sh) | sort -z >"$root/lint.expected" + sort -zu "$root/lint.calls" >"$root/lint.actual" + cmp "$root/lint.expected" "$root/lint.actual" + [[ $(cat "$root/lint.active") == 0 ]] + [[ $(wc -l <"$root/lint.workers") -eq $(jq length "$lint_groups") ]] +} +reset_lint +run_lint >"$root/lint.output" +check_lint_coverage +peak=$(cat "$root/lint.peak") +(( peak > 1 && peak <= 4 )) +reset_lint +if run_lint test-trading-ecs-host-contract.sh >"$root/lint.output" 2>&1; then + echo 'parallel lint failure did not fail the static gate' >&2; exit 1 +fi +check_lint_coverage +reset_lint +jq '.[0] -= ["../deployment/aliyun/test-recovery-retention-fixture.sh"]' "$lint_groups" >"$root/groups.new" +mv "$root/groups.new" "$lint_groups" +if run_lint >"$root/lint.output" 2>&1; then + echo 'split source companions passed the static gate' >&2; exit 1 +fi +grep -q 'missing retention source companion' "$root/lint.output" +cp "$root/groups.saved" "$lint_groups" +reset_lint +printf '#!/usr/bin/env bash\n:\n' >"$root/deployment/aliyun/unregistered.sh" +if run_lint >"$root/lint.output" 2>&1; then + echo 'unregistered collector script passed the static gate' >&2; exit 1 +fi +grep -q 'exact collector script set' "$root/lint.output" +[[ ! -s $root/lint.calls ]] +rm "$root/deployment/aliyun/unregistered.sh" +reset_lint +printf 'if\n' >"$root/deployment/aliyun/test-trading-ecs-host-contract.sh" +if run_lint >"$root/lint.output" 2>&1; then + echo 'invalid shell syntax passed the static gate' >&2; exit 1 +fi +[[ ! -s $root/lint.calls ]] +printf 'collector scheduling: coverage, isolation, failures, cancellation, fee admission and source-aware lint passed\n' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a23c7dd2..d5a03cb62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,10 +173,7 @@ jobs: - name: Verify collector release shell contracts working-directory: rust_hft run: | - for script in ../deployment/aliyun/*.sh; do - bash -n "$script" - done - shellcheck ../deployment/aliyun/*.sh + bash ../.github/scripts/check-collector-shells.sh bash ../.github/scripts/run-collector-control-contracts.sh monitor_contracts: