diff --git a/.github/workflows/fmt.yml b/.github/workflows/fmt.yml index aeb24704..f4d1a96a 100644 --- a/.github/workflows/fmt.yml +++ b/.github/workflows/fmt.yml @@ -3,35 +3,30 @@ name: Format on: [pull_request] jobs: - test: + fmt: name: fmt - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] + runs-on: ubuntu-latest steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Cache Cargo - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + # cargo fmt does not compile anything, so there is nothing worth caching. + # setup-rust-toolchain installs the rust-toolchain.toml-pinned rustfmt. + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-1 + cache: false - name: Run Cargo Fmt run: cargo fmt --all -- --check env: RUST_BACKTRACE: 1 + aggregate: name: fmt:required runs-on: ubuntu-latest if: ${{ always() }} - needs: test + needs: fmt steps: - name: Check fmt result - if: ${{ needs.test.result != 'success' }} + if: ${{ needs.fmt.result != 'success' }} run: exit 1 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f2e5b168..83dc05d0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,24 +2,54 @@ name: Lint on: [pull_request] +env: + # Retry dependency downloads on transient network errors. + CARGO_NET_RETRY: 10 + # Incremental compilation only pays off for repeated local rebuilds; in CI it + # just bloats the target dir (and the cache) for no benefit. + CARGO_INCREMENTAL: 0 + # Smaller debug info => smaller object files => faster linking and caches. + CARGO_PROFILE_DEV_DEBUG: line-tables-only + jobs: - test: + # Skip clippy/docs on docs-only changes; the required `lint:required` check + # passes via the skip (a skipped required check counts as passing). + changes: + runs-on: ubuntu-latest + outputs: + src: ${{ steps.filter.outputs.src }} + permissions: + pull-requests: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + id: filter + with: + predicate-quantifier: 'every' + filters: | + src: + - '**' + - '!**.md' + + lint: name: lint - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] + needs: changes + if: needs.changes.outputs.src == 'true' + runs-on: ubuntu-latest steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # rust-cache keys on all installed toolchains; the image's bundled `stable` + # drifts as the image updates, moving the key and causing misses. Remove it + # so only the rust-toolchain.toml-pinned version remains. + - name: Remove the runner's bundled Rust toolchain + run: rustup toolchain remove stable 2>/dev/null || true - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + cache-shared-key: ${{ runner.os }}-lint + cache-bin: false - uses: taiki-e/install-action@80e6af7a2ec7f280fffe2d0a9d3a12a9d11d86e9 # v2.75.1 with: @@ -44,15 +74,15 @@ jobs: rustup target add wasm32-unknown-unknown --toolchain nightly for package in $(cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | select(.publish == null and (.targets | any(.kind == ["lib"]))) | .name') do - CARGO_TARGET_DIR=target/nightly cargo +nightly docs-rs -p "$package" + CARGO_TARGET_DIR=target/nightly cargo +nightly docs-rs -p "$package" done aggregate: name: lint:required runs-on: ubuntu-latest - if: ${{ always() }} - needs: test + if: always() && needs.changes.outputs.src == 'true' + needs: [changes, lint] steps: - name: Check lint result - if: ${{ needs.test.result != 'success' }} + if: ${{ needs.lint.result != 'success' }} run: exit 1 diff --git a/.github/workflows/netlify.yml b/.github/workflows/netlify.yml deleted file mode 100644 index 56c0e179..00000000 --- a/.github/workflows/netlify.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Publish Cargo docs - -on: - push: - branches: - - main - pull_request_target: - -jobs: - publish: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install Rust - run: | - rustup update ${{ matrix.rust }} --no-self-update - rustup default ${{ matrix.rust }} - - name: Build Docs - run: | - cargo doc --no-deps - # Add an index.html for the root of the netlify docs. - rustdoc README.md --output target/doc && mv target/doc/README.html target/doc/index.html - - - if: github.ref == 'refs/heads/main' - name: Deploy to Netlify (main only) - uses: South-Paw/action-netlify-deploy@84b8cef9dab187f588f7f0be6f9affdb760b2ee1 # v1.0.4 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - netlify-auth-token: ${{ secrets.NETLIFY_AUTH_TOKEN }} - netlify-site-id: ${{ secrets.NETLIFY_SITE_ID }} - build-dir: target/doc/ - comment-on-commit: true - - - if: github.ref != 'refs/heads/main' - name: Deploy to Netlify (PR only) - id: deploy_docs - uses: netlify/actions/cli@6c34c3fcafc69ac2e1d6dbf226560329c6dfc51b - with: - args: deploy --dir=target/doc/ --prod - env: - NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} - NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} - - - if: github.ref != 'refs/heads/main' - name: Commenting on PR - uses: unsplash/comment-on-pr@d1a1d5dd1eb1bb657a01f4d92dd5e4d5bb7857d3 # v1.2.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: | - Netlify deployed agent-rust as draft - - Link: ${{ steps.deploy_docs.outputs.NETLIFY_URL }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 312be186..6a4ab0e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,39 +6,139 @@ on: - main pull_request: +env: + # Retry dependency downloads on transient network errors. + CARGO_NET_RETRY: 10 + # Incremental compilation only pays off for repeated local rebuilds; in CI it + # just bloats the target dir (and the cache) for no benefit. + CARGO_INCREMENTAL: 0 + # Smaller debug info => smaller object files => faster linking (the dominant + # cost on Windows MSVC) and smaller caches. line-tables-only keeps file:line + # numbers in backtraces. + CARGO_PROFILE_DEV_DEBUG: line-tables-only + # Link with LLVM's lld-link instead of the slow MSVC link.exe. LLVM ships at + # C:\Program Files\LLVM on the windows-2025 image. Only consumed when building + # the windows-msvc target, so it is a no-op on Linux/macOS. + CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: "C:/Program Files/LLVM/bin/lld-link.exe" + +defaults: + run: + shell: bash + jobs: + # Skip the (expensive) test jobs on docs-only changes. The heavy jobs gate on + # this output, so a docs-only PR skips them and the required `test:required` + # check passes via the skip. + changes: + runs-on: ubuntu-latest + outputs: + src: ${{ steps.filter.outputs.src }} + permissions: + pull-requests: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + id: filter + with: + # With 'every', a changed file matches only when it satisfies the + # positive pattern AND every negated pattern. + predicate-quantifier: 'every' + filters: | + src: + - '**' + - '!**.md' + + # Workspace tests for every crate except ref-tests, on all three OSes. Because + # each crate is tested from its own directory, the heavy pocket-ic dependency + # (only used by ref-tests) never compiles here, keeping this job's cache small. test: + name: Run Tests on ${{ matrix.os }} + needs: changes + if: needs.changes.outputs.src == 'true' runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + # windows is pinned to 2025 so the lld-link path above is guaranteed. + os: [ubuntu-latest, macos-latest, windows-2025] steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # Defender real-time scanning of every object/rlib slows file-heavy Rust + # builds and cache extraction considerably on Windows. + - name: Exclude build dirs from Windows Defender + if: ${{ contains(matrix.os, 'windows') }} + shell: pwsh + run: | + Add-MpPreference -ExclusionPath ` + "$env:GITHUB_WORKSPACE", ` + "$env:USERPROFILE\.cargo", ` + "$env:USERPROFILE\.rustup" -ErrorAction SilentlyContinue - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + # rust-cache keys on all installed toolchains; the image's bundled `stable` + # drifts as the image updates, moving the key and causing misses. Remove it + # so only the rust-toolchain.toml-pinned version remains. + - name: Remove the runner's bundled Rust toolchain + run: rustup toolchain remove stable 2>/dev/null || true + + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-1 - - - name: Install wasm-pack and chromedriver for WASM tests - if: ${{ matrix.os == 'ubuntu-latest' }} + cache-shared-key: ${{ runner.os }}-test + cache-bin: false + # setup-rust-toolchain defaults RUSTFLAGS to "-D warnings"; the tests + # were never built warnings-as-errors (that is lint's job), so clear it. + rustflags: "" + + # The ic-agent doctests spin up pocket-ic via ref-tests' helpers, so they + # need the downloaded assets. Skipped on Windows (the download script and + # those doctests are unix-only), matching the previous single-job setup. + - name: Download test assets + if: ${{ !contains(matrix.os, 'windows') }} + run: ./scripts/download_reftest_assets.sh + + - name: Run Tests run: | - curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - sudo apt-get install -y chromium-chromedriver + # Test all features and no features for each package. + # ref-tests is excluded here and runs in the ref-tests job with + # --test-threads=1 (pocket-ic spawns many threads per subnet). + for p in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | select(.name != "ref-tests") | .manifest_path'); do + pushd $(dirname $p) + cargo test --all-features --no-fail-fast + cargo test --no-default-features --no-fail-fast + popd + done + env: + RUST_BACKTRACE: 1 + + # ref-tests + SoftHSM run together: both need the compiled ref-tests binary, + # so co-locating them avoids compiling it (and pocket-ic) twice. Linux-only: + # the GitHub macOS runner's low per-process thread limit cannot be raised + # without root, and pocket-ic exhausts it; Windows lacks the assets. + ref_tests: + name: ref-tests + needs: changes + if: needs.changes.outputs.src == 'true' + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Remove the runner's bundled Rust toolchain + run: rustup toolchain remove stable 2>/dev/null || true + + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + cache-shared-key: ${{ runner.os }}-reftests + cache-bin: false + rustflags: "" - name: Install node for SoftHSM tests - if: ${{ matrix.os == 'ubuntu-latest' }} uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 20 - name: Install and Configure SoftHSM - if: ${{ matrix.os == 'ubuntu-latest' }} run: | set -ex sudo apt-get install -f libsofthsm2 opensc-pkcs11 opensc @@ -48,47 +148,16 @@ jobs: mkdir -p $HOME/softhsm/tokens - name: Download test assets - if: ${{ !contains(matrix.os, 'windows') }} - run: | - ./scripts/download_reftest_assets.sh + run: ./scripts/download_reftest_assets.sh - name: Run Tests (ref-tests) - # ref-tests are skipped on macOS CI: the GitHub Actions macOS runner has - # a low per-process thread limit (kern.maxthreadsperproc) that cannot be - # raised without root. pocket-ic spawns many OS threads per subnet, and - # tokio's blocking thread pool accumulates idle threads across sequential - # tests, exhausting the limit before all tests finish. This is a CI - # resource constraint — ref-tests work fine on a local macOS machine. - if: ${{ matrix.os == 'ubuntu-latest' }} - shell: bash run: | cd ref-tests cargo test --no-fail-fast -- --test-threads=1 env: RUST_BACKTRACE: 1 - - name: Run Tests - shell: bash - run: | - # Test all features and no features for each package. - # ref-tests is excluded here and run separately with --test-threads=1 - # to avoid exhausting OS thread limits when pocket-ic spawns many threads. - for p in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[] | select(.name != "ref-tests") | .manifest_path'); do - pushd $(dirname $p) - cargo test --all-features --no-fail-fast - cargo test --no-default-features --no-fail-fast - popd - done - env: - RUST_BACKTRACE: 1 - - - name: Run Tests (WASM) - if: ${{ matrix.os == 'ubuntu-latest' }} - run: | - CARGO_TARGET_DIR=target/wasm wasm-pack test --chrome --headless ic-agent --features wasm-bindgen - - name: Run Tests (SoftHSM) - if: ${{ matrix.os == 'ubuntu-latest' }} run: | set -ex softhsm2-util --init-token --slot $HSM_SLOT_INDEX --label "agent-rs-token" --so-pin $HSM_SO_PIN --pin $HSM_PIN @@ -105,12 +174,48 @@ jobs: HSM_SLOT_INDEX: 0 HSM_KEY_ID: abcdef + wasm: + name: WASM tests + needs: changes + if: needs.changes.outputs.src == 'true' + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Remove the runner's bundled Rust toolchain + run: rustup toolchain remove stable 2>/dev/null || true + + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + cache-shared-key: ${{ runner.os }}-wasm + cache-bin: false + rustflags: "" + + - name: Install wasm-pack and chromedriver + run: | + curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + sudo apt-get install -y chromium-chromedriver + + # CARGO_TARGET_DIR=target/wasm keeps wasm artifacts under ./target, so + # rust-cache (which caches ./target) still picks them up. + - name: Run Tests (WASM) + run: CARGO_TARGET_DIR=target/wasm wasm-pack test --chrome --headless ic-agent --features wasm-bindgen + aggregate: name: test:required + # Runs only when the test jobs ran; on docs-only changes this skips, and a + # skipped required check counts as passing for branch protection. + if: always() && needs.changes.outputs.src == 'true' runs-on: ubuntu-latest - if: ${{ always() }} - needs: test + needs: [changes, test, ref_tests, wasm] steps: - name: Check test result if: ${{ needs.test.result != 'success' }} run: exit 1 + - name: Check ref-tests result + if: ${{ needs.ref_tests.result != 'success' }} + run: exit 1 + - name: Check wasm result + if: ${{ needs.wasm.result != 'success' }} + run: exit 1 diff --git a/ic-agent/README.md b/ic-agent/README.md index 42d74411..bd4a0888 100644 --- a/ic-agent/README.md +++ b/ic-agent/README.md @@ -3,5 +3,4 @@ in Rust. It is the backend for [`dfx`](https://internetcomputer.org/docs/current ## Useful links -- [Documentation (master)](https://agent-rust.netlify.app/ic_agent) - [Documentation (published)](https://docs.rs/ic_agent) diff --git a/ic-agent/src/agent/route_provider/dynamic_routing/dynamic_route_provider.rs b/ic-agent/src/agent/route_provider/dynamic_routing/dynamic_route_provider.rs index 2ba5ef40..f1c908e2 100644 --- a/ic-agent/src/agent/route_provider/dynamic_routing/dynamic_route_provider.rs +++ b/ic-agent/src/agent/route_provider/dynamic_routing/dynamic_route_provider.rs @@ -639,7 +639,7 @@ mod tests { // Test 2: calls to route() return both seeds, as they become healthy. checker.overwrite_healthy_nodes(vec![node_1.clone(), node_2.clone()]); tokio::time::sleep(3 * check_interval).await; - let routed_domains = route_n_times(6, Arc::clone(&route_provider)); + let routed_domains = route_n_times(30, Arc::clone(&route_provider)); // Increased for probabilistic routing assert_routed_domains(routed_domains, vec![node_1.domain(), node_2.domain()]); } @@ -762,7 +762,7 @@ mod tests { // Test 2: calls to route() return two healthy seeds, as the unhealthy seed becomes healthy. checker.overwrite_healthy_nodes(vec![node_1.clone(), node_2.clone()]); tokio::time::sleep(2 * check_interval).await; - let routed_domains = route_n_times(6, Arc::clone(&route_provider)); + let routed_domains = route_n_times(30, Arc::clone(&route_provider)); // Increased for probabilistic routing assert_routed_domains(routed_domains, vec![node_1.domain(), node_2.domain()]); } diff --git a/ic-identity-hsm/README.md b/ic-identity-hsm/README.md index 045ef766..0a8838e3 100644 --- a/ic-identity-hsm/README.md +++ b/ic-identity-hsm/README.md @@ -2,5 +2,4 @@ ## Useful links -- [Documentation (master)](https://agent-rust.netlify.app/ic_identity_hsm) - [Documentation (published)](https://docs.rs/ic_identity_hsm)