diff --git a/.github/workflows/CICD.yml b/.github/workflows/CICD.yml deleted file mode 100644 index 312fdf2bc8f..00000000000 --- a/.github/workflows/CICD.yml +++ /dev/null @@ -1,1049 +0,0 @@ -name: CICD - -# spell-checker:ignore (abbrev/names) CACHEDIR CICD CodeCOV MacOS MinGW MSVC musl taiki -# spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic Dwarnings RUSTDOCFLAGS RUSTFLAGS Zpanic CARGOFLAGS CLEVEL nodocs -# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain fuzzers dedupe devel profdata profraw Cprofile -# spell-checker:ignore (people) Peltoche rivy Anson dawidd -# spell-checker:ignore (shell/tools) binutils choco clippy dmake esac fakeroot fdesc fdescfs gmake grcov halium lcov libclang libcrypto libfuse libssl limactl nextest nocross pacman popd printf pushd redoxer rsync rustc rustfmt rustup shopt sccache setfacl utmpdump xargs zstd -# spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils defconfig DESTDIR gecos getenforce gnueabihf issuecomment maint manpages msys multisize noconfirm nofeatures nullglob onexitbegin onexitend pell runtest tempfile testsuite toybox userns uutils libsystemd codspeed wasip libexecinfo - -env: - PROJECT_NAME: coreutils - PROJECT_DESC: "Core universal (cross-platform) utilities" - PROJECT_AUTH: "uutils" - RUST_MIN_SRV: "1.88.0" - CARGO_INCREMENTAL: "0" - # * style job configuration - STYLE_FAIL_ON_FAULT: true ## (bool) fail the build if a style job contains a fault (error or warning); may be overridden on a per-job basis - -on: - pull_request: - push: - tags: - - '*' - branches: - - '*' - -permissions: - contents: read # to fetch code (actions/checkout) - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - cargo-deny: - name: Style/cargo-deny - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: EmbarkStudios/cargo-deny-action@v2 - - style_deps: - ## ToDO: [2021-11-10; rivy] 'Style/deps' needs more informative output and better integration of results into the GHA dashboard - name: Style/deps - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - # note: `cargo-udeps` panics when processing stdbuf/libstdbuf ("uu_stdbuf_libstdbuf"); either b/c of the 'cpp' crate or 'libstdbuf' itself - # ... b/c of the panic, a more limited feature set is tested (though only excluding `stdbuf`) - - { os: ubuntu-latest , features: "feat_Tier1,feat_require_unix,feat_require_unix_utmpx" } - - { os: macos-latest , features: "feat_Tier1,feat_require_unix,feat_require_unix_utmpx" } - - { os: windows-latest , features: feat_os_windows } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: taiki-e/install-action@cargo-udeps - - uses: Swatinem/rust-cache@v2 - - name: Initialize workflow variables - id: vars - shell: bash - run: | - echo "RUSTC_BOOTSTRAP=1" >> "${GITHUB_ENV}" # Use -Z - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # failure mode - unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in - ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; - *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; - esac; - outputs FAIL_ON_FAULT FAULT_TYPE - # target-specific options - # * CARGO_FEATURES_OPTION - CARGO_FEATURES_OPTION='' ; - if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features "${{ matrix.job.features }}"' ; fi - outputs CARGO_FEATURES_OPTION - - name: Detect unused dependencies - shell: bash - run: | - ## Detect unused dependencies - unset fault - fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" - fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') - # - cargo udeps ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} --all-targets &> udeps.log || cat udeps.log - grep --ignore-case "all deps seem to have been used" udeps.log || { printf "%s\n" "::${fault_type} ::${fault_prefix}: \`cargo udeps\`: style violation (unused dependency found)" ; fault=true ; } - if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi - - doc_warnings: - name: Documentation/warnings - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: feat_os_unix } -# for now, don't build it on mac & windows because the doc is only published from linux -# + it needs a bunch of duplication for build -# and I don't want to add a doc step in the regular build to avoid long builds -# - { os: macos-latest , features: feat_os_unix } -# - { os: windows-latest , features: feat_os_windows } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Install/setup prerequisites - shell: bash - run: | - sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # failure mode - unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in - ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; - *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; - esac; - outputs FAIL_ON_FAULT FAULT_TYPE - # target-specific options - # * CARGO_FEATURES_OPTION - CARGO_FEATURES_OPTION='--all-features' ; - if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features ${{ matrix.job.features }}' ; fi - outputs CARGO_FEATURES_OPTION - - name: "`cargo doc` with warnings" - shell: bash - run: | - RUSTDOCFLAGS="-Dwarnings" cargo doc ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} --no-deps --workspace --document-private-items - - uses: DavidAnson/markdownlint-cli2-action@v24 - with: - fix: "true" - globs: | - *.md - docs/src/*.md - src/uu/*/*.md - - min_version: - name: MinRustV # Minimum supported rust version (aka, MinSRV or MSRV) - runs-on: ${{ matrix.job.os }} - strategy: - matrix: - job: - - { os: ubuntu-latest , features: feat_os_unix } - # rustix defaults to raw Linux syscalls; make sure the libc backend works too - - { os: ubuntu-latest , features: "feat_os_unix,rustix/use-libc" } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ env.RUST_MIN_SRV }} - components: rustfmt - - uses: taiki-e/install-action@nextest - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # target-specific options - # * CARGO_FEATURES_OPTION - unset CARGO_FEATURES_OPTION - if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features "${{ matrix.job.features }}"' ; fi - outputs CARGO_FEATURES_OPTION - - name: Confirm MinSRV compatible '*/Cargo.lock' - shell: bash - run: | - ## Confirm MinSRV compatible '*/Cargo.lock' - # * '*/Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38) - for dir in "." "fuzz"; do - ( cd "$dir" && cargo fetch --locked --quiet --target $(rustc --print host-tuple)) || { echo "::error file=$dir/Cargo.lock::Incompatible (or out-of-date) '$dir/Cargo.lock' file; update using \`cd '$dir' && cargo +${{ env.RUST_MIN_SRV }} update\`" ; exit 1 ; } - done - - name: Install/setup prerequisites - shell: bash - run: | - # Install a package for one of the tests - # Some tests are silently skipped if strace is missing - sudo apt-get -y update ; sudo apt-get -y install attr strace - # Some tests are silently skipped if unshare failed - # userns is enabled only on few jobs for security - sudo sysctl -w kernel.unprivileged_userns_clone=1 - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - - name: Info - shell: bash - run: | - ## Info - # environment - echo "## environment" - echo "CI='${CI}'" - # tooling info display - echo "## tooling" - which gcc >/dev/null 2>&1 && (gcc --version | head -1) || true - rustup -V 2>/dev/null - rustup show active-toolchain - cargo -V - rustc -V - cargo tree -V - # dependencies - echo "## dependency list" - ## * using the 'stable' toolchain is necessary to avoid "unexpected '--filter-platform'" errors - cargo +stable fetch --locked --quiet --target $(rustc --print host-tuple) - cargo +stable tree --no-dedupe --locked -e=no-dev --prefix=none ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} | grep -vE "$PWD" | sort --unique - - name: Test - run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} -p uucore -p coreutils - env: - RUSTFLAGS: "-Awarnings" - RUST_BACKTRACE: "1" - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/codecov-action@v7 - with: - token: ${{ secrets.CODECOV_TOKEN }} - report_type: test_results - files: target/nextest/ci/junit.xml - disable_search: true - flags: msrv,${{ matrix.job.os }} - fail_ci_if_error: false - - deps: - name: Dependencies - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: feat_os_unix } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: "`cargo update` testing" - shell: bash - run: | - ## `cargo update` testing - # * convert any errors/warnings to GHA UI annotations; ref: - for dir in "." "fuzz"; do - ( cd "$dir" && cargo fetch --locked --quiet --target host-tuple) || { echo "::error file=$dir/Cargo.lock::'$dir/Cargo.lock' file requires update (use \`cd '$dir' && cargo +${{ env.RUST_MIN_SRV }} update\`)" ; exit 1 ; } - done - - build_rust_stable: - # Keep the name independent of `features`: these checks are required by - # branch protection, so a renamed job would block every PR. - name: Build/stable (${{ matrix.job.os }}, ${{ matrix.job.name }}) - needs: [ min_version, deps ] - runs-on: ${{ matrix.job.os }} - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , name: feat_os_unix , features: feat_os_unix } - # Don't fail to build if SELinux specific utils are included by some reason. - # But we are not interested in existence of stubs - - { os: macos-latest , name: feat_os_unix , features: "feat_os_unix,chcon,runcon" } - - { os: windows-latest , name: feat_os_windows , features: "feat_os_windows,chcon,runcon" } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@nextest - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Test - run: cargo nextest run --hide-progress-bar --profile ci --features ${{ matrix.job.features }} - env: - RUST_BACKTRACE: "1" - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/codecov-action@v7 - with: - token: ${{ secrets.CODECOV_TOKEN }} - report_type: test_results - files: target/nextest/ci/junit.xml - disable_search: true - flags: stable,${{ matrix.job.os }} - fail_ci_if_error: false - - build_rust_nightly: - name: Build/nightly - needs: [ min_version, deps ] - runs-on: ${{ matrix.job.os }} - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: feat_os_unix } - - { os: macos-latest , features: feat_os_unix } - - { os: windows-latest , features: feat_os_windows } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@nightly - - uses: taiki-e/install-action@nextest - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Test - run: cargo nextest run --hide-progress-bar --profile ci --features ${{ matrix.job.features }} - env: - RUST_BACKTRACE: "1" - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/codecov-action@v7 - with: - token: ${{ secrets.CODECOV_TOKEN }} - report_type: test_results - files: target/nextest/ci/junit.xml - disable_search: true - flags: nightly,${{ matrix.job.os }} - fail_ci_if_error: false - - build: - permissions: - contents: write # to create GitHub release (softprops/action-gh-release) - - name: Build - needs: [ min_version, deps ] - runs-on: ${{ matrix.job.os }} - timeout-minutes: 90 - env: - DOCKER_OPTS: '--volume /etc/passwd:/etc/passwd --volume /etc/group:/etc/group' - strategy: - fail-fast: false - matrix: - job: - # - { os , target , default-features, features , use-cross , toolchain, skip-tests, workspace-tests, skip-package, skip-publish } - # `pgo: PGO` marks a published target whose instrumented binary the - # runner can execute, which is what training a profile needs. It is - # therefore off for the `cross` targets (no emulator) and the wasm - # ones (no runtime). The value is spelled `PGO` because GitHub appends - # the matrix values to the job title, so those jobs read as - # "Build (..., x86_64-unknown-linux-gnu, ..., PGO)". - # enable openssl for performance if min_version: already tests pore-Rust checksum crates - - { os: ubuntu-latest , target: arm-unknown-linux-gnueabihf , features: feat_os_unix_gnueabihf , use-cross: use-cross , skip-tests: true } - - { os: ubuntu-24.04-arm , target: aarch64-unknown-linux-gnu , features: feat_os_unix_gnueabihf, pgo: PGO } - - { os: ubuntu-latest , target: aarch64-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross , skip-tests: true } - - { os: ubuntu-latest , target: riscv64gc-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross , skip-tests: true } - - { os: ubuntu-latest , target: powerpc64le-unknown-linux-gnu , features: feat_os_unix , use-cross: use-cross , check-only: true } - # - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: feat_selinux , use-cross: use-cross } - - { os: ubuntu-latest , target: i686-unknown-linux-gnu , features: "feat_os_unix,test_risky_names", use-cross: use-cross } - - { os: ubuntu-latest , target: i686-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } - - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: "feat_os_unix,test_risky_names", use-cross: use-cross, skip-publish: true } - - { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: "feat_os_unix,openssl,uudoc" , use-cross: no, workspace-tests: true, pgo: PGO } - - { os: ubuntu-latest , target: x86_64-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross } - - { os: ubuntu-latest , target: x86_64-unknown-netbsd, features: "feat_os_unix", use-cross: use-cross , skip-tests: true , check-only: true } - # - { os: ubuntu-latest , target: x86_64-unknown-redox , features: feat_os_unix_redox , use-cross: redoxer , skip-tests: true , check-only: true } - - { os: ubuntu-latest , target: wasm32-wasip1, default-features: false, features: feat_wasm, skip-tests: true } - - { os: ubuntu-latest , target: wasm32-wasip2, default-features: false, features: feat_wasm, skip-tests: true } - - { os: macos-latest , target: aarch64-apple-darwin , features: "feat_os_unix,openssl", workspace-tests: true, pgo: PGO } # M1 CPU - - { os: macos-latest , target: x86_64-apple-darwin , features: feat_os_unix, workspace-tests: true, pgo: PGO } - - { os: windows-latest , target: i686-pc-windows-msvc , features: feat_os_windows } - # msvc and gnu works on the same target. So publishing msvc is enough. - - { os: windows-latest , target: x86_64-pc-windows-gnu , features: feat_os_windows, skip-publish: true } - - { os: windows-latest , target: x86_64-pc-windows-msvc , features: "feat_os_windows,openssl", pgo: PGO } - # No `pgo` here: on aarch64-pc-windows-msvc the instrumented binary - # writes .profraw files whose name table llvm-profdata rejects with - # "symbol name is empty", so every profile is unusable. Upstream bugs: - # https://github.com/rust-lang/rust/issues/150123 (same message with - # `-C instrument-coverage`) and https://github.com/rust-lang/rust/issues/156675. - - { os: windows-11-arm , target: aarch64-pc-windows-msvc , features: feat_os_windows } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Avoid no space left on device - shell: bash - run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android & - - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ env.RUST_MIN_SRV }} - targets: ${{ matrix.job.target }} - - uses: Swatinem/rust-cache@v2 - with: - key: "${{ matrix.job.os }}_${{ matrix.job.target }}" - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # toolchain - TOOLCHAIN="stable" ## default to "stable" toolchain - # * specify alternate/non-default TOOLCHAIN for *-pc-windows-gnu targets; gnu targets on Windows are broken for the standard *-pc-windows-msvc toolchain (refs: GH:rust-lang/rust#47048, GH:rust-lang/rust#53454, GH:rust-lang/cargo#6754) - case ${{ matrix.job.target }} in *-pc-windows-gnu) TOOLCHAIN="stable-${{ matrix.job.target }}" ;; esac; - # * use requested TOOLCHAIN if specified - if [ -n "${{ matrix.job.toolchain }}" ]; then TOOLCHAIN="${{ matrix.job.toolchain }}" ; fi - outputs TOOLCHAIN - # staging directory - STAGING='_staging' - outputs STAGING - # parse commit reference info - echo GITHUB_REF=${GITHUB_REF} - echo GITHUB_SHA=${GITHUB_SHA} - REF_NAME=${GITHUB_REF#refs/*/} - unset REF_BRANCH ; case "${GITHUB_REF}" in refs/heads/*) REF_BRANCH=${GITHUB_REF#refs/heads/} ;; esac; - unset REF_TAG ; case "${GITHUB_REF}" in refs/tags/*) REF_TAG=${GITHUB_REF#refs/tags/} ;; esac; - REF_SHAS=${GITHUB_SHA:0:10} - outputs REF_NAME REF_BRANCH REF_TAG REF_SHAS - # parse target - unset TARGET_ARCH - case '${{ matrix.job.target }}' in - aarch64-*) TARGET_ARCH=arm64 ;; - riscv64gc-*) TARGET_ARCH=riscv64 ;; - arm-*-*hf) TARGET_ARCH=armhf ;; - i686-*) TARGET_ARCH=i686 ;; - x86_64-*) TARGET_ARCH=x86_64 ;; - esac; - unset TARGET_OS - case '${{ matrix.job.target }}' in - *-linux-*) TARGET_OS=linux ;; - *-apple-*) TARGET_OS=macos ;; - *-windows-*) TARGET_OS=windows ;; - *-redox*) TARGET_OS=redox ;; - esac - outputs TARGET_ARCH TARGET_OS - # package name - PKG_suffix=".tar.gz" ; case '${{ matrix.job.target }}' in *-pc-windows-*) PKG_suffix=".zip" ;; esac; - # Some 3rd party utils need version at file names - # But we remove it from tag/latest-commit - test ${REF_TAG} \ - && PKG_BASENAME=${PROJECT_NAME}-${REF_TAG}-${{ matrix.job.target }} \ - || PKG_BASENAME=${PROJECT_NAME}-${{ matrix.job.target }} - PKG_NAME=${PKG_BASENAME}${PKG_suffix} - outputs PKG_suffix PKG_BASENAME PKG_NAME - # deployable tag? (ie, leading "vM" or "M"; M == version number) - unset DEPLOY ; if [[ $REF_TAG =~ ^[vV]?[0-9].* ]]; then DEPLOY='true' ; fi - outputs DEPLOY - # target-specific options - # * CARGO_FEATURES_OPTION - CARGO_FEATURES_OPTION='' ; - if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features=${{ matrix.job.features }}' ; fi - outputs CARGO_FEATURES_OPTION - # * CARGO_DEFAULT_FEATURES_OPTION - CARGO_DEFAULT_FEATURES_OPTION='' ; - if [ "${{ matrix.job.default-features }}" == "false" ]; then CARGO_DEFAULT_FEATURES_OPTION='--no-default-features' ; fi - outputs CARGO_DEFAULT_FEATURES_OPTION - # * CARGO_CMD - CARGO_CMD='cross' - CARGO_CMD_OPTIONS='+${{ env.RUST_MIN_SRV }}' - # Added suffix for artifacts, needed when multiple jobs use the same target. - ARTIFACTS_SUFFIX='' - case '${{ matrix.job.use-cross }}' in - ''|0|f|false|n|no) - CARGO_CMD='cargo' - ARTIFACTS_SUFFIX='-nocross' - ;; - redoxer) - CARGO_CMD='redoxer' - CARGO_CMD_OPTIONS='' - ;; - esac - # needed for target "aarch64-apple-darwin". There are two jobs, and the difference between them is whether "features" is set - if [ -z "${{ matrix.job.features }}" ]; then ARTIFACTS_SUFFIX='-nofeatures' ; fi - outputs CARGO_CMD - outputs CARGO_CMD_OPTIONS - outputs ARTIFACTS_SUFFIX - CARGO_TEST_OPTIONS='' - case '${{ matrix.job.workspace-tests }}' in - 1|t|true|y|yes) - # This also runs tests in other packages in the source directory (e.g. uucore). - # We cannot enable this everywhere as some platforms are currently broken, and - # we cannot use `cross` as its Docker image is ancient (Ubuntu 16.04) and is - # missing required system dependencies (e.g. recent libclang-dev). - CARGO_TEST_OPTIONS='--workspace' - ;; - esac - outputs CARGO_TEST_OPTIONS - - uses: taiki-e/install-action@v2 - # `cross` v0.2.5 lacks `libexecinfo` on NetBSD. However, this has been added in main. - # See https://github.com/cross-rs/cross/blob/main/docker/netbsd.sh - # We are pulling cross from a specific commit hash rather than HEAD to avoid any potential breakage that may be introduced. - # Once `cross` v0.3 is out, these conditionals can be removed. - if: steps.vars.outputs.CARGO_CMD == 'cross' && matrix.job.target != 'x86_64-unknown-netbsd' - with: - tool: cross@0.2.5 - - name: Install cross from git (NetBSD) - if: steps.vars.outputs.CARGO_CMD == 'cross' && matrix.job.target == 'x86_64-unknown-netbsd' - run: cargo install cross --git https://github.com/cross-rs/cross --rev f86fd03bb70b4c6802847c18087e21391498b0b4 - - name: Create all needed build/work directories - shell: bash - run: | - ## Create build/work space - mkdir -p '${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}' - - name: Install/setup prerequisites - shell: bash - run: | - ## Install/setup prerequisites - case '${{ matrix.job.target }}' in - arm-unknown-linux-gnueabihf) - sudo apt-get -y update - sudo apt-get -y install gcc-arm-linux-gnueabihf - ;; - aarch64-unknown-linux-*) - sudo apt-get -y update - sudo apt-get -y install gcc-aarch64-linux-gnu - ;; - riscv64gc-unknown-linux-*) - sudo apt-get -y update - sudo apt-get -y install gcc-riscv64-linux-gnu - ;; - *-redox*) - sudo apt-get -y update - sudo apt-get -y install fuse3 libfuse-dev - ;; - esac - case '${{ matrix.job.os }}' in - ubuntu-*) - # selinux and systemd headers needed to build tests - sudo apt-get -y update - sudo apt-get -y install libselinux1-dev libsystemd-dev - # pinky is a tool to show logged-in users from utmp, and gecos fields from /etc/passwd. - # In GitHub Action *nix VMs, no accounts log in, even the "runner" account that runs the commands, and "system boot" entry is missing. - # The account also has empty gecos fields. - # To work around these issues for pinky (and who) tests, we create a fake utmp file with a - # system boot entry and a login entry for the GH runner account. - FAKE_UTMP_2='[2] [00000] [~~ ] [reboot] [~ ] [6.0.0-test] [0.0.0.0] [2022-02-22T22:11:22,222222+00:00]' - FAKE_UTMP_7='[7] [999999] [tty2] [runner] [tty2] [ ] [0.0.0.0] [2022-02-22T22:22:22,222222+00:00]' - (echo "$FAKE_UTMP_2" ; echo "$FAKE_UTMP_7") | sudo utmpdump -r -o /var/run/utmp - # ... and add a full name to each account with a gecos field but no full name. - sudo sed -i 's/:,/:runner name,/' /etc/passwd - # We also create a couple optional files pinky looks for - touch /home/runner/.project - echo "foo" > /home/runner/.plan - # add user with digital username for testing with issue #7787 - echo 200:x:2000:2000::/home/200:/bin/bash | sudo tee -a /etc/passwd - echo 200:x:2000: | sudo tee -a /etc/group - ;; - esac - - uses: taiki-e/install-action@v2 - if: steps.vars.outputs.CARGO_CMD == 'redoxer' - with: - tool: redoxer@0.2.56 - - name: Initialize toolchain-dependent workflow variables - id: dep_vars - shell: bash - run: | - ## Dependent VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # * determine sub-crate utility list - CARGO_UTILITY_LIST_OPTIONS="$(cargo tree --depth 1 --format '{lib}' ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} --prefix none | sed -n 's/^uu_/-p &/p' | tr '\n' ' ')" - outputs CARGO_UTILITY_LIST_OPTIONS - - name: Info - shell: bash - run: | - ## Info - # commit info - echo "## commit" - echo GITHUB_REF=${GITHUB_REF} - echo GITHUB_SHA=${GITHUB_SHA} - # environment - echo "## environment" - echo "CI='${CI}'" - # tooling info display - echo "## tooling" - which gcc >/dev/null 2>&1 && (gcc --version | head -1) || true - rustup -V 2>/dev/null - rustup show active-toolchain - cargo -V - rustc -V - cargo tree -V - # dependencies - echo "## dependency list" - cargo fetch --locked --quiet --target $(rustc --print host-tuple) - cargo tree --locked --target=${{ matrix.job.target }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} --no-dedupe -e=no-dev --prefix=none | grep -vE "$PWD" | sort --unique - - name: Check - shell: bash - if: matrix.job.skip-publish != true && matrix.job.check-only == true - run: | - ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} check \ - --target=${{ matrix.job.target }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} - - name: Test - if: matrix.job.skip-tests != true && matrix.job.check-only != true - shell: bash - run: | - ## Test - ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} test --target=${{ matrix.job.target }} \ - ${{ steps.vars.outputs.CARGO_TEST_OPTIONS}} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} \ - ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} -p coreutils - env: - RUST_BACKTRACE: "1" - - name: Install llvm-tools (PGO) - if: matrix.job.pgo - shell: bash - run: rustup component add llvm-tools - - name: Train PGO profiles - if: matrix.job.pgo - shell: bash - run: | - ## The target dir is relative: on Windows `github.workspace` is a - ## backslash path that git-bash would not resolve. The script writes the - ## profile's absolute path (natively spelled) to profdata-path.txt. - ./util/build-pgo.sh \ - --target "${{ matrix.job.target }}" \ - --target-dir target/coreutils-pgo \ - --features "${{ matrix.job.features }}" \ - --train-only - echo "RUSTFLAGS=${RUSTFLAGS:+${RUSTFLAGS} }-Cprofile-use=$(cat target/coreutils-pgo/profdata-path.txt)" >> "$GITHUB_ENV" - - name: Verify PGO is applied to the published build - if: matrix.job.pgo - shell: bash - run: | - ## The release artifact must be the PGO build: if RUSTFLAGS did not - ## survive GITHUB_ENV we would silently publish an unoptimized binary. - echo "RUSTFLAGS=${RUSTFLAGS}" - case "${RUSTFLAGS}" in - *-Cprofile-use=*) ;; - *) echo "::error::-Cprofile-use missing from RUSTFLAGS" ; exit 1 ;; - esac - test -s "$(cat target/coreutils-pgo/profdata-path.txt)" - - name: Build coreutils - shell: bash - if: matrix.job.skip-publish != true && matrix.job.check-only != true && matrix.job.target != 'x86_64-pc-windows-msvc' - run: | - ## Build - ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} build --release --config=profile.release.strip=true \ - --target=${{ matrix.job.target }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} - # We don't want to have many duplicated long jobs at here - # So we build individual binaries (including coreutils) for few platforms until we deduplicate many release build for Linux - - name: Build all binaries - if: matrix.job.skip-tests != true && matrix.job.target == 'x86_64-pc-windows-msvc' - shell: bash - run: | - ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} build --release --config=profile.release.strip=true \ - --target=${{ matrix.job.target }} \ - ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} \ - -p coreutils --features=${{ matrix.job.features }} - - name: Package - if: matrix.job.skip-publish != true && matrix.job.check-only != true - shell: bash - run: | - ## Package artifact(s) - find target/${{ matrix.job.target }}/release -maxdepth 1 -type f \ - \( -name "*.wasm" -o -name "*.exe" -o -perm -u+x \) \ - -exec ln -v {} "${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}/" \; - # README and LICENSE - # * spell-checker:ignore EADME ICENSE - (shopt -s nullglob; for f in [R]"EADME"{,.*}; do cp $f '${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}/' ; done) - (shopt -s nullglob; for f in [L]"ICENSE"{-*,}{,.*}; do cp $f '${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}/' ; done) - # core compressed package - pushd '${{ steps.vars.outputs.STAGING }}/' >/dev/null - case '${{ matrix.job.target }}' in - *-pc-windows-*) 7z -y a '${{ steps.vars.outputs.PKG_NAME }}' '${{ steps.vars.outputs.PKG_BASENAME }}'/* | tail -2 ;; - *) tar czf '${{ steps.vars.outputs.PKG_NAME }}' '${{ steps.vars.outputs.PKG_BASENAME }}'/* ;; - esac - popd >/dev/null - - name: Package manpages and completions - if: matrix.job.target == 'x86_64-unknown-linux-gnu' && matrix.job.features == 'feat_os_unix,uudoc' - run: | - mkdir -p share/{man/man1,bash-completion/completions,fish/vendor_completions.d,zsh/site-functions,elvish/lib} - _uudoc=target/${{ matrix.job.target }}/release/uudoc - for bin in $('target/${{ matrix.job.target }}/release/coreutils' --list) coreutils;do - ${_uudoc} manpage ${bin} > share/man/man1/${bin}.1 - ${_uudoc} completion ${bin} bash > share/bash-completion/completions/${bin}.bash - ${_uudoc} completion ${bin} fish > share/fish/vendor_completions.d/${bin}.fish - ${_uudoc} completion ${bin} zsh > share/zsh/site-functions/_${bin} - ${_uudoc} completion ${bin} elvish > share/elvish/lib/${bin}.elv - done - rm share/zsh/site-functions/_[ # not supported - tar --zstd -cf docs.tar.zst share - - name: Publish - uses: softprops/action-gh-release@v3 - if: steps.vars.outputs.DEPLOY && matrix.job.skip-publish != true && matrix.job.check-only != true - with: - draft: true - overwrite_files: true - files: | - ${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_NAME }} - docs.tar.zst - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Publish latest commit - uses: softprops/action-gh-release@v3 - if: github.event_name == 'push' && github.ref == 'refs/heads/main' && matrix.job.skip-publish != true && matrix.job.check-only != true - with: - tag_name: latest-commit - body: | - commit: ${{ github.sha }} - draft: false - prerelease: true - files: | - ${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_NAME }} - docs.tar.zst - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - coverage: - name: Code Coverage - runs-on: ${{ matrix.job.os }} - timeout-minutes: 90 - env: - RUSTC_BOOTSTRAP: 1 - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: unix } - # FIXME: Re-enable macos code coverage - # - { os: macos-latest , features: macos } - # FIXME: Re-enable Code Coverage on windows, which currently fails due to "profiler_builtins". See #6686. - # - { os: windows-latest , features: windows, toolchain: stable-x86_64-pc-windows-gnu } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: taiki-e/install-action@v2 - with: - tool: nextest,grcov@0.8.24 - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - # - name: Reattach HEAD ## may be needed for accurate code coverage info - # run: git checkout ${{ github.head_ref }} - - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - - # * specify gnu-type TOOLCHAIN for windows; `grcov` requires gnu-style code coverage data files - case ${{ matrix.job.os }} in windows-*) TOOLCHAIN="$TOOLCHAIN-x86_64-pc-windows-gnu" ;; esac; - - # * use requested TOOLCHAIN if specified - if [ -n "${{ matrix.job.toolchain }}" ]; then TOOLCHAIN="${{ matrix.job.toolchain }}" ; fi - outputs TOOLCHAIN - - # target-specific options - - # * CARGO_FEATURES_OPTION - CARGO_FEATURES_OPTION='--all-features' ; ## default to '--all-features' for code coverage - if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features=${{ matrix.job.features }}' ; fi - outputs CARGO_FEATURES_OPTION - - # * CODECOV_FLAGS - CODECOV_FLAGS=$( echo "${{ matrix.job.os }}" | sed 's/[^[:alnum:]]/_/g' ) - outputs CODECOV_FLAGS - - - name: Install/setup prerequisites - shell: bash - run: | - ## Install/setup prerequisites - case '${{ matrix.job.os }}' in - ubuntu-latest) - # selinux and systemd headers needed to build tests - # acl: setfacl for coverage profile-trace dir default ACL (umask 0777 tests) - sudo apt-get -y update - sudo apt-get -y install libselinux1-dev libsystemd-dev acl - # pinky is a tool to show logged-in users from utmp, and gecos fields from /etc/passwd. - # In GitHub Action *nix VMs, no accounts log in, even the "runner" account that runs the commands, and "system boot" entry is missing. - # The account also has empty gecos fields. - # To work around these issues for pinky (and who) tests, we create a fake utmp file with a - # system boot entry and a login entry for the GH runner account. - FAKE_UTMP_2='[2] [00000] [~~ ] [reboot] [~ ] [6.0.0-test] [0.0.0.0] [2022-02-22T22:11:22,222222+00:00]' - FAKE_UTMP_7='[7] [999999] [tty2] [runner] [tty2] [ ] [0.0.0.0] [2022-02-22T22:22:22,222222+00:00]' - (echo "$FAKE_UTMP_2" ; echo "$FAKE_UTMP_7") | sudo utmpdump -r -o /var/run/utmp - # ... and add a full name to each account with a gecos field but no full name. - sudo sed -i 's/:,/:runner name,/' /etc/passwd - # We also create a couple optional files pinky looks for - touch /home/runner/.project - echo "foo" > /home/runner/.plan - # add user with digital username for testing with issue #7787 - echo 200:x:2000:2000::/home/200:/bin/bash | sudo tee -a /etc/passwd - echo 200:x:2000: | sudo tee -a /etc/group - ;; - esac - - ## Install the llvm-tools component to get access to `llvm-profdata` - rustup component add llvm-tools - - - name: Run test and coverage - id: run_test_cov - run: | - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - - # Run the coverage script - ./util/build-run-test-coverage-linux.sh - - outputs REPORT_FILE - env: - COVERAGE_DIR: ${{ github.workspace }}/coverage - FEATURES_OPTION: ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} - # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }} - - - name: Upload coverage results (to Codecov.io) - uses: codecov/codecov-action@v7 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ${{ steps.run_test_cov.outputs.report }} - ## flags: IntegrationTests, UnitTests, ${{ steps.vars.outputs.CODECOV_FLAGS }} - flags: ${{ steps.vars.outputs.CODECOV_FLAGS }} - name: codecov-umbrella - fail_ci_if_error: false - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/codecov-action@v7 - with: - token: ${{ secrets.CODECOV_TOKEN }} - report_type: test_results - files: target/nextest/coverage/junit.xml - disable_search: true - flags: coverage,${{ matrix.job.os }} - fail_ci_if_error: false - - test_openssl_dynamic: - name: Build/OpenSSL (dynamic) - needs: [ min_version, deps ] - runs-on: ${{ matrix.job.os }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - job: - # Linux: exercises OPENSSL_NO_VENDOR=1 (dynamic against system libcrypto). ubuntu-latest ships libssl-dev - # binaries with OPENSSL_NO_VENDOR=0 are tested and published at build job - - { os: ubuntu-latest, features: "md5sum,sha1sum,sha224sum,sha256sum,sha384sum,sha512sum,cksum,openssl" } - # macOS: vendored only — system libcrypto needs OPENSSL_DIR - # pointing at Homebrew, which isn't worth wiring up for a smoke test. - # MinGW(UCRT): done at make.yml as actual consumer is MSYS2 only - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - uses: taiki-e/install-action@nextest - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Test checksum utilities with OpenSSL (system / dynamic) - shell: bash - env: - OPENSSL_NO_VENDOR: "1" - RUST_BACKTRACE: "1" - run: | - cargo nextest run --hide-progress-bar --profile ci \ - --no-default-features --features "${{ matrix.job.features }}" - - test_no_diagnostics: - name: Build/No-diagnostics - needs: [ min_version, deps ] - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - uses: taiki-e/install-action@nextest - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - # The caret diagnostics (`feat_diagnostics`) are on by default; packagers - # who want the smaller dependency tree build with them off, so that build - # must stay green. The caret snapshot tests are gated on the feature and - # compile out; everything else runs as usual against the plain messages. - - name: Test without the diagnostics feature - shell: bash - env: - RUST_BACKTRACE: "1" - run: | - cargo nextest run --hide-progress-bar --profile ci \ - --no-default-features --features feat_os_unix - # Confirm the feature actually compiles the renderer out. If ariadne is - # still in the tree, some crate re-grew a hard dependency on it. - # (`cargo tree -i` exits non-zero when the crate is absent from the graph.) - - name: Verify ariadne is compiled out - shell: bash - run: | - if cargo tree -p coreutils --no-default-features --features feat_os_unix -e normal -i ariadne 2>/dev/null; then - echo "ERROR: ariadne is still in the dependency tree without feat_diagnostics" - exit 1 - fi - echo "OK: ariadne is not in the dependency tree" - - test_selinux: - name: Build/SELinux - needs: [ min_version, deps ] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Setup Lima - uses: lima-vm/lima-actions/setup@v1 - id: lima-actions-setup - - name: Cache ~/.cache/lima - uses: actions/cache@v6 - with: - path: ~/.cache/lima - key: lima-${{ steps.lima-actions-setup.outputs.version }} - - name: Start Fedora VM with SELinux - run: limactl start --plain --name=default --cpus=4 --disk=30 --memory=4 --network=lima:user-v2 template:fedora - - run: rsync -v -a -e ssh . lima-default:~/work/ - - name: Setup Rust and other build deps in VM - run: | - lima sudo dnf install --nodocs cargo clippy gcc g++ git rust libselinux-devel clang-devel attr -y - - name: Verify SELinux Status - run: | - lima getenforce - lima ls -laZ /etc/selinux - - name: Build and Test with SELinux - run: | - lima ls - lima bash -c "cd work && cargo test --features 'feat_selinux' --no-default-features" - - name: Lint with SELinux - run: lima bash -c "cd work && cargo clippy --all-targets --features 'feat_selinux' --no-default-features -- -D warnings" - - test_safe_traversal: - name: Safe Traversal Security Check - runs-on: ubuntu-latest - needs: [ min_version, deps ] - - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Install strace and libselinux headers (needed to build chcon) - run: sudo apt-get update && sudo apt-get install -y strace libselinux1-dev - - name: Build utilities with safe traversal - run: cargo build --profile=release-small -p uu_rm -p uu_chmod -p uu_chown -p uu_chgrp -p uu_mv -p uu_du -p uu_split -p uu_chcon - - name: Run safe traversal verification - run: ./util/check-safe-traversal.sh - - test_libc_interposition: - name: libc Interposition Check - runs-on: ubuntu-latest - needs: [ min_version, deps ] - - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Install fakeroot - run: sudo apt-get update && sudo apt-get install -y fakeroot - - name: Build utilities that must stay interposable - run: cargo build --profile=release-small -p uu_chmod -p uu_chown - - name: Run libc interposition verification - run: ./util/check-libc-interposition.sh - - test_open_once: - name: Redundant Open Check - runs-on: ubuntu-latest - needs: [ min_version, deps ] - - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Install strace - run: sudo apt-get update && sudo apt-get install -y strace - - name: Build utilities for redundant open checks - run: cargo build --profile=release-small -p uu_cp -p uu_mv - - name: Run redundant open verification - run: ./util/check-open-once.sh - - test_toctou: - name: TOCTOU Security Check - runs-on: ubuntu-latest - needs: [ min_version, deps ] - - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Install strace - run: sudo apt-get update && sudo apt-get install -y strace - - name: Build utilities for TOCTOU checks - run: cargo build --profile=release-small -p uu_mkfifo -p uu_touch -p uu_head - - name: Run TOCTOU verification - run: ./util/check-toctou.sh diff --git a/.github/workflows/CheckScripts.yml b/.github/workflows/CheckScripts.yml deleted file mode 100644 index 61d9cbf6bc1..00000000000 --- a/.github/workflows/CheckScripts.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: CheckScripts - -# spell-checker:ignore ludeeus mfinelli shellcheck scandir shfmt - -env: - SCRIPT_DIR: 'util' - -on: - push: - branches: - - '*' - paths: - - 'util/**/*.sh' - pull_request: - branches: - - main - paths: - - 'util/**/*.sh' - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - shell_check: - name: ShellScript/Check - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Run ShellCheck - uses: ludeeus/action-shellcheck@master - env: - SHELLCHECK_OPTS: -s bash - with: - severity: warning - scandir: ${{ env.SCRIPT_DIR }} - format: tty - - shell_fmt: - name: ShellScript/Format - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Setup shfmt - uses: mfinelli/setup-shfmt@v4 - - name: Run shfmt - shell: bash - run: | - # fmt options: bash syntax, 4 spaces indent, indent for switch-case - echo "## show the differences between formatted and original scripts..." - find ${{ env.SCRIPT_DIR }} -name "*.sh" -print0 | xargs -0 shfmt -ln=bash -i 4 -ci -d || true diff --git a/.github/workflows/FixPR.yml b/.github/workflows/FixPR.yml deleted file mode 100644 index a38a76a4432..00000000000 --- a/.github/workflows/FixPR.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: FixPR - -# spell-checker:ignore dedupe - -# Trigger automated fixes for PRs being merged (with associated commits) - -env: - BRANCH_TARGET: main - -permissions: - contents: write # FixPR commits Cargo.lock back to main - -on: - # * only trigger on pull request closed to specific branches - # ref: https://github.community/t/trigger-workflow-only-on-pull-request-merge/17359/9 - pull_request: - branches: - - main # == env.BRANCH_TARGET ## unfortunately, env context variables are only available in jobs/steps (see ) - types: [ closed ] - -jobs: - code_deps: - # Refresh dependencies (ie, 'Cargo.lock') and show updated dependency tree - if: github.event.pull_request.merged == true ## only for PR merges - name: Update/dependencies - runs-on: ${{ matrix.job.os }} - strategy: - matrix: - job: - - { os: ubuntu-latest , features: feat_os_unix } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Initialize job variables - id: vars - shell: bash - run: | - # surface MSRV from CICD workflow - RUST_MIN_SRV=$(grep -P "^\s+RUST_MIN_SRV:" .github/workflows/CICD.yml | grep -Po "(?<=\x22)\d+[.]\d+(?:[.]\d+)?(?=\x22)" ) - echo "RUST_MIN_SRV=${RUST_MIN_SRV}" >> $GITHUB_OUTPUT - - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ steps.vars.outputs.RUST_MIN_SRV }} - - uses: Swatinem/rust-cache@v2 - - name: Ensure updated 'Cargo.lock' - shell: bash - run: | - # Ensure updated '*/Cargo.lock' - # * '*/Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38) - for dir in "." "fuzz"; do - ( cd "$dir" && (cargo fetch --locked --quiet --target $(rustc --print host-tuple) || cargo +${{ steps.vars.outputs.RUST_MIN_SRV }} update) ) - done - - name: Info - shell: bash - run: | - # Info - ## environment - echo "## environment" - echo "CI='${CI}'" - ## tooling info display - echo "## tooling" - which gcc >/dev/null 2>&1 && (gcc --version | head -1) || true - rustup -V 2>/dev/null - rustup show active-toolchain - cargo -V - rustc -V - cargo tree -V - ## dependencies - echo "## dependency list" - cargo fetch --locked --quiet --target $(rustc --print host-tuple) - ## * using the 'stable' toolchain is necessary to avoid "unexpected '--filter-platform'" errors - cargo +stable tree --locked --no-dedupe -e=no-dev --prefix=none --features ${{ matrix.job.features }} | grep -vE "$PWD" | sort --unique - - name: Commit any changes (to '${{ env.BRANCH_TARGET }}') - uses: EndBug/add-and-commit@v11 - with: - new_branch: ${{ env.BRANCH_TARGET }} - default_author: github_actions - message: "maint ~ refresh 'Cargo.lock' 'fuzz/Cargo.lock'" - add: Cargo.lock fuzz/Cargo.lock - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/GnuComment.yml b/.github/workflows/GnuComment.yml deleted file mode 100644 index ed5deb0505c..00000000000 --- a/.github/workflows/GnuComment.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: GnuComment - -# spell-checker:ignore zizmor backquote - -on: - workflow_run: - workflows: ["GnuTests"] - types: - - completed # zizmor: ignore[dangerous-triggers] - -permissions: {} -jobs: - post-comment: - permissions: - actions: read # to list workflow runs artifacts - pull-requests: write # to comment on pr - - runs-on: ubuntu-latest - if: > - github.event.workflow_run.event == 'pull_request' && - (github.event.workflow_run.conclusion == 'success' || - github.event.workflow_run.conclusion == 'failure') - steps: - - name: 'Download artifact' - uses: actions/github-script@v9 - with: - script: | - // List all artifacts from GnuTests - var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{ github.event.workflow_run.id }}, - }); - - // Download the "comment" artifact, which contains a PR number (NR) and result.txt - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "comment" - })[0]; - if (!matchArtifact) { - core.info("No comment artifact found; skipping."); - return; - } - var download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data)); - - run: unzip comment.zip - - - name: 'Comment on PR' - uses: actions/github-script@v9 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const fs = require('fs'); - if (!fs.existsSync('./NR') || !fs.existsSync('./result.txt')) { - core.info("No GNU comment payload to post."); - return; - } - const issue_number = Number(fs.readFileSync('./NR')); - if (!issue_number) { - core.info('No PR number; skipping.'); - return; - } - const content = fs.readFileSync('./result.txt').toString(); - - const marker = ''; - const body = `${marker}\nGNU testsuite comparison:\n\`\`\`\n${content}\`\`\``; - - const { data: comments } = await github.rest.issues.listComments({ - ...context.repo, - issue_number - }); - - const existing = comments.find(c => - c.user.login === 'github-actions[bot]' && - c.body.includes(marker) - ); - - if (existing) { - await github.rest.issues.updateComment({ - ...context.repo, - comment_id: existing.id, - body - }); - } else { - if (content.trim().length <= 7) { // 7 because we have backquote + \n - return; - } - await github.rest.issues.createComment({ - ...context.repo, - issue_number, - body - }); - } diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml deleted file mode 100644 index 71c3c8f05cf..00000000000 --- a/.github/workflows/GnuTests.yml +++ /dev/null @@ -1,706 +0,0 @@ -name: GnuTests - -# spell-checker:ignore (abbrev/names) CodeCov gnulib GnuTests -# spell-checker:ignore (jargon) submodules devel -# spell-checker:ignore (libs/utils) chksum dpkg getenforce gperf lcov libexpect limactl pyinotify setenforce shopt valgrind libattr libcap taiki-e zstd cpio -# spell-checker:ignore (options) Ccodegen Coverflow Cpanic Zpanic -# spell-checker:ignore (vars) FILESET SUBDIRS XPASS -# spell-checker:ignore userns nodocs - -# * note: to run a single test => `REPO/util/run-gnu-test.sh PATH/TO/TEST/SCRIPT` - -on: - pull_request: - push: - branches: - - '*' - ## the reference branch carries no code, only published test results - - '!gnu-reference' - -permissions: - contents: read - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -env: - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - ## Machine-maintained branch holding the latest GNU test results of the - ## default branch; every PR run is compared against what it carries. - REFERENCE_BRANCH: 'gnu-reference' - TEST_FULL_SUMMARY_FILE: 'gnu-full-result.json' - TEST_ROOT_FULL_SUMMARY_FILE: 'gnu-root-full-result.json' - TEST_STTY_FULL_SUMMARY_FILE: 'gnu-stty-full-result.json' - TEST_SELINUX_FULL_SUMMARY_FILE: 'selinux-gnu-full-result.json' - TEST_SELINUX_ROOT_FULL_SUMMARY_FILE: 'selinux-root-gnu-full-result.json' - TEST_QEMU_FULL_SUMMARY_FILE: 'qemu-gnu-full-result.json' - -jobs: - native: - name: Run GNU tests (native) - runs-on: ubuntu-26.04 # some strace? test needs newer glibc - steps: - #### Get the code - - name: Checkout code (uutils) - uses: actions/checkout@v7.0.1 - with: - path: 'uutils' - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - with: - workspaces: "./uutils -> target" - - name: Checkout code (GNU coreutils) - run: (mkdir -p gnu && cd gnu && bash ../uutils/util/fetch-gnu.sh) - #### Build environment setup - - name: Install dependencies - shell: bash - run: | - ## Install dependencies - sudo apt-get update - ## Check that build-gnu.sh works on the non SELinux system by installing libselinux only on lima - sudo apt-get install -y gperf gdb python3-pyinotify valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev attr quilt - - name: Use GNU as the system coreutils - shell: bash - run: | - if dpkg -s coreutils-from-uutils >/dev/null 2>&1; then - sudo apt-get install -y --allow-remove-essential coreutils-from-gnu coreutils-from-uutils- - fi - - name: Add various locales - shell: bash - run: | - ## Add various locales - echo "Before:" - locale -a - ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' - ## Some others need a French locale - sudo locale-gen - sudo locale-gen --keep-existing fr_FR - sudo locale-gen --keep-existing fr_FR.UTF-8 - sudo locale-gen --keep-existing es_ES.UTF-8 - sudo locale-gen --keep-existing sv_SE - sudo locale-gen --keep-existing sv_SE.UTF-8 - sudo locale-gen --keep-existing en_US - sudo locale-gen --keep-existing en_US.UTF-8 - sudo locale-gen --keep-existing ru_RU.KOI8-R - sudo locale-gen --keep-existing fa_IR.UTF-8 # Iran - sudo locale-gen --keep-existing am_ET.UTF-8 # Ethiopia - sudo locale-gen --keep-existing th_TH.UTF-8 # Thailand - sudo locale-gen --keep-existing zh_CN.GB18030 # China - - sudo update-locale - echo "After:" - locale -a - - ### Build - - name: Build binaries - shell: bash - run: | - ## Build binaries - cd 'uutils' - env PROFILE=release-small bash util/build-gnu.sh - - ### Run tests as user - - name: Run GNU tests - shell: bash - run: | - ## Use unshare - sudo sysctl -w kernel.unprivileged_userns_clone=1 - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - ## Run GNU tests - path_GNU='gnu' - path_UUTILS='uutils' - bash "uutils/util/run-gnu-test.sh" - - name: Extract testing info from individual logs into JSON - shell: bash - run : | - path_UUTILS='uutils' - python uutils/util/gnu-json-result.py gnu/tests > ${{ env.TEST_FULL_SUMMARY_FILE }} - - ### This shell has been changed from "bash" to this command - ### "script" will start a pty and the -q command removes the "script" initiation log - ### the -e flag makes it propagate the error code and -c runs the command in a pty - ### the primary purpose of this change is to run the tty GNU tests - ### The reason its separated from the rest of the tests is because one test can corrupt the other - ### tests through the use of the shared terminal and it changes the environment that the other - ### tests are run in, which can cause different results. - - name: Run GNU stty tests - shell: 'script -q -e -c "bash {0}"' - run: | - ## Run GNU root tests - path_GNU='gnu' - path_UUTILS='uutils' - bash "uutils/util/run-gnu-test.sh" run-tty - - - name: Extract testing info from individual logs (stty) into JSON - shell: bash - run : | - path_UUTILS='uutils' - python uutils/util/gnu-json-result.py gnu/tests > ${{ env.TEST_STTY_FULL_SUMMARY_FILE }} - - ### Run tests as root - - name: Run GNU root tests - shell: bash - run: | - ## Root tests drop to nobody via `chroot --user`. /home/runner is - ## 0750, so that user cannot reach our build dir and PATH lookup silently - ## falls back to the runner's own /usr/bin coreutils. Allow traversal. - dir="${GITHUB_WORKSPACE}" - while [ "${dir}" != / ]; do sudo chmod o+x "${dir}"; dir=$(dirname "${dir}"); done - ## Fail here if that user still cannot run our build, instead of letting the - ## tests pass or fail on whatever coreutils the runner image ships. - sudo -u nobody "${GITHUB_WORKSPACE}/uutils/target/release-small/id" -u - ## Run GNU root tests - path_GNU='gnu' - path_UUTILS='uutils' - bash "uutils/util/run-gnu-test.sh" run-root - - - name: Extract testing info from individual logs (run as root) into JSON - shell: bash - run : | - path_UUTILS='uutils' - python uutils/util/gnu-json-result.py gnu/tests > ${{ env.TEST_ROOT_FULL_SUMMARY_FILE }} - - ### Upload artifacts - - name: Upload full json results - uses: actions/upload-artifact@v7 - with: - name: gnu-full-result - path: ${{ env.TEST_FULL_SUMMARY_FILE }} - - name: Upload root json results - uses: actions/upload-artifact@v7 - with: - name: gnu-root-full-result - path: ${{ env.TEST_ROOT_FULL_SUMMARY_FILE }} - - name: Upload stty json results - uses: actions/upload-artifact@v7 - with: - name: gnu-stty-full-result - path: ${{ env.TEST_STTY_FULL_SUMMARY_FILE }} - - - name: Compress test logs - shell: bash - run : | - # Compress logs before upload (fails otherwise) - gzip gnu/tests/*/*.log - - name: Upload test logs - uses: actions/upload-artifact@v7 - with: - name: test-logs - path: | - gnu/tests/*.log - gnu/tests/*/*.log.gz - - selinux: - name: Run GNU tests (SELinux) - runs-on: ubuntu-latest # using VM's glibc - steps: - #### Get the code - - name: Checkout code (uutils) - uses: actions/checkout@v7.0.1 - with: - path: 'uutils' - persist-credentials: false - - name: Checkout code (GNU coreutils) - run: (mkdir -p gnu && cd gnu && bash ../uutils/util/fetch-gnu.sh) - - #### Lima build environment setup - - name: Setup Lima - uses: lima-vm/lima-actions/setup@v1 - id: lima-actions-setup - - name: Cache ~/.cache/lima - uses: actions/cache@v6 - with: - path: ~/.cache/lima - key: lima-${{ steps.lima-actions-setup.outputs.version }} - - name: Start Fedora VM with SELinux - run: limactl start --plain --name=default --cpus=4 --disk=40 --memory=8 --network=lima:user-v2 template:fedora - - name: Verify SELinux Status and Configuration - run: | - lima getenforce - lima ls -laZ /etc/selinux - lima sudo sestatus - - - name: Install dependencies in VM - run: | - lima sudo dnf -y install --nodocs autoconf bison cargo clippy gperf gcc gdb jq libacl-devel libattr-devel libcap-devel libselinux-devel attr rust clang-devel automake patch quilt - - name: Copy the sources to VM - run: | - rsync -a -e ssh . lima-default:~/work/ - - ### Build - - name: Build binaries - run: | - lima bash -c "cd ~/work/uutils/ && SELINUX_ENABLED=1 PROFILE=release-small bash util/build-gnu.sh" - - ### Run tests as user - - name: Generate SELinux tests list - run: | - # Find and list all tests that require SELinux - lima bash -c "cd ~/work/gnu/ && grep -l 'require_selinux_' -r tests/ > ~/work/uutils/selinux-tests.txt" - lima bash -c "cd ~/work/uutils/ && cat selinux-tests.txt" - - # Count the tests - lima bash -c "cd ~/work/uutils/ && echo 'Found SELinux tests:'; wc -l selinux-tests.txt" - - name: Run GNU SELinux tests - run: | - # Ensure we're running in enforcing mode - lima sudo setenforce 1 - lima getenforce - - # Create test files with SELinux contexts for testing - lima sudo mkdir -p /var/test_selinux - lima sudo touch /var/test_selinux/test_file - lima sudo chcon -t etc_t /var/test_selinux/test_file - lima ls -Z /var/test_selinux/test_file # Verify context - - lima cat /proc/filesystems - lima bash -c "cd ~/work/uutils/ && bash util/run-gnu-test.sh \$(cat selinux-tests.txt)" - - name: Extract testing info from individual logs into JSON - shell: bash - run : | - lima bash -c "cd ~/work/gnu/ && python3 ../uutils/util/gnu-json-result.py tests > ~/work/${{ env.TEST_SELINUX_FULL_SUMMARY_FILE }}" - - ### Run tests as root - - name: Run GNU SELinux root tests - run: | - lima bash -c "cd ~/work/uutils/ && CI=1 bash util/run-gnu-test.sh run-root \$(cat selinux-tests.txt)" - - name: Extract testing info from individual logs (run as root) into JSON - shell: bash - run : | - lima bash -c "cd ~/work/gnu/ && python3 ../uutils/util/gnu-json-result.py tests > ~/work/${{ env.TEST_SELINUX_ROOT_FULL_SUMMARY_FILE }}" - - ### Upload artifacts - - name: Collect test logs and test results from VM - run: | - mkdir -p gnu/tests-selinux - - # Copy the json output back from the Lima VM to the host - rsync -v -a -e ssh lima-default:~/work/*.json ./ - # Copy the test directory now - rsync -v -a -e ssh lima-default:~/work/gnu/tests/ ./gnu/tests-selinux/ - - name: Upload SELinux json results - uses: actions/upload-artifact@v7 - with: - name: selinux-gnu-full-result - path: ${{ env.TEST_SELINUX_FULL_SUMMARY_FILE }} - - name: Upload SELinux root json results - uses: actions/upload-artifact@v7 - with: - name: selinux-root-gnu-full-result - path: ${{ env.TEST_SELINUX_ROOT_FULL_SUMMARY_FILE }} - - name: Compress SELinux test logs - shell: bash - run : | - # Compress logs before upload (fails otherwise) - gzip gnu/tests-selinux/*/*.log - - name: Upload SELinux test logs - uses: actions/upload-artifact@v7 - with: - name: selinux-test-logs - path: | - gnu/tests-selinux/*.log - gnu/tests-selinux/*/*.log.gz - - qemu: - name: Run GNU tests (SMACK/ROOTFS) - runs-on: ubuntu-latest # using Arch glibc - steps: - - name: Checkout code (uutils) - uses: actions/checkout@v7.0.1 - with: - path: 'uutils' - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - with: - workspaces: "./uutils -> target" - - name: Checkout code (GNU coreutils) - run: (mkdir -p gnu && cd gnu && bash ../uutils/util/fetch-gnu.sh) - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y qemu-system-x86 zstd cpio - - name: Run GNU SMACK/ROOTFS tests - run: | - cd uutils - bash util/run-gnu-tests-smack-ci.sh "$GITHUB_WORKSPACE/gnu" "$GITHUB_WORKSPACE/gnu/tests-qemu" - - name: Extract testing info into JSON - run: | - python3 uutils/util/gnu-json-result.py gnu/tests-qemu > ${{ env.TEST_QEMU_FULL_SUMMARY_FILE }} - - name: Upload SMACK/ROOTFS json results - uses: actions/upload-artifact@v7 - with: - name: qemu-gnu-full-result - path: ${{ env.TEST_QEMU_FULL_SUMMARY_FILE }} - - name: Compress SMACK/ROOTFS test logs - run: gzip gnu/tests-qemu/*/*.log 2>/dev/null || true - - name: Upload SMACK/ROOTFS test logs - uses: actions/upload-artifact@v7 - with: - name: qemu-test-logs - path: | - gnu/tests-qemu/*.log - gnu/tests-qemu/*/*.log.gz - - aggregate: - needs: [native, selinux, qemu] - permissions: - actions: read # to download this run's own result artifacts - contents: read # for actions/checkout to fetch code, and to read the reference branch - name: Aggregate GNU test results - runs-on: ubuntu-latest - steps: - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # - TEST_SUMMARY_FILE='gnu-result.json' - AGGREGATED_SUMMARY_FILE='aggregated-result.json' - - outputs TEST_SUMMARY_FILE AGGREGATED_SUMMARY_FILE - - name: Checkout code (uutils) - uses: actions/checkout@v7.0.1 - with: - path: 'uutils' - persist-credentials: false - - name: Fetch reference results (default branch) - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - ## Fetch the reference results published by the last GnuTests run on the - ## default branch. - ## They live on a dedicated, machine-maintained branch rather than in a - ## workflow artifact: an artifact is addressable only by run id, so - ## finding one meant listing workflow runs, and the server-side 'branch' - ## filter of that listing is served by a search index that is only - ## eventually consistent (observed returning nothing newer than a month - ## old while the unfiltered listing was current). A run resolved from a - ## stale listing made every test fixed on the default branch since then - ## look like a fix contributed by the PR, and every test broken since - ## then like a regression introduced by it. A branch has no such index, - ## and no 90-day expiry either. - ## A missing reference is not fatal: the branch does not exist until the - ## first run on the default branch has published to it. - mkdir -p reference/aggregated-result reference/test-summary - ## Resolve the branch tip once and read every file at that commit. The - ## branch is force-pushed by 'publish-reference', so reading each file - ## by branch name could straddle a publish and pair one file with - ## another reference's, comparing against a baseline that never existed. - ## Only a 404 means "no reference yet"; any other failure must not be - ## mistaken for one, or a transient API error would silently skip both - ## comparisons and let a regression through unnoticed. - REFERENCE_SHA='' - ref_err="${RUNNER_TEMP:-/tmp}/reference-ref.err" - for attempt in 1 2 3; do - if REFERENCE_SHA=$(gh api "repos/${{ github.repository }}/git/refs/heads/${REFERENCE_BRANCH}" \ - --jq '.object.sha' 2>"${ref_err}"); then - break - fi - REFERENCE_SHA='' - if grep -q 'HTTP 404' "${ref_err}"; then - ## The branch does not exist until the first run on the default - ## branch has published to it. - break - fi - if [[ "${attempt}" -eq 3 ]]; then - cat "${ref_err}" >&2 - echo "::error ::Could not resolve the '${REFERENCE_BRANCH}' branch; refusing to report an API failure as a missing baseline." - exit 1 - fi - sleep $(( attempt * 5 )) - done - fetch() { - gh api "repos/${{ github.repository }}/contents/${1}?ref=${REFERENCE_SHA}" \ - -H 'Accept: application/vnd.github.raw' > "${2}" 2>/dev/null \ - && [[ -s "${2}" ]] - } - if [[ -n "${REFERENCE_SHA}" ]] \ - && fetch aggregated-result.json reference/aggregated-result/aggregated-result.json \ - && fetch gnu-result.json reference/test-summary/gnu-result.json; then - ## 'meta.json' says which commit the reference was built from; it is - ## informational, so tolerate it being absent or unparsable. - meta=$(gh api "repos/${{ github.repository }}/contents/meta.json?ref=${REFERENCE_SHA}" \ - -H 'Accept: application/vnd.github.raw' 2>/dev/null || true) - published_sha=$(jq -r '.sha // empty' <<< "${meta}" 2>/dev/null || true) - published_at=$(jq -r '.published_at // empty' <<< "${meta}" 2>/dev/null || true) - echo "Reference: ${{ env.DEFAULT_BRANCH }}@${published_sha:-} (published ${published_at:-}, ${REFERENCE_BRANCH}@${REFERENCE_SHA})" - published_epoch='' - if [[ -n "${published_at}" ]]; then - published_epoch=$(date -u -d "${published_at}" +%s 2>/dev/null || true) - fi - if [[ -n "${published_epoch}" ]]; then - age_days=$(( ( $(date -u +%s) - published_epoch ) / 86400 )) - if [[ "${age_days}" -gt 2 ]]; then - echo "::warning ::The reference results are ${age_days} days old (${published_at}); the comparison below may report unrelated fixes and regressions." - fi - fi - else - rm -f reference/aggregated-result/aggregated-result.json reference/test-summary/gnu-result.json - echo "::warning ::No reference results found on the '${REFERENCE_BRANCH}' branch; the comparisons below will be skipped." - fi - - name: Download full json results - uses: actions/download-artifact@v8 - with: - name: gnu-full-result - path: results - merge-multiple: true - - name: Download root json results - uses: actions/download-artifact@v8 - with: - name: gnu-root-full-result - path: results - merge-multiple: true - - name: Download stty json results - uses: actions/download-artifact@v8 - with: - name: gnu-stty-full-result - path: results - merge-multiple: true - - - name: Download selinux json results - uses: actions/download-artifact@v8 - with: - name: selinux-gnu-full-result - path: results - merge-multiple: true - - name: Download selinux root json results - uses: actions/download-artifact@v8 - with: - name: selinux-root-gnu-full-result - path: results - merge-multiple: true - - name: Download SMACK/ROOTFS json results - uses: actions/download-artifact@v8 - with: - name: qemu-gnu-full-result - path: results - merge-multiple: true - - name: Extract/summarize testing info - id: summary - shell: bash - run: | - ## Extract/summarize testing info - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - - path_UUTILS='uutils' - - json_count=$(ls -l results/*.json | wc -l) - if [[ "$json_count" -ne 6 ]]; then - echo "::error ::Failed to download all results json files (expected 6 files, found $json_count); failing early" - ls -lR results || true - exit 1 - fi - - # Look at all individual results and summarize - eval $(python3 uutils/util/analyze-gnu-results.py -o=${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }} results/*.json) - - if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then - echo "::error ::Failed to parse test results from '${{ env.TEST_FULL_SUMMARY_FILE }}'; failing early" - exit 1 - fi - - output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR / SKIP: $SKIP" - echo "${output}" - - if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then - echo "::warning ::${output}" - fi - - jq -n \ - --arg date "$(date --rfc-email)" \ - --arg sha "$GITHUB_SHA" \ - --arg total "$TOTAL" \ - --arg pass "$PASS" \ - --arg skip "$SKIP" \ - --arg fail "$FAIL" \ - --arg xpass "$XPASS" \ - --arg error "$ERROR" \ - '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' - HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) - outputs HASH - - name: Upload SHA1/ID of 'test-summary' - uses: actions/upload-artifact@v7 - with: - name: "${{ steps.summary.outputs.HASH }}" - path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - - name: Upload test results summary - uses: actions/upload-artifact@v7 - with: - name: test-summary - path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - - name: Upload aggregated json results - uses: actions/upload-artifact@v7 - with: - name: aggregated-result - path: ${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }} - - name: Compare test failures VS reference - shell: bash - run: | - ## Compare test failures VS reference using JSON files - REF_SUMMARY_FILE='reference/aggregated-result/aggregated-result.json' - CURRENT_SUMMARY_FILE='${{ steps.vars.outputs.AGGREGATED_SUMMARY_FILE }}' - REPO_DEFAULT_BRANCH='${{ env.DEFAULT_BRANCH }}' - path_UUTILS='uutils' - - # Path to ignore file for intermittent issues - IGNORE_INTERMITTENT="uutils/.github/workflows/ignore-intermittent.txt" - - # Set up comment directory - COMMENT_DIR="reference/comment" - mkdir -p ${COMMENT_DIR} - echo ${{ github.event.number }} > ${COMMENT_DIR}/NR - COMMENT_LOG="${COMMENT_DIR}/result.txt" - - COMPARISON_RESULT=0 - if test -f "${CURRENT_SUMMARY_FILE}"; then - if test -f "${REF_SUMMARY_FILE}"; then - echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" - echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")" - - python3 uutils/util/compare_test_results.py \ - --ignore-file "${IGNORE_INTERMITTENT}" \ - --output "${COMMENT_LOG}" \ - "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" - - COMPARISON_RESULT=$? - else - echo "::warning ::Skipping test comparison; no prior reference summary is available at '${REF_SUMMARY_FILE}'." - fi - else - echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early" - exit 1 - fi - - if [ ${COMPARISON_RESULT} -eq 1 ]; then - echo "ONLY_INTERMITTENT=false" >> $GITHUB_ENV - echo "::error ::Found new non-intermittent test failures" - exit 1 - else - echo "ONLY_INTERMITTENT=true" >> $GITHUB_ENV - echo "::notice ::No new test failures detected" - fi - - name: Upload comparison log (for GnuComment workflow) - if: success() || failure() # run regardless of prior step success/failure - uses: actions/upload-artifact@v7 - with: - name: comment - path: reference/comment/ - - name: Compare test summary VS reference - if: success() || failure() # run regardless of prior step success/failure - shell: bash - run: | - ## Compare test summary VS reference - REF_SUMMARY_FILE='reference/test-summary/gnu-result.json' - if test -f "${REF_SUMMARY_FILE}"; then - echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" - mv "${REF_SUMMARY_FILE}" main-gnu-result.json - python uutils/util/compare_gnu_result.py - else - echo "::warning ::Skipping test summary comparison; no prior reference summary is available." - fi - - publish-reference: - ## Publish this run's results as the reference that every later PR run is - ## compared against. Only runs on a push to the default branch, so a fork - ## PR can never reach it. - needs: aggregate - if: > - always() - && github.event_name == 'push' - && github.ref_name == github.event.repository.default_branch - && needs.aggregate.result != 'cancelled' - && needs.aggregate.result != 'skipped' - permissions: - actions: read # to download this run's own result artifacts - contents: write # to push to the reference branch - name: Publish reference GNU test results - runs-on: ubuntu-latest - ## Only one run may publish at a time. The step below reads the currently - ## published run id and then force-pushes; two overlapping runs could - ## interleave those, letting an older one overwrite a newer reference. - concurrency: - group: publish-gnu-reference - cancel-in-progress: false - steps: - - name: Download aggregated json results - continue-on-error: true ## 'aggregate' may have failed before uploading - uses: actions/download-artifact@v8 - with: - name: aggregated-result - path: publish - - name: Download test results summary - continue-on-error: true ## 'aggregate' may have failed before uploading - uses: actions/download-artifact@v8 - with: - name: test-summary - path: publish - - name: Publish to the reference branch - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - ## Publish to the reference branch - ## The branch is a single-commit orphan history, force-pushed on every - ## run, so it never accumulates history. Runs on the default branch can - ## finish out of order, so refuse to overwrite a reference published by a - ## *newer* run (run ids are monotonic). - if [[ ! -s publish/aggregated-result.json || ! -s publish/gnu-result.json ]]; then - echo "::warning ::This run produced no results to publish; leaving the '${REFERENCE_BRANCH}' branch untouched." - exit 0 - fi - ## Fail closed: only a 404 means nothing has been published yet. A - ## suppressed error read as run id 0 would let this run force-push over - ## a reference a newer run had already placed. - published_run=0 - meta_err="${RUNNER_TEMP:-/tmp}/published-meta.err" - for attempt in 1 2 3; do - if published_meta=$(gh api "repos/${{ github.repository }}/contents/meta.json?ref=${REFERENCE_BRANCH}" \ - -H 'Accept: application/vnd.github.raw' 2>"${meta_err}"); then - published_run=$(jq -r '.run_id // 0' <<< "${published_meta}" 2>/dev/null || echo 0) - break - fi - if grep -q 'HTTP 404' "${meta_err}"; then - break - fi - if [[ "${attempt}" -eq 3 ]]; then - cat "${meta_err}" >&2 - echo "::error ::Could not read the published reference; refusing to force-push over a possibly newer one." - exit 1 - fi - sleep $(( attempt * 5 )) - done - if [[ "${published_run}" =~ ^[0-9]+$ ]] && (( published_run > GITHUB_RUN_ID )); then - echo "::notice ::Run ${published_run} already published a newer reference; nothing to do." - exit 0 - fi - jq -n --arg sha "${GITHUB_SHA}" \ - --arg run_id "${GITHUB_RUN_ID}" \ - --arg branch "${GITHUB_REF_NAME}" \ - --arg published_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '{sha: $sha, run_id: $run_id, branch: $branch, published_at: $published_at}' \ - > publish/meta.json - printf '%s\n' \ - '# GNU test reference results' \ - '' \ - 'Machine-maintained branch: the GNU test results of the latest run of' \ - 'the `GnuTests` workflow on the default branch. Every pull request run' \ - 'is compared against the files here, so that it reports only the test' \ - 'changes the pull request is itself responsible for.' \ - '' \ - 'Do not commit here by hand; the branch is force-pushed on every run.' \ - > publish/README.md - cd publish - git init -q -b "${REFERENCE_BRANCH}" - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git add -A - git commit -q -m "GNU test results for ${GITHUB_SHA}" - git push -q --force \ - "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ - "HEAD:refs/heads/${REFERENCE_BRANCH}" - echo "::notice ::Published reference results for ${GITHUB_SHA}." diff --git a/.github/workflows/NotifyTracking.yml b/.github/workflows/NotifyTracking.yml deleted file mode 100644 index ef7edd7d604..00000000000 --- a/.github/workflows/NotifyTracking.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: NotifyTracking - -# spell-checker:ignore zizmor -# spell-checker:ignore (misc) busybox toybox multisize - -## Tell uutils/coreutils-tracking to refresh as soon as a workflow that -## produces one of its inputs has finished on the default branch, so its graphs -## follow the data rather than wait for its scheduled refresh. -## -## 'GnuTests' publishes 'test-summary' and 'aggregated-result'; 'make' publishes -## 'busybox-result.json', 'toybox-result.json', 'size-result' and -## 'individual-size-result'. The tracking workflow picks up whatever is newest -## for each of them, so a notification from either is enough to run it. - -on: - workflow_run: - workflows: ["GnuTests", "make"] - types: - - completed # zizmor: ignore[dangerous-triggers] - -permissions: {} - -jobs: - dispatch: - name: Trigger the tracking refresh - ## Nothing is checked out and no result is read here, so the untrusted - ## content a 'workflow_run' trigger can carry is never executed. - ## - ## A failed run still counts. 'GnuTests' uploads its results and only then - ## fails the comparison when it finds a new failure, and 'publish-reference' - ## publishes regardless; requiring success here would leave the graphs - ## un-refreshed for exactly the regressions they exist to show. - if: > - github.repository == 'uutils/coreutils' - && github.event.workflow_run.event == 'push' - && github.event.workflow_run.head_branch == github.event.repository.default_branch - && github.event.workflow_run.conclusion != 'cancelled' - && github.event.workflow_run.conclusion != 'skipped' - runs-on: ubuntu-latest - ## The 'secrets' context is not available in a step's 'if:', so fold the - ## check into a job-level env var, which is. - env: - HAVE_APP_CREDENTIALS: ${{ secrets.TRACKING_APP_ID != '' && secrets.TRACKING_APP_PRIVATE_KEY != '' }} - steps: - ## A GITHUB_TOKEN is scoped to the repository it runs in, so writing to a - ## sibling repository needs a credential of its own even within the same - ## organization. This uses an organization-owned GitHub App rather than a - ## personal access token: it is not tied to an individual account and does - ## not silently expire. Install it on uutils/coreutils-tracking only, with - ## 'contents: write', which is what the repository_dispatch endpoint needs. - - name: Mint a token for the tracking repository - id: app-token - if: env.HAVE_APP_CREDENTIALS == 'true' - uses: actions/create-github-app-token@v3 - with: - app-id: ${{ secrets.TRACKING_APP_ID }} - private-key: ${{ secrets.TRACKING_APP_PRIVATE_KEY }} - owner: uutils - repositories: coreutils-tracking - - - name: Send the repository_dispatch - shell: bash - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - SOURCE_WORKFLOW: ${{ github.event.workflow_run.name }} - SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} - SOURCE_SHA: ${{ github.event.workflow_run.head_sha }} - run: | - ## Send the repository_dispatch - ## A missing token is not an error: the tracking repository still - ## refreshes on its own schedule, this only makes it prompt. - if [[ -z "${GH_TOKEN}" ]]; then - echo "::warning ::TRACKING_APP_ID / TRACKING_APP_PRIVATE_KEY are not set; leaving the tracking repository to its scheduled refresh." - exit 0 - fi - gh api --method POST 'repos/uutils/coreutils-tracking/dispatches' \ - -f 'event_type=refresh-data' \ - -f "client_payload[workflow]=${SOURCE_WORKFLOW}" \ - -f "client_payload[run_id]=${SOURCE_RUN_ID}" \ - -f "client_payload[sha]=${SOURCE_SHA}" - echo "::notice ::Asked uutils/coreutils-tracking to refresh after '${SOURCE_WORKFLOW}' (${SOURCE_SHA})." diff --git a/.github/workflows/SizeComment.yml b/.github/workflows/SizeComment.yml deleted file mode 100644 index 032c85d013d..00000000000 --- a/.github/workflows/SizeComment.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: SizeComment - -# spell-checker:ignore zizmor backquote - -on: - workflow_run: - workflows: ["make"] - types: - - completed # zizmor: ignore[dangerous-triggers] - -permissions: {} -jobs: - post-comment: - permissions: - actions: read # to list workflow runs artifacts - pull-requests: write # to comment on pr - - runs-on: ubuntu-latest - if: > - github.event.workflow_run.event == 'pull_request' - steps: - - name: 'Download artifact' - uses: actions/github-script@v9 - with: - script: | - // List all artifacts from the make workflow run - var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{ github.event.workflow_run.id }}, - }); - - // Download the "size-comment" artifact, which contains a PR - // number (NR) and result.txt produced by compare_size_results.py. - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "size-comment" - })[0]; - if (!matchArtifact) { - core.info("No size-comment artifact found; nothing to post."); - return; - } - var download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{ github.workspace }}/size-comment.zip', Buffer.from(download.data)); - - name: 'Unzip artifact' - run: | - if [ -f size-comment.zip ]; then - unzip size-comment.zip - fi - - - name: 'Comment on PR' - uses: actions/github-script@v9 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const fs = require('fs'); - if (!fs.existsSync('./NR') || !fs.existsSync('./result.txt')) { - core.info("No size comment payload to post."); - return; - } - const issue_number = Number(fs.readFileSync('./NR')); - if (!issue_number) { - core.info("No PR number; skipping."); - return; - } - const content = fs.readFileSync('./result.txt').toString(); - - const marker = ''; - const body = `${marker}\nBinary size comparison:\n\`\`\`\n${content}\`\`\``; - - const { data: comments } = await github.rest.issues.listComments({ - ...context.repo, - issue_number - }); - - const existing = comments.find(c => - c.user.login === 'github-actions[bot]' && - c.body.includes(marker) - ); - - if (existing) { - await github.rest.issues.updateComment({ - ...context.repo, - comment_id: existing.id, - body - }); - } else { - // Only comment when there is something meaningful to say. - // compare_size_results.py only writes the file when there is a - // significant change, so an empty/short body is treated as - // "nothing to report". - if (content.trim().length == 0) { - return; - } - await github.rest.issues.createComment({ - ...context.repo, - issue_number, - body - }); - } diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml deleted file mode 100644 index 0d20ba79ee6..00000000000 --- a/.github/workflows/android.yml +++ /dev/null @@ -1,202 +0,0 @@ -name: Android - -# spell-checker:ignore (people) reactivecircus juliangruber -# spell-checker:ignore (shell/tools) TERMUX nextest udevadm pkill -# spell-checker:ignore (misc) swiftshader playstore DATALOSS noaudio - -on: - pull_request: - push: - branches: - - '*' - - -permissions: - contents: read # to fetch code (actions/checkout) - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -env: - TERMUX: v0.118.0 - KEY_POSTFIX: nextest+rustc-hash+adb+sshd+upgrade+XGB+inc18 - COMMON_EMULATOR_OPTIONS: -no-metrics -no-window -noaudio -no-boot-anim -camera-back none -gpu off - EMULATOR_DISK_SIZE: 12GB - EMULATOR_HEAP_SIZE: 2048M - EMULATOR_BOOT_TIMEOUT: 1200 # 20min - -jobs: - test_android: - name: Test builds - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] # , macos-latest - cores: [4] # , 6 - ram: [4096] - api-level: [28] - target: [google_apis_playstore] - arch: [x86_64] # ,x86 ,arm64-v8a - runs-on: ${{ matrix.os }} - env: - EMULATOR_RAM_SIZE: ${{ matrix.ram }} - EMULATOR_CORES: ${{ matrix.cores }} - RUNNER_OS: ${{ matrix.os }} - AVD_CACHE_KEY: "set later due to limitations of github actions not able to concatenate env variables" - steps: - - name: Concatenate values to environment file - run: | - echo "AVD_CACHE_KEY=${{ matrix.os }}-${{ matrix.cores }}-${{ matrix.ram }}-${{ matrix.api-level }}-${{ matrix.target }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+${{ env.KEY_POSTFIX }}" >> $GITHUB_ENV - - name: Collect information about runner - if: always() - continue-on-error: true - run: | - hostname - uname -a - free -mh - df -Th - cat /proc/cpuinfo - - name: (Linux) create links from home to data partition - if: ${{ runner.os == 'Linux' }} - continue-on-error: true - run: | - ls -lah /mnt/ - cat /mnt/DATALOSS_WARNING_README.txt - sudo mkdir /mnt/data - sudo chmod a+rwx /mnt/data - mkdir /mnt/data/.android && ln -s /mnt/data/.android ~/.android - mkdir /mnt/data/work && ln -s /mnt/data/work ~/work - - name: Enable KVM group perms (linux hardware acceleration) - if: ${{ runner.os == 'Linux' }} - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Collect information about runner - if: always() - continue-on-error: true - run: | - free -mh - df -Th - - name: Restore AVD cache - uses: actions/cache/restore@v6 - id: avd-cache - continue-on-error: true - with: - path: | - ~/.android/avd/* - ~/.android/avd/*/snapshots/* - ~/.android/adb* - ~/__rustc_hash__ - key: avd-${{ env.AVD_CACHE_KEY }} - - name: Collect information about runner after AVD cache - if: always() - continue-on-error: true - run: | - free -mh - df -Th - ls -lah /mnt/data - du -sch /mnt/data - - name: Delete AVD Lockfile when run from cache - if: steps.avd-cache.outputs.cache-hit == 'true' - run: | - rm -f \ - ~/.android/avd/*.avd/*.lock \ - ~/.android/avd/*/*.lock - - name: Create and cache emulator image - if: steps.avd-cache.outputs.cache-hit != 'true' - uses: reactivecircus/android-emulator-runner@v2.38.0 - with: - api-level: ${{ matrix.api-level }} - target: ${{ matrix.target }} - arch: ${{ matrix.arch }} - ram-size: ${{ env.EMULATOR_RAM_SIZE }} - heap-size: ${{ env.EMULATOR_HEAP_SIZE }} - disk-size: ${{ env.EMULATOR_DISK_SIZE }} - cores: ${{ env.EMULATOR_CORES }} - force-avd-creation: true - emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-load - emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }} - script: | - util/android-commands.sh init "${{ matrix.arch }}" "${{ matrix.api-level }}" "${{ env.TERMUX }}" - - name: Save AVD cache - if: steps.avd-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v6 - with: - path: | - ~/.android/avd/* - ~/.android/avd/*/snapshots/* - ~/.android/adb* - ~/__rustc_hash__ - key: avd-${{ env.AVD_CACHE_KEY }} - - uses: juliangruber/read-file-action@v1 - id: read_rustc_hash - with: - # ~ expansion didn't work - path: ${{ runner.os == 'Linux' && '/home/runner/__rustc_hash__' || '/Users/runner/__rustc_hash__' }} - trim: true - - name: Restore rust cache - id: rust-cache - uses: actions/cache/restore@v6 - with: - path: ~/__rust_cache__ - # The version vX at the end of the key is just a development version to avoid conflicts in - # the github cache during the development of this workflow - key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3 - - name: Collect information about runner resources - if: always() - continue-on-error: true - run: | - free -mh - df -Th - - name: Build and Test - uses: reactivecircus/android-emulator-runner@v2.38.0 - with: - api-level: ${{ matrix.api-level }} - target: ${{ matrix.target }} - arch: ${{ matrix.arch }} - ram-size: ${{ env.EMULATOR_RAM_SIZE }} - heap-size: ${{ env.EMULATOR_HEAP_SIZE }} - disk-size: ${{ env.EMULATOR_DISK_SIZE }} - cores: ${{ env.EMULATOR_CORES }} - force-avd-creation: false - emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-save -snapshot ${{ env.AVD_CACHE_KEY }} - emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }} - # This is not a usual script. Every line is executed in a separate shell with `sh -c`. If - # one of the lines returns with error the whole script is failed (like running a script with - # set -e) and in consequences the other lines (shells) are not executed. - script: | - util/android-commands.sh sync_host - util/android-commands.sh build - util/android-commands.sh tests - if [ "${{ steps.rust-cache.outputs.cache-hit }}" != 'true' ]; then util/android-commands.sh sync_image; fi; exit 0 - - name: Collect information about runner resources - if: always() - continue-on-error: true - run: | - free -mh - df -Th - - name: Save rust cache - if: steps.rust-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v6 - with: - path: ~/__rust_cache__ - key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3 - - name: archive any output (error screenshots) - if: always() - uses: actions/upload-artifact@v7 - with: - name: test_output_${{ env.AVD_CACHE_KEY }} - path: output - - name: Collect information about runner resources - if: always() - continue-on-error: true - run: | - free -mh - df -Th diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml deleted file mode 100644 index fb0caec6f45..00000000000 --- a/.github/workflows/audit.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Security audit - -# spell-checker:ignore (misc) rustsec - -on: - schedule: - - cron: "0 0 * * *" - -permissions: - contents: read - issues: write - -jobs: - audit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: rustsec/audit-check@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml deleted file mode 100644 index ac8e6e4afa4..00000000000 --- a/.github/workflows/benchmarks.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Benchmarks - -# spell-checker:ignore (people) taiki-e -# spell-checker:ignore (misc) codspeed sccache - -on: - pull_request: - paths: &bench_paths - - 'src/uu/**' - - 'src/uucore/**' - - 'src/uucore_procs/**' - - 'Cargo.toml' - - 'Cargo.lock' - push: - branches: - - main - paths: *bench_paths - -permissions: - contents: read # to fetch code (actions/checkout) - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - detects-benchmarks: - name: Detect changes on crates - runs-on: ubuntu-latest - outputs: - package: ${{ steps.detect.outputs.package }} - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - id: detect - shell: bash - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - base="${{ github.event.pull_request.base.sha }}" - else - # for push diff of the previous - base="${{ github.event.before }}" - fi - - changed=$(git diff --name-only "$base" "${{ github.sha }}") - - if echo "$changed" | grep -qE '^(Cargo\.(toml|lock)|src/(uucore|uucore_procs)/)'; then - candidates=$(ls src/uu) - else - candidates=$(echo "$changed" | awk -F'/' '/^src\/uu\// {print $3}' | sort -u) - fi - - package=$(for pkg in $candidates; - do - if [ -d "src/uu/$pkg/benches" ]; then - echo "uu_$pkg" - fi - done | sort -u | jq -R . | jq -sc .) - - echo "package=$package" >> "$GITHUB_OUTPUT" - - benchmarks: - needs: detects-benchmarks - if: needs.detects-benchmarks.outputs.package != '[]' - name: Run ${{ matrix.type }} benchmarks for ${{ matrix.package }} (CodSpeed) - runs-on: ubuntu-latest - timeout-minutes: 90 - env: - CARGO_INCREMENTAL: 0 - strategy: - # Stagger the jobs: 70 concurrent runners hitting the GitHub cache API - # (rust-cache + sccache + CodSpeed) at once triggers HTTP 429s. - max-parallel: 12 - matrix: - type: [simulation, memory] - package: ${{ fromJson(needs.detects-benchmarks.outputs.package) }} - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - - uses: Swatinem/rust-cache@v2 - - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - - name: Install locales - shell: bash - run: | - sudo locale-gen fr_FR.UTF-8 - sudo update-locale - - - name: Install tools - uses: taiki-e/install-action@v2 - with: - tool: cargo-codspeed - - - name: Build benchmarks for ${{ matrix.package }} (${{ matrix.type }}) - shell: bash - run: | - echo "Building ${{ matrix.type }} benchmarks for ${{ matrix.package }}" - cargo codspeed build -m ${{ matrix.type }} -p ${{ matrix.package }} - - - name: Run ${{ matrix.type }} benchmarks for ${{ matrix.package }} - uses: CodSpeedHQ/action@v5 - env: - CODSPEED_LOG: debug - with: - mode: ${{ matrix.type }} - run: | - echo "Running ${{ matrix.type }} benchmarks for ${{ matrix.package }}" - cargo codspeed run -p ${{ matrix.package }} > /dev/null - token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/.github/workflows/check-toml.yml b/.github/workflows/check-toml.yml deleted file mode 100644 index ff130fb7693..00000000000 --- a/.github/workflows/check-toml.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Check TOML files - -# spell-checker:ignore (people) taiki-e -# spell-checker:ignore (misc) taplo tombi - -on: - push: - paths: - - "**.toml" - pull_request: - paths: - - "**.toml" - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Install Tombi - uses: tombi-toml/setup-tombi@v1 - - name: Install taplo-cli - uses: taiki-e/install-action@v2 - with: - tool: taplo-cli - - name: Lint TOML files - run: tombi lint --error-on-warnings - - name: Format TOML files - run: taplo fmt --check --diff diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml deleted file mode 100644 index 806dfec0e06..00000000000 --- a/.github/workflows/code-quality.yml +++ /dev/null @@ -1,264 +0,0 @@ -name: Code Quality - -# spell-checker:ignore (people) juliangruber pell reactivecircus taiki-e taplo -# spell-checker:ignore (misc) TERMUX noaudio pkill swiftshader esac sccache pcoreutils shopt subshell dequote libsystemd wasi wasip - -on: - pull_request: - push: - branches: - - '*' - -env: - CARGO_INCREMENTAL: "0" - # * style job configuration - STYLE_FAIL_ON_FAULT: true ## (bool) fail the build if a style job contains a fault (error or warning); may be overridden on a per-job basis - -permissions: - contents: read # to fetch code (actions/checkout) - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - - style_format: - name: Style/format - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: feat_os_unix } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # failure mode - unset FAIL_ON_FAULT ; case "$STYLE_FAIL_ON_FAULT" in - ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; - *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; - esac; - outputs FAIL_ON_FAULT FAULT_TYPE - - name: "`cargo fmt` testing" - shell: bash - run: | - ## `cargo fmt` testing - unset fault - fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" - fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') - # * convert any errors/warnings to GHA UI annotations; ref: - S=$(cargo fmt -- --check) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s\n" "$S" | sed -E -n -e "s/^Diff[[:space:]]+in[[:space:]]+${PWD//\//\\/}\/(.*)[[:space:]]+at[[:space:]]+[^0-9]+([0-9]+).*$/::${fault_type} file=\1,line=\2::${fault_prefix}: \`cargo fmt\`: style violation (file:'\1', line:\2; use \`cargo fmt -- \"\1\"\`)/p" ; fault=true ; } - if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi - - name: "cargo fmt on fuzz dir" - shell: bash - run: | - cd fuzz - cargo fmt --check - - style_lint: - name: Style/lint - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: all , workspace: true } - - { os: macos-latest , features: all , workspace: true } - - { os: windows-latest , features: feat_os_windows } - - { os: ubuntu-latest , features: feat_wasm , target: wasm32-wasip1 } - - { os: ubuntu-latest , features: feat_wasm , target: wasm32-wasip2 } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy - targets: ${{ matrix.job.target || '' }} - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # failure mode - unset FAIL_ON_FAULT ; case "$STYLE_FAIL_ON_FAULT" in - ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; - *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; - esac; - outputs FAIL_ON_FAULT FAULT_TYPE - - name: Install/setup prerequisites - if: ${{ ! matrix.job.target }} - shell: bash - run: | - ## Install/setup prerequisites - case '${{ matrix.job.os }}' in - ubuntu-*) - sudo apt-get -y update - # selinux and systemd headers needed to enable all features - sudo apt-get -y install libselinux1-dev libsystemd-dev - ;; - esac - - name: "`cargo clippy` lint testing" - uses: nick-fields/retry@v4 - with: - max_attempts: 3 - retry_on: error - timeout_minutes: 90 - shell: bash - command: | - ## `cargo clippy` lint testing - ARGS="--features ${{ matrix.job.features }}" - ARGS="${ARGS} --fault-type ${{ steps.vars.outputs.FAULT_TYPE }}" - if [[ "${{ matrix.job.workspace }}" =~ ^(1|t|true|y|yes)$ ]]; then - ARGS="${ARGS} --workspace" - fi - if [[ -n "${{ matrix.job.target }}" ]]; then - ARGS="${ARGS} --target ${{ matrix.job.target }}" - fi - if [[ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ]]; then - ARGS="${ARGS} --fail-on-fault" - fi - python3 util/run-clippy.py ${ARGS} - - name: "cargo clippy on fuzz dir" - if: runner.os != 'Windows' && !matrix.job.target - shell: bash - run: | - cd fuzz - cargo clippy --workspace --all-targets --all-features -- -D warnings - - style_spellcheck: - name: Style/spelling - runs-on: ${{ matrix.job.os }} - env: - # Spelling is enforced locally by the cspell pre-commit hook, where it is - # cheap to fix. Here it is advisory: unknown words show up as annotations - # on the diff instead of blocking the merge. - STYLE_FAIL_ON_FAULT: false - strategy: - matrix: - job: - - { os: ubuntu-latest , features: feat_os_unix } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - # failure mode - unset FAIL_ON_FAULT ; case "$STYLE_FAIL_ON_FAULT" in - ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; - *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; - esac; - outputs FAIL_ON_FAULT FAULT_TYPE - - name: Install/setup prerequisites - shell: bash - run: | - sudo npm install cspell -g ; - - name: Run `cspell` - shell: bash - run: | - ## Run `cspell` - unset fault - fault_type="${{ steps.vars.outputs.FAULT_TYPE }}" - fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]') - # * find cspell configuration ; note: avoid quotes around ${cfg_file} b/c `cspell` (v4) doesn't correctly dequote the config argument (or perhaps a subshell expansion issue?) - cfg_files=($(shopt -s nullglob ; echo {.vscode,.}/{,.}c[sS]pell{.json,.config{.js,.cjs,.json,.yaml,.yml},.yaml,.yml} ;)) - cfg_file=${cfg_files[0]} - unset CSPELL_CFG_OPTION ; if [ -n "$cfg_file" ]; then CSPELL_CFG_OPTION="--config $cfg_file" ; fi - S=$(cspell ${CSPELL_CFG_OPTION} --no-summary --no-progress .) && printf "%s\n" "$S" || { printf "%s\n" "$S" ; printf "%s" "$S" | sed -E -n "s/^(.*):([0-9]+):([0-9]+) - Unknown word \(([^)]+)\).*$/::${fault_type} file=\1,line=\2,col=\3::${fault_type^^}: \`cspell\`: Unknown word '\4' (file:'\1', line:\2)/p" ; fault=true ; true ; } - if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi - - style_empty_assertions: - name: Style/Empty assertions check - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Fail if empty stdout/stderr assertions are found - shell: bash - run: | - if grep -rnE '(stdout_is|stderr_is)\(""\)' tests/; then - echo "::error::Found empty stdout/stderr assertions (use specific checks instead of \"\") in tests/" - exit 1 - fi - - python: - name: Style/Python - runs-on: ubuntu-latest - steps: - - name: Clone repository - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - - name: ruff - uses: astral-sh/ruff-action@v4.1.0 - with: - src: "./util" - - - name: ruff - format - uses: astral-sh/ruff-action@v4.1.0 - with: - src: "./util" - args: format --check - - name: Run Python unit tests - shell: bash - run: | - python3 -m unittest util/test_compare_test_results.py util/test_compare_size_results.py - - pre_commit: - name: Pre-commit hooks - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - - - name: Install prek - run: pip install prek - - - name: Cache prek environments - uses: actions/cache@v6 - with: - path: ~/.cache/prek - key: prek-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} - restore-keys: | - prek-${{ runner.os }}- - - - name: Run prek - env: - # cspell is enforced locally and reported by Style/spelling; the rust - # hooks are already covered by the Style/format and Style/lint jobs. - SKIP: cspell,rust-linting,rust-clippy,cargo-lock-check - run: prek run --all-files --show-diff-on-failure diff --git a/.github/workflows/copilot-code-review.yml b/.github/workflows/copilot-code-review.yml deleted file mode 100644 index f6c5fdb4966..00000000000 --- a/.github/workflows/copilot-code-review.yml +++ /dev/null @@ -1,23 +0,0 @@ -# spell-checker:ignore Swatinem -name: Copilot Code Review - -# Declares the runner used by Copilot code review. Without this file, the -# review can fail to start before its timeout because no runner is assigned. -on: - workflow_dispatch: - pull_request: - paths: - - .github/workflows/copilot-code-review.yml - -permissions: - contents: read - -jobs: - copilot-setup-steps: - # Copilot code review only supports Ubuntu x64 runners. - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml deleted file mode 100644 index 1418c4e8187..00000000000 --- a/.github/workflows/devcontainer.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Devcontainer - -# spell-checker:ignore devcontainers nextest - -on: - pull_request: - push: - branches: - - '*' - -permissions: - contents: read # to fetch code (actions/checkout) - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - test: - name: Verify devcontainer - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Run test in devcontainer - uses: devcontainers/ci@v0.3 - with: - push: never - runCmd: | - curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin - cargo nextest run --hide-progress-bar --profile ci diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml deleted file mode 100644 index 67393fa7add..00000000000 --- a/.github/workflows/documentation.yml +++ /dev/null @@ -1,138 +0,0 @@ -# spell-checker:ignore libsystemd libattr libcap gsub - -name: Check uudoc Documentation Generation - -on: - pull_request: - paths: - - 'src/bin/uudoc.rs' - - 'src/uu/*/locales/en-US.ftl' - - 'Cargo.toml' - - '.github/workflows/documentation.yml' - push: - branches: - - main - paths: - - 'src/bin/uudoc.rs' - - 'src/uu/*/locales/en-US.ftl' - - 'Cargo.toml' - -permissions: - contents: read - -jobs: - check-doc: - name: Verify uudoc generates correct documentation - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - - name: Install/setup prerequisites - shell: bash - run: sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev libsystemd-dev libacl1-dev libattr1-dev libcap-dev - - - name: Download tldr - run: curl -L https://github.com/tldr-pages/tldr/releases/latest/download/tldr.zip -o docs/tldr.zip - - - name: Generate documentation - run: cargo run --bin uudoc --all-features - - - name: Get current version from Cargo.toml - id: version - run: | - VERSION=$(awk '/\[workspace\.package\]/{flag=1; next} flag && /^version = /{gsub(/.*= "/, ""); gsub(/".*/, ""); print; exit}' Cargo.toml) - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "Detected version: $VERSION" - - - name: Check for --repeated option in uniq.md - run: | - if [ ! -f "docs/src/utils/uniq.md" ]; then - echo "docs/src/utils/uniq.md does not exist" - exit 1 - fi - - if ! grep -q -- "--repeated" docs/src/utils/uniq.md; then - echo "'--repeated' option not found in docs/src/utils/uniq.md" - echo "Content of uniq.md:" - head -50 docs/src/utils/uniq.md - exit 1 - fi - - - name: Check for correct version in ls.md - run: | - VERSION="${{ steps.version.outputs.version }}" - - if [ ! -f "docs/src/utils/ls.md" ]; then - echo "docs/src/utils/ls.md does not exist" - exit 1 - fi - - if ! grep -Fqx "
v$VERSION
" docs/src/utils/ls.md; then - echo "Version '$VERSION' not found in docs/src/utils/ls.md" - echo "Found version info:" - grep -F '
' docs/src/utils/ls.md || echo "No version info found" - echo "Full version section:" - grep -A2 -B2 "version" docs/src/utils/ls.md || echo "No version section found" - exit 1 - fi - - - name: Verify usage information is present - run: | - if [ ! -f "docs/src/utils/cat.md" ]; then - echo "docs/src/utils/cat.md does not exist" - exit 1 - fi - - if ! grep -q "cat \[OPTION\].*\[FILE\]" docs/src/utils/cat.md; then - echo "Usage information missing from cat.md" - echo "Content around usage:" - grep -A5 -B5 "cat" docs/src/utils/cat.md | head -20 - exit 1 - fi - - - name: Verify help text is properly resolved - run: | - if [ ! -f "docs/src/utils/cat.md" ]; then - echo "docs/src/utils/cat.md does not exist" - exit 1 - fi - - if grep -q "cat-help-" docs/src/utils/cat.md; then - echo "Found unresolved Fluent keys in cat.md - help text not properly translated" - echo "Unresolved Fluent keys found:" - grep "cat-help-" docs/src/utils/cat.md - exit 1 - fi - - - name: Verify about text is present - run: | - if [ ! -f "docs/src/utils/cat.md" ]; then - echo "docs/src/utils/cat.md does not exist" - exit 1 - fi - - if ! grep -q "Concatenate FILE(s)" docs/src/utils/cat.md; then - echo "About text missing from cat.md" - echo "Content of cat.md:" - head -30 docs/src/utils/cat.md - exit 1 - fi - - - name: Verify tldr examples integration - run: | - if [ ! -f "docs/src/utils/cp.md" ]; then - echo "docs/src/utils/cp.md does not exist" - exit 1 - fi - - if ! grep -q "The examples are provided by" docs/src/utils/cp.md; then - echo "tldr examples integration missing from cp.md" - echo "Expected to find 'The examples are provided by' text" - echo "Content of cp.md:" - tail -20 docs/src/utils/cp.md - exit 1 - fi diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml deleted file mode 100644 index 178b4b49ea7..00000000000 --- a/.github/workflows/freebsd.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: FreeBSD - -# spell-checker:ignore sshfs usesh vmactions taiki esac fdescfs fdesc nextest copyback logind - -env: - # * style job configuration - STYLE_FAIL_ON_FAULT: true ## (bool) fail the build if a style job contains a fault (error or warning); may be overridden on a per-job basis - -on: - pull_request: - push: - branches: - - '*' - -permissions: - contents: read # to fetch code (actions/checkout) - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - style: - name: Style and Lint - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Prepare, build and test - uses: vmactions/freebsd-vm@v1.5.6 - with: - usesh: true - sync: rsync - copyback: false - # We need bash for inline shell string replacement - prepare: pkg install -y rust sudo bash python3 - run: | - ## Prepare, build, and test - # implementation modelled after ref: - # * NOTE: All steps need to be run in this block, otherwise, we are operating back on the mac host - set -e - # - TEST_USER=tester - REPO_NAME=${GITHUB_WORKSPACE##*/} - WORKSPACE_PARENT="/home/runner/work/${REPO_NAME}" - WORKSPACE="${WORKSPACE_PARENT}/${REPO_NAME}" - # - pw adduser -n ${TEST_USER} -d /root/ -g wheel -c "Coreutils user to build" -w random - chown -R ${TEST_USER}:wheel /root/ "${WORKSPACE_PARENT}"/ - whoami - # - # Further work needs to be done in a sudo as we are changing users - sudo -i -u ${TEST_USER} bash << EOF - set -e - whoami - ## VARs setup - cd "${WORKSPACE}" - unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in - ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; - *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; - esac; - FAULT_PREFIX=\$(echo "\${FAULT_TYPE}" | tr '[:lower:]' '[:upper:]') - ## Info - # environment - echo "## environment" - echo "CI='${CI}'" - echo "REPO_NAME='${REPO_NAME}'" - echo "TEST_USER='${TEST_USER}'" - echo "WORKSPACE_PARENT='${WORKSPACE_PARENT}'" - echo "WORKSPACE='${WORKSPACE}'" - echo "FAULT_PREFIX='\${FAULT_PREFIX}'" - env | sort - # tooling info - echo "## tooling info" - cargo -V - rustc -V - # - # To ensure that files are cleaned up, we don't want to exit on error - set +e - unset FAULT - export CARGO_INCREMENTAL=0 - ## cargo fmt testing - echo "## cargo fmt testing" - # * convert any errors/warnings to GHA UI annotations; ref: - S=\$(cargo fmt -- --check) && printf "%s\n" "\$S" || { printf "%s\n" "\$S" ; printf "%s\n" "\$S" | sed -E -n -e "s/^Diff[[:space:]]+in[[:space:]]+\${PWD//\//\\\\/}\/(.*)[[:space:]]+at[[:space:]]+[^0-9]+([0-9]+).*\$/::\${FAULT_TYPE} file=\1,line=\2::\${FAULT_PREFIX}: \\\`cargo fmt\\\`: style violation (file:'\1', line:\2; use \\\`cargo fmt -- \"\1\"\\\`)/p" ; FAULT=true ; } - ## cargo clippy lint testing - if [ -z "\${FAULT}" ]; then - echo "## cargo clippy lint testing" - CLIPPY_ARGS="--features feat_os_unix --fault-type \${FAULT_TYPE}" - if [ -n "\${FAIL_ON_FAULT}" ]; then CLIPPY_ARGS="\${CLIPPY_ARGS} --fail-on-fault"; fi - python3 util/run-clippy.py \${CLIPPY_ARGS} || FAULT=true - fi - # Clean to avoid to rsync back the files - cargo clean - if [ -n "\${FAIL_ON_FAULT}" ] && [ -n "\${FAULT}" ]; then exit 1 ; fi - EOF - - test: - name: Tests - runs-on: ubuntu-latest - timeout-minutes: 45 - env: - mem: 4096 - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Avoid no space left on device (Ubuntu runner) - shell: bash - run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android & - - name: Prepare, build and test - uses: vmactions/freebsd-vm@v1.3.9 - with: - usesh: true - sync: rsync - copyback: false - prepare: pkg install -y gmake nextest rust sudo - run: | - ## Prepare, build, and test - # implementation modelled after ref: - # * NOTE: All steps need to be run in this block, otherwise, we are operating back on the mac host - set -e - # - # We need a file-descriptor file system to test test_ls::test_ls_io_errors - mount -t fdescfs fdesc /dev/fd - # - TEST_USER=tester - REPO_NAME=${GITHUB_WORKSPACE##*/} - WORKSPACE_PARENT="/home/runner/work/${REPO_NAME}" - WORKSPACE="${WORKSPACE_PARENT}/${REPO_NAME}" - # - pw adduser -n ${TEST_USER} -d /root/ -g wheel -c "Coreutils user to build" -w random - chown -R ${TEST_USER}:wheel /root/ "${WORKSPACE_PARENT}"/ - whoami - # - # Further work needs to be done in a sudo as we are changing users - sudo -i -u ${TEST_USER} sh << EOF - set -e - whoami - ## Info - # environment - echo "## environment" - echo "CI='${CI}'" - echo "REPO_NAME='${REPO_NAME}'" - echo "TEST_USER='${TEST_USER}'" - echo "WORKSPACE_PARENT='${WORKSPACE_PARENT}'" - echo "WORKSPACE='${WORKSPACE}'" - env | sort - # tooling info - echo "## tooling info" - cargo -V - cargo nextest --version - rustc -V - # - # To ensure that files are cleaned up, we don't want to exit on error - set +e - cd "${WORKSPACE}" - unset FAULT - export CARGO_INCREMENTAL=0 - export RUSTFLAGS="-C strip=symbols" # for disk space - # vendored OpenSSL takes many times to build. Use system one (LibreSSL is supported). - export OPENSSL_NO_VENDOR=1 OPENSSL_DIR=/usr - cargo build || FAULT=1 - export PATH=~/.cargo/bin:${PATH} - export RUST_BACKTRACE=1 - export CARGO_TERM_COLOR=always - if (test -z "\$FAULT"); then cargo nextest run --hide-progress-bar --profile ci --features feat_os_unix || FAULT=1 ; fi - if (test -z "\$FAULT"); then cargo nextest run --hide-progress-bar --profile ci --all-features -p uucore || FAULT=1 ; fi - # Test building with make - if (test -z "\$FAULT"); then make || FAULT=1 ; fi - # Clean to avoid to rsync back the files - cargo clean - if (test -n "\$FAULT"); then exit 1 ; fi - EOF diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml deleted file mode 100644 index 9786baf684b..00000000000 --- a/.github/workflows/fuzzing.yml +++ /dev/null @@ -1,392 +0,0 @@ -name: Fuzzing - -# spell-checker:ignore (people) bamf taiki-e -# spell-checker:ignore (misc) fuzzer PIPESTATUS - -env: - CARGO_INCREMENTAL: "0" - -on: - pull_request: - paths: &fuzz_paths - - 'fuzz/**' - - 'src/uu/**' - - 'src/uucore/**' - - 'src/uucore_procs/**' - - 'Cargo.toml' - - 'Cargo.lock' - - '.github/workflows/fuzzing.yml' - push: - branches: - - '*' - paths: *fuzz_paths - -permissions: - contents: read # to fetch code (actions/checkout) - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - detect: - name: Detect changes on crates - runs-on: ubuntu-latest - outputs: - targets: ${{ steps.detect.outputs.targets }} - all: ${{ steps.detect.outputs.all }} - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - id: detect - shell: bash - run: | - # target:should_pass:owning crate (a src/uu package, or "uucore" for - # the ones that fuzz shared parsing code) - TARGETS=" - fuzz_test:true:test - fuzz_date:true:date - fuzz_expr:true:expr - fuzz_printf:true:printf - fuzz_echo:true:echo - fuzz_parse_glob:true:uucore - fuzz_parse_size:false:uucore - fuzz_parse_time:false:uucore - fuzz_seq_parse_number:false:seq - fuzz_seq:false:seq - fuzz_sort:false:sort - fuzz_wc:false:wc - fuzz_cut:false:cut - fuzz_split:false:split - fuzz_tr:false:tr - fuzz_env:false:env - fuzz_cksum:false:cksum - fuzz_non_utf8_paths:true:uucore - fuzz_dirname:true:dirname - " - - if [ "${{ github.event_name }}" = "pull_request" ]; then - base="${{ github.event.pull_request.base.sha }}" - else - # for push diff of the previous - base="${{ github.event.before }}" - fi - - if ! git cat-file -e "$base^{commit}" 2>/dev/null; then - # new branch, force push, ...: no usable base, fuzz everything - changed="" - all=true - else - changed=$(git diff --name-only "$base" "${{ github.sha }}") - # anything shared (uucore, the fuzzers themselves, the manifests or - # this workflow) can affect every target - if echo "$changed" | grep -qE '^(fuzz/|Cargo\.(toml|lock)$|src/(uucore|uucore_procs)/|\.github/workflows/fuzzing\.yml$)'; then - all=true - else - all=false - fi - fi - - if [ "$all" = true ]; then - selected=$(echo "$TARGETS" | awk -F: 'NF==3') - else - crates=$(echo "$changed" | awk -F/ '/^src\/uu\// {print $3}' | sort -u) - selected=$(echo "$TARGETS" | awk -F: -v crates="$crates" ' - BEGIN { n = split(crates, c, "\n"); for (i = 1; i <= n; i++) want[c[i]] = 1 } - NF == 3 && ($3 in want)') - fi - - targets=$(echo "$selected" | jq -Rsc ' - split("\n") | map(select(length > 0) | split(":")) - | map({name: .[0], should_pass: .[1]})') - - echo "all=$all" >> "$GITHUB_OUTPUT" - echo "targets=$targets" >> "$GITHUB_OUTPUT" - echo "Fuzzing everything: $all" - echo "Targets: $targets" - - uufuzz-examples: - name: Build and test uufuzz examples - needs: detect - if: needs.detect.outputs.all == 'true' - runs-on: ubuntu-26.04 # Use GNU linked against newer glibc which enables some feature e.g. stty 51, and fuzz against it - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - with: - workspaces: "fuzz -> target" - - name: Use latest GNU coreutils as system coreutils - run: | - sudo add-apt-repository -y ppa:bamf0/coreutils-reference - sudo apt-get update - apt-cache policy gnu-coreutils - sudo apt-get install -y gnu-coreutils=9.12-0ubuntu1~ppa1 coreutils-from-gnu coreutils-from-uutils- --allow-remove-essential - stty --version | grep 9.12 - - name: Build uufuzz library - run: | - cd fuzz/uufuzz - cargo build --release - - name: Run uufuzz tests - run: | - cd fuzz/uufuzz - cargo test --lib - - name: Build and run uufuzz examples - run: | - cd fuzz/uufuzz - echo "Building all examples..." - cargo build --examples --release - - # Run all examples except integration_testing (which has FD issues in CI) - for example in examples/*.rs; do - example_name=$(basename "$example" .rs) - if [ "$example_name" != "integration_testing" ]; then - cargo run --example "$example_name" --release - fi - done - - fuzz-build: - name: Build the fuzzers - needs: detect - if: needs.detect.outputs.all == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Install `cargo-fuzz` - uses: taiki-e/install-action@v2 - with: - tool: cargo-fuzz - - uses: Swatinem/rust-cache@v2 - with: - workspaces: "fuzz -> target" - - name: Emulate a nightly toolchain - run: | - echo "RUSTC_BOOTSTRAP=1" >> "${GITHUB_ENV}" - - name: Run `cargo-fuzz build` - # Force the correct target - # https://github.com/rust-fuzz/cargo-fuzz/issues/398 - run: cargo fuzz build --target host-tuple - - fuzz-run: - name: Fuzz (${{ matrix.target.name }}) - needs: detect - if: needs.detect.outputs.targets != '[]' - runs-on: ubuntu-26.04 # Use GNU linked against newer glibc which enables some feature e.g. stty 51, and fuzz against it - timeout-minutes: 15 - env: - RUN_FOR: 60 - strategy: - fail-fast: false - # Stagger the jobs: all the targets at once hammer the GitHub cache API. - max-parallel: 10 - matrix: - target: ${{ fromJson(needs.detect.outputs.targets) }} - - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Use latest GNU coreutils as system coreutils - run: | - sudo add-apt-repository -y ppa:bamf0/coreutils-reference - sudo apt-get update - sudo apt-get install -y gnu-coreutils=9.12-0ubuntu1~ppa1 coreutils-from-gnu coreutils-from-uutils- --allow-remove-essential - stty --version - - name: Install `cargo-fuzz` - uses: taiki-e/install-action@v2 - with: - tool: cargo-fuzz - - uses: Swatinem/rust-cache@v2 - with: - workspaces: "fuzz -> target" - - name: Emulate a nightly toolchain - run: | - echo "RUSTC_BOOTSTRAP=1" >> "${GITHUB_ENV}" - - name: Run ${{ matrix.target.name }} - shell: bash - env: - TARGET: ${{ matrix.target.name }} - SHOULD_PASS: ${{ matrix.target.should_pass }} - run: | - set -o pipefail - mkdir -p fuzz/stats - - STATS_FILE="fuzz/stats/${TARGET}.txt" - # Force the correct target - # https://github.com/rust-fuzz/cargo-fuzz/issues/398 - set +e - cargo fuzz run --target host-tuple "$TARGET" -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0 -print_final_stats=1 2>&1 | tee "$STATS_FILE" - FUZZ_STATUS=${PIPESTATUS[0]} - set -e - - # Save should_pass value to file for summary job to use - echo "$SHOULD_PASS" > "fuzz/stats/${TARGET}.should_pass" - - fuzz_stat() { - if grep -q "$1" "$STATS_FILE"; then - grep "$1" "$STATS_FILE" | awk '{print $2}' - else - echo "unknown" - fi - } - if grep -q "SUMMARY: " "$STATS_FILE"; then - STATUS=$(grep "SUMMARY: " "$STATS_FILE" | head -1) - else - STATUS="Completed" - fi - - # Print stats to job output for immediate visibility - echo "----------------------------------------" - echo "FUZZING STATISTICS FOR $TARGET" - echo "----------------------------------------" - echo "Runs: $(fuzz_stat stat::number_of_executed_units)" - echo "Execution Rate: $(fuzz_stat stat::average_exec_per_sec) execs/sec" - echo "New Units: $(fuzz_stat stat::new_units_added)" - echo "Expected: $SHOULD_PASS" - echo "Exit code: $FUZZ_STATUS" - echo "Status: $STATUS" - echo "----------------------------------------" - - # Add summary to GitHub step summary - { - echo "### Fuzzing Results for $TARGET" - echo "" - echo "| Metric | Value |" - echo "|--------|-------|" - echo "| Runs | $(fuzz_stat stat::number_of_executed_units) |" - echo "| Execution Rate | $(fuzz_stat stat::average_exec_per_sec) execs/sec |" - echo "| New Units | $(fuzz_stat stat::new_units_added) |" - echo "| Should pass | $SHOULD_PASS |" - echo "| Exit code | $FUZZ_STATUS |" - echo "| Status | $STATUS |" - echo "" - } >> "$GITHUB_STEP_SUMMARY" - - if [ "$SHOULD_PASS" = "true" ] && [ "$FUZZ_STATUS" -ne 0 ]; then - exit 1 - fi - - name: Upload Stats - if: always() - uses: actions/upload-artifact@v7 - with: - name: fuzz-stats-${{ matrix.target.name }} - path: | - fuzz/stats/*.txt - fuzz/stats/*.should_pass - retention-days: 5 - fuzz-summary: - needs: [detect, fuzz-run] - name: Fuzzing Summary - runs-on: ubuntu-latest - if: always() && needs.detect.outputs.targets != '[]' - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Download all stats - uses: actions/download-artifact@v8 - with: - path: fuzz/stats-artifacts - pattern: fuzz-stats-* - merge-multiple: true - - name: Prepare stats directory - run: | - mkdir -p fuzz/stats - # Debug: List content of stats-artifacts directory - echo "Contents of stats-artifacts directory:" - find fuzz/stats-artifacts -type f | sort - - # Extract files from the artifact directories - handle nested directories - find fuzz/stats-artifacts -type f -name "*.txt" -exec cp {} fuzz/stats/ \; - find fuzz/stats-artifacts -type f -name "*.should_pass" -exec cp {} fuzz/stats/ \; - - # Debug information - echo "Contents of stats directory after extraction:" - ls -la fuzz/stats/ - echo "Contents of should_pass files (if any):" - cat fuzz/stats/*.should_pass 2>/dev/null || echo "No should_pass files found" - - name: Generate Summary - run: | - echo "# Fuzzing Summary" > fuzzing_summary.md - echo "" >> fuzzing_summary.md - echo "| Target | Runs | Exec/sec | New Units | Should pass | Status |" >> fuzzing_summary.md - echo "|--------|------|----------|-----------|-------------|--------|" >> fuzzing_summary.md - - TOTAL_RUNS=0 - TOTAL_NEW_UNITS=0 - - for stat_file in fuzz/stats/*.txt; do - TARGET=$(basename "$stat_file" .txt) - SHOULD_PASS_FILE="${stat_file%.*}.should_pass" - - # Get expected status - if [ -f "$SHOULD_PASS_FILE" ]; then - EXPECTED=$(cat "$SHOULD_PASS_FILE") - else - EXPECTED="unknown" - fi - - # Extract runs - if grep -q "stat::number_of_executed_units" "$stat_file"; then - RUNS=$(grep "stat::number_of_executed_units" "$stat_file" | awk '{print $2}') - TOTAL_RUNS=$((TOTAL_RUNS + RUNS)) - else - RUNS="unknown" - fi - - # Extract execution rate - if grep -q "stat::average_exec_per_sec" "$stat_file"; then - EXEC_RATE=$(grep "stat::average_exec_per_sec" "$stat_file" | awk '{print $2}') - else - EXEC_RATE="unknown" - fi - - # Extract new units added - if grep -q "stat::new_units_added" "$stat_file"; then - NEW_UNITS=$(grep "stat::new_units_added" "$stat_file" | awk '{print $2}') - if [[ "$NEW_UNITS" =~ ^[0-9]+$ ]]; then - TOTAL_NEW_UNITS=$((TOTAL_NEW_UNITS + NEW_UNITS)) - fi - else - NEW_UNITS="unknown" - fi - - # Extract status - if grep -q "SUMMARY: " "$stat_file"; then - STATUS=$(grep "SUMMARY: " "$stat_file" | head -1) - else - STATUS="Completed" - fi - - echo "| $TARGET | $RUNS | $EXEC_RATE | $NEW_UNITS | $EXPECTED | $STATUS |" >> fuzzing_summary.md - done - - echo "" >> fuzzing_summary.md - echo "## Overall Statistics" >> fuzzing_summary.md - echo "" >> fuzzing_summary.md - echo "- **Total runs:** $TOTAL_RUNS" >> fuzzing_summary.md - echo "- **Total new units discovered:** $TOTAL_NEW_UNITS" >> fuzzing_summary.md - echo "- **Average execution rate:** $(grep -h "stat::average_exec_per_sec" fuzz/stats/*.txt | awk '{sum += $2; count++} END {if (count > 0) print sum/count " execs/sec"; else print "unknown"}')" >> fuzzing_summary.md - - # Add count by expected status - echo "- **Tests expected to pass:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "true")" >> fuzzing_summary.md - echo "- **Tests expected to fail:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "false")" >> fuzzing_summary.md - - # Write to GitHub step summary - cat fuzzing_summary.md >> $GITHUB_STEP_SUMMARY - - name: Show Summary - run: | - cat fuzzing_summary.md - - name: Upload Summary - uses: actions/upload-artifact@v7 - with: - name: fuzzing-summary - path: fuzzing_summary.md - retention-days: 5 diff --git a/.github/workflows/ignore-intermittent.txt b/.github/workflows/ignore-intermittent.txt deleted file mode 100644 index 82761434c8f..00000000000 --- a/.github/workflows/ignore-intermittent.txt +++ /dev/null @@ -1,24 +0,0 @@ -tests/basenc/bounded-memory -tests/cut/bounded-memory -tests/date/resolution -tests/expand/bounded-memory -tests/pr/bounded-memory -tests/tail/inotify-dir-recreate -tests/tail/overlay-headers -tests/tail/pid-pipe -tests/tail/retry -tests/tail/tail-n0f -tests/tail/pipe-f -tests/timeout/timeout -tests/tail/symlink -tests/shuf/shuf-reservoir -tests/sort/sort-stale-thread-mem -tests/timeout/timeout-group -tests/tty/tty-eof -tests/misc/stdbuf -tests/misc/tty-eof -tests/misc/usage_vs_getopt -tests/misc/tee -tests/tail/follow-name -tests/rm/isatty -tests/misc/io-errors diff --git a/.github/workflows/l10n.yml b/.github/workflows/l10n.yml deleted file mode 100644 index e5aac9e5dd0..00000000000 --- a/.github/workflows/l10n.yml +++ /dev/null @@ -1,1367 +0,0 @@ -name: L10n (Localization) - -# spell-checker: disable - -on: - pull_request: - push: - branches: - - '*' - -env: - # * style job configuration - STYLE_FAIL_ON_FAULT: true ## (bool) fail the build if a style job contains a fault (error or warning); may be overridden on a per-job basis - CARGO_INCREMENTAL: "0" - RUST_BACKTRACE: "1" -permissions: - contents: read # to fetch code (actions/checkout) - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - - l10n_build_test: - name: L10n/Build and Test - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: "feat_os_unix" } - - { os: macos-latest , features: "feat_os_unix" } - - { os: windows-latest , features: "feat_os_windows" } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: taiki-e/install-action@nextest - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Install/setup prerequisites - shell: bash - run: | - ## Install/setup prerequisites - case '${{ matrix.job.os }}' in - ubuntu-*) - # selinux headers needed for testing - sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - # Locales required by tests/by-util/test_date.rs and - # tests/by-util/test_dd.rs (fr_FR is the ISO-8859-1 variant - # used by test_iso8859_1_case_conversion). - sudo locale-gen --keep-existing fr_FR - sudo locale-gen --keep-existing fr_FR.UTF-8 - sudo locale-gen --keep-existing es_ES.UTF-8 - sudo locale-gen --keep-existing en_US.UTF-8 - sudo locale-gen --keep-existing en_GB.UTF-8 - sudo locale-gen --keep-existing bg_BG.UTF-8 # Bulgaria - sudo locale-gen --keep-existing fa_IR.UTF-8 # Iran - sudo locale-gen --keep-existing am_ET.UTF-8 # Ethiopia - sudo locale-gen --keep-existing th_TH.UTF-8 # Thailand - sudo locale-gen --keep-existing hu_HU.UTF-8 # Hungary - sudo update-locale - ;; - esac - - name: Test l10n functionality - shell: bash - run: | - ## Test l10n functionality - cargo test -p uucore locale - cargo test - - l10n_fluent_syntax: - name: L10n/Fluent Syntax Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Install Mozilla Fluent Linter - shell: bash - run: | - ## Install Mozilla Fluent Linter - pip install moz-fluent-linter - - name: Find and validate Fluent files - shell: bash - run: | - - fluent_dirs=$(find . -name "*.ftl" -type f -exec dirname {} \; | sort | uniq 2>/dev/null || :) - - if [ -n "$fluent_dirs" ]; then - echo "Found Fluent directories:" - echo "$fluent_dirs" - - # Count total files for informational purposes - total_files=$(find . -name "*.ftl" -type f | wc -l) - echo "Total Fluent files: $total_files" - else - echo "::notice::No Fluent (.ftl) files found in the repository" - exit 0 - fi - - echo "Running Fluent Linter on directories..." - - has_errors=false - - while IFS= read -r dir; do - echo "Checking directory $dir with Fluent Linter..." - - if ! moz-fluent-lint --config .github/fluent_linter_config.yml "$dir"; then - echo "::error::Fluent syntax errors found in $dir" - exit 1 - else - echo "✓ Fluent syntax check passed for directory $dir" - fi - - done <<< "$fluent_dirs" - - echo "::notice::All Fluent directories passed Fluent Linter validation" - - l10n_clap_error_localization: - name: L10n/Clap Error Localization Test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Install/setup prerequisites - shell: bash - run: | - sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - sudo locale-gen --keep-existing fr_FR.UTF-8 - locale -a | grep -i fr || exit 1 - - name: Test English clap error localization - shell: bash - run: | - export LANG=en_US.UTF-8 - export LC_ALL=en_US.UTF-8 - - # Test invalid argument error - should show colored error message - echo "Testing invalid argument error..." - error_output=$(cargo run --features feat_os_unix --bin coreutils -- cp --invalid-arg 2>&1 || echo "Expected error occurred") - echo "Error output: $error_output" - - # Check for expected English clap error patterns - english_errors_found=0 - - if echo "$error_output" | grep -q "error.*unexpected argument"; then - echo "✓ Found English clap error message pattern" - english_errors_found=$((english_errors_found + 1)) - fi - - if echo "$error_output" | grep -q "Usage:"; then - echo "✓ Found English usage pattern" - english_errors_found=$((english_errors_found + 1)) - fi - - if echo "$error_output" | grep -q "For more information.*--help"; then - echo "✓ Found English help suggestion" - english_errors_found=$((english_errors_found + 1)) - fi - - # Test typo suggestion - echo "Testing typo suggestion..." - typo_output=$(cargo run --features feat_os_unix --bin coreutils -- ls --verbos 2>&1 || echo "Expected error occurred") - echo "Typo output: $typo_output" - - if echo "$typo_output" | grep -q "similar.*verbose"; then - echo "✓ Found English typo suggestion" - english_errors_found=$((english_errors_found + 1)) - fi - - echo "English clap errors found: $english_errors_found" - if [ "$english_errors_found" -ge 2 ]; then - echo "✓ SUCCESS: English clap error localization working" - else - echo "✗ ERROR: English clap error localization not working properly" - exit 1 - fi - - - name: Test French clap error localization - shell: bash - run: | - export LANG=fr_FR.UTF-8 - export LC_ALL=fr_FR.UTF-8 - - # Test invalid argument error - should show French colored error message - echo "Testing invalid argument error in French..." - error_output=$(cargo run --features feat_os_unix --bin coreutils -- cp --invalid-arg 2>&1 || echo "Expected error occurred") - echo "French error output: $error_output" - - # Check for expected French clap error patterns - french_errors_found=0 - - if echo "$error_output" | grep -q "erreur.*argument inattendu"; then - echo "✓ Found French clap error message: 'erreur: argument inattendu'" - french_errors_found=$((french_errors_found + 1)) - fi - - if echo "$error_output" | grep -q "conseil.*pour passer.*comme valeur"; then - echo "✓ Found French tip message: 'conseil: pour passer ... comme valeur'" - french_errors_found=$((french_errors_found + 1)) - fi - - if echo "$error_output" | grep -q "Utilisation:"; then - echo "✓ Found French usage pattern: 'Utilisation:'" - french_errors_found=$((french_errors_found + 1)) - fi - - if echo "$error_output" | grep -q "Pour plus d'informations.*--help"; then - echo "✓ Found French help suggestion: 'Pour plus d'informations'" - french_errors_found=$((french_errors_found + 1)) - fi - - # Test typo suggestion in French - echo "Testing typo suggestion in French..." - typo_output=$(cargo run --features feat_os_unix --bin coreutils -- ls --verbos 2>&1 || echo "Expected error occurred") - echo "French typo output: $typo_output" - - if echo "$typo_output" | grep -q "conseil.*similaire.*verbose"; then - echo "✓ Found French typo suggestion with 'conseil'" - french_errors_found=$((french_errors_found + 1)) - fi - - echo "French clap errors found: $french_errors_found" - if [ "$french_errors_found" -ge 2 ]; then - echo "✓ SUCCESS: French clap error localization working - found $french_errors_found French patterns" - else - echo "✗ ERROR: French clap error localization not working properly" - echo "Note: This might be expected if French common locale files are not available" - # Don't fail the build - French clap localization might not be fully set up yet - echo "::warning::French clap error localization not working, but continuing" - fi - - # Test that colors are working (ANSI escape codes) - echo "Testing ANSI color codes in error output..." - if echo "$error_output" | grep -q $'\x1b\[3[0-7]m'; then - echo "✓ Found ANSI color codes in error output" - else - echo "✗ No ANSI color codes found - colors may not be working" - echo "::warning::ANSI color codes not detected in clap error output" - fi - - - name: Test clap localization with multiple utilities - shell: bash - run: | - export LANG=en_US.UTF-8 - export LC_ALL=en_US.UTF-8 - - utilities_to_test=("ls" "cat" "touch" "cp" "mv") - utilities_passed=0 - - for util in "${utilities_to_test[@]}"; do - echo "Testing $util with invalid argument..." - util_error=$(cargo run --features feat_os_unix --bin coreutils -- "$util" --nonexistent-flag 2>&1 || echo "Expected error occurred") - - if echo "$util_error" | grep -q "error.*unexpected argument"; then - echo "✓ $util: clap localization working" - utilities_passed=$((utilities_passed + 1)) - else - echo "✗ $util: clap localization not working" - echo "Output: $util_error" - fi - done - - echo "Utilities with working clap localization: $utilities_passed/${#utilities_to_test[@]}" - if [ "$utilities_passed" -ge 3 ]; then - echo "✓ SUCCESS: Clap localization working across multiple utilities" - else - echo "✗ ERROR: Clap localization not working for enough utilities" - exit 1 - fi - - l10n_french_integration: - name: L10n/French Integration Test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Install/setup prerequisites - shell: bash - run: | - ## Install/setup prerequisites - sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - - name: Generate French locale - shell: bash - run: | - ## Generate French locale for testing - sudo locale-gen --keep-existing fr_FR.UTF-8 - locale -a | grep -i fr || echo "French locale not found, continuing anyway" - - name: Test French localization - shell: bash - run: | - ## Test French localization with various commands - export LANG=fr_FR.UTF-8 - export LC_ALL=fr_FR.UTF-8 - - echo "Testing touch --help with French locale..." - help_output=$(cargo run --features feat_os_unix --bin coreutils -- touch --help 2>&1 || echo "Command failed") - echo "Help output: $help_output" - - # Check for specific French strings from touch fr-FR.ftl - french_strings_found=0 - if echo "$help_output" | grep -q "Mettre à jour les temps d'accès"; then - echo "✓ Found French description: 'Mettre à jour les temps d'accès'" - french_strings_found=$((french_strings_found + 1)) - fi - if echo "$help_output" | grep -q "changer seulement le temps d'accès"; then - echo "✓ Found French help text: 'changer seulement le temps d'accès'" - french_strings_found=$((french_strings_found + 1)) - fi - if echo "$help_output" | grep -q "FICHIER"; then - echo "✓ Found French usage pattern: 'FICHIER'" - french_strings_found=$((french_strings_found + 1)) - fi - - echo "Testing ls --help with French locale..." - ls_help=$(cargo run --features feat_os_unix --bin coreutils -- ls --help 2>&1 || echo "Command failed") - echo "ls help output: $ls_help" - - # Check for specific French strings from ls fr-FR.ftl - if echo "$ls_help" | grep -q "Lister le contenu des répertoires"; then - echo "✓ Found French ls description: 'Lister le contenu des répertoires'" - french_strings_found=$((french_strings_found + 1)) - fi - if echo "$ls_help" | grep -q "Afficher les informations d'aide"; then - echo "✓ Found French ls help text: 'Afficher les informations d'aide'" - french_strings_found=$((french_strings_found + 1)) - fi - - echo "Testing base64 --help with French locale..." - base64_help=$(cargo run --features feat_os_unix --bin coreutils -- base64 --help 2>&1 || echo "Command failed") - echo "base64 help output: $base64_help" - - # Check for specific French strings from base64 fr-FR.ftl - if echo "$base64_help" | grep -q "encoder/décoder les données"; then - echo "✓ Found French base64 description: 'encoder/décoder les données'" - french_strings_found=$((french_strings_found + 1)) - fi - - echo "Testing with error messages..." - error_output=$(cargo run --features feat_os_unix --bin coreutils -- ls /nonexistent 2>&1 || echo "Expected error occurred") - echo "Error output: $error_output" - - # Check for French error messages from ls fr-FR.ftl - if echo "$error_output" | grep -q "impossible d'accéder à"; then - echo "✓ Found French error message: 'impossible d'accéder à'" - french_strings_found=$((french_strings_found + 1)) - fi - if echo "$error_output" | grep -q "Aucun fichier ou répertoire de ce type"; then - echo "✓ Found French error text: 'Aucun fichier ou répertoire de ce type'" - french_strings_found=$((french_strings_found + 1)) - fi - - # Test that the binary works and doesn't crash with French locale - version_output=$(cargo run --features feat_os_unix --bin coreutils -- --version 2>&1 || echo "Version command failed") - echo "Version output: $version_output" - - # Final validation - ensure we found at least some French strings - echo "French strings found: $french_strings_found" - if [ "$french_strings_found" -gt 0 ]; then - echo "✓ SUCCESS: French locale integration test passed - found $french_strings_found French strings" - else - echo "✗ ERROR: No French strings were detected, but commands executed successfully" - exit 1 - fi - - l10n_multicall_binary_install: - name: L10n/Multi-call Binary Install Test - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: "feat_os_unix" } - - { os: macos-latest , features: "feat_os_unix" } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Install/setup prerequisites - shell: bash - run: | - ## Install/setup prerequisites - case '${{ matrix.job.os }}' in - ubuntu-*) - sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - ;; - esac - - name: Install via make and test multi-call binary - shell: bash - run: | - ## Install using make and test installed binaries - echo "Installing with make using DESTDIR..." - - # Create installation directory - INSTALL_DIR="$PWD/install-dir" - mkdir -p "$INSTALL_DIR" - mkdir -p "$INSTALL_DIR/usr/bin" - - # Build and install using make with DESTDIR - echo "Building multi-call binary with MULTICALL=y" - echo "Current directory: $PWD" - echo "Installation directory will be: $INSTALL_DIR" - - # First check if binary exists after build - echo "Checking if coreutils was built..." - target/release-small/coreutils --version || echo "No coreutils binary in target/release-small/" - - make FEATURES="${{ matrix.job.features }}" PROFILE=release-small MULTICALL=y - - echo "After build, checking target/release-small/:" - ls -la target/release-small/ | grep -E "(coreutils|^total)" || echo "Build may have failed" - - echo "Running make install..." - echo "Before install - checking what we have:" - target/release-small/coreutils --version 2>/dev/null || echo "No coreutils in target/release-small" - - # Run make install with verbose output to see what happens - echo "About to run: make install DESTDIR=\"$INSTALL_DIR\" PREFIX=/usr PROFILE=release-small MULTICALL=y" - echo "Expected install path: $INSTALL_DIR/usr/bin/coreutils" - - make install DESTDIR="$INSTALL_DIR" PREFIX=/usr PROFILE=release-small MULTICALL=y || { - echo "Make install failed! Exit code: $?" - echo "Let's see what happened:" - ls -la "$INSTALL_DIR" 2>/dev/null || echo "Install directory doesn't exist" - exit 1 - } - - echo "Make install completed successfully" - - # Debug: Show what was installed - echo "=== Installation Debug ===" - echo "Current directory: $(pwd)" - echo "INSTALL_DIR: $INSTALL_DIR" - echo "Checking if build succeeded..." - if target/release-small/coreutils --version; then - echo "✓ Build succeeded - coreutils binary exists in target/release-small/" - else - echo "✗ Build failed - no coreutils binary in target/release-small/" - echo "Contents of target/release-small/:" - ls -la target/release-small/ | head -20 - exit 1 - fi - - echo "Contents of installation directory:" - find "$INSTALL_DIR" -type f 2>/dev/null | head -20 || echo "No files found in $INSTALL_DIR" - - echo "Checking standard installation paths..." - ls -la "$INSTALL_DIR/usr/bin/" 2>/dev/null || echo "Directory $INSTALL_DIR/usr/bin/ not found" - ls -la "$INSTALL_DIR/usr/local/bin/" 2>/dev/null || echo "Directory $INSTALL_DIR/usr/local/bin/ not found" - ls -la "$INSTALL_DIR/bin/" 2>/dev/null || echo "Directory $INSTALL_DIR/bin/ not found" - - # Find where coreutils was actually installed - echo "Searching for coreutils binary in installation directory..." - # First try the expected location - if [ -x "$INSTALL_DIR/usr/bin/coreutils" ]; then - COREUTILS_BIN="$INSTALL_DIR/usr/bin/coreutils" - else - # Fallback: search for executable files named coreutils (excluding completion files) - COREUTILS_BIN=$(find "$INSTALL_DIR" -path "*/bin/*" -name "coreutils" -type f 2>/dev/null | head -1) - fi - if [ -n "$COREUTILS_BIN" ]; then - echo "Found coreutils at: $COREUTILS_BIN" - export COREUTILS_BIN - else - echo "ERROR: coreutils binary not found in installation directory!" - echo "Installation may have failed. Let's check the entire filesystem under install-dir:" - find "$INSTALL_DIR" -type f 2>/dev/null | head -50 - - # As a last resort, check if it's in the build directory - if "target/release/coreutils" --version; then - echo "Using binary from build directory as fallback" - COREUTILS_BIN="$(pwd)/target/release/coreutils" - export COREUTILS_BIN - else - exit 1 - fi - fi - - echo "Testing installed multi-call binary functionality..." - - # Test calling utilities through coreutils binary - echo "Testing: $COREUTILS_BIN ls --version" - "$COREUTILS_BIN" ls --version - - echo "Testing: $COREUTILS_BIN cat --version" - "$COREUTILS_BIN" cat --version - - echo "Testing: $COREUTILS_BIN touch --version" - "$COREUTILS_BIN" touch --version - - # Test individual binaries (if they exist) - LS_BIN="$INSTALL_DIR/usr/bin/ls" - if [ -f "$LS_BIN" ]; then - echo "Testing individual binary: $LS_BIN --version" - "$LS_BIN" --version - else - echo "Individual ls binary not found (multi-call only mode)" - # Check if symlinks exist - if [ -L "$LS_BIN" ]; then - echo "Found ls as symlink to: $(readlink -f "$LS_BIN")" - fi - fi - - echo "✓ Multi-call binary installation and functionality test passed" - - l10n_installation_test: - name: L10n/Installation Test (Make & Cargo) - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: "feat_os_unix" } - - { os: macos-latest , features: "feat_os_unix" } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Install/setup prerequisites - shell: bash - run: | - ## Install/setup prerequisites - case '${{ matrix.job.os }}' in - ubuntu-*) - sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - # Generate French locale for testing - sudo locale-gen --keep-existing fr_FR.UTF-8 - locale -a | grep -i fr || echo "French locale generation may have failed" - ;; - esac - - name: Test Make installation - shell: bash - run: | - ## Test installation via make with DESTDIR - echo "Testing make install with l10n features..." - - # Create installation directory - MAKE_INSTALL_DIR="$PWD/make-install-dir" - mkdir -p "$MAKE_INSTALL_DIR" - - # Build and install using make with DESTDIR - make FEATURES="${{ matrix.job.features }}" PROFILE=release-small MULTICALL=y - make install DESTDIR="$MAKE_INSTALL_DIR" PREFIX=/usr PROFILE=release-small MULTICALL=y - - # Verify installation - echo "Testing make-installed binaries..." - if "$MAKE_INSTALL_DIR/usr/bin/coreutils" --version; then - echo "✓ coreutils binary installed via make successfully" - else - echo "✗ coreutils binary not found after make install" - exit 1 - fi - - # Test utilities - echo "Testing make-installed utilities..." - "$MAKE_INSTALL_DIR/usr/bin/coreutils" ls --version - "$MAKE_INSTALL_DIR/usr/bin/coreutils" cat --version - "$MAKE_INSTALL_DIR/usr/bin/coreutils" touch --version - - # Test basic functionality - echo "test content" > test.txt - if "$MAKE_INSTALL_DIR/usr/bin/coreutils" cat test.txt | grep -q "test content"; then - echo "✓ Basic functionality works" - else - echo "✗ Basic functionality failed" - exit 1 - fi - - # Test French localization with make-installed binary (Ubuntu only) - if [ "${{ matrix.job.os }}" = "ubuntu-latest" ]; then - echo "Testing French localization with make-installed binary..." - - # Set French locale - export LANG=fr_FR.UTF-8 - export LC_ALL=fr_FR.UTF-8 - - echo "Testing ls --help with French locale..." - ls_help=$("$MAKE_INSTALL_DIR/usr/bin/coreutils" ls --help 2>&1 || echo "Command failed") - echo "ls help output (first 10 lines):" - echo "$ls_help" | head -10 - - # Check for specific French strings from ls fr-FR.ftl - french_strings_found=0 - - if echo "$ls_help" | grep -q "Lister le contenu des répertoires"; then - echo "✓ Found French ls description: 'Lister le contenu des répertoires'" - french_strings_found=$((french_strings_found + 1)) - fi - - if echo "$ls_help" | grep -q "Afficher les informations d'aide"; then - echo "✓ Found French ls help text: 'Afficher les informations d'aide'" - french_strings_found=$((french_strings_found + 1)) - fi - - if echo "$ls_help" | grep -q "FICHIER"; then - echo "✓ Found French usage pattern: 'FICHIER'" - french_strings_found=$((french_strings_found + 1)) - fi - - echo "Testing cat --help with French locale..." - cat_help=$("$MAKE_INSTALL_DIR/usr/bin/coreutils" cat --help 2>&1 || echo "Command failed") - echo "cat help output (first 5 lines):" - echo "$cat_help" | head -5 - - if echo "$cat_help" | grep -q "Concaténer"; then - echo "✓ Found French cat description containing: 'Concaténer'" - french_strings_found=$((french_strings_found + 1)) - fi - - echo "Testing error messages with French locale..." - error_output=$("$MAKE_INSTALL_DIR/usr/bin/coreutils" ls /nonexistent_test_directory 2>&1 || echo "Expected error occurred") - echo "Error output: $error_output" - - if echo "$error_output" | grep -q "impossible d'accéder à"; then - echo "✓ Found French error message: 'impossible d'accéder à'" - french_strings_found=$((french_strings_found + 1)) - fi - - # Final validation - echo "French strings found: $french_strings_found" - if [ "$french_strings_found" -gt 0 ]; then - echo "✓ SUCCESS: French localization test passed with make-installed binary - found $french_strings_found French strings" - else - echo "✗ ERROR: No French strings detected with make-installed binary" - exit 1 - fi - else - echo "Skipping French localization test on ${{ matrix.job.os }} (no French locale available)" - fi - - echo "✓ Make installation test passed" - - name: Test Cargo installation - shell: bash - run: | - ## Test installation via cargo install with DESTDIR-like approach - echo "Testing cargo install with l10n features..." - - # Create installation directory - CARGO_INSTALL_DIR="$PWD/cargo-install-dir" - mkdir -p "$CARGO_INSTALL_DIR" - - # Install using cargo with l10n features - cargo install --path . --features "ls,cat,touch" --root "$CARGO_INSTALL_DIR" --locked - - # Verify installation - echo "Testing cargo-installed binaries..." - if "$CARGO_INSTALL_DIR/bin/coreutils" --version; then - echo "✓ coreutils binary installed successfully" - else - echo "✗ coreutils binary not found after cargo install" - exit 1 - fi - - # Test utilities - echo "Testing installed utilities..." - "$CARGO_INSTALL_DIR/bin/coreutils" ls --version - "$CARGO_INSTALL_DIR/bin/coreutils" cat --version - "$CARGO_INSTALL_DIR/bin/coreutils" touch --version - - # Test basic functionality - echo "test content" > test.txt - if "$CARGO_INSTALL_DIR/bin/coreutils" cat test.txt | grep -q "test content"; then - echo "✓ Basic functionality works" - else - echo "✗ Basic functionality failed" - exit 1 - fi - - echo "✓ Cargo installation test passed" - - name: Download additional locales from coreutils-l10n - shell: bash - run: | - ## Download additional locale files from coreutils-l10n repository - echo "Downloading additional locale files from coreutils-l10n..." - git clone --depth=1 https://github.com/uutils/coreutils-l10n.git coreutils-l10n-repo - - # Create installation directory - CARGO_INSTALL_DIR="$PWD/cargo-install-dir" - - # Create locale directory for cargo install - LOCALE_DIR="$CARGO_INSTALL_DIR/share/locales" - mkdir -p "$LOCALE_DIR" - - # Debug: Check structure of l10n repo - echo "Checking structure of coreutils-l10n-repo:" - ls -la coreutils-l10n-repo/ | head -10 - echo "Looking for locales directory:" - find coreutils-l10n-repo -name "*.ftl" -type f 2>/dev/null | head -10 || : - echo "Checking specific utilities:" - ls -la coreutils-l10n-repo/src/uu/ls/locales/ 2>/dev/null || echo "No ls directory in correct location" - find coreutils-l10n-repo -path "*/ls/*.ftl" 2>/dev/null | head -5 || echo "No ls ftl files found" - - # Copy non-English locale files from l10n repo - for util_dir in src/uu/*/; do - util_name=$(basename "$util_dir") - l10n_util_dir="coreutils-l10n-repo/src/uu/$util_name/locales" - - if [ -d "$l10n_util_dir" ]; then - echo "Installing locales for $util_name..." - mkdir -p "$LOCALE_DIR/$util_name" - - for locale_file in "$l10n_util_dir"/*.ftl; do - if [ -f "$locale_file" ]; then - filename=$(basename "$locale_file") - # Skip English locale files (they are embedded) - if [ "$filename" != "en-US.ftl" ]; then - cp "$locale_file" "$LOCALE_DIR/$util_name/" - echo " Installed $filename to $LOCALE_DIR/$util_name/" - fi - fi - done - else - # Debug: Show what's not found - if [ "$util_name" = "ls" ] || [ "$util_name" = "cat" ]; then - echo "WARNING: No l10n directory found for $util_name at $l10n_util_dir" - fi - fi - done - - # Debug: Show what was actually installed - echo "Files installed in locale directory:" - find "$LOCALE_DIR" -name "*.ftl" 2>/dev/null | head -10 || : - - # Fallback: If no files were installed from l10n repo, try copying from main repo - if [ -z "$(find "$LOCALE_DIR" -name "*.ftl" 2>/dev/null)" ]; then - echo "No files found from l10n repo, trying fallback from main repository..." - for util_dir in src/uu/*/; do - util_name=$(basename "$util_dir") - if [ -d "$util_dir/locales" ]; then - echo "Copying locales for $util_name from main repo..." - mkdir -p "$LOCALE_DIR/$util_name" - cp "$util_dir/locales"/*.ftl "$LOCALE_DIR/$util_name/" 2>/dev/null || : - fi - done - echo "Files after fallback:" - find "$LOCALE_DIR" -name "*.ftl" 2>/dev/null | head -10 || : - fi - - echo "✓ Additional locale files installed" - - name: Test French localization after cargo install - shell: bash - run: | - ## Test French localization with cargo-installed binary and downloaded locales - echo "Testing French localization with cargo-installed binary..." - - # Set installation directories - CARGO_INSTALL_DIR="$PWD/cargo-install-dir" - LOCALE_DIR="$CARGO_INSTALL_DIR/share/locales" - - echo "Checking installed binary..." - if ! "$CARGO_INSTALL_DIR/bin/coreutils" --version; then - echo "✗ coreutils binary not found" - exit 1 - fi - - echo "Checking locale files..." - echo "LOCALE_DIR is: $LOCALE_DIR" - echo "Checking if locale directory exists:" - ls -la "$LOCALE_DIR" 2>/dev/null || echo "Locale directory not found" - echo "Contents of locale directory:" - find "$LOCALE_DIR" -name "*.ftl" 2>/dev/null | head -10 || echo "No locale files found" - echo "Looking for ls locale files specifically:" - ls -la "$LOCALE_DIR/ls/" 2>/dev/null || echo "No ls locale directory" - - # Test French localization - export LANG=fr_FR.UTF-8 - export LC_ALL=fr_FR.UTF-8 - - echo "Testing ls --help with French locale..." - ls_help=$("$CARGO_INSTALL_DIR/bin/coreutils" ls --help 2>&1 || echo "Command failed") - echo "ls help output (first 10 lines):" - echo "$ls_help" | head -10 - - # Check for specific French strings from ls fr-FR.ftl - french_strings_found=0 - - if echo "$ls_help" | grep -q "Lister le contenu des répertoires"; then - echo "✓ Found French ls description: 'Lister le contenu des répertoires'" - french_strings_found=$((french_strings_found + 1)) - fi - - if echo "$ls_help" | grep -q "Afficher les informations d'aide"; then - echo "✓ Found French ls help text: 'Afficher les informations d'aide'" - french_strings_found=$((french_strings_found + 1)) - fi - - if echo "$ls_help" | grep -q "FICHIER"; then - echo "✓ Found French usage pattern: 'FICHIER'" - french_strings_found=$((french_strings_found + 1)) - fi - - echo "Testing cat --help with French locale..." - cat_help=$("$CARGO_INSTALL_DIR/bin/coreutils" cat --help 2>&1 || echo "Command failed") - echo "cat help output (first 5 lines):" - echo "$cat_help" | head -5 - - if echo "$cat_help" | grep -q "Concaténer"; then - echo "✓ Found French cat description containing: 'Concaténer'" - french_strings_found=$((french_strings_found + 1)) - fi - - echo "Testing error messages with French locale..." - error_output=$("$CARGO_INSTALL_DIR/bin/coreutils" ls /nonexistent_test_directory 2>&1 || echo "Expected error occurred") - echo "Error output: $error_output" - - if echo "$error_output" | grep -q "impossible d'accéder à"; then - echo "✓ Found French error message: 'impossible d'accéder à'" - french_strings_found=$((french_strings_found + 1)) - fi - - # Verify the binary works in French locale - version_output=$("$CARGO_INSTALL_DIR/bin/coreutils" --version 2>&1) - if [ $? -eq 0 ]; then - echo "✓ Binary executes successfully with French locale" - echo "Version output: $version_output" - else - echo "✗ Binary failed to execute with French locale" - exit 1 - fi - - # Final validation - echo "French strings found: $french_strings_found" - if [ "$french_strings_found" -gt 0 ]; then - echo "✓ SUCCESS: French localization test passed after cargo install - found $french_strings_found French strings" - else - echo "✗ ERROR: No French strings detected with cargo-installed binary" - echo "This indicates an issue with locale loading from downloaded files" - exit 1 - fi - - echo "✓ French localization verification completed" - - l10n_locale_support_verification: - name: L10n/Locale Support Verification - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Install/setup prerequisites - shell: bash - run: | - ## Install/setup prerequisites including locale support - sudo apt-get -y update - sudo apt-get -y install libselinux1-dev - - # Generate multiple locales for testing - sudo locale-gen --keep-existing en_US.UTF-8 fr_FR.UTF-8 de_DE.UTF-8 es_ES.UTF-8 - locale -a | grep -E "(en_US|fr_FR|de_DE|es_ES)" || echo "Some locales may not be available" - - name: Install binaries with locale support - shell: bash - run: | - ## Install both multi-call and individual binaries using make - echo "Installing binaries with full locale support..." - - # Create installation directory - INSTALL_DIR="$PWD/install-dir" - mkdir -p "$INSTALL_DIR" - - # Build and install using make with DESTDIR - make FEATURES="feat_os_unix" PROFILE=release-small MULTICALL=y - make install DESTDIR="$INSTALL_DIR" PREFIX=/usr PROFILE=release-small MULTICALL=y - - # Debug: Show what was installed - echo "Contents of installation directory:" - find "$INSTALL_DIR" -type f -name "coreutils" -o -name "ls" 2>/dev/null | head -20 || : - echo "Looking for binaries in: $INSTALL_DIR/usr/bin/" - ls -la "$INSTALL_DIR/usr/bin/" || echo "Directory not found" - - echo "✓ Installation completed" - - name: Verify locale detection and startup - shell: bash - run: | - ## Test that installed binaries start correctly with different locales - echo "Testing locale detection and startup..." - - # Set installation directory path - INSTALL_DIR="$PWD/install-dir" - - # Test with different locales - locales_to_test=("C" "en_US.UTF-8" "fr_FR.UTF-8") - - for locale in "${locales_to_test[@]}"; do - echo "Testing with locale: $locale" - - # Test multi-call binary startup - if LC_ALL="$locale" "$INSTALL_DIR/usr/bin/coreutils" --version >/dev/null 2>&1; then - echo "✓ Multi-call binary starts successfully with locale: $locale" - else - echo "✗ Multi-call binary failed to start with locale: $locale" - exit 1 - fi - - # Test individual binary startup (if available) - if [ -f "$INSTALL_DIR/usr/bin/ls" ]; then - if LC_ALL="$locale" "$INSTALL_DIR/usr/bin/ls" --version >/dev/null 2>&1; then - echo "✓ Individual binary (ls) starts successfully with locale: $locale" - else - echo "✗ Individual binary (ls) failed to start with locale: $locale" - exit 1 - fi - else - echo "Individual ls binary not found (multi-call only mode)" - fi - - # Test that help text appears (even if not localized) - help_output=$(LC_ALL="$locale" "$INSTALL_DIR/usr/bin/coreutils" ls --help 2>&1) - if echo "$help_output" | grep -q -i "usage\|list"; then - echo "✓ Help text appears correctly with locale: $locale" - else - echo "✗ Help text missing or malformed with locale: $locale" - echo "Help output: $help_output" - exit 1 - fi - done - - echo "✓ All locale startup tests passed" - - name: Test locale-specific functionality - shell: bash - run: | - ## Test locale-specific behavior with installed binaries - echo "Testing locale-specific functionality..." - - # Set installation directory path - INSTALL_DIR="$PWD/install-dir" - - # Test with French locale (if available) - if locale -a | grep -q fr_FR.UTF-8; then - echo "Testing French locale functionality..." - - export LANG=fr_FR.UTF-8 - export LC_ALL=fr_FR.UTF-8 - - # Test that the program runs successfully with French locale - french_version_output=$("$INSTALL_DIR/usr/bin/coreutils" --version 2>&1) - if [ $? -eq 0 ]; then - echo "✓ Program runs successfully with French locale" - echo "Version output: $french_version_output" - else - echo "✗ Program failed with French locale" - echo "Error output: $french_version_output" - exit 1 - fi - - # Test basic functionality with French locale - temp_file=$(mktemp) - echo "test content" > "$temp_file" - - if "$INSTALL_DIR/usr/bin/coreutils" cat "$temp_file" | grep -q "test content"; then - echo "✓ Basic file operations work with French locale" - else - echo "✗ Basic file operations failed with French locale" - exit 1 - fi - - rm -f "$temp_file" - - # Test that French translations are actually working - echo "Testing French translations..." - french_strings_found=0 - - echo "Testing ls --help with French locale..." - ls_help=$("$INSTALL_DIR/usr/bin/coreutils" ls --help 2>&1 || echo "Command failed") - echo "ls help output (first 10 lines):" - echo "$ls_help" | head -10 - - # Check for actual French strings that appear in ls --help output - if echo "$ls_help" | grep -q "Lister le contenu des répertoires"; then - echo "✓ Found French ls description: 'Lister le contenu des répertoires'" - french_strings_found=$((french_strings_found + 1)) - fi - if echo "$ls_help" | grep -q "Ignorer les fichiers et répertoires commençant par"; then - echo "✓ Found French explanation: 'Ignorer les fichiers et répertoires commençant par'" - french_strings_found=$((french_strings_found + 1)) - fi - if echo "$ls_help" | grep -q "Afficher les informations d'aide"; then - echo "✓ Found French help text: 'Afficher les informations d'aide'" - french_strings_found=$((french_strings_found + 1)) - fi - if echo "$ls_help" | grep -q "FICHIER"; then - echo "✓ Found French usage pattern: 'FICHIER'" - french_strings_found=$((french_strings_found + 1)) - fi - if echo "$ls_help" | grep -q "Définir le format d'affichage"; then - echo "✓ Found French option description: 'Définir le format d'affichage'" - french_strings_found=$((french_strings_found + 1)) - fi - - echo "Testing cat --help with French locale..." - cat_help=$("$INSTALL_DIR/usr/bin/coreutils" cat --help 2>&1 || echo "Command failed") - echo "cat help output (first 5 lines):" - echo "$cat_help" | head -5 - - # Check for French strings in cat help - if echo "$cat_help" | grep -q "Concaténer"; then - echo "✓ Found French cat description containing: 'Concaténer'" - french_strings_found=$((french_strings_found + 1)) - fi - - echo "Testing with error messages..." - error_output=$("$INSTALL_DIR/usr/bin/coreutils" ls /nonexistent_directory_for_testing 2>&1 || echo "Expected error occurred") - echo "Error output: $error_output" - - # Check for French error messages - if echo "$error_output" | grep -q "impossible d'accéder à"; then - echo "✓ Found French error message: 'impossible d'accéder à'" - french_strings_found=$((french_strings_found + 1)) - fi - if echo "$error_output" | grep -q "Aucun fichier ou répertoire de ce type"; then - echo "✓ Found French error text: 'Aucun fichier ou répertoire de ce type'" - french_strings_found=$((french_strings_found + 1)) - fi - - # Test version output - echo "Testing --version with French locale..." - version_output=$("$INSTALL_DIR/usr/bin/coreutils" --version 2>&1) - echo "Version output: $version_output" - - # Final validation - ensure we found at least some French strings - echo "French strings found: $french_strings_found" - if [ "$french_strings_found" -gt 0 ]; then - echo "✓ SUCCESS: French locale translation test passed - found $french_strings_found French strings" - else - echo "✗ ERROR: No French strings were detected in installed binaries" - echo "This indicates that French translations are not working properly with installed binaries" - exit 1 - fi - else - echo "French locale not available, skipping French-specific tests" - fi - - # Test with standard build configuration - echo "Testing standard build configuration..." - cd "$GITHUB_WORKSPACE" - - # Create separate installation directory for standard build - STANDARD_BUILD_INSTALL_DIR="$PWD/standard-build-install-dir" - mkdir -p "$STANDARD_BUILD_INSTALL_DIR" - - # Clean and build standard version - make clean - make FEATURES="feat_os_unix" PROFILE=release-small MULTICALL=y - make install DESTDIR="$STANDARD_BUILD_INSTALL_DIR" PREFIX=/usr PROFILE=release-small MULTICALL=y - - # Verify standard build binary works - if "$STANDARD_BUILD_INSTALL_DIR/usr/bin/coreutils" --version >/dev/null 2>&1; then - echo "✓ Standard build works correctly" - else - echo "✗ Standard build failed" - exit 1 - fi - - echo "✓ All locale-specific functionality tests passed" - - l10n_embedded_locale_regression_test: - name: L10n/Embedded Locale Regression Test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Install/setup prerequisites - shell: bash - run: | - sudo apt-get -y update && sudo apt-get -y install libselinux1-dev - - name: Test embedded locale functionality for individual utilities - shell: bash - run: | - bash util/test_locale_regression.sh - - l10n_locale_embedding_cat: - name: L10n/Locale Embedding - Cat Utility - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - with: - # Use different cache key for each build to avoid conflicts - key: cat-locale-embedding - - name: Install prerequisites - run: sudo apt-get -y update && sudo apt-get -y install libselinux1-dev - - name: Build cat with targeted locale embedding - run: UUCORE_TARGET_UTIL=cat cargo build -p uu_cat --release - - name: Verify cat locale count - run: | - locale_file=$(find target/release/build -name "embedded_locales.rs" | head -1) - if [ -z "$locale_file" ]; then - echo "ERROR: Could not find embedded_locales.rs" - exit 1 - fi - locale_count=$(grep -c '/en-US\.ftl' "$locale_file") - echo "Cat binary has $locale_count embedded locales" - if [ "$locale_count" -le 5 ]; then - echo "✓ SUCCESS: Cat uses targeted locale embedding ($locale_count files)" - else - echo "✗ FAILURE: Cat has too many locale files ($locale_count). Expected ≤ 5" - exit 1 - fi - - l10n_locale_embedding_ls: - name: L10n/Locale Embedding - Ls Utility - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - with: - # Use different cache key for each build to avoid conflicts - key: ls-locale-embedding - - name: Install prerequisites - run: sudo apt-get -y update && sudo apt-get -y install libselinux1-dev - - name: Build ls with targeted locale embedding - run: UUCORE_TARGET_UTIL=ls cargo build -p uu_ls --release - - name: Verify ls locale count - run: | - locale_file=$(find target/release/build -name "embedded_locales.rs" | head -1) - if [ -z "$locale_file" ]; then - echo "ERROR: Could not find embedded_locales.rs" - exit 1 - fi - locale_count=$(grep -c '/en-US\.ftl' "$locale_file") - echo "Ls binary has $locale_count embedded locales" - if [ "$locale_count" -le 5 ]; then - echo "✓ SUCCESS: Ls uses targeted locale embedding ($locale_count files)" - else - echo "✗ FAILURE: Ls has too many locale files ($locale_count). Expected ≤ 5" - exit 1 - fi - - l10n_locale_embedding_multicall: - name: L10n/Locale Embedding - Multicall Binary - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - with: - # Use different cache key for each build to avoid conflicts - key: multicall-locale-embedding - - name: Install prerequisites - run: sudo apt-get -y update && sudo apt-get -y install libselinux1-dev - - name: Build multicall binary with all locales - run: cargo build --release - - name: Verify multicall locale count - run: | - locale_file=$(find target/release/build -name "embedded_locales.rs" | head -1) - if [ -z "$locale_file" ]; then - echo "ERROR: Could not find embedded_locales.rs" - exit 1 - fi - locale_count=$(grep -c '/en-US\.ftl' "$locale_file") - echo "Multicall binary has $locale_count embedded locales" - echo "First 10 locales:" - grep -o '[a-z_][a-z_0-9]*/en-US\.ftl' "$locale_file" | head -10 - if [ "$locale_count" -ge 80 ]; then - echo "✓ SUCCESS: Multicall has all locales ($locale_count files)" - else - echo "✗ FAILURE: Multicall has too few locale files ($locale_count). Expected ≥ 80" - exit 1 - fi - - l10n_locale_embedding_cargo_install: - name: L10n/Locale Embedding - Cargo Install - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - with: - key: cargo-install-locale-embedding - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Install prerequisites - run: | - sudo apt-get -y update - sudo apt-get -y install libselinux1-dev - # Generate French locale for testing - sudo locale-gen --keep-existing fr_FR.UTF-8 - locale -a | grep -i fr || exit 1 - - - name: Test English locale embedding (default) - run: | - export LANG=en_US.UTF-8 - export LC_ALL=en_US.UTF-8 - - echo "Building uu_yes with LANG=$LANG" - cargo build --package uu_yes --release - - # Find the generated embedded_locales.rs - locale_file=$(find target/release/build -path "*/uu_yes-*/out/embedded_locales.rs" -o -path "*/uucore-*/out/embedded_locales.rs" | head -1) - if [ -z "$locale_file" ]; then - echo "ERROR: Could not find embedded_locales.rs" - exit 1 - fi - - echo "Found embedded_locales.rs at: $locale_file" - echo "Checking embedded locales..." - - # Should contain en-US - if grep -q 'yes/en-US\.ftl' "$locale_file" || grep -q 'uucore/en-US\.ftl' "$locale_file"; then - echo "✓ Found en-US locale (fallback)" - else - echo "✗ ERROR: en-US locale not found" - exit 1 - fi - - # Should NOT contain fr-FR when building with en_US.UTF-8 - if grep -q 'yes/fr-FR\.ftl' "$locale_file" || grep -q 'uucore/fr-FR\.ftl' "$locale_file"; then - echo "✗ ERROR: Unexpectedly found fr-FR locale when LANG=en_US.UTF-8" - exit 1 - else - echo "✓ Correctly omitted fr-FR locale" - fi - - echo "✓ SUCCESS: English locale embedding working correctly" - - - name: Test French locale embedding (system locale) - run: | - export LANG=fr_FR.UTF-8 - export LC_ALL=fr_FR.UTF-8 - - # Clean previous build to ensure fresh compile - cargo clean -p uu_yes - cargo clean -p uucore - - echo "Building uu_yes with LANG=$LANG" - cargo build --package uu_yes --release - - # Find the generated embedded_locales.rs - locale_file=$(find target/release/build -path "*/uu_yes-*/out/embedded_locales.rs" -o -path "*/uucore-*/out/embedded_locales.rs" | head -1) - if [ -z "$locale_file" ]; then - echo "ERROR: Could not find embedded_locales.rs" - exit 1 - fi - - echo "Found embedded_locales.rs at: $locale_file" - echo "Checking embedded locales..." - - # Should contain en-US (fallback) - if grep -q 'yes/en-US\.ftl' "$locale_file" || grep -q 'uucore/en-US\.ftl' "$locale_file"; then - echo "✓ Found en-US locale (fallback)" - else - echo "✗ ERROR: en-US locale not found" - exit 1 - fi - - # Should contain fr-FR when building with fr_FR.UTF-8 - if grep -q 'yes/fr-FR\.ftl' "$locale_file" || grep -q 'uucore/fr-FR\.ftl' "$locale_file"; then - echo "✓ Found fr-FR locale (system locale from LANG)" - else - echo "Note: fr-FR locale not found - this is expected if French translation doesn't exist yet" - echo "::notice::French locale for 'yes' utility may not be available" - fi - - echo "✓ SUCCESS: System locale detection working correctly" - - - name: Test locale count is reasonable - run: | - export LANG=fr_FR.UTF-8 - cargo clean -p uu_yes - cargo build --package uu_yes --release - - locale_file=$(find target/release/build -path "*/uucore-*/out/embedded_locales.rs" | head -1) - if [ -z "$locale_file" ]; then - echo "ERROR: Could not find uucore embedded_locales.rs" - exit 1 - fi - - # Count embedded locales (should be en-US + system locale, not all locales) - locale_count=$(grep -c '/en-US\.ftl\|/fr-FR\.ftl' "$locale_file" || echo "0") - echo "uu_yes has $locale_count embedded locale entries for yes utility" - - # For a single utility build, should have minimal locales (en-US + optionally system locale) - # Not the full multicall set - total_match_count=$(grep -c '=> Some(r###' "$locale_file" || echo "0") - echo "Total embedded entries: $total_match_count" - - if [ "$total_match_count" -le 10 ]; then - echo "✓ SUCCESS: Locale embedding is targeted ($total_match_count entries)" - else - echo "::warning::More locales than expected ($total_match_count entries)" - echo "This might be expected for utility + uucore locales" - fi diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml deleted file mode 100644 index 70e043a8013..00000000000 --- a/.github/workflows/make.yml +++ /dev/null @@ -1,537 +0,0 @@ -name: make -# spell-checker:ignore (abbrev/names) CACHEDIR taiki -# spell-checker:ignore (env/flags) CHERE RUSTDOCFLAGS RUSTFLAGS CARGOFLAGS CLEVEL relocs EAGAIN -# spell-checker:ignore (jargon) deps softprops toolchain -# spell-checker:ignore (people) dawidd -# spell-checker:ignore (shell/tools) bsdtar nextest pacman sccache zstd -# spell-checker:ignore (misc) bindir busytest defconfig dynamicbase DESTDIR libopenssl manpages multisize noconfirm runtest testsuite toybox uutils - -env: - PROJECT_NAME: coreutils - PROJECT_DESC: "Core universal (cross-platform) utilities" - PROJECT_AUTH: "uutils" - CARGO_INCREMENTAL: 0 - # * style job configuration - STYLE_FAIL_ON_FAULT: true ## (bool) fail the build if a style job contains a fault (error or warning); may be overridden on a per-job basis - -on: - pull_request: - push: - tags: - - '*' - branches: - - '*' - -permissions: - contents: read # to fetch code (actions/checkout) - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - build_makefile: - name: Build/Makefile and measure binary size - env: - # to reduce the size: - RUSTFLAGS: "-C force-unwind-tables=no -C strip=symbols -Clink-arg=-z -Clink-arg=pack-relative-relocs" - permissions: - contents: write # Publish individual binaries - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest , features: feat_os_unix } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: taiki-e/install-action@nextest - - uses: Swatinem/rust-cache@v2 - # Test build on the system missing libselinux (don't install libselinux1-dev at here) - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: "`make build`" - # Also check that target/CACHEDIR.TAG is created on a fresh checkout - shell: bash - run: | - set -x - # Target directory must not exist to start with, otherwise cargo - # will not create target/CACHEDIR.TAG. - if [[ -d target ]]; then - mv -T target target.cache - fi - # Check that we don't cross-build uudoc - env CARGO_BUILD_TARGET=invalid-unknown-linux-gnu make install-manpages PREFIX=/tmp/usr UTILS=true - # We don't build coreutils without MULTICALL=y - ! test -e target/debug/coreutils - # build (host) - make build - echo "Check that target directory will be ignored by backup tools" - test -f target/CACHEDIR.TAG - # Restore cache for target/release (we only did a debug build) - mv -t target/ target.cache/release 2>/dev/null || true - # only check the GNUmakefile logic, full tests run in CICD.yml - - name: "`make nextest UTILS=stdbuf`" - shell: bash - run: make nextest PROFILE=ci UTILS=stdbuf CARGOFLAGS="--hide-progress-bar" - env: - RUST_BACKTRACE: "1" - - name: "`make install`" - shell: bash - run: | - set -x - DESTDIR=/tmp/ make PROFILE=release install - # Check that the utils are present - test -f /tmp/usr/local/bin/tty - # Check that the manpage is present - test -f /tmp/usr/local/share/man/man1/md5sum.1 - # Check that the completion is present - test -f /tmp/usr/local/share/zsh/site-functions/_b2sum - test -f /tmp/usr/local/share/bash-completion/completions/head.bash - test -f /tmp/usr/local/share/fish/vendor_completions.d/cat.fish - env: - RUST_BACKTRACE: "1" - - name: "`make uninstall`" - shell: bash - run: | - set -x - DESTDIR=/tmp/ make uninstall - # Check that the utils are not present - ! test -f /tmp/usr/local/bin/tty - # Check that the manpage is not present - ! test -f /tmp/usr/local/share/man/man1/whoami.1 - # Check that the completion is not present - ! test -f /tmp/usr/local/share/zsh/site-functions/_install - ! test -f /tmp/usr/local/share/bash-completion/completions/head.bash - ! test -f /tmp/usr/local/share/fish/vendor_completions.d/cat.fish - - name: "`make install MULTICALL=n`" - shell: bash - run: | - set -x - DESTDIR=/tmp/individual make PROFILE=release MULTICALL=n install - # Check that *sum are present - for s in {md5,b2,sha1,sha224,sha256,sha384,sha512}sum - do test -e /tmp/individual/usr/local/bin/${s} - done - - name: "`make install MULTICALL=y LN=ln -svf`" - shell: bash - run: | - set -x - DESTDIR=/tmp/ make PROFILE=release MULTICALL=y LN="ln -svf" install - # Check that symlinks of *sum are present - for s in {md5,b2,sha1,sha224,sha256,sha384,sha512}sum - do test $(readlink /tmp/usr/local/bin/${s}) = coreutils - done - - name: Package - shell: bash - run: | - # Check that libstdbuf is present too - mv target/release/deps/libstdbuf.so -t /tmp/individual/usr/local/bin - ZSTD_CLEVEL=19 tar --zstd -caf individual-x86_64-unknown-linux-gnu.tar.zst -C /tmp/individual/usr/local bin - - name: Publish - uses: softprops/action-gh-release@v3 - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - with: - tag_name: latest-commit - draft: false - prerelease: true - files: | - individual-x86_64-unknown-linux-gnu.tar.zst - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Compute uutil release sizes - shell: bash - run: | - ## Compute uutil release sizes - DATE=$(date --rfc-email) - find /tmp/individual/usr/local/bin -type f -printf '%f\0' | sort -z | - while IFS= read -r -d '' name; do - size=$(du -s /tmp/individual/usr/local/bin/$name | awk '{print $1}') - echo "\"$name\"" - echo "$size" - done | \ - jq -n \ - --arg date "$DATE" \ - --arg sha "$GITHUB_SHA" \ - 'reduce inputs as $name ({}; . + { ($name): input }) | { ($date): {sha: $sha, sizes: map_values(.)} }' > individual-size-result.json - SIZE=$(cat individual-size-result.json | jq '[.[] | .sizes | .[]] | reduce .[] as $num (0; . + $num)') - SIZE_MULTI=$(du -s /tmp/usr/local/bin/coreutils | awk '{print $1}') - jq -n \ - --arg date "$DATE" \ - --arg sha "$GITHUB_SHA" \ - --arg size "$SIZE" \ - --arg multisize "$SIZE_MULTI" \ - '{($date): { sha: $sha, size: $size, multisize: $multisize, }}' > size-result.json - - name: Download the previous individual size result - uses: dawidd6/action-download-artifact@v27 - with: - workflow: make.yml - name: individual-size-result - repo: uutils/coreutils - path: dl - - name: Download the previous size result - uses: dawidd6/action-download-artifact@v27 - with: - workflow: make.yml - name: size-result - repo: uutils/coreutils - path: dl - - name: Compare individual binary sizes VS reference - shell: bash - run: | - ## Produce a per-binary size comparison and a PR comment body. - ## The comment body is consumed by the SizeComment workflow, which - ## mirrors GnuComment for the GNU testsuite results. - COMMENT_DIR="size-comment" - mkdir -p "${COMMENT_DIR}" - # `github.event.number` is empty on non-PR events; the downstream - # workflow only posts comments when triggered for a pull request. - echo "${{ github.event.number }}" > "${COMMENT_DIR}/NR" - COMMENT_LOG="${COMMENT_DIR}/result.txt" - - if test -f dl/individual-size-result.json; then - python3 util/compare_size_results.py \ - individual-size-result.json \ - dl/individual-size-result.json \ - --output "${COMMENT_LOG}" - else - echo "::warning ::Skipping individual size comparison; no reference is available." - fi - - name: Check uutil release multi-binary sizes - shell: bash - run: | - ## The aggregate `multiple binaries` and `multicall binary` totals - ## are still checked inline; they are simple scalar comparisons and - ## do not feed into the per-binary PR comment. - check() { - # Warn if the size increases by more than 5% - threshold='1.05' - - if [[ "$2" -eq 0 || "$3" -eq 0 ]]; then - echo "::warning file=$4::Invalid size for $1. Sizes cannot be 0." - return - fi - - ratio=$(jq -n "$2 / $3") - echo "$1: size=$2, previous_size=$3, ratio=$ratio, threshold=$threshold" - if [[ "$(jq -n "$ratio > $threshold")" == 'true' ]]; then - echo "::warning file=$4::Size of $1 increases by more than 5%" - fi - } - ## Check size result - size=$(cat size-result.json | jq -r '.[] | .size') - previous_size=$(cat dl/size-result.json | jq -r '.[] | .size') - check 'multiple binaries' "$size" "$previous_size" 'size-result.json' - multisize=$(cat size-result.json | jq -r '.[] | .multisize') - previous_multisize=$(cat dl/size-result.json | jq -r '.[] | .multisize') - check 'multicall binary' "$multisize" "$previous_multisize" 'size-result.json' - - name: Upload size comparison comment (for SizeComment workflow) - if: success() || failure() # run regardless of prior step success/failure - uses: actions/upload-artifact@v7 - with: - name: size-comment - path: size-comment/ - - name: Upload the individual size result - uses: actions/upload-artifact@v7 - with: - name: individual-size-result - path: individual-size-result.json - - name: Upload the size result - uses: actions/upload-artifact@v7 - with: - name: size-result - path: size-result.json - - name: "`make UTILS=XXX`" - shell: bash - run: | - set -x - # Regression-test for https://github.com/uutils/coreutils/issues/8701 - make UTILS="rm chmod chown chgrp mv du" - # Verifies that - # 1. there is no "error: none of the selected packages contains this - # feature: feat_external_libstdbuf" - # 2. the makefile doesn't try to install libstdbuf even though stdbuf is skipped - DESTDIR=/tmp/ make SKIP_UTILS="stdbuf" install - - # keep this job minimal to avoid have many duplicated build with CICD - build_makefile-windows-cygwin: - name: Build MSYS - permissions: - contents: write # Publish cygwin binary - runs-on: ${{ matrix.job.os }} - env: - CARGO_INCREMENTAL: 0 - RUSTFLAGS: "-C link-arg=-Wl,--dynamicbase -C link-arg=-Wl,--enable-auto-image-base" - strategy: - fail-fast: false - matrix: - job: - - { os: windows-latest , features: feat_os_windows } - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Disable Mandatory ASLR (required for Cygwin/MSYS2 bash on GitHub runners) - shell: powershell - run: | - # exit code 254 from bash.exe is caused by Windows Mandatory ASLR - # See: https://github.com/appveyor/ci/issues/3777 - Set-ProcessMitigation -System -Disable ForceRelocateImages - - name: Update MSYS2 packages - # msys2-runtime must be upgraded in a separate bash session before the build. - # Upgrading it mid-session changes the shared memory layout (60344 -> 59320), - # which causes subsequent fork() calls to fail with EAGAIN (exit code 254). - shell: 'C:\msys64\usr\bin\bash.exe --login -eo pipefail {0}' - # test Cygwin & MinGW(UCRT) instead of pre-installed Rust. - run: | - pacman -Sy --noconfirm --needed base-devel libopenssl make openssl-devel pkgconf rust \ - mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-rust - env: - CHERE_INVOKING: 1 - - name: "`(x86_64-pc-cygwin) make install PROG_PREFIX=uu- PROFILE=release-small COMPLETIONS=n MANPAGES=n LOCALES=n`" - shell: 'C:\msys64\usr\bin\bash.exe --login -eo pipefail {0}' - run: | - set -x - export CARGOFLAGS="--features openssl" - DESTDIR=/tmp/c make install PROG_PREFIX=uu- PROFILE=release-small COMPLETIONS=n MANPAGES=n LOCALES=n - # Check that cksum is linked with openssl (e.g. msys-crypto-*.dll) - # Also check that utils are built with given profile - ldd target/release-small/cksum | grep crypto - # Check that the progs have prefix - test -f /tmp/c/usr/local/bin/uu-tty - test -f /tmp/c/usr/local/libexec/uu-coreutils/libstdbuf.dll - # Check that the manpage is not present - ! test -f /tmp/c/usr/local/share/man/man1/uu-whoami.1 - # Check that the completion is not present - ! test -f /tmp/c/usr/local/share/zsh/site-functions/_uu-install - ! test -f /tmp/c/usr/local/share/bash-completion/completions/uu-head.bash - ! test -f /tmp/c/usr/local/share/fish/vendor_completions.d/uu-cat.fish - # package - mv target/release-small/deps/stdbuf.dll -t /tmp/c/usr/local/bin - bsdtar -caf individual-x86_64-pc-cygwin.tar.zst -C /tmp/c/usr/local bin - env: - CHERE_INVOKING: 1 - OPENSSL_NO_VENDOR: "1" - RUST_BACKTRACE: "1" - - name: Publish - uses: softprops/action-gh-release@v3 - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - with: - tag_name: latest-commit - draft: false - prerelease: true - files: | - individual-x86_64-pc-cygwin.tar.zst - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: "`make install` (MinGW UCRT)" - shell: 'C:\msys64\usr\bin\bash.exe --login -eo pipefail {0}' - env: - CHERE_INVOKING: 1 - MSYSTEM: "UCRT64" - OPENSSL_DIR: "/ucrt64" - OPENSSL_NO_VENDOR: "1" - OPENSSL_STATIC: "0" - run: | - set -x - export CARGOFLAGS="--features openssl" - # Check that we exclude unix programs to avoid build failure - # Windows's stdbuf depends on Cygwin's libstdbuf. So make should not try to install libstdbuf - make install DESTDIR=/tmp/w PREFIX=/tmp/usr MULTICALL=y COMPLETIONS=n MANPAGES=n LOCALES=n \ - SKIP_UTILS="arch b2sum base32 base64 basename basenc cat comm cp csplit cut date dd df dir dircolors dirname du echo env expand expr factor false fmt fold head hostname hostid join link ln ls md5sum mkdir mktemp more mv nl nproc numfmt od paste pathchk pr printenv printf ptx pwd readlink realpath rm rmdir seq sha1sum sha224sum sha256sum sha384sum sha512sum shred shuf sleep sort split sum sync tac tail tee test touch tr true truncate tsort tty uname unexpand uniq unlink vdir wc whoami yes" - target/release/coreutils.exe stdbuf --version - - test_busybox: - name: Tests/BusyBox test suite - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest } - steps: - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - echo "TEST_SUMMARY_FILE=busybox-result.json" >> $GITHUB_OUTPUT - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Install/setup prerequisites - shell: bash - run: | - sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - ## Install/setup prerequisites - make prepare-busytest - - name: Run BusyBox test suite - id: summary - shell: bash - run: | - ## Run BusyBox test suite - set -v - cp .busybox-config target/debug/.config - ## Run BusyBox test suite - bindir=$(pwd)/target/debug - cd tmp/busybox-*/testsuite - output=$(bindir=$bindir ./runtest 2>&1 || true) - printf "%s\n" "${output}" - FAIL=$(echo "$output" | grep "^FAIL:\s" | wc --lines) - PASS=$(echo "$output" | grep "^PASS:\s" | wc --lines) - SKIP=$(echo "$output" | grep "^SKIPPED:\s" | wc --lines) - TOTAL=`expr $FAIL + $PASS + $SKIP` - echo "FAIL $FAIL" - echo "SKIP $SKIP" - echo "PASS $PASS" - echo "TOTAL $TOTAL" - cd - - output="Busybox tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP" - echo "${output}" - if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then echo "::warning ::${output}" ; fi - jq -n \ - --arg date "$(date --rfc-email)" \ - --arg sha "$GITHUB_SHA" \ - --arg total "$TOTAL" \ - --arg pass "$PASS" \ - --arg skip "$SKIP" \ - --arg fail "$FAIL" \ - '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' - HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) - echo "HASH=${HASH}" >> $GITHUB_OUTPUT - - name: Reserve SHA1/ID of 'test-summary' - uses: actions/upload-artifact@v7 - with: - name: "${{ steps.summary.outputs.HASH }}" - path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - - name: Reserve test results summary - uses: actions/upload-artifact@v7 - with: - name: busybox-test-summary - path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - - name: Upload json results - uses: actions/upload-artifact@v7 - with: - name: busybox-result.json - path: ${{ steps.vars.outputs.TEST_SUMMARY_FILE }} - - test_toybox: - name: Tests/Toybox test suite - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - job: - - { os: ubuntu-latest } - steps: - - name: Initialize workflow variables - id: vars - shell: bash - run: | - ## VARs setup - outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } - TEST_SUMMARY_FILE="toybox-result.json" - outputs TEST_SUMMARY_FILE - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: Swatinem/rust-cache@v2 - - name: Run sccache-cache - id: sccache-setup - uses: mozilla-actions/sccache-action@v0.0.11 - continue-on-error: true - - name: Export sccache - if: steps.sccache-setup.outcome == 'success' - run: | - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - - name: Install/setup prerequisites - shell: bash - run: | - sudo apt-get -y update ; sudo apt-get -y install libselinux1-dev - - name: Build coreutils - shell: bash - run: | - set -v - make MULTICALL=y && (cd target/debug && for binary in $(./coreutils --list);do ln -sf coreutils ${binary};done) - - name: Run toybox src - shell: bash - run: | - make toybox-src - - name: Run Toybox test suite - id: summary - shell: bash - run: | - ## Run Toybox test suite - set -v - cd tmp/toybox-*/ - make defconfig - make tests &> tmp.log || true - cat tmp.log - FAIL=$(grep "FAIL" tmp.log | wc --lines) - PASS=$(grep "PASS:" tmp.log| wc --lines) - SKIP=$(grep " disabled$" tmp.log| wc --lines) - TOTAL=`expr $FAIL + $PASS + $SKIP` - echo "FAIL $FAIL" - echo "SKIP $SKIP" - echo "PASS $PASS" - echo "TOTAL $TOTAL" - cd - - jq -n \ - --arg date "$(date --rfc-email)" \ - --arg sha "$GITHUB_SHA" \ - --arg total "$TOTAL" \ - --arg pass "$PASS" \ - --arg skip "$SKIP" \ - --arg fail "$FAIL" \ - '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' - output="Toybox tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP" - echo "${output}" - if [[ "$FAIL" -gt 0 || "$ERROR" -gt 0 ]]; then echo "::warning ::${output}" ; fi - HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) - echo "HASH=${HASH}" >> $GITHUB_OUTPUT - - name: Reserve SHA1/ID of 'test-summary' - uses: actions/upload-artifact@v7 - with: - name: "${{ steps.summary.outputs.HASH }}" - path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - - name: Reserve test results summary - uses: actions/upload-artifact@v7 - with: - name: toybox-test-summary - path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" - - name: Upload json results - uses: actions/upload-artifact@v7 - with: - name: toybox-result.json - path: ${{ steps.vars.outputs.TEST_SUMMARY_FILE }} diff --git a/.github/workflows/manpage-lint.yml b/.github/workflows/manpage-lint.yml deleted file mode 100644 index 58ad8eb9b04..00000000000 --- a/.github/workflows/manpage-lint.yml +++ /dev/null @@ -1,111 +0,0 @@ -# spell-checker:ignore mandoc uudoc manpages libsystemd libattr libcap DESTDIR - -name: Manpage Validation - -on: - pull_request: - paths: - - 'src/bin/uudoc.rs' - - 'src/uu/*/locales/*.ftl' - - 'src/uu/*/src/*.rs' - - 'Cargo.toml' - - 'GNUmakefile' - - '.github/workflows/manpage-lint.yml' - push: - branches: - - main - paths: - - 'src/bin/uudoc.rs' - - 'src/uu/*/locales/*.ftl' - - 'src/uu/*/src/*.rs' - - 'Cargo.toml' - - 'GNUmakefile' - - '.github/workflows/manpage-lint.yml' - -permissions: - contents: read - -jobs: - manpage-lint: - name: Validate manpages with mandoc - runs-on: ubuntu-latest - strategy: - matrix: - locale: [en_US.UTF-8, fr_FR.UTF-8] - - steps: - - name: Checkout repository - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - - name: Install prerequisites - shell: bash - run: | - sudo apt-get update - sudo apt-get install -y mandoc locales-all - sudo apt-get install -y libselinux1-dev libsystemd-dev libacl1-dev libattr1-dev libcap-dev - - - name: Build manpages (${{ matrix.locale }}) - run: | - # Create temporary directory for manpages - MANPAGE_DIR=$(mktemp -d) - echo "MANPAGE_DIR=${MANPAGE_DIR}" >> $GITHUB_ENV - - # Set locale for manpage generation - export LANG=${{ matrix.locale }} - - # Build and install manpages to temporary directory - make install-manpages DESTDIR="${MANPAGE_DIR}" - - - name: Validate manpages with mandoc (${{ matrix.locale }}) - run: | - # Find all generated manpages - MANPAGE_PATH="${MANPAGE_DIR}/usr/local/share/man/man1" - - # Check if manpages were generated - if [ ! -d "${MANPAGE_PATH}" ]; then - echo "Error: No manpages found at ${MANPAGE_PATH}" - exit 1 - fi - - # Initialize error tracking - ERRORS_FOUND=0 - ERROR_LOG=$(mktemp) - - echo "Validating ${{ matrix.locale }} manpages with mandoc..." - echo "==========================================" - - # Validate each manpage - for manpage in "${MANPAGE_PATH}"/*.1; do - if [ -f "$manpage" ]; then - filename=$(basename "$manpage") - - # Run mandoc lint and capture output (only errors, not style warnings) - if ! mandoc -T lint -W error "$manpage" 2>&1 | tee -a "$ERROR_LOG"; then - echo "Errors found in $filename" - ERRORS_FOUND=1 - else - # Check if mandoc produced any output (errors only, not style warnings) - if mandoc -T lint -W error "$manpage" 2>&1 | grep -q .; then - echo "Warnings found in $filename" - ERRORS_FOUND=1 - else - echo "$filename is valid" - fi - fi - fi - done - - echo "" - echo "==================================" - - # Summary and exit - if [ "$ERRORS_FOUND" -eq 1 ]; then - echo "Manpage validation failed. Issues found:" - echo "" - cat "$ERROR_LOG" - exit 1 - else - echo "All manpages validated successfully!" - fi diff --git a/.github/workflows/openbsd.yml b/.github/workflows/openbsd.yml deleted file mode 100644 index 8322e6725bc..00000000000 --- a/.github/workflows/openbsd.yml +++ /dev/null @@ -1,219 +0,0 @@ -name: OpenBSD - -# spell-checker:ignore sshfs usesh vmactions taiki esac fdescfs fdesc sccache nextest copyback logind - -env: - # * style job configuration - STYLE_FAIL_ON_FAULT: true ## (bool) fail the build if a style job contains a fault (error or warning); may be overridden on a per-job basis - -on: - pull_request: - push: - branches: - - '*' - -permissions: - contents: read # to fetch code (actions/checkout) - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - style: - name: Style and Lint - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Prepare, build and test - uses: vmactions/openbsd-vm@v1 - with: - usesh: true - sync: rsync - copyback: false - mem: 4096 - # We need bash for inline shell string replacement - # Use sudo-- to get the default sudo package without ambiguity - prepare: | - # Clean up disk space before installing packages - df -h - pkg_add curl sudo-- bash rust rust-clippy rust-rustfmt python3 - rm -rf /usr/share/relink/* /usr/X11R6/* /usr/share/doc/* /usr/share/man/* & - # Clean up package cache after installation - pkg_delete -a & - df -h - run: | - ## Prepare, build, and test - # implementation modelled after ref: - # * NOTE: All steps need to be run in this block, otherwise, we are operating back on the mac host - set -e - # - export CARGO_INCREMENTAL=0 - TEST_USER=tester - REPO_NAME=${GITHUB_WORKSPACE##*/} - WORKSPACE_PARENT="/home/runner/work/${REPO_NAME}" - WORKSPACE="${WORKSPACE_PARENT}/${REPO_NAME}" - # - useradd -m -G wheel ${TEST_USER} - chown -R ${TEST_USER}:wheel /root/ "${WORKSPACE_PARENT}"/ - whoami - # - # Further work needs to be done in a sudo as we are changing users - sudo -i -u ${TEST_USER} bash << EOF - set -e - whoami - ## VARs setup - cd "${WORKSPACE}" - unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in - ''|0|f|false|n|no|off) FAULT_TYPE=warning ;; - *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;; - esac; - FAULT_PREFIX=\$(echo "\${FAULT_TYPE}" | tr '[:lower:]' '[:upper:]') - ## Info - echo "::group::environment" - echo "CI='${CI}'" - echo "REPO_NAME='${REPO_NAME}'" - echo "TEST_USER='${TEST_USER}'" - echo "WORKSPACE_PARENT='${WORKSPACE_PARENT}'" - echo "WORKSPACE='${WORKSPACE}'" - echo "FAULT_PREFIX='\${FAULT_PREFIX}'" - env | sort - echo "::endgroup::" - echo "::group::tooling info" - cargo -V - rustc -V - echo "::endgroup::" - # - # To ensure that files are cleaned up, we don't want to exit on error - set +e - unset FAULT - ## cargo fmt testing - echo "::group::cargo fmt" - # * convert any errors/warnings to GHA UI annotations; ref: - S=\$(cargo fmt -- --check) && printf "%s\n" "\$S" || { printf "%s\n" "\$S" ; printf "%s\n" "\$S" | sed -E -n -e "s/^Diff[[:space:]]+in[[:space:]]+\${PWD//\//\\\\/}\/(.*)[[:space:]]+at[[:space:]]+[^0-9]+([0-9]+).*\$/::\${FAULT_TYPE} file=\1,line=\2::\${FAULT_PREFIX}: \\\`cargo fmt\\\`: style violation (file:'\1', line:\2; use \\\`cargo fmt -- \"\1\"\\\`)/p" ; FAULT=true ; } - echo "::endgroup::" - ## cargo clippy lint testing - if [ -z "\${FAULT}" ]; then - echo "::group::cargo clippy" - CLIPPY_ARGS="--features feat_os_unix --fault-type \${FAULT_TYPE}" - if [ -n "\${FAIL_ON_FAULT}" ]; then CLIPPY_ARGS="\${CLIPPY_ARGS} --fail-on-fault"; fi - python3 util/run-clippy.py \${CLIPPY_ARGS} || FAULT=true - echo "::endgroup::" - fi - # Clean to avoid to rsync back the files and free up disk space - cargo clean - if [ -n "\${FAIL_ON_FAULT}" ] && [ -n "\${FAULT}" ]; then exit 1 ; fi - EOF - - test: - name: Tests - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Avoid no space left on device # shared with guest? - shell: bash - run: | - mkdir /tmp/empty - sudo rsync -a --delete /tmp/empty/ /usr/lib/jvm/ - sudo rsync -a --delete /tmp/empty/ /usr/share/dotnet/ - sudo rsync -a --delete /tmp/empty/ /usr/local/lib/android/ - - name: Prepare, build and test - uses: vmactions/openbsd-vm@v1 - with: - usesh: true - sync: rsync - copyback: false - mem: 6144 - # Install rust and build dependencies - prepare: | - # Clean up disk space before installing packages - df -h - mkdir -p empty - rsync -a --delete empty/ /usr/share/relink/ - rsync -a --delete empty/ /usr/X11R6/ - rsync -a --delete empty/ /usr/share/doc/ - rsync -a --delete empty/ /usr/share/man/ - pkg_add curl gmake sudo-- rust - # Clean up package cache after installation - pkg_delete -a - df -h - run: | - ## Prepare, build, and test - # implementation modelled after ref: - # * NOTE: All steps need to be run in this block, otherwise, we are operating back on the mac host - set -e - # - export CARGO_INCREMENTAL=0 - TEST_USER=tester - REPO_NAME=${GITHUB_WORKSPACE##*/} - WORKSPACE_PARENT="/home/runner/work/${REPO_NAME}" - WORKSPACE="${WORKSPACE_PARENT}/${REPO_NAME}" - # - useradd -m -G wheel ${TEST_USER} - chown -R ${TEST_USER}:wheel /root/ "${WORKSPACE_PARENT}"/ - whoami - # Add fake host for reverse DNS lookup (needed for hostname test) - printf "10.0.2.15\topenbsd.my.domain openbsd\n" >> /etc/hosts - # - # Further work needs to be done in a sudo as we are changing users - sudo -i -u ${TEST_USER} sh << EOF - set -e - whoami - # Increase max open files (512 by default) - ulimit -n 1024 - # todo: install nextest if official OpenBSD package was provided - # Use regular cargo test currently - ## Info - echo "::group::environment" - echo "CI='${CI}'" - echo "REPO_NAME='${REPO_NAME}'" - echo "TEST_USER='${TEST_USER}'" - echo "WORKSPACE_PARENT='${WORKSPACE_PARENT}'" - echo "WORKSPACE='${WORKSPACE}'" - env | sort - echo "::endgroup::" - echo "::group::tooling info" - cargo -V - rustc -V - echo "::endgroup::" - # - # To ensure that files are cleaned up, we don't want to exit on error - set +e - cd "${WORKSPACE}" - unset FAULT - # openbsd is very slow. Omit duplicated cargo build and do test only - export RUST_BACKTRACE=1 - export CARGO_TERM_COLOR=always - # Avoid filling disk space - export RUSTFLAGS="-C strip=symbols" - # vendored OpenSSL takes many times to build. Use system one (LibreSSL is supported). - export OPENSSL_NO_VENDOR=1 OPENSSL_DIR=/usr - # Use cargo test since nextest might not support OpenBSD - if (test -z "\$FAULT"); then - echo "::group::cargo test (workspace)" - cargo test --features feat_os_unix || FAULT=1 - echo "::endgroup::" - fi - if (test -z "\$FAULT"); then - echo "::group::cargo test (uucore)" - cargo test --all-features -p uucore || FAULT=1 - echo "::endgroup::" - fi - # Test building with make - if (test -z "\$FAULT"); then - echo "::group::make MULTICALL=Y" - make MULTICALL=Y || FAULT=1 - echo "::endgroup::" - fi - # Clean to avoid to rsync back the files and free up disk space - cargo clean - # Additional cleanup to free disk space - if (test -n "\$FAULT"); then exit 1 ; fi - EOF diff --git a/.github/workflows/wsl2.yml b/.github/workflows/wsl2.yml deleted file mode 100644 index d682b4a3059..00000000000 --- a/.github/workflows/wsl2.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: WSL2 - -# spell-checker:ignore nextest noprofile norc - -on: - pull_request: - push: - branches: - - '*' - -permissions: - contents: read # to fetch code (actions/checkout) - -# End the current execution if there is a new changeset in the PR. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - test: - name: Test - runs-on: ${{ matrix.job.os }} - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - job: - - { os: windows-latest, distribution: Ubuntu-24.04, features: feat_os_unix} - steps: - - name: Install WSL2 - uses: Vampire/setup-wsl@v7.2.0 - with: - additional-packages: build-essential - distribution: ${{ matrix.job.distribution }} - use-cache: 'true' - wsl-version: 2 - - name: Set up WSL2 user - shell: wsl-bash {0} - run: | - useradd -m -p password user - - name: Set up WSL2 shell command - uses: Vampire/setup-wsl@v7.2.0 - with: - distribution: ${{ matrix.job.distribution }} - wsl-shell-command: bash -c "sudo -u user bash --noprofile --norc -euo pipefail '{0}'" - # it is required to use WSL2's linux file system - - name: Checkout source in WSL2 - shell: wsl-bash {0} - run: | - mkdir -p "$HOME/src" - curl -L https://github.com/uutils/coreutils/archive/${{ github.sha }}.tar.gz | tar -xzf - --strip-components=1 -C "$HOME/src" - - name: Install rust and nextest - shell: wsl-bash {0} - run: | - curl https://sh.rustup.rs -sSf --output rustup.sh - sh rustup.sh -y --profile=minimal - curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C "$HOME/.cargo/bin" - - name: Test - shell: wsl-bash {0} - run: | - cd "$HOME/src" - # chmod tests expect umask 0022 - umask 0022 - . "$HOME/.cargo/env" - export CARGO_TERM_COLOR=always - export RUST_BACKTRACE=1 - CARGO_INCREMENTAL=0 - cargo nextest run --hide-progress-bar --profile ci --features '${{ matrix.job.features }}' diff --git a/Cargo.toml b/Cargo.toml index 6b646b58359..c1fe0510495 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -590,7 +590,7 @@ feat_Tier1 = [ ## (primary platforms) feature sets # "feat_wasm" == set of utilities which can be built for wasm target # We don't need to support all of wasm targets. So the ambiguous name is used at here -feat_wasm = ["feat_common_core", "nproc"] +feat_wasm = ["feat_common_core", "nproc", "sync"] # "feat_os_unix" == set of utilities which can be built/run on modern/usual *nix platforms. feat_os_unix = [ "feat_Tier1", diff --git a/src/uu/sync/locales/en-US.ftl b/src/uu/sync/locales/en-US.ftl index 272e5a822d2..a13c76250e2 100644 --- a/src/uu/sync/locales/en-US.ftl +++ b/src/uu/sync/locales/en-US.ftl @@ -3,7 +3,7 @@ sync-usage = sync [OPTION]... FILE... # Help messages sync-help-file-system = sync the file systems that contain the files (Linux and Windows only) -sync-help-data = sync only file data, no unneeded metadata (Linux only) +sync-help-data = sync only file data, no unneeded metadata # Error messages sync-error-data-needs-argument = --data needs at least one argument diff --git a/src/uu/sync/locales/fr-FR.ftl b/src/uu/sync/locales/fr-FR.ftl index 31c51aa4d6e..123ecdfb254 100644 --- a/src/uu/sync/locales/fr-FR.ftl +++ b/src/uu/sync/locales/fr-FR.ftl @@ -3,7 +3,7 @@ sync-usage = sync [OPTION]... FICHIER... # Messages d'aide sync-help-file-system = synchroniser les systèmes de fichiers qui contiennent les fichiers (Linux et Windows uniquement) -sync-help-data = synchroniser seulement les données des fichiers, pas les métadonnées inutiles (Linux uniquement) +sync-help-data = synchroniser seulement les données des fichiers, pas les métadonnées inutiles # Messages d'erreur sync-error-data-needs-argument = --data nécessite au moins un argument diff --git a/src/uu/sync/src/sync.rs b/src/uu/sync/src/sync.rs index 9339031d2fa..8bfc78c973a 100644 --- a/src/uu/sync/src/sync.rs +++ b/src/uu/sync/src/sync.rs @@ -18,7 +18,7 @@ pub mod options { static ARG_FILES: &str = "files"; -#[cfg(unix)] +#[cfg(any(unix, target_os = "wasi"))] mod platform { use std::fs::{File, OpenOptions}; use std::os::unix::fs::OpenOptionsExt; @@ -33,6 +33,7 @@ mod platform { reason = "fn sig must match on all platforms" )] pub fn do_sync() -> UResult<()> { + #[cfg(unix)] // todo: recursive sync on wasi rustix::fs::sync(); Ok(()) } @@ -248,7 +249,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { syncfs(&files)?; } } else if matches.get_flag(options::DATA) { - #[cfg(unix)] + #[cfg(any(unix, target_os = "wasi"))] fdatasync(&files)?; } else { sync()?;