diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 0d20ba79ee6..2ba9e84dcb8 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -1,7 +1,7 @@ name: Android -# spell-checker:ignore (people) reactivecircus juliangruber -# spell-checker:ignore (shell/tools) TERMUX nextest udevadm pkill +# spell-checker:ignore (people) reactivecircus dtolnay +# spell-checker:ignore (shell/tools) TERMUX nextest udevadm pkill sccache # spell-checker:ignore (misc) swiftshader playstore DATALOSS noaudio on: @@ -10,7 +10,6 @@ on: branches: - '*' - permissions: contents: read # to fetch code (actions/checkout) @@ -21,35 +20,26 @@ concurrency: env: TERMUX: v0.118.0 - KEY_POSTFIX: nextest+rustc-hash+adb+sshd+upgrade+XGB+inc18 COMMON_EMULATOR_OPTIONS: -no-metrics -no-window -noaudio -no-boot-anim -camera-back none -gpu off - EMULATOR_DISK_SIZE: 12GB - EMULATOR_HEAP_SIZE: 2048M - EMULATOR_BOOT_TIMEOUT: 1200 # 20min + EMULATOR_DISK_SIZE: 6GB + EMULATOR_HEAP_SIZE: 1024M + EMULATOR_BOOT_TIMEOUT: 600 + EMULATOR_RAM_SIZE: 3072M + EMULATOR_CORES: 4 jobs: test_android: name: Test builds - timeout-minutes: 90 + timeout-minutes: 45 strategy: fail-fast: false matrix: - os: [ubuntu-latest] # , macos-latest - cores: [4] # , 6 - ram: [4096] + os: [ubuntu-latest] api-level: [28] target: [google_apis_playstore] - arch: [x86_64] # ,x86 ,arm64-v8a + arch: [x86_64] runs-on: ${{ matrix.os }} - env: - EMULATOR_RAM_SIZE: ${{ matrix.ram }} - EMULATOR_CORES: ${{ matrix.cores }} - RUNNER_OS: ${{ matrix.os }} - AVD_CACHE_KEY: "set later due to limitations of github actions not able to concatenate env variables" steps: - - name: Concatenate values to environment file - run: | - echo "AVD_CACHE_KEY=${{ matrix.os }}-${{ matrix.cores }}-${{ matrix.ram }}-${{ matrix.api-level }}-${{ matrix.target }}-${{ matrix.arch }}+termux-${{ env.TERMUX }}+${{ env.KEY_POSTFIX }}" >> $GITHUB_ENV - name: Collect information about runner if: always() continue-on-error: true @@ -78,84 +68,39 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - name: Collect information about runner - if: always() - continue-on-error: true - run: | - free -mh - df -Th - - name: Restore AVD cache - uses: actions/cache/restore@v6 - id: avd-cache - continue-on-error: true - with: - path: | - ~/.android/avd/* - ~/.android/avd/*/snapshots/* - ~/.android/adb* - ~/__rustc_hash__ - key: avd-${{ env.AVD_CACHE_KEY }} - - name: Collect information about runner after AVD cache - if: always() + - name: Run sccache-cache + id: sccache-setup + uses: mozilla-actions/sccache-action@v0.0.11 continue-on-error: true + - name: Export sccache + if: steps.sccache-setup.outcome == 'success' run: | - free -mh - df -Th - ls -lah /mnt/data - du -sch /mnt/data - - name: Delete AVD Lockfile when run from cache - if: steps.avd-cache.outputs.cache-hit == 'true' - run: | - rm -f \ - ~/.android/avd/*.avd/*.lock \ - ~/.android/avd/*/*.lock - - name: Create and cache emulator image - if: steps.avd-cache.outputs.cache-hit != 'true' - uses: reactivecircus/android-emulator-runner@v2.38.0 - with: - api-level: ${{ matrix.api-level }} - target: ${{ matrix.target }} - arch: ${{ matrix.arch }} - ram-size: ${{ env.EMULATOR_RAM_SIZE }} - heap-size: ${{ env.EMULATOR_HEAP_SIZE }} - disk-size: ${{ env.EMULATOR_DISK_SIZE }} - cores: ${{ env.EMULATOR_CORES }} - force-avd-creation: true - emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-load - emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }} - script: | - util/android-commands.sh init "${{ matrix.arch }}" "${{ matrix.api-level }}" "${{ env.TERMUX }}" - - name: Save AVD cache - if: steps.avd-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v6 - with: - path: | - ~/.android/avd/* - ~/.android/avd/*/snapshots/* - ~/.android/adb* - ~/__rustc_hash__ - key: avd-${{ env.AVD_CACHE_KEY }} - - uses: juliangruber/read-file-action@v1 - id: read_rustc_hash - with: - # ~ expansion didn't work - path: ${{ runner.os == 'Linux' && '/home/runner/__rustc_hash__' || '/Users/runner/__rustc_hash__' }} - trim: true - - name: Restore rust cache - id: rust-cache - uses: actions/cache/restore@v6 + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable with: - path: ~/__rust_cache__ - # The version vX at the end of the key is just a development version to avoid conflicts in - # the github cache during the development of this workflow - key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3 - - name: Collect information about runner resources - if: always() - continue-on-error: true + targets: x86_64-linux-android + - name: Install cargo-nextest on host + uses: taiki-e/install-action@cargo-nextest + - name: Cross-compile coreutils and tests for Android run: | - free -mh - df -Th - - name: Build and Test + export ANDROID_NDK_ROOT=${ANDROID_NDK_LATEST_HOME:-$ANDROID_NDK_HOME} + export TOOLCHAIN=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64 + export CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER=$TOOLCHAIN/bin/x86_64-linux-android28-clang + export CARGO_TARGET_X86_64_LINUX_ANDROID_AR=$TOOLCHAIN/bin/llvm-ar + export CC_x86_64_linux_android=$TOOLCHAIN/bin/x86_64-linux-android28-clang + export CXX_x86_64_linux_android=$TOOLCHAIN/bin/x86_64-linux-android28-clang++ + export AR_x86_64_linux_android=$TOOLCHAIN/bin/llvm-ar + + mkdir -p target/nextest-target + + echo "=== Cross-compiling cargo-nextest for Android ===" + cargo install cargo-nextest --target x86_64-linux-android --root target/nextest-target --locked + + echo "=== Cross-compiling coreutils & archiving tests with nextest ===" + cargo nextest archive --target x86_64-linux-android --features feat_os_unix_android --archive-file target/test-archive.tar.zst + - name: Run Android Emulator and Tests uses: reactivecircus/android-emulator-runner@v2.38.0 with: api-level: ${{ matrix.api-level }} @@ -166,33 +111,17 @@ jobs: disk-size: ${{ env.EMULATOR_DISK_SIZE }} cores: ${{ env.EMULATOR_CORES }} force-avd-creation: false - emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-save -snapshot ${{ env.AVD_CACHE_KEY }} + emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }} - # This is not a usual script. Every line is executed in a separate shell with `sh -c`. If - # one of the lines returns with error the whole script is failed (like running a script with - # set -e) and in consequences the other lines (shells) are not executed. script: | - util/android-commands.sh sync_host - util/android-commands.sh build + util/android-commands.sh init "${{ matrix.arch }}" "${{ matrix.api-level }}" "${{ env.TERMUX }}" + util/android-commands.sh sync_tests target/test-archive.tar.zst target/nextest-target/bin/cargo-nextest util/android-commands.sh tests - if [ "${{ steps.rust-cache.outputs.cache-hit }}" != 'true' ]; then util/android-commands.sh sync_image; fi; exit 0 - - name: Collect information about runner resources - if: always() - continue-on-error: true - run: | - free -mh - df -Th - - name: Save rust cache - if: steps.rust-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v6 - with: - path: ~/__rust_cache__ - key: ${{ matrix.arch }}_${{ matrix.target}}_${{ steps.read_rustc_hash.outputs.content }}_${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}_v3 - name: archive any output (error screenshots) if: always() uses: actions/upload-artifact@v7 with: - name: test_output_${{ env.AVD_CACHE_KEY }} + name: test_output_${{ matrix.arch }}_${{ matrix.api-level }} path: output - name: Collect information about runner resources if: always() diff --git a/util/android-commands.sh b/util/android-commands.sh index 63adf0ec422..0be462c123e 100755 --- a/util/android-commands.sh +++ b/util/android-commands.sh @@ -326,19 +326,17 @@ init() { api_level="$2" termux="$3" - snapshot_name="${AVD_CACHE_KEY}" - - # shellcheck disable=SC2015 - curl -sLO "https://github.com/termux/termux-app/releases/download/${termux}/termux-app_${termux}+github-debug_${arch}.apk" && - snapshot "termux-app_${termux}+github-debug_${arch}.apk" && - hash_rustc && - exit_termux && - adb -s emulator-5554 emu avd snapshot save "$snapshot_name" && - echo "Emulator image created. Name: $snapshot_name" || { - pkill -9 qemu-system-x86_64 - return 1 - } - pkill -9 qemu-system-x86_64 || true + echo "Downloading and installing Termux..." + curl -sLO "https://github.com/termux/termux-app/releases/download/${termux}/termux-app_${termux}+github-debug_${arch}.apk" + adb install -g "termux-app_${termux}+github-debug_${arch}.apk" + + echo "Initializing SSH connection..." + reinit_ssh_connection || return 1 + + echo "Installing minimal runtime tools (tar, mount-utils)..." + install_packages_via_ssh_using_apt "tar mount-utils" + + echo "Termux initialized successfully." } reinit_ssh_connection() { @@ -587,6 +585,21 @@ ls -la ${cache_dest}" echo "Finished sync image -> host: ${repo}" } +sync_tests() { + archive="$1" + nextest_bin="$2" + + reinit_ssh_connection || return 1 + + echo "Running sync tests -> device: archive=${archive}, nextest=${nextest_bin}" + run_command_via_ssh "mkdir -p ~/coreutils/target" + copy_file_or_dir_to_device_via_ssh "$archive" "$dev_home_dir/coreutils/test-archive.tar.zst" + copy_file_or_dir_to_device_via_ssh "$nextest_bin" "$dev_home_dir/coreutils/cargo-nextest" + run_command_via_ssh "chmod +x ~/coreutils/cargo-nextest" + + echo "Finished sync tests -> device" +} + build() { echo "Running build" @@ -675,6 +688,17 @@ elif [ $# -eq 2 ]; then exit 1 ;; esac +elif [ $# -eq 3 ]; then + case "$1" in + sync_tests) + sync_tests "$2" "$3" + exit_code=$? + ;; + *) + help + exit 1 + ;; + esac elif [ $# -eq 4 ]; then case "$1" in init) diff --git a/util/android-scripts/run-tests.sh b/util/android-scripts/run-tests.sh index 17eed8808e0..b4fa72a85a4 100755 --- a/util/android-scripts/run-tests.sh +++ b/util/android-scripts/run-tests.sh @@ -7,26 +7,9 @@ echo "PATH: $PATH" export PATH=$HOME/.cargo/bin:$PATH export RUST_BACKTRACE=full export CARGO_TERM_COLOR=always -export CARGO_INCREMENTAL=0 echo "PATH: $PATH" -run_with_retry() { - tries=$1 - shift 1 - - for i in $(seq 1 $tries); do - echo "Try #$i of $tries: run $*" - "$@" && echo "Done in try#$i" && return 0 - done - - exit_code=$? - - echo "Still failing after $tries. Code: $exit_code" - - return $exit_code -} - run_tests_in_subprocess() ( # limit virtual memory to 3GB to avoid that OS kills sshd @@ -41,21 +24,17 @@ run_tests_in_subprocess() ( } kill_all_background_jobs() { - jobs -p | xargs -I{} kill -- {} + jobs -p | xargs -I{} kill -- {} 2>/dev/null || true } - # observe (log) every 2 seconds the system resource usage to judge if we are at a limit - watchplus 2 df -h & - watchplus 2 free -hm & - - nextest_params=(--profile ci --hide-progress-bar --features feat_os_unix_android) + # observe (log) every 5 seconds the system resource usage to judge if we are at a limit + watchplus 5 df -h & + watchplus 5 free -hm & - # run tests + # run pre-compiled nextest archive on device cd ~/coreutils && \ - run_with_retry 3 timeout --preserve-status --verbose -k 1m 10m \ - cargo nextest run --no-run "${nextest_params[@]}" && timeout --preserve-status --verbose -k 1m 60m \ - cargo nextest run "${nextest_params[@]}" + ./cargo-nextest nextest run --archive-file test-archive.tar.zst --profile ci --hide-progress-bar result=$?