uucore: detect CPU features on Android

.github/workflows/android.yml

@@ -1,8 +1,8 @@
 name: Android
 
 # spell-checker:ignore (people) reactivecircus juliangruber
-# spell-checker:ignore (shell/tools) TERMUX nextest udevadm pkill
-# spell-checker:ignore (misc) swiftshader playstore DATALOSS noaudio
+# spell-checker:ignore (shell/tools) TERMUX nextest udevadm pkill cmdline commandlinetools sdkmanager
+# spell-checker:ignore (misc) swiftshader playstore noaudio
 
 on:
   pull_request:
@@ -29,17 +29,24 @@ env:
 
 jobs:
   test_android:
-    name: Test builds
+    name: ${{ matrix.test-suite == 'all' && 'Test builds' || 'Checksum and wc tests' }} (API ${{ matrix.api-level }}, ${{ matrix.arch }})
     timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest] # , macos-latest
         cores: [4] # , 6
         ram: [4096]
-        api-level: [28]
+        api-level: [28, "37.0"]
         target: [google_apis_playstore]
         arch: [x86_64] # ,x86 ,arm64-v8a
+        include:
+          - api-level: 28
+            test-suite: all
+          # Keep full-suite coverage on API 28. API 37 has separate filesystem
+          # and libc compatibility failures; cover the CPU-feature consumers here.
+          - api-level: "37.0"
+            test-suite: cksum-wc
     runs-on: ${{ matrix.os }}
     env:
       EMULATOR_RAM_SIZE: ${{ matrix.ram }}
@@ -64,7 +71,6 @@ jobs:
       continue-on-error: true
       run: |
         ls -lah /mnt/
-        cat /mnt/DATALOSS_WARNING_README.txt
         sudo mkdir /mnt/data
         sudo chmod a+rwx /mnt/data
         mkdir /mnt/data/.android && ln -s /mnt/data/.android ~/.android
@@ -78,6 +84,14 @@ jobs:
     - uses: actions/checkout@v7.0.1
       with:
         persist-credentials: false
+    - name: Install Android SDK command-line tools 22.0
+      run: |
+        curl --fail --location --retry 3 --output "$RUNNER_TEMP/commandlinetools.zip" \
+          https://dl.google.com/android/repository/commandlinetools-linux-15859902_latest.zip
+        unzip -q "$RUNNER_TEMP/commandlinetools.zip" -d "$RUNNER_TEMP/android-tools"
+        mv "$ANDROID_HOME/cmdline-tools/latest" "$RUNNER_TEMP/android-tools-previous"
+        mv "$RUNNER_TEMP/android-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/latest"
+        "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --version
     - name: Collect information about runner
       if: always()
       continue-on-error: true
@@ -121,6 +135,7 @@ jobs:
         disk-size: ${{ env.EMULATOR_DISK_SIZE }}
         cores: ${{ env.EMULATOR_CORES }}
         force-avd-creation: true
+        disable-animations: false
         emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-load
         emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }}
         script: |
@@ -166,15 +181,16 @@ jobs:
         disk-size: ${{ env.EMULATOR_DISK_SIZE }}
         cores: ${{ env.EMULATOR_CORES }}
         force-avd-creation: false
+        disable-animations: false
         emulator-options: ${{ env.COMMON_EMULATOR_OPTIONS }} -no-snapshot-save -snapshot ${{ env.AVD_CACHE_KEY }}
         emulator-boot-timeout: ${{ env.EMULATOR_BOOT_TIMEOUT }}
         # This is not a usual script. Every line is executed in a separate shell with `sh -c`. If
         # one of the lines returns with error the whole script is failed (like running a script with
         # set -e) and in consequences the other lines (shells) are not executed.
         script: |
-          util/android-commands.sh sync_host
+          timeout --verbose --kill-after=30s 10m util/android-commands.sh sync_host
           util/android-commands.sh build
-          util/android-commands.sh tests
+          util/android-commands.sh tests "${{ matrix.test-suite }}"
           if [ "${{ steps.rust-cache.outputs.cache-hit }}" != 'true' ]; then util/android-commands.sh sync_image; fi; exit 0
     - name: Collect information about runner resources
       if: always()

src/uucore/src/lib/features/hardware.rs

@@ -243,12 +243,8 @@ impl HasHardwareFeatures for SimdPolicy {
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 fn detect_avx512() -> bool {
-    if cfg!(target_os = "android") {
-        false
-    } else {
-        std::arch::is_x86_feature_detected!("avx512f")
-            && std::arch::is_x86_feature_detected!("avx512bw")
-    }
+    std::arch::is_x86_feature_detected!("avx512f")
+        && std::arch::is_x86_feature_detected!("avx512bw")
 }
 
 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
@@ -258,11 +254,7 @@ fn detect_avx512() -> bool {
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 fn detect_avx2() -> bool {
-    if cfg!(target_os = "android") {
-        false
-    } else {
-        std::arch::is_x86_feature_detected!("avx2")
-    }
+    std::arch::is_x86_feature_detected!("avx2")
 }
 
 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
@@ -272,11 +264,7 @@ fn detect_avx2() -> bool {
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 fn detect_pclmulqdq() -> bool {
-    if cfg!(target_os = "android") {
-        false
-    } else {
-        std::arch::is_x86_feature_detected!("pclmulqdq")
-    }
+    std::arch::is_x86_feature_detected!("pclmulqdq")
 }
 
 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
@@ -286,11 +274,7 @@ fn detect_pclmulqdq() -> bool {
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 fn detect_sse2() -> bool {
-    if cfg!(target_os = "android") {
-        false
-    } else {
-        std::arch::is_x86_feature_detected!("sse2")
-    }
+    std::arch::is_x86_feature_detected!("sse2")
 }
 
 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
@@ -300,11 +284,7 @@ fn detect_sse2() -> bool {
 
 #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
 fn detect_asimd() -> bool {
-    if cfg!(target_os = "android") {
-        false
-    } else {
-        std::arch::is_aarch64_feature_detected!("asimd")
-    }
+    std::arch::is_aarch64_feature_detected!("asimd")
 }
 
 #[cfg(not(all(target_arch = "aarch64", target_endian = "little")))]
@@ -315,11 +295,7 @@ fn detect_asimd() -> bool {
 #[cfg(target_arch = "aarch64")]
 fn detect_pmull() -> bool {
     // PMULL is an optional extension. ASIMD alone is not enough.
-    if cfg!(target_os = "android") {
-        false
-    } else {
-        std::arch::is_aarch64_feature_detected!("pmull")
-    }
+    std::arch::is_aarch64_feature_detected!("pmull")
 }
 
 #[cfg(not(target_arch = "aarch64"))]

tests/by-util/test_cksum.rs

@@ -3338,7 +3338,6 @@ mod format_mix {
     }
 }
 
-#[cfg(not(target_os = "android"))]
 mod debug_flag {
     use super::*;
 

tests/by-util/test_wc.rs

@@ -673,6 +673,20 @@ fn test_invalid_byte_sequence_word_count() {
         .stdout_is("      1       3       6\n");
 }
 
+#[test]
+#[cfg(all(target_arch = "x86_64", not(wasi_runner)))]
+fn test_debug_reports_sse2_on_x86_64() {
+    // SSE2 is part of the x86_64 baseline
+    new_ucmd!()
+        .args(&["-l", "--debug"])
+        .env("GLIBC_TUNABLES", "")
+        .pipe_in("line\n")
+        .succeeds()
+        .stdout_is("1\n")
+        .stderr_contains("using hardware support")
+        .stderr_contains("SSE2");
+}
+
 #[cfg(unix)]
 #[test]
 #[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")]

util/android-commands.sh

@@ -86,8 +86,7 @@ where COMMAND is one of:
                 host; by default, REPO is: $this_repo
   build         run \`cargo build --features feat_os_unix_android\` on the
                 device
-  tests         run \`cargo test --features feat_os_unix_android\` on the
-                device
+  tests [SUITE] run tests on the device (SUITE: all or cksum-wc; default: all)
 
 If you have multiple devices, use the ANDROID_SERIAL environment variable to
 specify which to connect to."
@@ -121,8 +120,9 @@ take_screen_shot() {
 
 get_app_user() {
     app="$1"
-    app_user="$(adb shell dumpsys package $app | grep 'userId=' | cut -d= -f2 | sort -u)"
-    if [[ -z "$app_user" ]]; then
+    app_user="$(adb shell cmd package list packages -U "$app" | tr -d '\r' |
+        awk -v app="$app" '$1 == "package:" app { sub(/^uid:/, "", $2); print $2 }')"
+    if [[ ! "$app_user" =~ ^[0-9]+$ ]]; then
         echo "Couldn't find user for app: $app">&2
         exit 1
     fi
@@ -393,7 +393,14 @@ test_ssh_connection() {
 # takes a local (on runner side) script file and runs it via ssh on the virtual android device. forwards return code.
 # adds a timestamp to every line to be able to see where delays are
 run_script_file_via_ssh() {
-    ssh -p 9022 "$(termux_user)@127.0.0.1" -o StrictHostKeyChecking=accept-new "bash -s" < "$1" 2>&1 | add_timestamp_to_lines
+    local script=$1
+    shift
+    local command="bash -s --"
+    local arg
+    for arg in "$@"; do
+        printf -v command '%s %q' "$command" "$arg"
+    done
+    ssh -p 9022 "$(termux_user)@127.0.0.1" -o StrictHostKeyChecking=accept-new "$command" < "$script" 2>&1 | add_timestamp_to_lines
     return "${PIPESTATUS[0]}"
 }
 
@@ -552,14 +559,18 @@ sync_host() {
     cache_home="${HOME}/${cache_dir_name}"
     cache_dest="$dev_home_dir/${cache_dir_name}"
 
-    reinit_ssh_connection
+    # Keep Termux in the foreground so Android does not freeze its SSH processes.
+    adb shell am start -n com.termux/.HomeActivity || return
+    reinit_ssh_connection || return
 
     echo "Running sync host -> image: ${repo}"
 
     # run_command_via_ssh "mkdir $dev_home_dir/coreutils"
 
-    copy_file_or_dir_to_device_via_ssh "$repo" "$dev_home_dir"
-    [[ -e "$cache_home" ]] && copy_file_or_dir_to_device_via_ssh "$cache_home" "$cache_dest"
+    copy_file_or_dir_to_device_via_ssh "$repo" "$dev_home_dir" || return
+    if [[ -e "$cache_home" ]]; then
+        copy_file_or_dir_to_device_via_ssh "$cache_home" "$cache_dest" || return
+    fi
 
     echo "Finished sync host -> image: ${repo}"
 }
@@ -609,7 +620,7 @@ tests() {
 
     run_script_file_via_ssh "$this_repo/util/android-scripts/collect-info.sh"
 
-    run_script_file_via_ssh "$this_repo/util/android-scripts/run-tests.sh" || return
+    run_script_file_via_ssh "$this_repo/util/android-scripts/run-tests.sh" "${1:-all}" || return
 
     echo "Finished tests"
 }
@@ -658,6 +669,10 @@ if [ $# -eq 1 ]; then
     esac
 elif [ $# -eq 2 ]; then
     case "$1" in
+        tests)
+            tests "$2"
+            exit_code=$?
+            ;;
         snapshot)
             snapshot "$2"
             exit_code=$?

util/android-scripts/run-tests.sh

@@ -8,30 +8,38 @@ export PATH=$HOME/.cargo/bin:$PATH
 export RUST_BACKTRACE=full
 export CARGO_TERM_COLOR=always
 export CARGO_INCREMENTAL=0
+export CARGO_BUILD_JOBS=2
 
 echo "PATH: $PATH"
 
+case "${1:-all}" in
+    all) test_filter=() ;;
+    cksum-wc) test_filter=(-E 'test(test_cksum::) | test(test_wc::)') ;;
+    *) echo "Unknown test suite: $1" >&2; exit 1 ;;
+esac
+
 run_with_retry() {
-    tries=$1
+    local tries=$1
     shift 1
+    local exit_code=0
 
-    for i in $(seq 1 $tries); do
+    for ((i = 1; i <= tries; i++)); do
         echo "Try #$i of $tries: run $*"
-        "$@" && echo "Done in try#$i" && return 0
+        if "$@"; then
+            echo "Done in try#$i"
+            return 0
+        else
+            exit_code=$?
+        fi
     done
 
-    exit_code=$?
-
     echo "Still failing after $tries. Code: $exit_code"
 
     return $exit_code
 }
 
 run_tests_in_subprocess() (
 
-    # limit virtual memory to 3GB to avoid that OS kills sshd
-    ulimit -v $((1024 * 1024 * 3))
-
     watchplus() {
         # call: watchplus <interval> <command>
         while true; do
@@ -48,7 +56,9 @@ run_tests_in_subprocess() (
     watchplus 2 df -h &
     watchplus 2 free -hm &
 
-    nextest_params=(--profile ci --hide-progress-bar --features feat_os_unix_android)
+    # Recent Android allocators reserve more virtual memory than a 3 GiB limit
+    # allows. Limit parallelism to reduce memory pressure without capping addresses.
+    nextest_params=(--profile ci --hide-progress-bar --test-threads 2 --features feat_os_unix_android "${test_filter[@]}")
 
     # run tests
     cd ~/coreutils && \
@@ -64,5 +74,5 @@ run_tests_in_subprocess() (
     return $result
 )
 
-# run sub-shell to be able to use ulimit without affecting the sshd
+# Keep the resource monitors and their cleanup in a separate shell.
 run_tests_in_subprocess