From 61849c09b8533f5e0ee227e54a7430763872f655 Mon Sep 17 00:00:00 2001 From: Josef Haupt Date: Thu, 27 Aug 2026 17:07:24 +0200 Subject: [PATCH 1/2] Prune GitHubg cache + bump actions --- .github/workflows/ci.yml | 66 +++++++++++++---------------- .github/workflows/docker-build.yml | 46 ++++++++++---------- .github/workflows/documentation.yml | 13 +++--- .github/workflows/lint.yml | 9 ++-- .github/workflows/publish.yml | 4 +- .github/workflows/test-publish.yml | 4 +- 6 files changed, 66 insertions(+), 76 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dab13784f..d7a35a57b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,8 @@ name: Tests on: pull_request: - # No `edited`: that fires on title/description edits too, which re-runs the - # whole 3-OS x 3-Python matrix on an unchanged commit. `synchronize` already - # covers every code change. + # `edited` fires on title/description edits too, re-running the whole + # 3-OS x 3-Python matrix with no new commit behind it. types: [opened, synchronize, reopened] branches: [main] paths: @@ -16,67 +15,62 @@ on: concurrency: group: "${{ github.event.pull_request.number }}-${{ github.ref_name }}-${{ github.workflow }}" - # Cancel superseded PR runs when new commits are pushed, but let pushes to main - # run to completion so a merge's CI is never cancelled by the next merge. + # Superseded PR runs are safe to cancel; a merge's CI on main is not. cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: running-tests: runs-on: ${{ matrix.os }} - # Backstop for a wedged run (e.g. a hung model download or a native deadlock - # that pytest-timeout can't interrupt). Healthy jobs finish in ~2-6 min, plus a - # few for a cold-cache pre-download, so 30 caps a hang's cost with wide margin. + # Backstop for a wedged run - a hung download or a native deadlock that + # pytest-timeout cannot interrupt. Healthy jobs finish in ~2-6 min. timeout-minutes: 30 env: - # Fixed, cacheable location for birdnet's model/label downloads. The acoustic - # (3.0 ONNX ~520 MB, 2.4 TF ~120 MB), geo (~15 MB) and perch-v2 (~380 MB) - # models all land here, and - # birdnet reads this env var at import time and skips the download when the - # files already exist - so a cache hit avoids re-fetching them from tuc.cloud - # on every job. Placed under the workspace because the `runner` context isn't - # available in job-level env but `github.workspace` is; checkout runs before - # the cache is restored, so it never wipes the restored models. + # birdnet reads this at import time and skips models already present, so a + # restored cache avoids re-fetching them from tuc.cloud. Under the workspace + # because job-level env can use `github.workspace` but not `runner`. BIRDNET_APP_DATA: ${{ github.workspace }}/.birdnet-app-data strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: submodules: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - name: Set up uv - uses: astral-sh/setup-uv@v9.0.0 + uses: astral-sh/setup-uv@v10.0.1 with: - enable-cache: true - cache-dependency-glob: "pyproject.toml" - # Cache the downloaded models across runs. Keyed on the birdnet dependency - # (via pyproject.toml) and the OS only - the model files are identical across - # Python versions, so all matrix jobs on an OS share one cache. birdnet - # re-downloads only missing/updated models, so restoring an older cache is safe. + # Must be explicit: the default ("auto") caches on hosted runners. + # Restoring ~0.7 GB costs about what re-downloading the wheels does. + enable-cache: false + - name: Resolve birdnet pin + id: bn + shell: bash + run: | + pin=$(python -c "import tomllib; print(next(d for d in tomllib.load(open('pyproject.toml','rb'))['project']['dependencies'] if d.startswith('birdnet')))") + echo "pin=${pin//[^A-Za-z0-9._-]/_}" >> "$GITHUB_OUTPUT" - name: Cache birdnet models - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ${{ env.BIRDNET_APP_DATA }} - key: birdnet-models-${{ runner.os }}-${{ hashFiles('pyproject.toml') }} + # The birdnet pin alone decides the model set, and the files are + # identical across Python versions, so every matrix job on an OS shares + # one entry. birdnet re-fetches only what is missing, so a prefix hit + # on an older entry is safe. + key: birdnet-models-${{ runner.os }}-${{ steps.bn.outputs.pin }} restore-keys: | birdnet-models-${{ runner.os }}- - name: Install dependencies run: uv pip install --system .[embeddings,train,tests,gui-tests] - # Pre-download the models outside pytest's per-test 120s timeout. On a cold - # cache the acoustic-model download from tuc.cloud can exceed 120s on slower - # (Windows/macOS) runners and trip the timeout mid-download - which also means - # the model cache never captures a complete model, so the next run is cold - # again and the flake is permanent. Fetching them here (no per-test timeout) - # breaks that cycle and lets the cache actually populate. Mirrors the models - # baked into the Docker image. + # Outside pytest: a cold download can exceed the per-test 120s timeout, and + # being killed mid-download would cache a truncated model. - name: Pre-download birdnet models - # Match what the suite loads: acoustic 3.0 ONNX + geo 3.0 ONNX (the default - # analyze/geomodel path) and acoustic 2.4 TF (embeddings/custom-classifier). + # acoustic 3.0 + geo 3.0 ONNX are the default analyze path; acoustic 2.4 + # TF is what embeddings and custom classifiers load. run: python -c "import birdnet; birdnet.load('acoustic', '3.0', 'onnx', lang='en_us'); birdnet.load('acoustic', '2.4', 'tf', lang='en_us'); birdnet.load('geo', '3.0', 'onnx', lang='en_us')" - name: Run tests run: | diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 526ecb923..01a4c0cb8 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -35,39 +35,40 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Build image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . load: true tags: birdnet:test cache-from: type=gha - cache-to: type=gha,mode=max - - # linux/arm64 has no tensorflow-cpu wheel, so the arm64 leg breaks - # differently from amd64. The publish job builds it, but only on release — - # validate it here (build only, no load/smoke test under emulation) so a - # broken arm64 build fails the PR instead of the release. + # A PR's cache is scoped to its merge ref, unreadable by main or other + # PRs, so exporting it only consumes quota. The non-empty value has to + # stay in the true branch: '' is falsy, so the inverse spelling exports + # unconditionally. + cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max' || '' }} + + # linux/arm64 has no tensorflow-cpu wheel, so it breaks differently from + # amd64. Build-only (no load or smoke test under emulation) so a broken + # arm64 fails the PR rather than the release. - name: Validate arm64 build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . platforms: linux/arm64 cache-from: type=gha - cache-to: type=gha,mode=max + cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max' || '' }} - name: Smoke test image run: | - # Run inference end-to-end: mount the example audio read-only and - # write results to a separate directory so nothing is polluted. mkdir -p out docker run --rm \ -v "$PWD/birdnet_analyzer/example:/audio:ro" \ @@ -76,14 +77,12 @@ jobs: output_file="out/BirdNET_SelectionTable.txt" - # The container must produce a non-empty Raven selection table. if [ ! -s "$output_file" ]; then echo "::error::Expected output '$output_file' was not created or is empty" ls -la out exit 1 fi - # It must contain a header plus at least one detection row. line_count=$(wc -l < "$output_file") if [ "$line_count" -lt 2 ]; then echo "::error::Output table has no detection rows ($line_count lines)" @@ -91,8 +90,7 @@ jobs: exit 1 fi - # Sanity-check the header exposes the expected Raven columns - # (structure only — exact model scores are covered by the pytest suite). + # Structure only - exact model scores are covered by the pytest suite. header=$(head -n 1 "$output_file") for col in "Selection" "Begin Time (s)" "Confidence" "Begin Path"; do case "$header" in @@ -115,16 +113,16 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -132,7 +130,7 @@ jobs: - name: Extract image metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} # A release tagged v2.4.0 produces the tags 2.4.0, 2.4 and latest @@ -141,7 +139,7 @@ jobs: type=semver,pattern={{major}}.{{minor}} - name: Build and push image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 11026c400..9141592a4 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -2,7 +2,7 @@ name: documentation on: pull_request: - types: [opened, synchronize, reopened, edited] + types: [opened, synchronize, reopened] branches: [main] paths: [docs/**, .github/workflows/documentation.yml, birdnet_analyzer/cli.py] @@ -19,15 +19,16 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.13" - name: Set up uv - uses: astral-sh/setup-uv@v9.0.0 + uses: astral-sh/setup-uv@v10.0.1 with: - enable-cache: true - cache-dependency-glob: "pyproject.toml" + # Re-enabling needs a cache-suffix, or this key collides with the + # ubuntu/3.13 test job's. + enable-cache: false - name: Install dependencies run: uv pip install --system .[docs] - name: Sphinx build diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8b95176e2..24eb2a1a7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,8 +12,6 @@ on: ] pull_request: branches: [main] - # See the note in ci.yml: `edited` would re-run this on title/description - # edits, with no commit behind them. types: [opened, synchronize, reopened] paths: [ @@ -27,10 +25,9 @@ jobs: ruff: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - # ruff is a static analyzer and doesn't import the package, so skip the - # (heavy) dependency install entirely. Keep the version in sync with the - # ruff pin in pyproject.toml's [dev] extra. + - uses: actions/checkout@v7 + # ruff never imports the package, so no dependency install is needed. + # Keep this version in sync with the ruff pin in pyproject.toml's [dev]. - name: Lint with Ruff uses: astral-sh/ruff-action@v4.1.0 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cc3dff91c..cbd5ed893 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,10 +15,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.13" diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml index 67c5804b9..6689ccc93 100644 --- a/.github/workflows/test-publish.yml +++ b/.github/workflows/test-publish.yml @@ -16,10 +16,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.13" From d90521a19ca9e9af1fb1536f282aa912c468b2e6 Mon Sep 17 00:00:00 2001 From: Josef Haupt Date: Thu, 27 Aug 2026 18:49:11 +0200 Subject: [PATCH 2/2] . --- .github/workflows/ci.yml | 37 +++++++++++++++++++----------- .github/workflows/docker-build.yml | 7 +++--- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7a35a57b..1cae7fed8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,8 +2,6 @@ name: Tests on: pull_request: - # `edited` fires on title/description edits too, re-running the whole - # 3-OS x 3-Python matrix with no new commit behind it. types: [opened, synchronize, reopened] branches: [main] paths: @@ -29,6 +27,10 @@ jobs: # restored cache avoids re-fetching them from tuc.cloud. Under the workspace # because job-level env can use `github.workspace` but not `runner`. BIRDNET_APP_DATA: ${{ github.workspace }}/.birdnet-app-data + # acoustic 3.0 + geo 3.0 ONNX are the default analyze path; acoustic 2.4 TF + # is what embeddings and custom classifiers load. Hashed into the cache key + # below, so editing this list invalidates the entry rather than freezing it. + PREDOWNLOAD: "import birdnet; birdnet.load('acoustic', '3.0', 'onnx', lang='en_us'); birdnet.load('acoustic', '2.4', 'tf', lang='en_us'); birdnet.load('geo', '3.0', 'onnx', lang='en_us')" strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] @@ -47,21 +49,29 @@ jobs: # Must be explicit: the default ("auto") caches on hosted runners. # Restoring ~0.7 GB costs about what re-downloading the wheels does. enable-cache: false - - name: Resolve birdnet pin - id: bn + - name: Resolve model cache key + id: models shell: bash run: | - pin=$(python -c "import tomllib; print(next(d for d in tomllib.load(open('pyproject.toml','rb'))['project']['dependencies'] if d.startswith('birdnet')))") - echo "pin=${pin//[^A-Za-z0-9._-]/_}" >> "$GITHUB_OUTPUT" + python - <<'EOF' >> "$GITHUB_OUTPUT" + import hashlib, os, re, tomllib + + deps = tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"] + pin = next(d for d in deps if re.match(r"birdnet\s*[=<>!~;\[]", d)) + models = hashlib.sha256(os.environ["PREDOWNLOAD"].encode()).hexdigest()[:12] + print("key=" + re.sub(r"[^A-Za-z0-9._-]", "_", pin) + "-" + models) + EOF + # Must stay after checkout: the path is gitignored, and checkout's default + # `clean` (`git clean -ffdx`) would delete the restored models. - name: Cache birdnet models uses: actions/cache@v6 with: path: ${{ env.BIRDNET_APP_DATA }} - # The birdnet pin alone decides the model set, and the files are - # identical across Python versions, so every matrix job on an OS shares - # one entry. birdnet re-fetches only what is missing, so a prefix hit - # on an older entry is safe. - key: birdnet-models-${{ runner.os }}-${{ steps.bn.outputs.pin }} + # An exact hit skips the save, so the key has to cover everything that + # decides the contents - the birdnet pin and the model list. The files + # are identical across Python versions, so all matrix jobs on an OS + # share one entry, and birdnet re-fetches only what a prefix hit missed. + key: birdnet-models-${{ runner.os }}-${{ steps.models.outputs.key }} restore-keys: | birdnet-models-${{ runner.os }}- - name: Install dependencies @@ -69,9 +79,8 @@ jobs: # Outside pytest: a cold download can exceed the per-test 120s timeout, and # being killed mid-download would cache a truncated model. - name: Pre-download birdnet models - # acoustic 3.0 + geo 3.0 ONNX are the default analyze path; acoustic 2.4 - # TF is what embeddings and custom classifiers load. - run: python -c "import birdnet; birdnet.load('acoustic', '3.0', 'onnx', lang='en_us'); birdnet.load('acoustic', '2.4', 'tf', lang='en_us'); birdnet.load('geo', '3.0', 'onnx', lang='en_us')" + shell: bash + run: python -c "$PREDOWNLOAD" - name: Run tests run: | python -m pytest diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 01a4c0cb8..fc4368cb7 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -32,6 +32,8 @@ jobs: test: name: Build and test image runs-on: ubuntu-latest + # The emulated arm64 leg dominates; observed worst case ~18 min for the job. + timeout-minutes: 60 steps: - name: Checkout repository @@ -50,10 +52,6 @@ jobs: load: true tags: birdnet:test cache-from: type=gha - # A PR's cache is scoped to its merge ref, unreadable by main or other - # PRs, so exporting it only consumes quota. The non-empty value has to - # stay in the true branch: '' is falsy, so the inverse spelling exports - # unconditionally. cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max' || '' }} # linux/arm64 has no tensorflow-cpu wheel, so it breaks differently from @@ -106,6 +104,7 @@ jobs: needs: test if: github.event_name == 'release' runs-on: ubuntu-latest + timeout-minutes: 90 permissions: contents: read