Fix remote eval reruns by preserving upsert IDs (#763) #1420
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: checks | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| permissions: | |
| contents: read | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| fetch-depth: 0 # Fetch full history for proper diff | |
| - name: Set up mise | |
| uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 | |
| with: | |
| cache: true | |
| experimental: true | |
| - name: Run pre-commit | |
| run: | | |
| mise exec -- pre-commit run --from-ref origin/${{ github.base_ref || 'main' }} --to-ref HEAD | |
| ensure-pinned-actions: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - run: bash scripts/ensure-pinned-actions.sh | |
| api-codegen: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Setup Python environment | |
| uses: ./.github/actions/setup-python-env | |
| - name: Test generator and check committed output | |
| run: | | |
| mise exec -- make -C py test-api-codegen check-api-client-codegen | |
| static_checks: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Setup Python environment | |
| uses: ./.github/actions/setup-python-env | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Run pylint and type tests | |
| shell: bash | |
| run: | | |
| mise exec python@${{ matrix.python-version }} -- uv run --project ./py nox -f ./py/noxfile.py -s pylint test_types | |
| smoke: | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] | |
| os: [ubuntu-24.04, windows-2025] | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Setup Python environment | |
| uses: ./.github/actions/setup-python-env | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Test whether the Python SDK can be installed | |
| run: | | |
| # This is already done by make install-dev, but we're keeping this as a separate step | |
| # to explicitly verify that installation works | |
| mise exec python@${{ matrix.python-version }} -- uv sync --project ./py --all-extras | |
| - name: Test whether the Python SDK can be imported | |
| run: | | |
| mise exec python@${{ matrix.python-version }} -- uv run --project ./py python -c 'import braintrust' | |
| nox: | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] | |
| os: [ubuntu-24.04, windows-2025] | |
| shard: [0, 1, 2, 3, 4, 5] | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Setup Python environment | |
| uses: ./.github/actions/setup-python-env | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache Temporal test server binaries | |
| if: runner.os == 'Linux' | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| # The temporalio Python SDK downloads its test-server binary into | |
| # this directory (filename keyed by SDK version), and reuses any | |
| # existing binary on subsequent calls. Caching the directory lets | |
| # CI shards share binaries and avoid temporal.download 429s. | |
| path: ~/.cache/braintrust/temporal-test-server | |
| key: temporal-test-server-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('py/pyproject.toml') }} | |
| restore-keys: | | |
| temporal-test-server-${{ runner.os }}-${{ runner.arch }}- | |
| - name: Configure Temporal test server cache dir | |
| if: runner.os == 'Linux' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| dir="$HOME/.cache/braintrust/temporal-test-server" | |
| mkdir -p "$dir" | |
| echo "BRAINTRUST_TEMPORAL_TEST_SERVER_DIR=$dir" >> "$GITHUB_ENV" | |
| - name: Cache LiveKit server binaries | |
| if: runner.os == 'Linux' | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| # The LiveKit Agents nox session downloads a pinned standalone | |
| # livekit-server binary here when livekit-server is not already on | |
| # PATH. Caching avoids repeated GitHub release downloads across CI shards. | |
| path: ~/.cache/braintrust/livekit-server | |
| key: livekit-server-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('py/noxfile.py') }} | |
| restore-keys: | | |
| livekit-server-${{ runner.os }}-${{ runner.arch }}- | |
| - name: Configure LiveKit server cache dir | |
| if: runner.os == 'Linux' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| dir="$HOME/.cache/braintrust/livekit-server" | |
| mkdir -p "$dir" | |
| echo "BRAINTRUST_LIVEKIT_SERVER_DIR=$dir" >> "$GITHUB_ENV" | |
| - name: Cache Hugging Face models | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: ~/.cache/huggingface/hub | |
| key: huggingface-transformers-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}-${{ hashFiles('py/src/braintrust/integrations/transformers/test_transformers.py') }} | |
| restore-keys: | | |
| huggingface-transformers-${{ runner.os }}-${{ runner.arch }}- | |
| - name: Run nox tests (shard ${{ matrix.shard }}/6) | |
| shell: bash | |
| run: | | |
| mise exec python@${{ matrix.python-version }} -- python ./py/scripts/nox-matrix.py ${{ matrix.shard }} 6 \ | |
| --exclude-static-checks \ | |
| --exclude-session test_api_codegen | |
| adk-py: | |
| uses: ./.github/workflows/adk-py-test.yaml | |
| langchain-py: | |
| uses: ./.github/workflows/langchain-py-test.yaml | |
| upload-wheel: | |
| needs: | |
| - smoke | |
| - nox | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Set up mise | |
| uses: jdx/mise-action@7e36c90d9ab29c415a2384db3006f3ec8a8cc654 # v4.2.4 | |
| with: | |
| cache: true | |
| experimental: true | |
| - name: Install build dependencies and build wheel | |
| run: | | |
| mise exec -- make -C py install-build-deps build | |
| - name: Upload wheel as artifact | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: python-wheel | |
| path: py/dist/*.whl | |
| retention-days: 5 | |
| checks-passed: | |
| name: checks-passed | |
| needs: | |
| - lint | |
| - ensure-pinned-actions | |
| - api-codegen | |
| - static_checks | |
| - smoke | |
| - nox | |
| - adk-py | |
| - langchain-py | |
| - upload-wheel | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| if: always() | |
| steps: | |
| - name: Verify all required checks passed | |
| run: | | |
| FAILED=0 | |
| check_result() { | |
| local job_name="$1" | |
| local job_result="$2" | |
| if [ "$job_result" != "success" ] && [ "$job_result" != "skipped" ]; then | |
| echo "Job '$job_name' finished with result: $job_result" | |
| FAILED=1 | |
| fi | |
| } | |
| check_result "lint" "${{ needs.lint.result }}" | |
| check_result "ensure-pinned-actions" "${{ needs['ensure-pinned-actions'].result }}" | |
| check_result "api-codegen" "${{ needs['api-codegen'].result }}" | |
| check_result "static_checks" "${{ needs.static_checks.result }}" | |
| check_result "smoke" "${{ needs.smoke.result }}" | |
| check_result "nox" "${{ needs.nox.result }}" | |
| check_result "adk-py" "${{ needs['adk-py'].result }}" | |
| check_result "langchain-py" "${{ needs['langchain-py'].result }}" | |
| check_result "upload-wheel" "${{ needs['upload-wheel'].result }}" | |
| if [ "$FAILED" -ne 0 ]; then | |
| echo "One or more required checks failed" | |
| exit 1 | |
| fi | |
| echo "All required checks passed" |