From 7dd443aebc95bbb9fd09349497a87068dfe82014 Mon Sep 17 00:00:00 2001 From: Joshua Dias Barreto Date: Mon, 13 Jul 2026 19:12:28 +0000 Subject: [PATCH 1/2] Add label-triggered external-PR benchmark gate Adds .github/workflows/benchmark-external.yml so a maintainer can run the gate on an EXTERNAL (fork) PR. The default pull_request gate can't -- fork PRs get no secrets -- so this uses pull_request_target to access KIRO_API_KEY, guarded so it only runs when a maintainer opts a PR in. Control model: - Triggers ONLY on the "labeled" event; the job runs ONLY if the label is "run-benchmark". A fork contributor cannot self-add the label, so nothing runs until a maintainer reviews the PR and applies it. - A later push does NOT re-run it -- the maintainer must re-label (re-review). - permissions: contents: read (drops the default write GITHUB_TOKEN). The maintainer who labels a PR is taking responsibility for it: the job runs the PR's contributed skill while holding KIRO_API_KEY, so only label PRs you've reviewed. Use a service-account key so it can be rotated independently. On its own branch (not main) so it can be adopted separately from the internal gate. --- .github/workflows/benchmark-external.yml | 82 ++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/benchmark-external.yml diff --git a/.github/workflows/benchmark-external.yml b/.github/workflows/benchmark-external.yml new file mode 100644 index 0000000..ca535b3 --- /dev/null +++ b/.github/workflows/benchmark-external.yml @@ -0,0 +1,82 @@ +name: NAD Benchmark (external PRs) + +# Gates EXTERNAL (fork) PRs. The default pull_request gate (benchmark.yml) can't, +# because fork PRs receive no secrets. This uses pull_request_target so a +# fork-triggered run can access KIRO_API_KEY, and is guarded so it runs ONLY when +# a maintainer explicitly opts a PR in. +# +# How it's controlled: +# - Triggers ONLY on the `labeled` event, and the job runs ONLY if the label is +# `run-benchmark`. A fork contributor cannot add the label (no write access), +# so nothing runs until a MAINTAINER reviews the PR and applies the label. +# - A later push does NOT re-run it — a maintainer must re-apply the label, i.e. +# re-review, before new commits are scored. +# - permissions: contents: read (drops the default write-capable GITHUB_TOKEN). +# +# The maintainer who applies the label is vouching for the PR: this job checks out +# and runs the PR's contributed skill in a job that holds KIRO_API_KEY, so only +# label PRs whose changes you've looked at. Use a service-account key (not a +# personal one) so it can be rotated independently. + +on: + pull_request_target: + # ONLY the label event — never `synchronize`/`opened`, so untrusted code + # never auto-runs on push. A maintainer must (re-)apply the label each time. + types: [labeled] + +# Least privilege: the base-context GITHUB_TOKEN is write-capable by default under +# pull_request_target. Restrict it to read so untrusted code can't use it to push, +# open releases, or modify the repo. +permissions: + contents: read + +concurrency: + group: nad-benchmark-external-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + gate: + # Run ONLY when a maintainer added the `run-benchmark` label. This `if` is the + # trust gate — without it, pull_request_target would run untrusted code with + # secrets on every PR event. + if: github.event.label.name == 'run-benchmark' + name: NAD external benchmark (labeled) + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + # Check out the PR's HEAD — i.e. the contributor's UNTRUSTED code. This is + # required (we must score their skill) and is the crux of the risk above. + # The workflow FILE itself still comes from base (pull_request_target), so + # the contributor cannot alter these steps — only the skill/agent/test/ + # grader content that the steps then run. + - name: Checkout PR head (untrusted) + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Run NAD benchmark + env: + KIRO_API_KEY: ${{ secrets.KIRO_API_KEY }} + run: | + set -euo pipefail + python benchmark/run_skill_tests.py \ + --nad-root=. \ + --subset=all \ + --count=1 \ + --workers=4 \ + --judge \ + --gate=90 + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: nad-benchmark-results + path: benchmark/build/nad_benchmark_results.json + if-no-files-found: warn From 894ba5b24e9d0b2adcf05f4dcb496bb9017feb91 Mon Sep 17 00:00:00 2001 From: Joshua Dias Barreto Date: Mon, 13 Jul 2026 19:26:06 +0000 Subject: [PATCH 2/2] test: harmless edit to trigger the external benchmark gate --- skills/neuron-nki-writing/SKILL.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skills/neuron-nki-writing/SKILL.md b/skills/neuron-nki-writing/SKILL.md index fc64d5c..fcd1995 100644 --- a/skills/neuron-nki-writing/SKILL.md +++ b/skills/neuron-nki-writing/SKILL.md @@ -431,3 +431,5 @@ else: | `/neuron-nki-debugging` | Debug compiler errors on device | | `/neuron-nki-profiling` | Profile kernel performance | | `/neuron-nki-profile-querying` | Query and analyze kernel profile data | + +