From 7e1e44e2d12fd678ac8a71f2a68868e486cc5eb2 Mon Sep 17 00:00:00 2001 From: Hasan Demirkiran Date: Tue, 18 Aug 2026 15:34:37 +0300 Subject: [PATCH] chore: rename classifier to Kestrel --- README.md | 6 +++--- results/README.md | 4 ++-- .../{k-cl-1.predictions.jsonl => kestrel.predictions.jsonl} | 0 results/results.json | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename results/{k-cl-1.predictions.jsonl => kestrel.predictions.jsonl} (100%) diff --git a/README.md b/README.md index 5c280ea..146a1dc 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ The approximately 20:1 test mix is a constructed operating point for comparing p | System | Precision | Recall | F1 | Mean latency | |---|---:|---:|---:|---:| -| k-cl-1 | 0.947 | 0.922 | 0.934 | 22 µs | +| Kestrel | 0.947 | 0.922 | 0.934 | 22 µs | | Claude Opus 4.8 | 0.515 | 0.549 | 0.531 | 1.33 s | | Claude Sonnet 5 | 0.547 | 0.456 | 0.497 | 2.64 s | | Kimi K3 | 0.469 | 0.518 | 0.493 | 8.65 s | @@ -56,7 +56,7 @@ The approximately 20:1 test mix is a constructed operating point for comparing p | Shieldstral 1.0 (3B, local) | 0.406 | 0.269 | 0.324 | 186 ms | | Llama Guard 4 (12B) | 0.023 | 0.285 | 0.042 | 1.1 s | -All systems were scored on the same 4,194 commands. Hosted-model latency was measured sequentially and includes the API round trip. Shieldstral used its default 0.5 threshold; Llama Guard counted any unsafe category as risky. Quality results, prompts, and the k-cl-1 per-example verdicts are under [`results/`](results/). The k-cl-1 implementation and weights are not part of this benchmark repository. +All systems were scored on the same 4,194 commands. Hosted-model latency was measured sequentially and includes the API round trip. Shieldstral used its default 0.5 threshold; Llama Guard counted any unsafe category as risky. Quality results, prompts, and the Kestrel per-example verdicts are under [`results/`](results/). The Kestrel implementation and weights are not part of this benchmark repository. ## Build @@ -81,7 +81,7 @@ After building the fixed split, score a JSONL prediction file: ```bash .venv/bin/python -m shellrisk_bench.score \ --gold data/splits/test.jsonl \ - --predictions results/k-cl-1.predictions.jsonl + --predictions results/kestrel.predictions.jsonl ``` Prediction rows contain a stable command hash and a binary verdict: diff --git a/results/README.md b/results/README.md index 8500584..9bcfbe9 100644 --- a/results/README.md +++ b/results/README.md @@ -1,6 +1,6 @@ # Published result artifacts -`results.json` contains the aggregate comparison reported with ShellRisk-Bench v0.1. `k-cl-1.predictions.jsonl` contains one binary k-cl-1 verdict for each item in the deterministic test split, keyed by the SHA-256 identifier generated by `shellrisk_bench.prepare`. +`results.json` contains the aggregate comparison reported with ShellRisk-Bench v0.1. `kestrel.predictions.jsonl` contains one binary Kestrel verdict for each item in the deterministic test split, keyed by the SHA-256 identifier generated by `shellrisk_bench.prepare`. The classifier implementation and weights are intentionally not included. The prediction artifact makes its reported confusion matrix and quality metrics independently auditable after rebuilding the upstream data. @@ -24,6 +24,6 @@ Shieldstral 1.0 used its native yes/no classification format with the equivalent Hosted-model latency was measured on a fixed balanced sample of 40 test commands, strictly sequentially with one request in flight. The measurement includes the provider API round trip. It used sufficient output budget for each model to emit a verdict, with no retries or failures in the reported pass. -k-cl-1 latency was measured locally over all 4,194 test commands using a warm native implementation on an Apple M5 Pro MacBook Pro with 24 GB memory. Mean latency was 21.5 µs per command, p50 18.4 µs, p99 92.2 µs, and maximum 352 µs. Shieldstral was also measured locally on the same Apple Silicon system after warmup. +Kestrel latency was measured locally over all 4,194 test commands using a warm native implementation on an Apple M5 Pro MacBook Pro with 24 GB memory. Mean latency was 21.5 µs per command, p50 18.4 µs, p99 92.2 µs, and maximum 352 µs. Shieldstral was also measured locally on the same Apple Silicon system after warmup. Measurements were collected on 2026-08-18. Hosted model identifiers are recorded exactly as exposed by the providers at that time. diff --git a/results/k-cl-1.predictions.jsonl b/results/kestrel.predictions.jsonl similarity index 100% rename from results/k-cl-1.predictions.jsonl rename to results/kestrel.predictions.jsonl diff --git a/results/results.json b/results/results.json index ef71317..d36612a 100644 --- a/results/results.json +++ b/results/results.json @@ -9,7 +9,7 @@ "seed": 13 }, "systems": { - "k-cl-1": {"precision": 0.947, "recall": 0.922, "f1": 0.934, "mean_latency_ms": 0.0215}, + "kestrel": {"precision": 0.947, "recall": 0.922, "f1": 0.934, "mean_latency_ms": 0.0215}, "claude-opus-4-8": {"precision": 0.515, "recall": 0.549, "f1": 0.531, "mean_latency_ms": 1334.4}, "claude-sonnet-5": {"precision": 0.547, "recall": 0.456, "f1": 0.497, "mean_latency_ms": 2641.3}, "kimi-k3": {"precision": 0.469, "recall": 0.518, "f1": 0.493, "mean_latency_ms": 8654.0},