diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..f6bebf0 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,44 @@ +{ + "name": "inferencekit", + "owner": { + "name": "Stellon Labs" + }, + "metadata": { + "description": "Make your model serve faster on your GPU, and prove it. Skills for Claude Code, Codex and OpenCode, plus a CLI and MCP server: your agent picks and tunes inference engines, can write GPU kernels, and backs every number with client-timed benchmarks, anti-cheat gates and quality checks.", + "version": "0.1.0" + }, + "plugins": [ + { + "name": "inferencekit", + "source": "./plugin", + "description": "Make your model serve faster on your GPU, and prove it. Skills for Claude Code, Codex and OpenCode, plus a CLI and MCP server: your agent picks and tunes inference engines, can write GPU kernels, and backs every number with client-timed benchmarks, anti-cheat gates and quality checks.", + "version": "0.1.0", + "author": { + "name": "Stellon Labs" + }, + "license": "Apache-2.0", + "category": "development", + "tags": [ + "inference", + "llm", + "serving", + "benchmarking", + "capacity", + "profiling", + "quantization", + "roofline", + "gpu", + "nvidia", + "cuda", + "mcp", + "nsight-systems", + "nsight-compute", + "correctness", + "vllm", + "sglang", + "llama.cpp", + "tensorrt-llm" + ] + } + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4a8bd53 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,120 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: tests (Python ${{ matrix.python }}) + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python: ["3.10", "3.12"] + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v10.2.0 + with: + version: "0.12.7" + python-version: ${{ matrix.python }} + enable-cache: true + - name: Versions agree (pyproject, __version__, plugin, marketplace, MCP pin) + run: uv run --no-project python scripts/release/check_versions.py + - name: Install (uv.lock must be current) + run: uv sync --locked --all-extras + - name: Tests + run: uv run --locked --all-extras pytest -q -rs + + plugin: + name: Claude Code plugin manifests + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@v7.0.0 + with: + node-version: "22" + - name: Validate the plugin + run: npx -y @anthropic-ai/claude-code@2.1.296 plugin validate ./plugin + - name: Validate the marketplace + run: npx -y @anthropic-ai/claude-code@2.1.296 plugin validate . + + codex: + # Codex reads the Claude-compatible marketplace (.claude-plugin/marketplace.json) and plugin + # manifests. Advisory: a Codex CLI change must not block a merge, but it should be visible. + name: Codex plugin install (advisory) + runs-on: ubuntu-24.04 + timeout-minutes: 10 + continue-on-error: true + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@v7.0.0 + with: + node-version: "22" + - name: Use a throwaway Codex home + run: | + mkdir -p "$RUNNER_TEMP/codex-home" + echo "CODEX_HOME=$RUNNER_TEMP/codex-home" >> "$GITHUB_ENV" + - name: Add the marketplace and install the plugin + run: | + npx -y @openai/codex@0.162.1 plugin marketplace add "$GITHUB_WORKSPACE" + npx -y @openai/codex@0.162.1 plugin add inferencekit@inferencekit + npx -y @openai/codex@0.162.1 plugin list | tee plugins.txt + grep -E '^inferencekit@inferencekit +installed, enabled' plugins.txt + - name: The plugin's MCP server is registered with the pinned launcher + run: | + npx -y @openai/codex@0.162.1 mcp list | tee mcp.txt + version="$(python3 -c 'import json; print(json.load(open("plugin/.claude-plugin/plugin.json"))["version"])')" + grep -F "inferencekit-cli[mcp]==$version inferencekit mcp serve" mcp.txt + + opencode: + # OpenCode has no marketplace for plugins: plugin/opencode/install.py wires the skills, commands, subagents and + # MCP server into its config. Advisory, like the Codex job: an OpenCode change must not block a merge. + name: OpenCode install (advisory) + runs-on: ubuntu-24.04 + timeout-minutes: 10 + continue-on-error: true + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@v7.0.0 + with: + node-version: "22" + - name: Install into a throwaway config home + run: | + echo "XDG_CONFIG_HOME=$RUNNER_TEMP/opencode-config" >> "$GITHUB_ENV" + XDG_CONFIG_HOME="$RUNNER_TEMP/opencode-config" python3 plugin/opencode/install.py + - name: OpenCode loads the skills, commands, subagents and MCP entry + run: | + npx -y opencode-ai@1.19.0 debug skill > skills.json + npx -y opencode-ai@1.19.0 debug config > config.json + python3 - <<'PY' + import json + skills = {s["name"] for s in json.load(open("skills.json"))} + cfg = json.load(open("config.json")) + version = json.load(open("plugin/.claude-plugin/plugin.json"))["version"] + assert {"inference-triage", "optimizing-inference", "benchmarking", "profiling", "optimizing-kernels", + "proving-correctness", "evaluating-quantization", "reporting-perf-claims"} <= skills, skills + assert {"optimize-inference", "bench", "profile", "perf-report"} <= set(cfg.get("command", {})) + assert {"engine-scout", "quant-scout", "profile-analyst", "claims-skeptic"} <= set(cfg.get("agent", {})) + assert cfg["mcp"]["inferencekit"]["command"][:3] == ["uvx", "--from", f"inferencekit-cli[mcp]=={version}"] + print("OpenCode sees", sorted(skills)) + PY diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b2cdc65 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.venv/ +__pycache__/ +*.pyc +runs/ +*.nsys-rep +*.ncu-rep +*.sqlite diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d67b4dc --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,73 @@ +# AGENTS.md: always-on rules for inference performance work + +These rules apply to any agent doing inference performance work with this repository or the inferencekit tools, including agents that read only this file. Procedures live in `plugin/skills//SKILL.md`; each rule's reason is in `plugin/skills/inference-triage/references/evidence-rules.md` (same numbers). + +## Tools + +- CLI `inferencekit `; MCP server `inferencekit mcp serve` (stdio; every command is also a tool with the same arguments; launcher `uvx --from 'inferencekit-cli[mcp]==0.1.0' inferencekit mcp serve`). Commands and flags: `inferencekit --help` and `inferencekit --help`. +- Measure: `doctor`, `ceiling bw`, `roofline`, `bench rounds` (runs `bench serve` per engine per round, fresh process, balanced order: published numbers), `bench serve` (one client-timed call), `bench decode` (engine-timed; engines without a server), `bench image`, `bench audio` (client-timed media screening; contracts in `docs/media-benchmarks.md`), `compare` (matching benchmark kinds; load records are refused). +- Check: `gates run`, `prompts verify|show`, `quality trajectory|score|gsm8k|sampling`, `correctness`. +- Diagnose: `nsys capture|analyze`, `ncu capture|summarize`. +- Record and report: `intake template|validate`, `manifest validate`, `evidence list|show`, `ledger trailers`, `history`, `review build`, `report`. +- Evidence lands in `runs/`; `runs/`, `*.nsys-rep`, `*.ncu-rep` and `*.sqlite` are git-ignored. Never commit them. +- Hardware routing starts with `doctor`: NVIDIA uses Nsight Systems then Nsight Compute; AMD ROCm uses rocprofv3 with architecture-specific counter support; Apple Silicon uses Instruments, Metal/Xcode and MLX captures. `doctor` reads AMD DRM sysfs and Apple system facts; unknown details stay unknown. `ceiling bw` and nsys/ncu capture remain NVIDIA-only. Retrieve current official docs when architecture, runtime, flags or profiler support is uncertain; see `plugin/skills/inference-triage/references/hardware.md`. +- Subagents (`plugin/agents/`): `engine-scout` (one engine's support, sourced), `quant-scout` (one model's quantized checkpoints, bytes per token and fit from headers), `profile-analyst` (one profile's bottleneck), `claims-skeptic` (refutes a report's claims), `log-reader` (one question about a big log or doc, answered in at most 40 lines). They do GPU-free work only; any agent can follow the files as checklists. + +## Which skill + +| Need | Skill | +|---|---| +| Make a model fast end to end; which engine; hit a target | `optimizing-inference` | +| Find current optimizations and turn sources into experiments | `researching-optimizations` | +| Not sure where to start | `inference-triage` | +| tok/s, TTFT, capacity `N*`, ceilings, engine vs engine, before vs after | `benchmarking` | +| Where the time goes; why one kernel is slow | `profiling` | +| Custom engine, compiler inspection or kernel work for the task | `optimizing-kernels` | +| Is it still the same model; did quality regress | `proving-correctness` | +| Fit consumer memory while meeting latency and quality; minimize memory | `optimizing-inference` (affordability reference) | +| Should I quantize; which quant; will it fit; a big model on a small GPU | `evaluating-quantization` | +| Writing up or auditing a claim | `reporting-perf-claims` | + +## Evidence rules (numbers are stable: skills and evals cite them) + +1. Label every number **observed** (cite the evidence id), **derived** (show the formula) or **inferred** (name the measurement that would confirm it). +2. Claim verdicts are **PASS / FAIL / UNPROVEN / NOT RUN**. +3. Missing data is UNPROVEN (attempted, insufficient) or NOT RUN (never attempted); never zero, blank or PASS. +4. Combine checks with the precedence FAIL > UNPROVEN > PASS: any FAIL fails the whole; else any UNPROVEN makes it UNPROVEN. A check that does not apply is NOT RUN and is skipped; one that applies but ended NOT RUN counts as UNPROVEN; if nothing ran, the whole is NOT RUN. +5. Write the claim (metric, threshold, workload, bucket, decision rule) before measuring. +6. A speedup needs interleaved rounds and a paired ratio CI95 that excludes 1 (`bench rounds` `_ratio_to_best` for served engines, `compare` for `bench decode`); quote the conservative bound. +7. Comparison verdicts `improved | regressed | within_noise | inconclusive` come only from the tools: `bench rounds`' paired ratio CI for served engines, or `inferencekit compare` for two records of the same kind and workload (`bench decode`, single-user `bench serve`, `bench clip`, `bench image` or `bench audio`). Media timing alone leaves quality and lifecycle proof UNPROVEN. +8. Never compare speed across precisions (weights, activations, KV cache, lm_head). A format trade-off (`evaluating-quantization`) reports speed only paired with quality, labelled cross-precision. A numerics change the task explicitly allows is ranked, declared and labelled "numerics changed". +9. Compare against each baseline's best documented configuration for the goal measured, at a pinned version. +10. Quote the settings actually in effect (the engine's resolved config and logs), not the intended config; prove each knob took effect. +11. Report decode against a measured ceiling, not the datasheet: `inferencekit ceiling bw` on NVIDIA; on other hardware, a vendor or STREAM-style read-bandwidth measurement saved with the run (`plugin/skills/inference-triage/references/hardware.md`). +12. Profiles rank where time went; they never prove a speedup. Nsight Compute durations are never end-to-end evidence. +13. Every published claim and the final result needs a correctness or quality proof, an interleaved A/B with a CI and, for each claimed change, a profile showing the mechanism; the final candidate and any commit you hand over need the full proof, once (`bench rounds`, all gates, quality, all of GSM8K for LLMs, `quality sampling` when token sampling is timed; image and speech need their pre-registered modality checks, never LLM proxy gates). Inside the change loop each candidate gets one fast correctness check (greedy token identity, or logprob agreement for a numerics change, vs the previous build) and a screen; the first candidate that screens at >= 1.2x and the first at >= 1.5x of the baseline (or the task's target) get a light check (`plugin/skills/optimizing-inference/references/state.md`, "Best so far"). Between milestones a best is screened, not verified. +14. Bitwise identity is expected only for pure refactors within one engine; otherwise judge quality against the A/A noise floor, with margins at divergence. +15. Compute the Amdahl bound before optimising a kernel. Size it with critical-path time per step (`critical_pct_of_step`), not summed kernel durations; `kernel_sum_us / span_us > 1` means kernels overlap. +16. Publish losses, refuted ideas and NOT RUN items with the wins. +17. Never extrapolate to GPUs, precisions, contexts, buckets or concurrency levels you did not measure. +18. Published LLM numbers come only from `inferencekit bench serve` (client-side timer); label every `bench decode` number "engine-timed". Images and speech require a pre-registered client-timed modality adapter including final image decoding or first playable audio. A token endpoint is not proof of image latency or speech TTFA. +19. Each round starts every engine as a fresh process, in a balanced order (`inferencekit bench rounds`); R is pre-registered (default: the smallest multiple of the Williams period that is at least 3, i.e. 4 for 2 engines, 6 for 3, 4 for 4; the period is the shortest run of rounds in which each engine follows every other one equally often; never below 3), or an adaptive plan is (`bench rounds --ci-target`, `--min-rounds`, a maximum `--rounds`; it stops at whole Williams periods); CIs are bootstrapped over rounds; warmups never count. +20. Run `inferencekit gates run --base-url --engine