diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index de99b2f..4a101bb 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -17,7 +17,8 @@ "./skills/local-ai-use", "./skills/local-ai-app-integration", "./skills/serving-llms-on-instinct", - "./skills/tracelens-analysis-orchestrator" + "./skills/tracelens-analysis-orchestrator", + "./skills/build-lemonade-router" ], "description": "AMD's verified Agent Skills in one plugin: route image/audio through local AI on Ryzen AI, serve LLMs on AMD Instinct GPUs with vLLM, and analyze GPU kernel and PyTorch trace performance." } diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index cd38ee1..2862568 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -21,7 +21,8 @@ "./skills/local-ai-use", "./skills/local-ai-app-integration", "./skills/serving-llms-on-instinct", - "./skills/tracelens-analysis-orchestrator" + "./skills/tracelens-analysis-orchestrator", + "./skills/build-lemonade-router" ], "interface": { "displayName": "AMD Skills", diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json index 62c7697..81c4d0f 100644 --- a/.cursor-plugin/marketplace.json +++ b/.cursor-plugin/marketplace.json @@ -16,7 +16,8 @@ "./skills/local-ai-use", "./skills/local-ai-app-integration", "./skills/serving-llms-on-instinct", - "./skills/tracelens-analysis-orchestrator" + "./skills/tracelens-analysis-orchestrator", + "./skills/build-lemonade-router" ], "description": "AMD's verified Agent Skills in one plugin: route image/audio through local AI on Ryzen AI, serve LLMs on AMD Instinct GPUs with vLLM, and analyze GPU kernel and PyTorch trace performance.", "author": { diff --git a/skills/build-lemonade-router/SKILL.md b/skills/build-lemonade-router/SKILL.md new file mode 100644 index 0000000..5144f5a --- /dev/null +++ b/skills/build-lemonade-router/SKILL.md @@ -0,0 +1,256 @@ +--- +name: build-lemonade-router +description: >- + Turns a natural-language description of routing intent into a valid Lemonade + `collection.router` policy JSON. The skill generates and validates the JSON + only - it does not register it or call the live server. + Use when the user wants to route requests between models ("route sensitive + queries to X and everything else to Y"), generate a router/hybrid-router + config or policy, author a collection.router JSON, split traffic between a + small local model and a big/cloud model, add PII/jailbreak/topic classifiers + to routing, or mentions Lemonade Router, routing rules, routing.router, + candidates/default_model, keywords_any, semantic_similarity, or LLM-as-router. + Fills every field the user did not specify with safe defaults. +--- + +# Lemonade Router Config Generator + +Generate a **`collection.router` policy JSON** from a plain-English description +of how requests should be routed. The skill produces and validates the JSON +only - it does not call the live server, register the policy, or run requests +through it. The JSON is accepted by the strict server-side parser on the first +try and stays editable in the desktop app's Hybrid Router editor. + +## Prerequisites + +- **Lemonade Server v10.1.0+** running locally (`lemonade server start`). + Required only to register and test the generated policy - the skill itself + (JSON generation + offline validation) works without a live server. +- **No GPU or ROCm dependency** for authoring. The router policy is a JSON + document; no hardware is needed to generate or validate it. +- **Python** (any 3.x) in PATH - used by the bundled offline validator + (`scripts/validate.py`). No extra packages required. + +The router picks one **candidate** model per request. Two authoring modes +exist, and choosing the right one is the first decision: + +| Mode | JSON shape | When | +|------|-----------|------| +| **LLM-as-router** | `routing.router` block | The user describes intent only by *meaning* ("sensitive", "hard questions", "creative writing") with no concrete signals. A small LLM reads each prompt and picks the candidate. | +| **Rules** | `routing.rules` (+ optional `routing.classifiers`) | The user names any concrete signal: keywords, regex, length, tools, images, metadata, PII/topic classifiers, thresholds, "first match", fallback logic. Deterministic, no extra LLM call for simple conditions. | + +`routing.router` is **mutually exclusive** with `routing.rules` and +`routing.classifiers` - never emit both. + +## Step 1 - Extract from the user's words + +- **Candidates**: the models that may *answer* requests. Verbatim model names + (e.g. `Gemma-3-4b-it-GGUF`). If the user names none, ask - never invent + model names. `lemonade list` or `GET /api/v1/models` shows what's available. +- **Default / fallback**: which candidate gets everything that matches nothing. + If unstated, use the model the user framed as "local", "small", or "safe"; + otherwise the first candidate mentioned. +- **Signals**: every condition mentioned (keywords, patterns, length, images, + tools, topics, PII, safety) and which model each one routes to. +- **Classifier models**: models named for *detection* rather than answering + (BERT-style encoders, embedding models, an LLM used as judge). + +## Step 2 - Scaffold + +Always exactly this envelope (the parser rejects unknown or missing keys): + +```json +{ + "version": "1", + "model_name": "user.MyHybridRouter", + "recipe": "collection.router", + "components": [], + "routing": { } +} +``` + +- `version` is the literal string `"1"`. +- `model_name` must start with `user.`; slug from the user's description if + they gave a name (`user.` using only `[A-Za-z0-9._-]`). If they didn't + name it, derive one from context instead of a fixed literal - e.g. + `user.-Router` - so two different policies don't + collide by default. **`/pull` is idempotent per `model_name`: registering a + second policy under the same name silently overwrites the first.** If this + conversation already produced an unnamed router, don't reuse the same + derived name for the next one - ask, or pick a visibly different name. + +## Step 3 - Candidates and default + +```json +"candidates": [""], +"default_model": "" +``` + +`default_model` MUST be listed in `candidates`. Candidates should be +chat-capable LLMs - not embedding, classification, or image models. + +## Step 4 - Mode A: LLM-as-router + +```json +"router": { + "type": "llm", + "model": "", + "prompt": "You route user requests to the best model. ." +} +``` + +- `model` defaults to the smallest candidate (it may be a candidate; it also + works as a separate small model). +- **Write intent only - never specify a reply format.** The engine + unconditionally appends its own contract after your prompt: it lists the + candidate names and demands a strict JSON reply `{"model": "", + "rationale": ""}`, then falls back to `default_model` on any + deviation. A prompt that also says "reply with ONLY the model name" (or + similar) is not just redundant, it's wrong about the wire format and can + confuse weaker judge models into replying with a bare string that then + fails to parse - silently falling back to `default_model` on every request, + with no visible error. Only describe *when to pick which candidate*. +- Emit **no** `rules` and **no** `classifiers` key in this mode. + +## Step 5 - Mode B: classifiers + +Only declare classifiers the rules actually reference. Three types: + +```json +{ "id": "clf-1", "type": "classifier", "model": "", + "labels": ["PII", "Jailbreak"], "default_label": "PII", "on_error": "match_false" } + +{ "id": "clf-2", "type": "semantic_similarity", "model": "", + "reference_phrases": { "shopping": ["I want to shop for pants", "add to cart"] }, + "default_label": "shopping", "on_error": "match_false" } + +{ "id": "clf-3", "type": "llm", "model": "", + "prompt": "Classify the request into only labels SAFE, RISKY", + "labels": ["SAFE", "RISKY"], "default_label": "SAFE", "on_error": "match_false" } +``` + +Hard constraints (parser-enforced - see `reference.md` for the full matrix): + +- `classifier` type: model should be a text-classification model (an + `onnxruntime` encoder like `Bert-Phishing-ONNX`); `labels` must match the + model's actual output labels. A chat LLM here is legal (LLM-as-classifier + via chat) but prefer `type: "llm"` for that - it is explicit and prompted. +- `semantic_similarity`: `reference_phrases` is `{concept: [phrases...]}`, + at least one concept, each with at least one phrase. Concept names ARE the + labels - a `labels` key is **rejected** for this type. Model must be an + embedding model. Give 3–5 varied phrases per concept when inventing them. +- `llm`: `prompt` AND non-empty `labels` are both required. **Write intent + only - never tell the model how to format its reply.** The engine appends + its own `{"model": "", "rationale": "..."}` contract after + your prompt (the same contract as `routing.router`). An authored line like + "Reply with exactly one label: SAFE or RISKY" causes weaker models to output + bare `SAFE`, which the parser rejects - the score comes back empty and the + rule silently never fires. Describe what makes a request belong to each + label; leave the reply format to the engine. +- `default_label`, when present, must be one of the labels/concepts. +- Defaults when unspecified: `id` = `clf-1`, `clf-2`, …; `on_error` = + `"match_false"` (fail-open: a broken classifier doesn't match, so requests + fall through - use `"match_true"` only when the user wants fail-closed + safety); `default_label` = the first label. + +## Step 6 - Mode B: rules + +```json +"rules": [ + { "id": "rule-1", "match": { ... }, "route_to": "", + "outputs": { "reason": "" } } +] +``` + +- **Order matters - first match wins.** Put the most specific / + privacy-critical rules first (a "sensitive stays local" rule must precede a + "code goes to the big model" rule, or coding prompts with PII leak). +- `route_to` MUST be a candidate. `id` uses only `[A-Za-z0-9._-]`; default + `rule-1`, `rule-2`, …. +- No rule for the "everything else" case - that is `default_model`. + +**Match conditions** - combine with `all` (AND), `any` (OR), `not`; one +condition per leaf object; nesting is allowed: + +| Leaf | Example | Notes | +|------|---------|-------| +| `keywords_any` / `keywords_all` | `{ "keywords_any": ["SSN", "Email"] }` | case-insensitive substring - `"hi"` matches inside `"this"`, `"shipping"`, `"high"`, etc. Use `regex` with `\b...\b` when word-boundary precision is needed | +| `regex` | `{ "regex": "\\b\\d{3}-?\\d{2}-?\\d{4}\\b" }` | ECMAScript flavor | +| `min_chars` / `max_chars` | `{ "min_chars": 4000 }` | input length, UTF-8 bytes, non-negative integer | +| `has_tools` / `has_images` | `{ "has_images": true }` | booleans | +| `classifier` | `{ "classifier": "clf-1", "label": "PII", "min_score": 0.5 }` | band test; `min_score`/`max_score` in [0,1]; default `min_score` 0.5; omit `label` only if the classifier has `default_label` | +| `metadata` | `{ "metadata": { "key": "consent", "equals": "denied" } }` | exactly one of `equals` / `any` / `exists`; note: not editable in the desktop UI yet - use only when the user asks for metadata routing | + +## Step 7 - Components + +`components` = union of: all `candidates` + every classifier `model` + the +`router.model` (Mode A). Deduplicate, keep order stable. The parser rejects +any referenced model that is not declared here. + +## Step 8 - Validate + +Write the JSON to a file, then **run the bundled offline validator** before +presenting anything to the user - it mechanically re-checks every rule in +`reference.md`'s validation checklist (score ranges, non-negative integer +char counts, mode exclusivity, label/classifier references, the `model_name` +collision, the router-prompt-contract mistake above, etc.) so you don't have +to re-derive them by eye every time: + +```bash +python scripts/validate.py router.json # Windows +python3 scripts/validate.py router.json # macOS/Linux +``` + +It exits 0 with `"ready": true` when there are no errors (warnings/advisories +may still be worth mentioning to the user). If it reports errors, fix the +JSON and re-run - don't present JSON that fails this check. It is pure +offline structural/numeric validation; it cannot verify a named model +actually exists or has the right capability (chat/embedding/classification). + +## Step 9 - Output curl commands + +**This step is mandatory — do not skip it even if the user did not ask.** + +Do not call the live server yourself — not even to check models or register +the policy. Instead, print the three curl commands below as plain text for +the user to copy and run. Fill in `` and `` from the +policy, and a short `` that should hit the first rule. Do not +execute these commands with Bash or any other tool. + +```bash +# 1. Check a model exists before registering +curl http://localhost:13305/api/v1/models/ + +# 2. Register the policy (idempotent - re-POST to update) +curl -X POST http://localhost:13305/api/v1/pull \ + -H "Content-Type: application/json" --data-binary @router.json + +# 3. Route a request and inspect the decision +curl -X POST http://localhost:13305/api/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "", "route_trace": true, + "messages": [{"role": "user", "content": ""}]}' +``` + +The `x-lemonade-route` response header carries the matched rule id (or +`default`). With `"route_trace": true` the body also carries +`x_lemonade_route`: `{ route_to, matched_rule, default_used, outputs, +trace[] }` - useful for verifying each rule fires as expected. + +## Defaults summary + +| Field | Default when the user doesn't say | +|-------|-----------------------------------| +| `model_name` | `user.-Router` (never reuse a name already used earlier in this conversation) | +| `default_model` | the "small/local/safe" candidate, else first mentioned | +| mode | rules if any concrete signal is named, else LLM-as-router | +| classifier `id` / rule `id` | `clf-N` / `rule-N` | +| `on_error` | `match_false` | +| `default_label` | first label / concept | +| `min_score` | `0.5` | +| `outputs` | omit | +| router prompt | intent only - no reply-format instruction (Step 4) | + +Worked NL → JSON pairs live in `examples.md`; the full schema, parser error +matrix, and model-capability table live in `reference.md`; the offline +validator is `scripts/validate.py` (run it - see Steps 8–9). diff --git a/skills/build-lemonade-router/evals/evals.py b/skills/build-lemonade-router/evals/evals.py new file mode 100644 index 0000000..06454cc --- /dev/null +++ b/skills/build-lemonade-router/evals/evals.py @@ -0,0 +1,111 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# See LICENSE for license information. + +"""Behavioral tests for the `build-lemonade-router` skill. + +Run locally (needs the `claude` CLI authenticated): + + cd eval/behavioral + python -m pytest -c pytest.ini -p conftest ../../skills/build-lemonade-router/evals/evals.py + +Each check on `run` prints a `[PASS]`/`[FAIL]` line and raises on failure. +`logs_contains` / `workspace_contains` are deterministic; `should` / +`should_not` are graded by an LLM judge over the captured evidence. +""" + +from harness import claude + + +def test_keyword_router_generation(): + """Trigger: user asks to route coding questions to a bigger model.""" + agent_configs = [(claude, "opus")] + for agent, model in agent_configs: + with agent(model, skill="build-lemonade-router") as run: + run = run.prompt( + "Route coding questions - anything mentioning functions, bugs, " + "or stack traces - to Qwen3.5-9B-GGUF. Everything else goes to " + "Qwen3.5-2B-GGUF." + ) + + # Deterministic: skill must produce a JSON file and validate it + run.logs_contains("build-lemonade-router") + run.workspace_contains("router.json") + + # Positive behavioral expectations + run.should("Produce a collection.router JSON with recipe collection.router") + run.should("Include keywords_any condition matching coding-related terms") + run.should("Set Qwen3.5-2B-GGUF as default_model") + run.should("Run the offline validator scripts/validate.py and report it passed") + run.should("Output curl commands for the user to register and test the policy") + + # Negative behavioral expectations + run.should_not("Execute POST /api/v1/pull by making an actual HTTP call with a tool") + run.should_not("Invent model names not provided by the user") + run.should_not("Include both router and rules keys in the same policy") + + +def test_pii_regex_router_generation(): + """Trigger: user asks to keep PII local using regex patterns.""" + agent_configs = [(claude, "opus")] + for agent, model in agent_configs: + with agent(model, skill="build-lemonade-router") as run: + run = run.prompt( + "Any message with a Social Security number or email address must " + "stay on Qwen3.5-9B-GGUF. Everything else can go to " + "Qwen3.5-9B-NoThinking." + ) + + run.logs_contains("build-lemonade-router") + run.workspace_contains("router.json") + + run.should("Include a regex condition matching SSN patterns") + run.should("Include a regex condition matching email addresses") + run.should("Place the PII rule before any other rules") + run.should("Run the offline validator and confirm the JSON is ready") + + run.should_not("Execute any live Lemonade server API call by making an actual HTTP call with a tool") + run.should_not("Use routing.router instead of routing.rules for this request") + + +def test_llm_as_router_generation(): + """Trigger: user describes intent only by meaning with no concrete signals.""" + agent_configs = [(claude, "opus")] + for agent, model in agent_configs: + with agent(model, skill="build-lemonade-router") as run: + run = run.prompt( + "I want sensitive queries to go to Qwen3.5-9B-GGUF and everything " + "else to Qwen3.5-9B-NoThinking. Use the local model as the router." + ) + + run.logs_contains("build-lemonade-router") + run.workspace_contains("router.json") + + run.should("Use routing.router block instead of routing.rules") + run.should("Set type to llm inside the router block") + run.should("Write a prompt that describes routing intent only, not reply format") + run.should_not("Include both routing.router and routing.rules in the output") + run.should_not("Tell the router model to reply with only the model name") + + +def test_non_trigger_general_question(): + """Non-trigger: general question unrelated to routing should not activate skill.""" + agent_configs = [(claude, "opus")] + for agent, model in agent_configs: + with agent(model, skill="build-lemonade-router") as run: + run = run.prompt("What is the capital of France?") + + run.should_not("Produce a collection.router JSON") + run.should_not("Run scripts/validate.py") + run.should_not("Mention routing.rules or routing.candidates") + + +def test_non_trigger_unrelated_code_task(): + """Non-trigger: coding task unrelated to Lemonade routing.""" + agent_configs = [(claude, "opus")] + for agent, model in agent_configs: + with agent(model, skill="build-lemonade-router") as run: + run = run.prompt("Write a Python function to reverse a linked list.") + + run.should_not("Generate a collection.router policy") + run.should_not("Ask about Lemonade model candidates") diff --git a/skills/build-lemonade-router/examples.md b/skills/build-lemonade-router/examples.md new file mode 100644 index 0000000..9bdc050 --- /dev/null +++ b/skills/build-lemonade-router/examples.md @@ -0,0 +1,472 @@ +# NL → router policy examples + +Each pair shows a user's natural-language request and the exact JSON the skill +should produce. Note what was defaulted: names, ids, thresholds, `on_error`, +`default_label`, and `model_name` all come from the defaults table in +`SKILL.md` when the user doesn't specify them - including `model_name`, which +is derived from the default candidate rather than a fixed literal, so the +examples below each get a distinct name. + + +## 1. Pure intent, no concrete signals → LLM-as-router + +> I want to route my sensitive queries to Gemma-3-4b-it-GGUF and everything +> else to Gemma-4-E4B-it-GGUF + +"Sensitive" is meaning, not a mechanical signal → Mode A. The small candidate +doubles as the router model and as `default_model` (fail-safe: on any router +hiccup, requests stay on the model the user trusts with sensitive data). + +```json +{ + "version": "1", + "model_name": "user.Gemma-3-4b-it-GGUF-Router", + "recipe": "collection.router", + "components": ["Gemma-3-4b-it-GGUF", "Gemma-4-E4B-it-GGUF"], + "routing": { + "candidates": ["Gemma-3-4b-it-GGUF", "Gemma-4-E4B-it-GGUF"], + "default_model": "Gemma-3-4b-it-GGUF", + "router": { + "type": "llm", + "model": "Gemma-3-4b-it-GGUF", + "prompt": "You route user requests to the best model. Prompts with sensitive information route to Gemma-3-4b-it-GGUF and everything else to Gemma-4-E4B-it-GGUF." + } + } +} +``` + +Note what's *not* in the prompt: no "reply with only the model name" or any +other reply-format instruction. The engine appends its own strict JSON +contract (`{"model": ..., "rationale": ...}`, listing the exact candidate +names) after whatever the author writes - an authored format instruction +would only contradict it. See `reference.md`'s +[Validation checklist](reference.md#validation-checklist) item 12. + +## 2. Concrete signals, no classifiers → deterministic rules + +> Send coding questions or anything longer than 4000 characters to +> Qwen3-32B-GGUF, requests with images stay on Qwen3-8B-GGUF, default to +> Qwen3-8B-GGUF + +Keywords/length/images are mechanical → Mode B, no classifiers needed. The +images rule is placed first (more specific; keeps image requests local even +when they are long or mention code). + +```json +{ + "version": "1", + "model_name": "user.Qwen3-8B-GGUF-Router", + "recipe": "collection.router", + "components": ["Qwen3-8B-GGUF", "Qwen3-32B-GGUF"], + "routing": { + "candidates": ["Qwen3-8B-GGUF", "Qwen3-32B-GGUF"], + "default_model": "Qwen3-8B-GGUF", + "rules": [ + { + "id": "rule-1", + "match": { "has_images": true }, + "route_to": "Qwen3-8B-GGUF", + "outputs": { "reason": "images-stay-local" } + }, + { + "id": "rule-2", + "match": { + "any": [ + { "keywords_any": ["def ", "function", "stack trace", "compile"] }, + { "min_chars": 4000 } + ] + }, + "route_to": "Qwen3-32B-GGUF", + "outputs": { "reason": "coding-or-long" } + } + ] + } +} +``` + +## 3. Classifiers + nested conditions → full rules mode + +> Route requests containing personally identifiable information (PII), such as +> Social Security numbers or email addresses, to Gemma-3-4b-it-GGUF. Detect +> PII using both classification and pattern matching where appropriate. Also +> send requests that involve tools or images together with PII to the same +> model. Use Bert-Phishing-ONNX to identify PII and jailbreak attempts. Use +> nomic-embed-text-v2-moe-GGUF to recognize shopping- and apparel-related +> requests through semantic similarity. For requests related to clothing or +> apparel that include images, use the semantic classifier together with +> additional context, such as an LLM safety assessment or the length of the +> request, before routing. Use Gemma-3-4b-it-GGUF as the LLM classifier for +> SAFE and RISKY classifications. If none of the routing rules apply, or if a +> classifier fails, fall back to Gemma-3-4b-it-GGUF. + +All three classifier types, nested `any` inside `all`, an SSN regex, and +"classifier fails → fall back" mapping to `on_error: match_false` + +`default_model`. Components include the classifier models even though they +never answer requests. + +```json +{ + "version": "1", + "model_name": "user.Gemma-3-4b-it-GGUF-PII-Router", + "recipe": "collection.router", + "components": [ + "Gemma-3-4b-it-GGUF", + "Gemma-4-E4B-it-GGUF", + "Bert-Phishing-ONNX", + "nomic-embed-text-v2-moe-GGUF" + ], + "routing": { + "candidates": ["Gemma-3-4b-it-GGUF", "Gemma-4-E4B-it-GGUF"], + "default_model": "Gemma-3-4b-it-GGUF", + "classifiers": [ + { + "id": "clf-1", + "type": "classifier", + "model": "Bert-Phishing-ONNX", + "labels": ["PII", "Jailbreak"], + "default_label": "PII", + "on_error": "match_false" + }, + { + "id": "clf-2", + "type": "semantic_similarity", + "model": "nomic-embed-text-v2-moe-GGUF", + "reference_phrases": { + "shopping": [ + "I want to shop for pants", + "find me a jacket in medium", + "add these shoes to my cart" + ] + }, + "default_label": "shopping", + "on_error": "match_false" + }, + { + "id": "clf-3", + "type": "llm", + "model": "Gemma-3-4b-it-GGUF", + "prompt": "Classify the request into only labels SAFE, RISKY", + "labels": ["SAFE", "RISKY"], + "default_label": "SAFE", + "on_error": "match_false" + } + ], + "rules": [ + { + "id": "rule-1", + "match": { + "all": [ + { "classifier": "clf-1", "min_score": 0.5 }, + { "keywords_any": ["SSN", "Email"] }, + { "has_tools": true }, + { + "any": [ + { "regex": "\\b\\d{3}-?\\d{2}-?\\d{4}\\b" }, + { "has_images": true } + ] + } + ] + }, + "route_to": "Gemma-3-4b-it-GGUF" + }, + { + "id": "rule-2", + "match": { + "all": [ + { "classifier": "clf-2", "min_score": 0.5 }, + { "keywords_any": ["apparel", "clothes", "clothing"] }, + { "has_images": true }, + { + "any": [ + { "classifier": "clf-3", "min_score": 0.5 }, + { "min_chars": 500 } + ] + } + ] + }, + "route_to": "Gemma-3-4b-it-GGUF" + } + ] + } +} +``` + +## 4. Negation and metadata opt-out + +> Anything without tool calls goes to Phi-4-mini-GGUF; if the request metadata +> has consent=denied it must stay on Phi-4-mini-GGUF no matter what; the rest +> to Qwen3-32B-GGUF + +```json +{ + "version": "1", + "model_name": "user.Phi-4-mini-GGUF-Router", + "recipe": "collection.router", + "components": ["Phi-4-mini-GGUF", "Qwen3-32B-GGUF"], + "routing": { + "candidates": ["Phi-4-mini-GGUF", "Qwen3-32B-GGUF"], + "default_model": "Qwen3-32B-GGUF", + "rules": [ + { + "id": "rule-1", + "match": { "metadata": { "key": "consent", "equals": "denied" } }, + "route_to": "Phi-4-mini-GGUF", + "outputs": { "reason": "privacy" } + }, + { + "id": "rule-2", + "match": { "not": { "has_tools": true } }, + "route_to": "Phi-4-mini-GGUF" + } + ] + } +} +``` + +Tell the user: the metadata rule is honored by the server but is not yet +editable in the desktop Hybrid Router editor (it will warn about a lossy edit +if they open this policy there). + +--- + +## 5. HR chatbot - LLM-as-router with a privacy-first prompt + +> Build an HR assistant router. Any request with PII - names with salaries, +> SSNs, bank accounts, equity details, dates of birth - must stay on the local +> model. Everything else can go to the cloud model. When in doubt, keep it +> local. + +"PII" is a meaning judgment, not a regex - the boundary is fuzzy and context- +dependent (a name alone is fine; a name + salary is not). That ambiguity is +exactly what Mode A is designed for: a small LLM reads each request and +decides. The router model doubles as `default_model` so a router hiccup never +leaks to the cloud. + +The prompt describes *when* to pick each candidate. It does **not** tell the +model how to format its reply - the engine appends its own `{"model": ..., +"rationale": ...}` contract. + +```json +{ + "version": "1", + "model_name": "user.HR-Admin-Router", + "recipe": "collection.router", + "components": ["Qwen3.5-9B-GGUF", "fireworks.kimi-k2p6"], + "routing": { + "candidates": ["Qwen3.5-9B-GGUF", "fireworks.kimi-k2p6"], + "default_model": "Qwen3.5-9B-GGUF", + "router": { + "type": "llm", + "model": "Qwen3.5-9B-GGUF", + "prompt": "You are a routing assistant for an AI company. Your job is to choose which model should handle each request.\n\nUse Qwen3.5-9B-GGUF (local, private) when:\n- The request contains personally identifiable information (PII), such as names with salaries, Social Security numbers (SSNs), bank account numbers, email addresses, compensation data, equity details, or dates of birth.\n- Data privacy is paramount - anything that should never leave the local machine.\n\nUse fireworks.kimi-k2p6 (cloud, powerful) for all other requests.\n\nIf the request is ambiguous, default to Qwen3.5-9B-GGUF. When in doubt, prioritize privacy over capability." + } + } +} +``` + +**When to prefer Mode A over regex for PII**: a regex catches `123-45-6789` +but misses "Alice's salary is sixty thousand". If the domain has natural- +language PII, Mode A catches it; if the domain has structured PII (form +inputs, database exports), regex is more reliable and cheaper (no extra LLM +call per request). Use both together when both forms appear - put regex rules +first (cheaper, no latency), then fall through to an LLM router or classifier +for the fuzzy residual. + +--- + +## 6. Benefits chatbot - three-tier routing with max_chars and rich outputs + +> Route a benefits chatbot. Any request containing PII (email, salary, SSN, +> equity, compensation, bank account, date of birth) must stay local on +> `Qwen3.5-9B-GGUF`. Complex analysis (comparison, benchmarking, +> optimization, or anything longer than 800 characters) goes to +> `fireworks.kimi-k2p6`. Short, simple benefit lookups (401k, PTO, healthcare, +> open enrollment, etc. under 400 characters) also stay local. Default to +> cloud for anything else. + +Three tiers: **PII fence** first (privacy-critical, `on_error: match_true` +would be appropriate but this config uses the default fail-open for +simplicity), **complexity escalation** second, **domain fast-path** third. +`outputs` carries two fields (`reason` + `data_class`/`tier`) - useful for +downstream logging and observability. + +New patterns not shown in examples 1–4: +- `max_chars` to cap a rule to *short* prompts only +- An `all` combining `max_chars` + `keywords_any` (domain keyword on a short + prompt = cheap local RAG; same keyword on a long prompt falls through to + cloud) +- Multiple key/value pairs in `outputs` for structured tagging + +```json +{ + "version": "1", + "model_name": "user.Benefits-Router", + "recipe": "collection.router", + "components": ["Qwen3.5-9B-GGUF", "fireworks.kimi-k2p6"], + "routing": { + "candidates": ["Qwen3.5-9B-GGUF", "fireworks.kimi-k2p6"], + "default_model": "fireworks.kimi-k2p6", + "rules": [ + { + "id": "pii-stays-local", + "match": { + "any": [ + { "regex": "[A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z]{2,}" }, + { "regex": "\\$[0-9,]+(K|k|M|m)?\\b" }, + { "regex": "\\b[0-9]{3}-[0-9]{2}-[0-9]{4}\\b" }, + { "keywords_any": ["salary", "equity", "ssn", "social security", "bank account", "date of birth", "compensation"] } + ] + }, + "route_to": "Qwen3.5-9B-GGUF", + "outputs": { "reason": "pii-detected", "data_class": "sensitive" } + }, + { + "id": "complex-benefits-analysis", + "match": { + "any": [ + { "keywords_any": ["compare", "benchmark", "analyze", "optimize", "recommend", "industry standard", "strategy"] }, + { "min_chars": 800 } + ] + }, + "route_to": "fireworks.kimi-k2p6", + "outputs": { "reason": "complex-benefits-analysis", "tier": "cloud" } + }, + { + "id": "simple-benefits-rag", + "match": { + "all": [ + { "max_chars": 400 }, + { "keywords_any": ["401k", "401(k)", "vesting", "parental leave", "PTO", "vacation", "healthcare", "dental", "vision", "FSA", "HSA", "COBRA", "open enrollment", "qualifying life event", "copay", "deductible", "premium", "handbook", "policy", "onboard"] } + ] + }, + "route_to": "Qwen3.5-9B-GGUF", + "outputs": { "reason": "simple-benefits-rag", "tier": "local-fast" } + } + ] + } +} +``` + +**Rule ordering matters here**: `pii-stays-local` fires first. A message like +"what's the copay on my plan - my salary is $95K" would match both rule 1 +(salary keyword) and rule 3 (copay + short). Because rule 1 comes first, it +routes local - correct. Reordering would leak PII to the cloud. + +--- + +## 7. Finance chatbot - semantic similarity + LLM classifier in tandem + +> Build a finance assistant router for a startup. PII-flavored finance queries +> (individual salaries, equity by person, payroll details) stay on the local +> model. Deep modeling requests (Monte Carlo, cap table, cohort forecasting, +> sensitivity analysis) go to the cloud. Simple metric lookups (burn rate, MRR, +> runway, ARR) stay local. As a fallback, use a local LLM to judge whether a +> request is COMPLEX or SIMPLE, routing COMPLEX to cloud. Default to local. + +The most advanced pattern: **two classifiers working in tandem** - a fast +embedding classifier (`semantic_similarity`) fires first to catch known +patterns, then an LLM classifier acts as a catch-all judge for requests that +didn't match any semantic bucket. Four rules, two classifier types, three +score thresholds tuned independently per rule. + +What's unique here not shown elsewhere: +- Two classifiers declared; rules reference each independently +- `semantic_similarity` with three distinct concept buckets (not just two) +- An `llm` classifier used as a safety net *after* semantic matching, not as + the primary signal - keeps latency low for the common case +- Per-rule score thresholds tuned to the domain (0.65 for PII, 0.72 for deep + modeling, 0.60 for simple lookups) rather than the default 0.5 +- `outputs` carries a `classifier` field to tell downstream code *which* + classifier fired + +```json +{ + "version": "1", + "model_name": "user.Finance-Router", + "recipe": "collection.router", + "components": [ + "fireworks.kimi-k2p6", + "Qwen3.5-9B-GGUF", + "embeddinggemma-300m-qat-q8_0-GGUF-Q8_0" + ], + "routing": { + "candidates": ["Qwen3.5-9B-GGUF", "fireworks.kimi-k2p6"], + "default_model": "Qwen3.5-9B-GGUF", + "classifiers": [ + { + "id": "finance-topic", + "type": "semantic_similarity", + "model": "embeddinggemma-300m-qat-q8_0-GGUF-Q8_0", + "reference_phrases": { + "pii-finance": [ + "show me the salary breakdown by employee", + "what is the equity package for this specific person", + "list all compensation by individual", + "employee bank account for payroll", + "individual 401k contribution amounts", + "personal compensation details for staff member" + ], + "simple-lookup": [ + "what is our current burn rate", + "what is the MRR this month", + "how much runway do we have left", + "what is the current ARR", + "what is today's headcount", + "how much did we spend last quarter" + ], + "deep-modeling": [ + "run a Monte Carlo simulation on our runway", + "model the cap table after the next funding round", + "forecast revenue using cohort analysis", + "calculate waterfall distribution for an exit scenario", + "build a sensitivity analysis for burn rate assumptions", + "recommend an intervention strategy for high churn and retention", + "analyze cap table dilution under different term sheet scenarios" + ] + } + }, + { + "id": "complexity", + "type": "llm", + "model": "Qwen3.5-9B-GGUF", + "prompt": "You are a financial complexity classifier. Assess whether this finance request requires deep multi-step modeling and cloud-level reasoning, or is a simple metric lookup that a local model can handle.\n\nClassify as COMPLEX if the request involves: statistical modeling, simulations, multi-variable forecasting, cohort analysis, cap table calculations, or synthesizing multiple data sources.\n\nClassify as SIMPLE if the request is a direct data lookup, a single metric question, or a short factual query.", + "labels": ["COMPLEX", "SIMPLE"], + "default_label": "SIMPLE", + "on_error": "match_false" + } + ], + "rules": [ + { + "id": "pii-finance-semantic", + "match": { "classifier": "finance-topic", "label": "pii-finance", "min_score": 0.65 }, + "route_to": "Qwen3.5-9B-GGUF", + "outputs": { "reason": "pii-semantic-finance", "data_class": "sensitive", "classifier": "semantic_similarity" } + }, + { + "id": "deep-model-semantic", + "match": { "classifier": "finance-topic", "label": "deep-modeling", "min_score": 0.72 }, + "route_to": "fireworks.kimi-k2p6", + "outputs": { "reason": "deep-finance-model-semantic", "tier": "cloud", "classifier": "semantic_similarity" } + }, + { + "id": "deep-model-llm-judge", + "match": { "classifier": "complexity", "label": "COMPLEX", "min_score": 0.5 }, + "route_to": "fireworks.kimi-k2p6", + "outputs": { "reason": "llm-judged-complex-finance", "tier": "cloud", "classifier": "llm" } + }, + { + "id": "simple-metric-lookup", + "match": { "classifier": "finance-topic", "label": "simple-lookup", "min_score": 0.60 }, + "route_to": "Qwen3.5-9B-GGUF", + "outputs": { "reason": "simple-metric-lookup", "tier": "local-fast", "classifier": "semantic_similarity" } + } + ] + } +} +``` + +**Tuning guidance for `reference_phrases`**: aim for 5–8 varied phrases per +concept covering different vocabulary, register, and specificity. Phrases that +are too similar to each other (all formal, all the same length) produce a +tight cluster that fails on paraphrases. Include at least one informal and one +technical phrasing per concept. diff --git a/skills/build-lemonade-router/reference.md b/skills/build-lemonade-router/reference.md new file mode 100644 index 0000000..7a4e491 --- /dev/null +++ b/skills/build-lemonade-router/reference.md @@ -0,0 +1,206 @@ +# Lemonade Router Config: Reference + +Detailed contract for the `build-lemonade-router` skill. Read this when the +generation steps in `SKILL.md` leave a question open. The server-side parser +(`routing_policy_parser.cpp`) is strict: it rejects unknown keys at every +level, so never emit fields not listed here. + +## Contents + +- [Root document](#root-document) +- [routing block](#routing-block) +- [Classifier entries](#classifier-entries) +- [Rules and match expressions](#rules-and-match-expressions) +- [Model capability requirements](#model-capability-requirements) +- [Validation checklist](#validation-checklist) +- [Registering, invoking, tracing](#registering-invoking-tracing) +- [Desktop-editor compatibility](#desktop-editor-compatibility) + +--- + +## Root document + +Allowed keys: `version`, `model_name`, `recipe`, `components`, `models`, +`routing`. + +| Key | Required | Constraint | +|-----|----------|-----------| +| `version` | yes | literal `"1"` (string, not number) | +| `model_name` | yes | non-empty; prefix `user.`; chars `[A-Za-z0-9._-]` after the prefix | +| `recipe` | yes | literal `"collection.router"` | +| `components` | yes | non-empty array; every model the policy references | +| `models` | no | embedded component definitions (produced by exports; don't generate) | +| `routing` | yes | object, see below | + +## routing block + +Allowed keys: `candidates`, `default_model`, `router`, `classifiers`, `rules`. + +| Key | Required | Constraint | +|-----|----------|-----------| +| `candidates` | yes | non-empty array of unique names; each declared in `components` | +| `default_model` | yes | must be listed in `candidates` | +| `router` | either this or `rules` | `{ "type": "llm", "model": , "prompt": }` - no other keys. Mutually exclusive with `rules` AND `classifiers`. Desugars server-side into one llm classifier + identity rules. | +| `classifiers` | no (rules mode only) | array; every entry must be referenced by some rule ideally | +| `rules` | either this or `router` | non-empty array; first match wins | + +## Classifier entries + +Allowed keys: `id`, `type`, `model`, `prompt`, `labels`, `default_label`, +`reference_phrases`, `on_error`. + +| Field | `classifier` | `semantic_similarity` | `llm` | +|-------|--------------|----------------------|-------| +| `id` | required, unique | required, unique | required, unique | +| `model` | required | required (embedding model) | required (chat LLM) | +| `prompt` | - | - | **required**, non-empty | +| `labels` | optional (must match the model's real output labels) | **rejected** (concepts are the labels) | **required**, ≥ 1 entry | +| `reference_phrases` | - | **required**: `{concept: [phrase, ...]}`, ≥1 concept, ≥1 phrase each | - | +| `default_label` | optional; must be in `labels`; requires `labels` non-empty | optional; must be a concept name | optional; must be in `labels` | +| `on_error` | `"match_false"` (default) or `"match_true"` | same | same | + +Notes: + +- `on_error: match_false` = fail-open (classifier failure → condition doesn't + match → request falls through, usually to `default_model`). + `match_true` = fail-closed (failure counts as a match - for safety rules + where "can't check" should route to the restricted/local path). +- A label-less `classifier` entry is legal (single-score models); its rule + leaves then must not name a `label`. +- Scores are `{label: score}` with each score in `[0, 1]`. +- **`llm` classifier wire contract** - the engine appends its own JSON reply + contract after every `llm` classifier prompt, exactly as it does for + `routing.router`. The model must reply `{"model": "", + "rationale": ""}`. The engine then scores the chosen label + `1.0` and all other labels `0.0`. **Never tell the model how to format its + reply** - an authored instruction like "Reply with exactly one label: X or Y" + causes weaker models to output bare `X`, the parser rejects it, the score + comes back empty, and the rule silently never fires. Describe classification + intent only (what makes a request COMPLEX vs SIMPLE, SAFE vs RISKY, etc.). + +## Rules and match expressions + +Rule keys: `id`, `match`, `route_to`, `outputs`. + +- `id`: required, unique, charset `[A-Za-z0-9._-]`. +- `route_to`: required, must be a candidate. +- `outputs`: optional object, copied verbatim into the routing decision - the + engine never interprets it. Use for human-readable reasons or downstream + tags (`{"reason": "privacy"}`, `{"route_category": "cloud"}`). +- `match`: a match expression. + +Match-expression grammar: + +- Logical node: exactly ONE of `all: [...]`, `any: [...]`, `not: {...}` and + nothing else in that object. Arrays non-empty. Nesting allowed (depth ≤ 64). +- Leaf node: one or more condition keys (multiple keys in one leaf are an + implicit AND, but prefer one condition per leaf - see + [Desktop-editor compatibility](#desktop-editor-compatibility)). + +| Condition | Value | Semantics | +|-----------|-------|-----------| +| `keywords_any` | non-empty array of non-empty strings | case-insensitive substring, any - `"hi"` also matches inside `"this"`, `"shipping"`, `"high"`, etc. Use `regex` `\b...\b` for word-boundary matching | +| `keywords_all` | same | all must appear; same substring semantics as `keywords_any` | +| `regex` | non-empty string | ECMAScript regex over the input text | +| `min_chars` / `max_chars` | integer ≥ 0 | input length in **UTF-8 bytes** | +| `has_tools` / `has_images` | boolean | request carries `tools[]` / image parts. `false` is legal and means "must NOT have"; `{"not": {"has_tools": true}}` is the equivalent preferred spelling | +| `classifier` | classifier id (string) | band test on that classifier's score | +| `label` | string | only with `classifier`; must exist on that classifier; may be omitted only if the classifier has `default_label` | +| `min_score` / `max_score` | number in [0,1], min ≤ max | only with `classifier`; omitting both defaults to `min_score: 0.5` | +| `metadata` | `{ "key": , ... }` | plus exactly one of `equals` (string), `any` (non-empty string array), `exists` (boolean); evaluated over the request's OpenAI `metadata` object. Comparisons are case-sensitive. `any` comma-decodes the metadata value before matching: a caller sending `{"dept": "legal,compliance"}` (scalar) matches `any: ["legal", "compliance"]` - same as if two separate values were in the array. | + +The routed input text is the last user message (chat), the `prompt` +(completions), or the `input` (responses). + +## Model capability requirements + +Checked at registration time - a mismatch fails the `/pull`: + +| Role | Needs | Typical models | +|------|-------|----------------| +| candidate / `router.model` / `llm` classifier | chat-capable LLM | `*-GGUF` chat models, cloud models | +| `semantic_similarity` model | embeddings | `nomic-embed-text-*-GGUF` | +| `classifier` model | classification output (onnxruntime encoder) - or a chat LLM (LLM-as-classifier via chat; prefer `type: "llm"`) | `Bert-Phishing-ONNX`, `Phishing-Email-Detection-ONNX` | + +Cloud candidates (`fireworks.` etc.) are valid but only exist after the +provider is installed and authenticated - tell the user to run +`lemonade cloud install ` + auth **before** registering the policy. + +## Validation checklist + +Every generated policy must pass all of these before it is shown to the user. +Items 1–9 are mechanically enforced by `scripts/validate.py` (see `SKILL.md` +Step 8) - run it rather than re-deriving these by eye. Items 10–12 need +authoring judgment and aren't (fully) checkable by a script. + +1. Root has exactly `version "1"`, `model_name` (`user.` prefix), `recipe + "collection.router"`, non-empty `components`, `routing`. +2. `routing` has `candidates` (non-empty, unique) and `default_model` ∈ + `candidates`. +3. Exactly one of `router` / `rules` present; `classifiers` only with `rules`. +4. `components` ⊇ candidates ∪ classifier models ∪ router model. +5. Every rule: unique safe `id`, `route_to` ∈ candidates, non-empty `match`. +6. Every logical node has a single key; every leaf key is in the condition + table; no invented keys anywhere. +7. Every `classifier` leaf references a declared classifier id; its `label` + exists on that classifier, or is omitted and the classifier has + `default_label` (or is label-less). +8. `llm` classifiers have `prompt` + `labels`; `semantic_similarity` have + `reference_phrases` and NO `labels`; `default_label` ∈ labels/concepts. +9. Scores in [0,1] with min ≤ max; char counts are non-negative integers + (not a float, not a bool - Python/JSON `true`/`false` are not integers); + keyword arrays contain only non-empty strings. +10. Privacy/safety rules are ordered before broader routing rules. +11. `model_name` isn't a name already used earlier in this conversation + (`/pull` is idempotent per `model_name` - a collision silently overwrites + the earlier registration, not an error). +12. Every `llm` prompt (both `routing.router.prompt` and `type: "llm"` + classifier `prompt`) describes intent only - it never tells the model how + to format its reply. The engine appends its own JSON `{model, rationale}` + contract unconditionally to both. An authored instruction like "reply with + only the model name" or "reply with exactly one label: X or Y" contradicts + it: a weaker model obeys the authored line, outputs a bare string, the + parser rejects it, and the router silently falls through to `default_model` + on every request with no visible error. + +## Registering, invoking, tracing + +```bash +# register (idempotent - re-POST to update) +curl -X POST http://localhost:13305/api/v1/pull \ + -H "Content-Type: application/json" --data-binary @router.json + +# route: just name the collection as the model +curl -X POST http://localhost:13305/api/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "user.MyHybridRouter", "route_trace": true, + "messages": [{"role": "user", "content": "My SSN is 123-45-6789"}]}' + +# remove +curl -X POST http://localhost:13305/api/v1/delete \ + -H "Content-Type: application/json" -d '{"model_name": "user.MyHybridRouter"}' +``` + +Every routed response carries the `x-lemonade-route` header (matched rule id +or `default`). With `"route_trace": true` the body also carries +`x_lemonade_route`: `{ version, route_to, matched_rule, default_used, outputs, +trace[] }` - use it to demonstrate each rule to the user. `version` is always +`"1"`. `trace[]` entries include `score` for classifier conditions and omit it +for keyword/metadata conditions. Registration errors come back as descriptive +parser messages (e.g. `routing.default_model 'X' must be listed in +routing.candidates`); fix the field it names and re-POST. + +## Desktop-editor compatibility + +Users may open the generated policy in the Lemonade desktop app's Hybrid +Router editor (gear icon on the collection). To keep the JSON fully editable +there: + +- One condition per leaf object. Compound leaves (`{"min_chars": 5, + "has_tools": true}`) are valid server-side but the editor cannot display + them and will warn it must drop them on save. +- `metadata` conditions are JSON-only for now - the editor shows a lossy-edit + warning for rules containing them. Generate them only when the user + explicitly wants metadata routing, and say so in your reply. +- Everything else in this reference (nested `all`/`any`/`not`, all three + classifier types, negated booleans) round-trips cleanly. diff --git a/skills/build-lemonade-router/scripts/validate.py b/skills/build-lemonade-router/scripts/validate.py new file mode 100644 index 0000000..fd33602 --- /dev/null +++ b/skills/build-lemonade-router/scripts/validate.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python3 +""" +Validate a generated collection.router policy JSON against the same structural +and numeric rules the Lemonade server's C++ parser (routing_policy_parser.cpp) +enforces at POST /v1/pull - offline, no server or network required. + +Catches the class of mistake easy to make when hand-authoring nested JSON: an +out-of-range score, a negative char count, an unbalanced match expression, a +default_label that isn't in labels, a semantic_similarity classifier that +declares labels, an llm classifier missing labels, etc. + +This does NOT (and cannot, without a live server) check whether a named model +actually exists or has the right capability (chat/embedding/classification) - +that needs GET /api/v1/models/{id} against a running lemond; see SKILL.md +Step 8 for that optional live check. + +Usage: + python3 scripts/validate.py router.json + python3 scripts/validate.py < router.json + cat router.json | python3 scripts/validate.py - + +Exits 0 if no errors (warnings/advisories may still be present), 1 otherwise. +JSON to stdout. +""" + +import argparse +import json +import re +import sys + +LEAF_KEYS = { + "keywords_any", "keywords_all", "regex", "min_chars", "max_chars", + "has_tools", "has_images", "classifier", "label", "min_score", + "max_score", "metadata", +} +LOGICAL_KEYS = {"all", "any", "not"} +CLASSIFIER_TYPES = {"classifier", "semantic_similarity", "llm"} +ON_ERROR_VALUES = {"match_false", "match_true"} +SAFE_ID_RE = re.compile(r"^[A-Za-z0-9._-]+$") + +ROOT_KEYS = {"version", "model_name", "recipe", "components", "models", "routing"} +ROUTING_KEYS = {"candidates", "default_model", "router", "rules", "classifiers"} +CLASSIFIER_KEYS = {"id", "type", "model", "labels", "default_label", "on_error", + "prompt", "reference_phrases"} +BAD_PROMPT_RE = re.compile( + r"reply with only the exact model name|only the model name|" + r"reply with only.{0,20}model name|" + r"reply with (exactly )?one label|" + r"reply with only.{0,20}label|" + r"output only.{0,20}label|" + r"respond with only.{0,20}label", re.I) +# Keep the old name as an alias so existing code that references it still works +BAD_ROUTER_PROMPT_RE = BAD_PROMPT_RE + +# Nested unbounded quantifiers rejected by std::regex (ECMAScript) as ReDoS safeguard. +# Python re compiles these fine, so we catch them with a simple structural check. +_REDOS_RE = re.compile(r"\([^)]*[+*][^)]*\)[+*]|[+*]\{[^}]*\}[+*]") + + +# Constructs accepted by Node/V8 (ES2018+) but rejected by std::regex ECMAScript +# (the C++ engine the server uses, which implements ~ES5.1). +# This scan runs before Node so that Node's newer engine can't give a false pass. +_STDREGEX_UNSUPPORTED_RE = re.compile( + r"\(\?P[<']" # Python named group (?P / (?P'name' — not ECMAScript at all + r"|\(\?P=[^)]*\)" # Python backreference (?P=name) + r"|\(\?#[^)]*\)" # Python comment (?#...) + r"|\(\?<=[^)]*\)" # lookbehind (?<=...) — ES2018, not in std::regex + r"|\(\?...) — ES2018, not in std::regex + r"|\\[pP]\{" # Unicode property escape \p{...} / \P{...} — ES2018, not in std::regex +) +# Keep the old name so any external code that references it doesn't break. +_PYTHON_ONLY_RE = _STDREGEX_UNSUPPORTED_RE + + +def _check_ecmascript_regex(pattern): + """Return (error_str, skip_notice) for pattern against ECMAScript regex rules. + + Always runs the dialect scan first: rejects constructs that are valid in + Python re (and may be accepted by Node v8+) but are not supported by + std::regex with the ECMAScript flag (the C++ engine the server uses). + Lookbehinds ((?<=...) / (?…), Unicode property escapes \p{…}, etc. + m = _STDREGEX_UNSUPPORTED_RE.search(pattern) + if m: + return ( + f"regex construct '{m.group()}' is not supported by std::regex ECMAScript " + f"(the server's C++ engine, ~ES5.1); Node/V8 accepts this but the server will reject it", + None, + ) + + node = shutil.which("node") or shutil.which("nodejs") + if not node: + # Best-effort syntax check via Python re. + try: + re.compile(pattern) + except re.error as exc: + return f"regex syntax error: {exc}", None + return None, "node/nodejs not found - ECMAScript regex validation skipped; install Node.js to enable" + + # Wrap in try/catch so Node exits 0 on valid, 1 with the SyntaxError message on invalid. + script = ( + "try { new RegExp(" + json.dumps(pattern) + "); process.exit(0); } " + "catch(e) { process.stdout.write(e.message); process.exit(1); }" + ) + result = subprocess.run([node, "-e", script], capture_output=True, text=True, timeout=5) + if result.returncode != 0: + return result.stdout.strip() or "invalid ECMAScript regex", None + return None, None + + +def _err(issues, check, message, path=""): + issues.append({"check": check, "severity": "error", + "message": f"{path}: {message}" if path else message}) + + +def _warn(issues, check, message, path=""): + issues.append({"check": check, "severity": "warning", + "message": f"{path}: {message}" if path else message}) + + +def _is_non_negative_int(v): + return isinstance(v, int) and not isinstance(v, bool) and v >= 0 + + +def _is_score(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) and 0.0 <= v <= 1.0 + + +def validate_match_expr(expr, classifiers, path, issues, depth=0): + if depth > 64: + _err(issues, "match_depth", "exceeds max nesting depth (64)", path) + return + if not isinstance(expr, dict) or not expr: + _err(issues, "match_shape", "must be a non-empty object", path) + return + + logical = LOGICAL_KEYS & expr.keys() + if logical: + if len(expr) != 1: + _err(issues, "match_shape", + f"logical key ({', '.join(sorted(logical))}) cannot be mixed with other keys", path) + return + key = next(iter(logical)) + if key == "not": + validate_match_expr(expr["not"], classifiers, f"{path}.not", issues, depth + 1) + else: + children = expr[key] + if not isinstance(children, list) or not children: + _err(issues, "match_shape", f"'{key}' must be a non-empty array", path) + return + for i, child in enumerate(children): + validate_match_expr(child, classifiers, f"{path}.{key}[{i}]", issues, depth + 1) + return + + # Leaf node + unknown = set(expr.keys()) - LEAF_KEYS + if unknown: + _err(issues, "unknown_key", f"unknown condition key(s): {', '.join(sorted(unknown))}", path) + + condition_count = 0 + for key in ("keywords_any", "keywords_all"): + if key not in expr: + continue + condition_count += 1 + v = expr[key] + if not isinstance(v, list) or not v or not all(isinstance(x, str) and x for x in v): + _err(issues, "keywords", f"'{key}' must be a non-empty array of non-empty strings", path) + + if "regex" in expr: + condition_count += 1 + v = expr["regex"] + if not isinstance(v, str) or not v: + _err(issues, "regex", "'regex' must be a non-empty string", path) + else: + ecma_err, ecma_skip = _check_ecmascript_regex(v) + if ecma_err: + _err(issues, "regex_dialect", + f"invalid ECMAScript regex: {ecma_err}", path) + elif ecma_skip: + _warn(issues, "regex_dialect", ecma_skip, path) + if _REDOS_RE.search(v): + _err(issues, "regex_redos", + "pattern contains nested unbounded quantifiers (e.g. (X+)+) which the " + "server rejects at load time as a ReDoS safeguard", path) + + for key in ("min_chars", "max_chars"): + if key in expr: + condition_count += 1 + if not _is_non_negative_int(expr[key]): + _err(issues, "char_count", f"'{key}' must be a non-negative integer, got {expr[key]!r}", path) + + for key in ("has_tools", "has_images"): + if key in expr: + condition_count += 1 + if not isinstance(expr[key], bool): + _err(issues, "boolean_condition", f"'{key}' must be a boolean, got {expr[key]!r}", path) + + if "classifier" in expr: + condition_count += 1 + cid = expr["classifier"] + if not isinstance(cid, str) or not cid: + _err(issues, "classifier_ref", "'classifier' must be a non-empty string", path) + elif cid not in classifiers: + _err(issues, "classifier_ref", f"references undeclared classifier '{cid}'", path) + else: + clf = classifiers[cid] + pool = (list((clf.get("reference_phrases") or {}).keys()) + if clf.get("type") == "semantic_similarity" else (clf.get("labels") or [])) + label = expr.get("label") + if label is not None: + if not isinstance(label, str) or not label: + _err(issues, "classifier_label", "'label' must be a non-empty string", path) + elif pool and label not in pool: + _err(issues, "classifier_label", f"label '{label}' is not declared on classifier '{cid}'", path) + elif pool and not clf.get("default_label"): + _err(issues, "classifier_label", + f"omits 'label' but classifier '{cid}' has labels and no default_label", path) + + min_score = expr.get("min_score") + max_score = expr.get("max_score") + if min_score is not None and not _is_score(min_score): + _err(issues, "score_range", f"'min_score' must be in [0, 1], got {min_score!r}", path) + if max_score is not None and not _is_score(max_score): + _err(issues, "score_range", f"'max_score' must be in [0, 1], got {max_score!r}", path) + if (_is_score(min_score) if min_score is not None else False) and \ + (_is_score(max_score) if max_score is not None else False) and min_score > max_score: + _err(issues, "score_range", f"min_score ({min_score}) exceeds max_score ({max_score})", path) + elif "label" in expr or "min_score" in expr or "max_score" in expr: + _err(issues, "classifier_ref", "'label'/'min_score'/'max_score' require a 'classifier' key in the same leaf", path) + + if "metadata" in expr: + condition_count += 1 + md = expr["metadata"] + if not isinstance(md, dict) or not isinstance(md.get("key"), str) or not md.get("key"): + _err(issues, "metadata", "requires a non-empty string 'key'", path) + else: + unknown_md = set(md.keys()) - {"key", "equals", "any", "exists"} + if unknown_md: + _err(issues, "metadata", f"unknown key(s): {', '.join(sorted(unknown_md))}", path) + comparators = [c for c in ("equals", "any", "exists") if c in md] + if len(comparators) != 1: + _err(issues, "metadata", "must have exactly one of equals/any/exists", path) + elif comparators[0] == "equals" and not isinstance(md["equals"], str): + _err(issues, "metadata", "'equals' must be a string", path) + elif comparators[0] == "any" and ( + not isinstance(md["any"], list) or not md["any"] + or not all(isinstance(x, str) and x for x in md["any"])): + _err(issues, "metadata", "'any' must be a non-empty array of non-empty strings", path) + elif comparators[0] == "exists" and not isinstance(md["exists"], bool): + _err(issues, "metadata", "'exists' must be a boolean", path) + + if condition_count == 0: + _err(issues, "empty_leaf", "leaf has no recognized condition", path) + + +def validate(policy): + issues = [] + + if not isinstance(policy, dict): + _err(issues, "root", "policy must be a JSON object") + return issues + + unknown_root = set(policy.keys()) - ROOT_KEYS + if unknown_root: + _err(issues, "unknown_key", f"unknown root key(s): {', '.join(sorted(unknown_root))}") + + if policy.get("version") != "1": + _err(issues, "version", f"must be the string \"1\", got {policy.get('version')!r}") + + model_name = policy.get("model_name") + if not isinstance(model_name, str) or not model_name.startswith("user.") or \ + not SAFE_ID_RE.match(model_name[len("user."):] or ""): + _err(issues, "model_name", "must be a non-empty string starting with 'user.' using [A-Za-z0-9._-]") + elif model_name == "user.MyHybridRouter": + _warn(issues, "model_name", + "using the bare default name - if you register more than one router in this " + "session without renaming, later /pull calls will silently overwrite earlier " + "ones (model_name is the collection identity). Give it a distinct name.") + + if policy.get("recipe") != "collection.router": + _err(issues, "recipe", f"must be \"collection.router\", got {policy.get('recipe')!r}") + + components = policy.get("components") + if not isinstance(components, list) or not components or not all(isinstance(c, str) and c for c in components): + _err(issues, "components", "must be a non-empty array of non-empty strings") + components = components if isinstance(components, list) else [] + + routing = policy.get("routing") + if not isinstance(routing, dict): + _err(issues, "routing", "must be an object") + return issues + + unknown_routing = set(routing.keys()) - ROUTING_KEYS + if unknown_routing: + _err(issues, "unknown_key", f"unknown routing key(s): {', '.join(sorted(unknown_routing))}") + + candidates = routing.get("candidates") + if not isinstance(candidates, list) or not candidates or not all(isinstance(c, str) and c for c in candidates): + _err(issues, "candidates", "routing.candidates must be a non-empty array of non-empty strings") + candidates = candidates if isinstance(candidates, list) else [] + elif len(set(candidates)) != len(candidates): + _err(issues, "candidates", "routing.candidates contains duplicates") + + default_model = routing.get("default_model") + if default_model not in candidates: + _err(issues, "default_model", f"'{default_model}' must be listed in routing.candidates") + + has_router = "router" in routing + has_rules = "rules" in routing + has_classifiers = "classifiers" in routing + if has_router == has_rules: + _err(issues, "mode", "routing must contain exactly one of 'router' or 'rules'") + if has_router and has_classifiers: + _err(issues, "mode", "'router' cannot be combined with 'classifiers'") + + needed_components = set(candidates) + classifiers_by_id = {} + + if has_router: + router = routing["router"] + if not isinstance(router, dict): + _err(issues, "router", "routing.router must be an object") + else: + unknown = set(router.keys()) - {"type", "model", "prompt"} + if unknown: + _err(issues, "router", f"unknown key(s): {', '.join(sorted(unknown))}") + if router.get("type") != "llm": + _err(issues, "router", "routing.router.type must be \"llm\"") + model = router.get("model") + if not isinstance(model, str) or not model: + _err(issues, "router", "routing.router.model must be a non-empty string") + else: + needed_components.add(model) + prompt = router.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + _err(issues, "router", "routing.router.prompt must be a non-empty string") + elif BAD_ROUTER_PROMPT_RE.search(prompt): + _warn(issues, "router_prompt_contract", + "prompt tells the model to reply with a bare model name, but the engine " + "always demands a JSON {model, rationale} reply and appends its own " + "instruction saying so - this line is redundant/misleading; author only " + "routing intent and let the engine own the reply format.") + + if has_classifiers: + clfs = routing["classifiers"] + if not isinstance(clfs, list): + _err(issues, "classifiers", "routing.classifiers must be an array") + clfs = [] + seen_ids = set() + for i, clf in enumerate(clfs): + p = f"routing.classifiers[{i}]" + if not isinstance(clf, dict): + _err(issues, "classifier", "must be an object", p) + continue + unknown_clf = set(clf.keys()) - CLASSIFIER_KEYS + if unknown_clf: + _err(issues, "unknown_key", f"unknown key(s): {', '.join(sorted(unknown_clf))}", p) + + cid = clf.get("id") + if not isinstance(cid, str) or not cid: + _err(issues, "classifier_id", "must be a non-empty string", p) + elif cid in seen_ids: + _err(issues, "classifier_id", f"duplicate id '{cid}'", p) + else: + seen_ids.add(cid) + classifiers_by_id[cid] = clf + + ctype = clf.get("type") + if ctype not in CLASSIFIER_TYPES: + _err(issues, "classifier_type", f"type must be one of {sorted(CLASSIFIER_TYPES)}, got {ctype!r}", p) + + model = clf.get("model") + if not isinstance(model, str) or not model: + _err(issues, "classifier_model", "model must be a non-empty string", p) + else: + needed_components.add(model) + + on_error = clf.get("on_error") + if on_error is not None and on_error not in ON_ERROR_VALUES: + _err(issues, "on_error", f"must be one of {sorted(ON_ERROR_VALUES)}, got {on_error!r}", p) + + labels = clf.get("labels") + if ctype == "llm": + prompt = clf.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + _err(issues, "llm_classifier", "requires a non-empty 'prompt'", p) + elif BAD_PROMPT_RE.search(prompt): + _warn(issues, "llm_classifier_prompt_contract", + "prompt tells the model to reply with a bare label, but the engine " + "appends its own JSON {model, rationale} contract and the classifier " + "scores the chosen label 1.0 — an authored reply-format instruction " + "causes weaker models to output a bare string the parser rejects, " + "making the rule silently never fire. Describe classification intent " + "only; do not instruct the model how to format its reply.", p) + if not isinstance(labels, list) or not labels or not all(isinstance(x, str) and x for x in labels): + _err(issues, "llm_classifier", "requires a non-empty 'labels' array", p) + elif ctype == "semantic_similarity": + if "labels" in clf: + _err(issues, "semantic_similarity", + "must not declare 'labels' - concept names in reference_phrases are the labels", p) + rp = clf.get("reference_phrases") + if not isinstance(rp, dict) or not rp: + _err(issues, "semantic_similarity", "requires a non-empty 'reference_phrases' object", p) + rp = {} + else: + for concept, phrases in rp.items(): + if not isinstance(phrases, list) or not phrases or \ + not all(isinstance(x, str) and x for x in phrases): + _err(issues, "semantic_similarity", + f"concept '{concept}' needs a non-empty array of non-empty phrases", p) + labels = list(rp.keys()) + elif ctype == "classifier": + if labels is not None and (not isinstance(labels, list) or not all(isinstance(x, str) and x for x in labels)): + _err(issues, "classifier", "'labels', if present, must be an array of non-empty strings", p) + + default_label = clf.get("default_label") + if default_label is not None: + if not isinstance(default_label, str) or not default_label: + _err(issues, "default_label", "must be a non-empty string", p) + elif not labels: + _err(issues, "default_label", "set but classifier declares no labels/concepts to validate against", p) + elif default_label not in labels: + _err(issues, "default_label", f"'{default_label}' is not in labels/concepts {labels}", p) + + if has_rules: + rules = routing["rules"] + if not isinstance(rules, list) or not rules: + _err(issues, "rules", "routing.rules must be a non-empty array") + rules = [] + seen_ids = set() + for i, rule in enumerate(rules): + p = f"routing.rules[{i}]" + if not isinstance(rule, dict): + _err(issues, "rule", "must be an object", p) + continue + unknown = set(rule.keys()) - {"id", "match", "route_to", "outputs"} + if unknown: + _err(issues, "rule", f"unknown key(s): {', '.join(sorted(unknown))}", p) + + rid = rule.get("id") + if not isinstance(rid, str) or not rid or not SAFE_ID_RE.match(rid): + _err(issues, "rule_id", "must be a non-empty string matching [A-Za-z0-9._-]", p) + elif rid in seen_ids: + _err(issues, "rule_id", f"duplicate id '{rid}'", p) + else: + seen_ids.add(rid) + + route_to = rule.get("route_to") + if route_to not in candidates: + _err(issues, "route_to", f"'{route_to}' must be listed in routing.candidates", p) + + if "outputs" in rule and not isinstance(rule["outputs"], dict): + _err(issues, "outputs", "must be an object", p) + + match = rule.get("match") + if not isinstance(match, dict) or not match: + _err(issues, "match", "must be a non-empty object", p) + else: + validate_match_expr(match, classifiers_by_id, f"{p}.match", issues) + + missing = needed_components - set(components) + if missing: + _err(issues, "components", f"referenced but not declared in components: {sorted(missing)}") + + return issues + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("path", nargs="?", default="-", help="policy JSON file, or '-'/omitted for stdin") + args = p.parse_args() + + raw = sys.stdin.read() if args.path == "-" else open(args.path, encoding="utf-8").read() + try: + policy = json.loads(raw) + except json.JSONDecodeError as e: + print(json.dumps({"ready": False, "errors": [{"check": "json", "severity": "error", "message": str(e)}], + "warnings": [], "advisories": []}, indent=2)) + sys.exit(1) + + issues = validate(policy) + errors = [i for i in issues if i["severity"] == "error"] + warnings = [i for i in issues if i["severity"] == "warning"] + advisories = [i for i in issues if i["severity"] == "advisory"] + result = {"ready": len(errors) == 0, "errors": errors, "warnings": warnings, "advisories": advisories} + print(json.dumps(result, indent=2)) + sys.exit(0 if len(errors) == 0 else 1) + + +if __name__ == "__main__": + main() diff --git a/skills/build-lemonade-router/skill-card.md b/skills/build-lemonade-router/skill-card.md new file mode 100644 index 0000000..e89beb8 --- /dev/null +++ b/skills/build-lemonade-router/skill-card.md @@ -0,0 +1,13 @@ +# Skill Card + +## Description + +Generate a valid Lemonade `collection.router` policy JSON from a natural-language description of routing intent, with safe defaults for everything unspecified. + +## Owner + +AMD + +## License + +MIT diff --git a/walkthroughs/README.md b/walkthroughs/README.md index 39976d4..7684e62 100644 --- a/walkthroughs/README.md +++ b/walkthroughs/README.md @@ -8,6 +8,7 @@ Participatns using other are still encouraged to participate. Just please note t Please choose a skill to get started. +* [build-lemonade-router](./build-lemonade-router.md): Generate a valid Lemonade router policy JSON from a plain-English description of routing intent. * [local-ai-use](./local-ai-use.md): Teach your agent how to run image generation locally. * [local-ai-app-integration](./local-ai-app-integration.md): Add a local AI mode to a cloud-only app. * [tracelens-analysis-orchestrator](./tracelens-analysis-orchestrator.md): Run agentic PyTorch profiler trace analysis and produce a prioritized performance report. \ No newline at end of file diff --git a/walkthroughs/build-lemonade-router.md b/walkthroughs/build-lemonade-router.md new file mode 100644 index 0000000..7d8a217 --- /dev/null +++ b/walkthroughs/build-lemonade-router.md @@ -0,0 +1,92 @@ +# AMD Skills Walkthroughs: `build-lemonade-router` + +The goal of this skill is to teach your AI agent to turn a plain-English +description of routing intent into a valid Lemonade `collection.router` policy +JSON, ready for you to register and use. + +## Prerequisites + +- [Lemonade Server](https://github.com/lemonade-sdk/lemonade) installed and + running (`lemonade server start`), with at least two chat-capable models + downloaded (`lemonade list` to check). +- Claude Code installed and authenticated. + +## Step 1 - Confirm the skill is visible + +```bash +claude "Which skills can you see?" --model sonnet +``` + +You should see `build-lemonade-router` in the list. If not, install it: + +```bash +npx skills add amd/skills --skill build-lemonade-router --agent claude-code +``` + +## Step 2 - Generate a simple keyword router + +Open Claude and run: + +``` +Route coding questions - anything mentioning functions, bugs, or stack traces - +to Qwen3.5-9B-GGUF. Everything else goes to Qwen3.5-2B-GGUF. +``` + +The agent should: +1. Ask which models you want if you haven't specified them (or proceed if you + have). +2. Generate a `collection.router` JSON with a `keywords_any` rule. +3. Run `scripts/validate.py` against the JSON and confirm it passes. +4. Output the JSON in a fenced block, plus curl commands for you to register + and test it. + +## Step 3 - Register and test the router yourself + +Copy the curl commands the agent produced and run them in your terminal: + +```bash +# Register +curl -X POST http://localhost:13305/api/v1/pull \ + -H "Content-Type: application/json" --data-binary @router.json + +# Test - should match the keyword rule +curl -X POST http://localhost:13305/api/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "user.", "route_trace": true, + "messages": [{"role": "user", "content": "How do I fix this bug?"}]}' +``` + +Check the `x-lemonade-route` response header - it should show the matched rule +id. With `"route_trace": true` the body also contains `x_lemonade_route` with +the full per-condition trace. + +## Step 4 - Try a PII privacy router + +``` +Any message containing a Social Security number or email address must stay on +Qwen3.5-9B-GGUF. Everything else can go to Qwen3.5-9B-NoThinking. +``` + +The agent should produce a rules-mode policy with two `regex` conditions and +place the PII rule first. Validate by sending a test message with a fake SSN +(`123-45-6789`) and confirming the header shows `pii-stays-local`. + +## Step 5 - Try an LLM-as-router (intent-only) + +``` +I want sensitive queries to stay on Qwen3.5-9B-GGUF and everything else to go +to Qwen3.5-9B-NoThinking. Use the local model as the router. +``` + +The agent should choose Mode A (`routing.router` block) rather than rules, +because "sensitive" is a meaning judgment with no concrete signal. The +generated prompt should describe routing intent only - no reply-format +instructions. + +## Step 6 - (Optional) Try to get things done without the skill + +Remove the skill and ask the same routing questions. Without the skill, the +agent is likely to produce JSON that fails the server-side parser on the first +try, get the mode exclusivity wrong (`router` + `rules` together), or omit +required fields like `components`. This demonstrates the value of the skill's +structured generation and offline validation step.