fix(skill): back off crystallization retries after repeated failures

apps/memos-local-plugin/core/skill/eligibility.ts

@@ -66,6 +66,18 @@ function decide(
   existing: SkillRow | null,
   cfg: SkillConfig,
 ): EligibilityDecision {
+  // WHY: skill_eligible=false can mean either the crystallize-failure
+  // backoff (3 consecutive failures) or a manual toggle. Reporting it as
+  // its own skip reason keeps the two sources distinguishable from the
+  // experience-type success-anchor check below.
+  if (policy.skillEligible === false) {
+    return {
+      policy,
+      existingSkill: existing,
+      action: "skip",
+      reason: "policy.skillEligible=false (backoff or manual)",
+    };
+  }
   if (policy.status !== "active") {
     return {
       policy,
@@ -125,7 +137,6 @@ function decide(
 }
 
 function hasSuccessAnchor(policy: PolicyRow): boolean {
-  if (policy.skillEligible === false) return false;
   const type = policy.experienceType ?? "success_pattern";
   if (type === "failure_avoidance" || type === "repair_instruction" || type === "preference") {
     return false;

apps/memos-local-plugin/core/skill/skill.ts

@@ -28,6 +28,7 @@ import type { Repos } from "../storage/repos/index.js";
 import { now as nowMs } from "../time.js";
 import { ids } from "../id.js";
 import type {
+  PolicyId,
   PolicyRow,
   SkillId,
   SkillRow,
@@ -154,6 +155,12 @@ export async function runSkill(
         reason: crystResult.skippedReason,
         modelRefusal: crystResult.modelRefusal,
       });
+      // WHY: "llm-disabled" is a global configuration state, not a failure of
+      // this policy. Counting it would trip the whole candidate pool while
+      // the LLM is merely switched off — with no recovery path back.
+      if (crystResult.skippedReason !== "llm-disabled") {
+        bumpFailureBackoff(deps, decision.policy.id);
+      }
       continue;
     }
 
@@ -176,6 +183,7 @@ export async function runSkill(
         skillId: "sk_placeholder" as SkillId,
         reason: verdict.reason ?? "verify-failed",
       });
+      bumpFailureBackoff(deps, decision.policy.id);
       continue;
     }
 
@@ -225,6 +233,10 @@ export async function runSkill(
     }
     timings.persist += nowMs() - tPersist;
 
+    // A successful crystallization resets the failure counter so later
+    // failures start counting from zero again.
+    deps.repos.kv.del(failCountKey(decision.policy.id));
+
     if (decision.action === "rebuild") rebuilt += 1;
     else crystallized += 1;
 
@@ -338,6 +350,29 @@ export function applySkillFeedback(
 
 // ─── Helpers ──────────────────────────────────────────────────────────────
 
+// WHY: a crystallize/verify failure leaves the policy untouched, so every
+// trigger retries it forever (2026-08-28 audit: 2640 repeated failures over
+// 25 days, the bulk of wasted skill invocations). Count consecutive failures
+// in kv; after 3, flip skill_eligible=false so the policy is skipped. A
+// successful crystallization clears the counter.
+const SKILL_FAILURE_BACKOFF_LIMIT = 3;
+
+function failCountKey(policyId: string): string {
+  return `skill.failCount:${policyId}`;
+}
+
+function bumpFailureBackoff(deps: RunSkillDeps, policyId: PolicyId): void {
+  const count = deps.repos.kv.get<number>(failCountKey(policyId), 0) + 1;
+  deps.repos.kv.set(failCountKey(policyId), count);
+  if (count >= SKILL_FAILURE_BACKOFF_LIMIT) {
+    deps.repos.policies.setSkillEligible(policyId, false);
+    // Clear the counter on trip: a later manual re-enable must get a fresh
+    // window, not instant re-trip on the next single failure.
+    deps.repos.kv.del(failCountKey(policyId));
+    deps.log.warn("skill.backoff.exhausted", { policyId, count });
+  }
+}
+
 function gatherPolicies(input: RunSkillInput, repos: Repos): PolicyRow[] {
   if (input.policyId) {
     const single = repos.policies.getById(input.policyId);

apps/memos-local-plugin/core/storage/repos/policies.ts

@@ -73,6 +73,12 @@ export function makePoliciesRepo(db: StorageDb) {
       columns: ["id", "support", "gain", "status", "updated_at"],
     }),
   );
+  // WHY: the skill-crystallize backoff flips eligibility for a single policy.
+  // updated_at is deliberately left untouched so the rebuild heuristic for
+  // existing skills is not triggered as a side effect.
+  const setSkillEligibleStmt = db.prepare<{ id: string; eligible: number }>(
+    `UPDATE policies SET skill_eligible=@eligible WHERE id=@id`,
+  );
   const selectById = db.prepare<{ id: string }, RawPolicyRow>(
     `SELECT ${COLUMNS.join(", ")} FROM policies WHERE id=@id`,
   );
@@ -104,6 +110,13 @@ export function makePoliciesRepo(db: StorageDb) {
       });
     },
 
+    // WHY: the skill-crystallize backoff needs to close crystallization
+    // eligibility for one policy without bumping updated_at (which would
+    // trigger the rebuild heuristic for an existing skill).
+    setSkillEligible(id: PolicyId, eligible: boolean): void {
+      setSkillEligibleStmt.run({ id, eligible: eligible ? 1 : 0 });
+    },
+
     getById(id: PolicyId): PolicyRow | null {
       const r = selectById.get({ id });
       if (!r) return null;

apps/memos-local-plugin/tests/unit/skill/backoff.test.ts

@@ -0,0 +1,150 @@
+import { describe, it, expect, afterEach } from "vitest";
+
+import { rootLogger } from "../../../core/logger/index.js";
+import { runSkill, type RunSkillDeps } from "../../../core/skill/index.js";
+import { fakeLlm } from "../../helpers/fake-llm.js";
+import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js";
+import type { EpisodeId, PolicyId } from "../../../core/types.js";
+import { makeDraft, makeSkillConfig, seedPolicy, seedSessionOnly, seedTrace } from "./_helpers.js";
+
+let handle: TmpDbHandle | null = null;
+
+function open(): TmpDbHandle {
+  handle = makeTmpDb();
+  return handle;
+}
+
+afterEach(() => {
+  handle?.cleanup();
+  handle = null;
+});
+
+function makeDeps(h: TmpDbHandle, llm: RunSkillDeps["llm"]): RunSkillDeps {
+  return {
+    repos: h.repos,
+    embedder: null,
+    llm,
+    log: rootLogger.child({ channel: "core.skill.backoff-test" }),
+    bus: { emit: () => {} } as unknown as RunSkillDeps["bus"],
+    config: makeSkillConfig(),
+  };
+}
+
+// Scripted model refusal — crystallize returns { ok:false, skippedReason:"llm-refusal" }.
+// (We deliberately do NOT drive failures with llm=null: that path returns
+// "llm-disabled", a global config state the backoff must not count.)
+function refusingLlm(): RunSkillDeps["llm"] {
+  // NB: the refusal detector scans the raw JSON string, so a plain-string
+  // response is what actually trips it (an object response starts with "{").
+  return fakeLlm({ completeJson: { "skill.crystallize": "I cannot assist with this request." } });
+}
+
+function seedCandidate(h: TmpDbHandle): PolicyId {
+  const sessionId = "s_backoff";
+  seedSessionOnly(h, sessionId);
+  const episodeId = "ep_backoff" as EpisodeId;
+  seedTrace(h, {
+    episodeId,
+    sessionId,
+    userText: "pip install cryptography failing",
+    agentText: "apk add openssl-dev libffi-dev, retry pip install",
+    reflection: "install system libs before pip",
+    value: 0.9,
+  });
+  seedTrace(h, {
+    episodeId,
+    sessionId,
+    userText: "cryptography install retry",
+    agentText: "apk add openssl-dev && pip install cryptography",
+    reflection: "musl wheels need system libs",
+    value: 0.8,
+  });
+  return seedPolicy(h, { sourceEpisodeIds: [episodeId] }).id;
+}
+
+function failCount(h: TmpDbHandle, policyId: PolicyId): number {
+  return h.repos.kv.get<number>(`skill.failCount:${policyId}`, 0);
+}
+
+describe("skill crystallize failure backoff", () => {
+  it("flips skill_eligible off after SKILL_FAILURE_BACKOFF_LIMIT consecutive failures", async () => {
+    const h = open();
+    const policyId = seedCandidate(h);
+    const deps = makeDeps(h, refusingLlm());
+
+    for (let i = 1; i <= 2; i++) {
+      const r = await runSkill({ trigger: "manual", policyId }, deps);
+      expect(r.crystallized).toBe(0);
+      expect(failCount(h, policyId)).toBe(i);
+      // below the limit the policy stays eligible
+      expect(h.repos.policies.getById(policyId)!.skillEligible).not.toBe(false);
+    }
+
+    await runSkill({ trigger: "manual", policyId }, deps);
+    // the counter is cleared on trip — the re-enable path gets a fresh window
+    expect(failCount(h, policyId)).toBe(0);
+    expect(h.repos.policies.getById(policyId)!.skillEligible).toBe(false);
+  });
+
+  it("gives a manually re-enabled policy a fresh backoff window", async () => {
+    const h = open();
+    const policyId = seedCandidate(h);
+    const deps = makeDeps(h, refusingLlm());
+    for (let i = 0; i < 3; i++) await runSkill({ trigger: "manual", policyId }, deps);
+    expect(h.repos.policies.getById(policyId)!.skillEligible).toBe(false);
+    expect(failCount(h, policyId)).toBe(0);
+
+    // admin re-enables the policy: the next failure must count from 0,
+    // not instantly re-trip off the stale counter
+    h.repos.policies.setSkillEligible(policyId, true);
+    await runSkill({ trigger: "manual", policyId }, deps);
+    expect(h.repos.policies.getById(policyId)!.skillEligible).not.toBe(false);
+    expect(failCount(h, policyId)).toBe(1);
+  });
+
+  it("reports the tripped backoff as its own skip reason on later runs", async () => {
+    const h = open();
+    const policyId = seedCandidate(h);
+    const deps = makeDeps(h, refusingLlm());
+    for (let i = 0; i < 3; i++) await runSkill({ trigger: "manual", policyId }, deps);
+
+    const r = await runSkill({ trigger: "manual", policyId }, deps);
+    // the eligibility gate now skips before crystallize — the run counts
+    // the policy as skipped (not evaluated) and the failure counter
+    // stays cleared at 0 (cleared when the backoff tripped)
+    expect(r.evaluated).toBe(0);
+    expect(r.crystallized).toBe(0);
+    // the counter was cleared when the backoff tripped
+    expect(failCount(h, policyId)).toBe(0);
+    expect(h.repos.policies.getById(policyId)!.skillEligible).toBe(false);
+  });
+
+  it("clears the counter after a successful crystallization", async () => {
+    const h = open();
+    const policyId = seedCandidate(h);
+
+    await runSkill({ trigger: "manual", policyId }, makeDeps(h, refusingLlm()));
+    expect(failCount(h, policyId)).toBe(1);
+
+    await runSkill({ trigger: "manual", policyId }, makeDeps(h, fakeLlm({ completeJson: { "skill.crystallize": makeDraft() } })));
+    expect(failCount(h, policyId)).toBe(0);
+    // (fewer-than-limit failures not tripping the backoff is covered by
+    // the second run of the first test above)
+  });
+
+  it("does not count llm-disabled toward the backoff", async () => {
+    const h = open();
+    const policyId = seedCandidate(h);
+    // llm=null => crystallize returns { ok:false, skippedReason:"llm-disabled" }:
+    // a global configuration state. Even after many such ticks the policy
+    // must stay eligible and the counter untouched.
+    const deps = makeDeps(h, null);
+
+    for (let i = 0; i < 4; i++) {
+      const r = await runSkill({ trigger: "manual", policyId }, deps);
+      expect(r.crystallized).toBe(0);
+    }
+    expect(failCount(h, policyId)).toBe(0);
+    expect(h.repos.policies.getById(policyId)!.skillEligible).not.toBe(false);
+  });
+});