Record a flow in Chrome, narrated, straight into Intelligence with the OpenBot Recorder

.claude/skills/openbot-screen-layout/SKILL.md

@@ -143,7 +143,7 @@ itself when the dialog changes something.
 
 ## The rem Trap
 
-`app/src/styles.css` sets `html { font-size: 15px }`. Every rem-based Tailwind token is therefore
+`app/src/theme.css` sets `html { font-size: 15px }`. Every rem-based Tailwind token is therefore
 **0.9375×** its documented pixel value:
 
 | Token | Documented | Actual here |

.env.example

@@ -166,6 +166,11 @@ COPILOTKIT_LICENSE_TOKEN=
 # CPK_INTELLIGENCE_SKILLS_REVISION=exact-revision-id
 # See docs/automatic-learning.md for managed/self-hosted setup and permissions.
 
+# OpenBot accepts the OpenBot Recorder extension only from its own extension id. A fork, or a build
+# signed with another key, has another id: list it here, comma separated, each 32 letters from a to
+# p. Any other value stops the server at boot.
+# OPENBOT_RECORDER_EXTRA_EXTENSION_IDS=
+
 # How long a Bot's stream may say nothing before this deployment gives up on the turn, in
 # milliseconds. A Bot is any AG-UI endpoint, which means it will be redeployed mid-answer, its own
 # upstream will time out, and it will sometimes accept a connection and then write nothing at all.
@@ -207,6 +212,11 @@ OPENAI_API_KEY=
 #   BOT_MODEL=openai/gpt-5.5
 #
 # OPENAI_BASE_URL=
+# Behind an OPENAI_BASE_URL that is not OpenAI or Azure, built-in Bots call the endpoint's Chat
+# Completions API, which is what most compatible endpoints serve. A gateway that also serves the
+# Responses API keeps Responses-only features, such as reasoning summaries, with `responses`.
+# `chat` forces Chat Completions; any other value is an error when a built-in Bot runs.
+# COPILOTKIT_OPENAI_API=responses
 
 # Optional dictation, independent of every agent/chat provider. Set provider, base URL,
 # and model together. The endpoint must implement POST /audio/transcriptions under this

.github/workflows/desktop.yml

@@ -9,10 +9,10 @@ name: Desktop
 # it at runtime with an `expect`, so a bad row panics the window rather than failing a type check.
 on:
   pull_request:
-    paths: ["desktop/**", "shared/harnesses.json", "shared/marks/**", "package.json", ".github/workflows/desktop.yml"]
+    paths: ["desktop/**", "shared/harnesses.json", "shared/marks/**", "shared/recorder-extension.json", "package.json", ".github/workflows/desktop.yml"]
   push:
     branches: [main]
-    paths: ["desktop/**", "shared/harnesses.json", "shared/marks/**", "package.json", ".github/workflows/desktop.yml"]
+    paths: ["desktop/**", "shared/harnesses.json", "shared/marks/**", "shared/recorder-extension.json", "package.json", ".github/workflows/desktop.yml"]
   workflow_call:
     secrets:
       OPENBOT_GOOGLE_MODEL_OAUTH_CLIENT_SECRET:
@@ -143,6 +143,18 @@ jobs:
       - name: Rust regression tests
         run: cargo test --locked --lib --bins
         working-directory: desktop/src-tauri
+      # The binary run as the Recorder's browsers run it: the native messaging host Chrome starts, and
+      # the install that writes its manifests. The host name and extension id come from
+      # shared/recorder-extension.json, which the Rust reads at compile time.
+      - name: Recorder native host tests
+        run: cargo test --locked --test recorder_host
+        working-directory: desktop/src-tauri
+      # Only a release build is a GUI-subsystem executable on Windows, and only a release build proves
+      # the debug-only test seam is absent.
+      - name: Recorder native host tests (release)
+        if: matrix.platform.name == 'windows'
+        run: cargo test --locked --release --test recorder_host
+        working-directory: desktop/src-tauri
       # Hosted Windows runners are administrators. Exercise setup again with an actual Users-only
       # account: DISM's feature reads worked as CI's administrator but refused ordinary app users.
       - name: Windows standard-user setup regression

.gitignore

@@ -53,3 +53,6 @@ venv/
 .pytest_cache/
 .ruff_cache/
 .mypy_cache/
+
+# The Recorder extension's signing key never enters the repository (Decision 9).
+extension/**/*.pem

CHANGELOG.md

@@ -8,7 +8,7 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
 
 ## Unreleased
 
-**Before upgrading.** Five things change for an existing deployment:
+**Before upgrading.** Seven things change for an existing deployment:
 - A hosted computer's browser is now headed by default. It uses a little more memory than headless
   did; set `COMPUTER_BROWSER_MODE=headless` (Helm: `computers.browserMode: headless`) to keep the old
   behaviour.
@@ -31,6 +31,62 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
 - Migration `0056_sign_in_code_step` widens the check on `sign_in_requests.status` to allow
   `code_required` (below). A server and a computer must be upgraded together: an older computer has
   no `/sign-in/code`.
+- Migration `0057_recorder_extension` adds six tables for the OpenBot Recorder and each person's own
+  Learning space (below): `recorder_connect_codes`, `recorder_connections`,
+  `recorder_trajectory_bindings`, `learning_person_spaces`, `learning_skill_sharing` and
+  `learning_open_skills`. Nothing existing changes, and the tables stay empty until somebody
+  connects a browser or changes Settings → Learning.
+- The CopilotKit runtime moves to 1.79.1. A built-in Bot whose `openai/...` model is reached
+  through an `OPENAI_BASE_URL` that is not OpenAI or Azure now calls that endpoint's Chat
+  Completions API (`/chat/completions`) instead of the Responses API. That is what most
+  OpenAI-compatible endpoints serve, so Bots behind one of them now work; a gateway that also serves
+  the Responses API loses Responses-only features, such as reasoning summaries, without an error.
+  `COPILOTKIT_OPENAI_API=responses` keeps the Responses API (`chat` forces Chat Completions); any
+  other value is an error when a built-in Bot runs. Deployments on OpenAI or Azure are unchanged.
+
+### Teach a Bot by doing the task in your own browser: the OpenBot Recorder
+
+The OpenBot Recorder is a Chrome and Edge extension. A person names the task, does it in their own
+browser, and the Recorder sends what they did, step by step (each click, entry, choice, submit,
+Enter and page change, with what an agent needs to find that element again, and the one request a
+step sent when it saved something), masked in the browser first, straight to Intelligence as a
+trajectory in that person's own Learning space; the page's own traffic is never recorded. Every Bot
+they use then gets the Skills learned there, beside the Bot's own. OpenBot connects the browser and
+hands out each capture pass; it receives nothing that is recorded.
+
+- **Settings → Browsers** connects the Recorder in this browser and lists every browser connected
+  as you, each with a Disconnect. OpenBot Desktop can connect the Recorder on its own computer too.
+- **Only a connected browser gets a capture pass.** The runtime's `trajectory/<id>/connect` route
+  answers only the Recorder's own bearer, and binds the trajectory to that person; no Bot is
+  picked or named. A signed-in session alone gets no pass, and the Recorder's bearer cannot run a
+  Bot or read a conversation.
+- **One Learning space per person.** Every recording joins the person's own Intelligence Learning
+  container, `user-` and 32 hex digits derived from their OpenBot user id. OpenBot makes it when
+  the browser connects (and again on a capture pass if it is still missing) with the same
+  credential that makes the `openbot` container: a service token on a CopilotKit local evaluation
+  install, or a CopilotKit CLI login against managed Intelligence. Intelligence's runtime key
+  cannot make one. With neither, a missing space cannot be made, and the Recorder's popup says
+  Learning will not see the recording; the recording still goes to Intelligence.
+- **Every Bot a person uses draws on their space.** A run gets the Bot's own learned Skills, then
+  the person's own, then any another person opened (below). On a name clash the Bot's own wins,
+  then the person's own; two people's opened Skills with one name are both kept, each named
+  `<their space>/<name>`. A person's space that is missing or unreachable never fails a run: it
+  goes on with the Skills it has. Admin → Automatic Learning off, or a Bot excluded there, still
+  means no learned Skills at all.
+- **Settings → Learning** chooses who else's Bots use a person's Skills: only their own (the
+  default), every Bot in the deployment, or the Skills they open one by one. Runs read what is
+  open from memory, refreshed at once on the replica that saved a change and within 30 seconds on
+  the others.
+- **Intelligence must allow trajectories:** the `trajectories` licence feature,
+  `FF_LEARNING_TRAJECTORIES=true`, and an `INTELLIGENCE_GATEWAY_WS_URL` that people's browsers can
+  reach.
+- **`OPENBOT_RECORDER_EXTRA_EXTENSION_IDS`** lists other extension ids to trust besides the
+  Recorder's own, for a fork or a build signed with another key. Each must be 32 letters from a to
+  p, comma separated; **a value that is not stops the server at boot.**
+- Connecting, disconnecting and refusals are audited as `recorder.connected`, `recorder.revoked`,
+  `recorder.connect_refused`, `recorder.bearer_refused` and `recorder.trajectory_refused`; changing
+  Settings → Learning as `learning.sharing_changed`, `learning.skill_opened` and
+  `learning.skill_closed`.
 
 ### The Bot's screen opens only when there is something to watch
 

Dockerfile

@@ -57,6 +57,7 @@ COPY server/package.json server/package.json
 COPY worker/package.json worker/package.json
 COPY shared/package.json shared/package.json
 COPY toolkit/package.json toolkit/package.json
+COPY extension/package.json extension/package.json
 RUN bun install --frozen-lockfile
 
 # The lockfile travels with the manifest, because `--frozen-lockfile` with no lockfile in the context
@@ -70,12 +71,13 @@ RUN cd agent-computer && bun install --frozen-lockfile
 # A second tree with the build-time dependencies left out, for the runtime stage to take. Vite,
 # biome and the test tooling are a gigabyte that nothing in a running container imports.
 RUN mkdir -p /prod && cp package.json bun.lock /prod/ \
-  && cd /prod && mkdir -p app server worker shared toolkit \
+  && cd /prod && mkdir -p app server worker shared toolkit extension \
   && cp /src/app/package.json app/package.json \
   && cp /src/server/package.json server/package.json \
   && cp /src/worker/package.json worker/package.json \
   && cp /src/shared/package.json shared/package.json \
   && cp /src/toolkit/package.json toolkit/package.json \
+  && cp /src/extension/package.json extension/package.json \
   && bun install --frozen-lockfile --production
 
 
@@ -127,6 +129,8 @@ WORKDIR /app
 COPY --from=deps /prod/node_modules node_modules
 COPY --from=deps /prod/server/node_modules server/node_modules
 COPY --from=deps /prod/toolkit/node_modules toolkit/node_modules
+# shared files import zod at runtime, and the isolated install resolves it from shared/node_modules
+COPY --from=deps /prod/shared/node_modules shared/node_modules
 COPY --from=deps /src/package.json package.json
 COPY --from=deps /src/bun.lock bun.lock
 # The browser's tree is a separate install root with its own lockfile rather than a workspace of the

README.md

@@ -64,7 +64,7 @@ A Bot is any endpoint speaking [AG-UI](https://github.com/ag-ui-protocol/ag-ui),
 ## Requirements
 
 - Docker, for PostgreSQL and the shipped Bots.
-- [Bun](https://bun.sh) 1.3+, for the app and API server.
+- [Bun](https://bun.sh) 1.4+, for the app, the API server and the tests (some tests fail on 1.3).
 - A CopilotKit Intelligence project and license. A free plan is available, and Intelligence can be self-hosted.
 - A model key. The proof-of-concept Bot uses OpenAI; the LangGraph Bot can use OpenAI, Anthropic, or Google.
 

agent-computer/Dockerfile

@@ -29,8 +29,10 @@ RUN bun install --frozen-lockfile
 
 COPY agent-computer/src ./src
 # index.ts imports ../../shared/file-download at runtime (the other shared imports are type-only and
-# erased), so the one file it needs goes where that path resolves from /app/src.
+# erased), and demonstration.ts imports ../../shared/demonstration-rules at runtime too, so the files
+# they need go where those paths resolve from /app/src.
 COPY shared/file-download.ts /shared/file-download.ts
+COPY shared/demonstration-rules.ts /shared/demonstration-rules.ts
 
 # Durable working directory. Bot-produced files that must survive replacement live here.
 RUN mkdir -p /workspace

agent-computer/src/demonstration.ts

@@ -1,63 +1,11 @@
 import type { Page } from "playwright";
+import {
+  describeDemonstrationTarget,
+  gestureForRecording,
+  safeDemonstrationUrl,
+} from "../../shared/demonstration-rules";
 import type { InputMessage } from "./screencast";
-export type RecordedGesture = {
-  kind: "click" | "type" | "key" | "scroll";
-  key?: string;
-  deltaY?: number;
-};
-export function gestureForRecording(
-  input: InputMessage,
-): RecordedGesture | null {
-  if (input.type === "mouse")
-    return input.event === "released" && input.button !== "right"
-      ? { kind: "click" }
-      : null;
-  if (input.type === "wheel")
-    return {
-      kind: "scroll",
-      deltaY: Math.max(-2000, Math.min(2000, input.deltaY)),
-    };
-  if (input.type === "text") return { kind: "type" };
-  if (input.event !== "down") return null;
-  if (input.key.length === 1 || (input.text && input.key !== "Enter"))
-    return { kind: "type" };
-  if (
-    [
-      "Enter",
-      "Tab",
-      "Escape",
-      "Backspace",
-      "Delete",
-      "ArrowDown",
-      "ArrowUp",
-      "ArrowLeft",
-      "ArrowRight",
-      "Home",
-      "End",
-      "PageDown",
-      "PageUp",
-    ].includes(input.key)
-  )
-    return { kind: "key", key: input.key };
-  return null;
-}
-export function safeDemonstrationUrl(raw: string): string {
-  try {
-    const url = new URL(raw);
-    if (!["http:", "https:"].includes(url.protocol)) return "about:blank";
-    const path = url.pathname
-      .split("/")
-      .map((part) =>
-        part.length > 40 || /token|secret|password|credential/i.test(part)
-          ? "[redacted]"
-          : part,
-      )
-      .join("/");
-    return `${url.origin}${path}`;
-  } catch {
-    return "about:blank";
-  }
-}
+
 /** Never reads a field's value, page body, clipboard or image. All text entry is parameterized. */
 export async function describeHumanGesture(page: Page, input: InputMessage) {
   const gesture = gestureForRecording(input);
@@ -66,77 +14,6 @@ export async function describeHumanGesture(page: Page, input: InputMessage) {
     input.type === "mouse" || input.type === "wheel"
       ? { x: input.x, y: input.y }
       : null;
-  const target = await page.evaluate((point) => {
-    const hit = point
-      ? document.elementFromPoint(point.x, point.y)
-      : document.activeElement;
-    const closest =
-      hit?.closest(
-        "button,a,input,textarea,select,label,[role],[contenteditable=true]",
-      ) ?? hit;
-    // A click on a label's text is a click on the control it names: "Small", a radio, not a label.
-    const element =
-      closest instanceof HTMLLabelElement
-        ? (closest.control ?? closest)
-        : closest;
-    if (!element) return { role: "page", name: "Page", sensitive: false };
-    const tag = element.tagName.toLowerCase();
-    const inputType =
-      element instanceof HTMLInputElement ? element.type.toLowerCase() : "";
-    const role =
-      element.getAttribute("role") ||
-      (
-        {
-          checkbox: "checkbox",
-          radio: "radio",
-          button: "button",
-          submit: "button",
-          reset: "button",
-          image: "button",
-          range: "slider",
-          file: "button",
-        } as Record<string, string>
-      )[inputType] ||
-      (
-        {
-          button: "button",
-          a: "link",
-          input: "textbox",
-          textarea: "textbox",
-          select: "combobox",
-        } as Record<string, string>
-      )[tag] ||
-      tag;
-    const label =
-      element.getAttribute("aria-label") ||
-      (element instanceof HTMLInputElement ||
-      element instanceof HTMLTextAreaElement ||
-      element instanceof HTMLSelectElement
-        ? Array.from(element.labels ?? [])
-            .map((entry) => entry.textContent ?? "")
-            .join(" ")
-        : "") ||
-      (tag === "button" || tag === "a" ? element.textContent : "") ||
-      element.getAttribute("placeholder") ||
-      role;
-    const sensitive =
-      element.getAttribute("type") === "password" ||
-      // Short words only as whole words: "pin" is in "topping" and "shipping", "otp" in "hotpot".
-      /password|secret|token|one.time|verification|credit.card|security.code|passcode|(?:^|[^a-z])(?:cc-|(?:cvc|cvv|pin|otp)(?![a-z]))/i.test(
-        [
-          label,
-          element.getAttribute("name"),
-          element.getAttribute("autocomplete"),
-        ].join(" "),
-      );
-    const name = sensitive
-      ? "[sensitive field]"
-      : label
-          .replace(/[A-Za-z0-9_+/=-]{32,}/g, "[redacted]")
-          .replace(/[^\s@]+@[^\s@]+/g, "[redacted]")
-          .trim()
-          .slice(0, 120);
-    return { role: role.slice(0, 40), name: name || role, sensitive };
-  }, point);
+  const target = await page.evaluate(describeDemonstrationTarget, { point });
   return { ...gesture, url: safeDemonstrationUrl(page.url()), target };
 }