Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
177 changes: 175 additions & 2 deletions .claude/skills/open-edit/SKILL.md

Large diffs are not rendered by default.

13 changes: 1 addition & 12 deletions .claude/skills/open-edit/scripts/preflight.sh
Original file line number Diff line number Diff line change
Expand Up @@ -155,38 +155,29 @@ else
REPOSITORY="${REPOSITORY_ARG:-$DEFAULT_REPOSITORY}"; REF="${REF_ARG:-$DEFAULT_REF}"
fi

brew_has() { have brew && brew list "$1" >/dev/null 2>&1; }
jpeg_ready() {
brew_has jpeg-turbo || [ -f /opt/homebrew/opt/jpeg-turbo/lib/libjpeg.8.dylib ] ||
[ -f /usr/local/opt/jpeg-turbo/lib/libjpeg.8.dylib ]
}

install_brew_formula() {
local formula="$1"
have brew || { say "Homebrew is required to install $formula; install Homebrew first"; FAILED=1; return; }
brew install "$formula" || FAILED=1
}

handle_global_dependencies() {
local missing_git=0 missing_node=0 missing_pnpm=0 missing_ffmpeg=0 missing_jpeg=0
local missing_git=0 missing_node=0 missing_pnpm=0 missing_ffmpeg=0
have git || missing_git=1
have node || missing_node=1
pnpm_ok || missing_pnpm=1
{ have "${VEED_ENGINE_FFMPEG:-ffmpeg}" && have "${VEED_ENGINE_FFPROBE:-ffprobe}"; } || missing_ffmpeg=1
jpeg_ready || missing_jpeg=1

if [ "$missing_git" -eq 1 ]; then need_approval "install Git globally: brew install git"; fi
if [ "$missing_node" -eq 1 ]; then need_approval "install Node globally: brew install node"; fi
if [ "$missing_pnpm" -eq 1 ]; then need_approval "install pnpm ${MIN_PNPM} or newer globally: npm install --global pnpm@${MIN_PNPM}"; fi
if [ "$missing_ffmpeg" -eq 1 ]; then need_approval "install FFmpeg globally: brew install ffmpeg"; fi
if [ "$missing_jpeg" -eq 1 ]; then need_approval "install jpeg-turbo globally: brew install jpeg-turbo"; fi

[ "$MODE" = "auto" ] || return
NEEDS_APPROVAL=0
[ "$missing_git" -eq 0 ] || install_brew_formula git
[ "$missing_node" -eq 0 ] || install_brew_formula node
[ "$missing_ffmpeg" -eq 0 ] || install_brew_formula ffmpeg
[ "$missing_jpeg" -eq 0 ] || install_brew_formula jpeg-turbo
if [ "$missing_pnpm" -eq 1 ]; then
have npm || { say "npm is unavailable after installing Node"; FAILED=1; return; }
npm install --global "pnpm@${MIN_PNPM}" || FAILED=1
Expand All @@ -195,7 +186,6 @@ handle_global_dependencies() {
have node || FAILED=1
pnpm_ok || FAILED=1
{ have "${VEED_ENGINE_FFMPEG:-ffmpeg}" && have "${VEED_ENGINE_FFPROBE:-ffprobe}"; } || FAILED=1
jpeg_ready || FAILED=1
}

handle_global_dependencies
Expand Down Expand Up @@ -276,7 +266,6 @@ handle_renderer() {
[ "$ROOT_KIND" != "missing" ] || return
engine="$(engine_path)"
if [ ! -x "$engine" ]; then
if ! jpeg_ready; then say "renderer installation is waiting for approved jpeg-turbo installation"; return; fi
if [ "$MODE" = "dry" ]; then say "WOULD APPLY LOCALLY — install the renderer in $ROOT/.veed-engine"; return; fi
say "installing the renderer locally"
bash "$ROOT/pipeline/scripts/install-veed-engine.sh" || die "renderer installation failed"
Expand Down
5 changes: 5 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ preview/transcribe-video.ts
# VEED login token caches (owner-only, never committed)
veed/.veed-token.json
veed/.veed-client.json
# The workspace the user chose to have billed (local state, names an account resource)
veed/.veed-workspace.json
# Speaking rates measured on THIS machine's paid runs. Local evidence, not shared state: the rates worth
# committing are seeded in veed/voice-rates.ts.
veed/.veed-voice-rates.json

# Harness installs skills per-checkout; track ONLY the product skill so a stray skill can't be
# committed (and then mirrored). The already-tracked SKILL.md stays tracked regardless.
Expand Down
7 changes: 4 additions & 3 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,8 @@ Run the isolated VEED tests without network access:

```sh
node --import tsx --test \
tests/cli-entry.test.ts tests/login-page.test.ts tests/oauth.test.ts tests/orchestrate.test.ts \
tests/token-store.test.ts \
tests/transcript-mapper.test.ts tests/workspace.test.ts
tests/charge-records.test.ts tests/cli-entry.test.ts tests/fabric.test.ts \
tests/generate.test.ts tests/generate-charge.test.ts tests/generate-set.test.ts tests/generate-validation.test.ts tests/login-page.test.ts tests/oauth.test.ts tests/orchestrate.test.ts tests/sample-presenter.test.ts \
tests/go.test.ts tests/token-store.test.ts \
tests/transcript-mapper.test.ts tests/voice-rates.test.ts tests/workspace.test.ts
```
6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ or MCP server when it helps. Source video is optional: stills, slides, generated
motion graphics are enough when the brief calls for it. Be creative and fluid — try new ideas
rather than collapsing every ask onto captions.

OpenEdit edits footage. Supply your own source files, or ask it to generate a talking-head clip with
VEED Fabric when you have none — that spends credits, and never without your approval.

<a href="https://github.com/veedstudio/open-edit/releases/download/launch-examples/OpenEdit-4x3-trim.mp4"><img src="docs/examples/openedit-launch.webp" alt="OpenEdit launch video" width="100%"></a>

*This launch video was made in OpenEdit — click it to watch with sound.*
Expand All @@ -42,7 +45,8 @@ rather than collapsing every ask onto captions.
| Intel Macs | Not supported — the renderer ships macOS-arm64 only |
| Older macOS | Built and tested on Tahoe 26.0. Nothing checks the version, so earlier releases may work — untested |
| Windows, Linux | Planned; prioritisation depends on demand |
| Transcription | needed only to caption speech — OpenEdit asks once and remembers: VEED ([sign up](https://www.veed.io/signup) / [login](https://www.veed.io/login)), WhisperX locally, or your own service |
| Transcription | needed only to caption speech — OpenEdit asks once and remembers: VEED ([sign up](https://www.veed.io/signup) / [login](https://www.veed.io/login)), WhisperX locally, or your own service. **VEED transcription consumes VEED credits**; WhisperX runs locally on your machine, and your own service is billed by whoever provides it |
| Generation | optional — a veed.io account, only if you ask OpenEdit to generate footage you do not have |

## Installation
Download this repo, or install via the command line with:
Expand Down
16 changes: 12 additions & 4 deletions SETUP.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,23 @@ Preflight has three modes: bare applies safe local setup, `--dry` reports withou

## Requirements
1. **macOS arm64**, **Git**, **Node**, and the pnpm version declared in `package.json`.
2. **jpeg-turbo** — `brew install jpeg-turbo`. It supplies `libjpeg.8.dylib`, which the renderer loads at launch.
3. **ffmpeg/ffprobe** (frame extraction + audio mux). On PATH or set `VEED_ENGINE_FFMPEG ` / `VEED_ENGINE_FFPROBE`.
4. **veed-engine-cli** — the render engine. Its first install is automatic and local; later updates require
2. **ffmpeg/ffprobe** (frame extraction + audio mux). On PATH or set `VEED_ENGINE_FFMPEG ` / `VEED_ENGINE_FFPROBE`.
3. **veed-engine-cli** — the render engine. Its first install is automatic and local; later updates require
approval. `pipeline/scripts/install-veed-engine.sh` downloads and verifies the latest macOS-arm64 release into `.veed-engine/`.
Set `VEED_ENGINE_BIN` to use an existing installation.
5. **A transcription provider** — the skill asks once and remembers the answer. Either
4. **A transcription provider** — the skill asks once and remembers the answer. Either
**VEED** (`node --import tsx veed/login.ts`, one-time OAuth, ~30-day refreshable token; a free account
covers about 10 minutes a month) or **WhisperX** locally (`bash pipeline/scripts/install-whisperx.sh`,
free and offline). You can also point it at your own service. See `README.md`.
5. **VEED credits.** **VEED transcription consumes VEED credits** (WhisperX runs locally on your
machine; your own service is billed by whoever provides it). Separately, with no source video the skill can GENERATE a talking-head clip
(`veed/generate.ts`), which spends a workspace's AI Playground credits. **Fabric reuses VEED
transcription's authentication** — same account, same login, same token; no connector, no second sign-in. Generating is TWO charges on two allowances: AI Playground
credits for Fabric One Lipsync (~4 a second of finished video) and text-to-speech SECONDS for the speech. The
script sets the duration at roughly 1,080 characters a minute, so a 60-second read is a couple of
hundred credits plus about a minute of TTS. It never picks the workspace for you and never spends
without your explicit approval of the estimate; what the balance moved by is reported and recorded
under `runs/<key>/`.
6. **config.ts** — optional machine-path overrides. Videos live anywhere:
invoke the skill with the video's path (absolute, or relative to your CWD).

Expand Down
17 changes: 12 additions & 5 deletions docs/FLOW.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,14 @@ SCRIPT steps are deterministic; AGENT steps need an LLM. The default (recipe-bac
end — the only agent is the opt-in analysis pass; creative face-1 (the user brought their own
reference/brand/concept) and creative iteration on a delivered result are authored inline by the orchestrator.
The whole thing is encoded executably in `.claude/skills/open-edit/SKILL.md` (the golden master) — this doc
is the conceptual map. One video → ONE captioned MP4. No per-shot intermediates, no user-approval gate.
is the conceptual map. One video → ONE captioned MP4. No per-shot intermediates, and no user-approval gate on
the captioning itself — the only gates are the FOOTAGE step's three, and they exist only when the user brought no
footage.

| step | kind | what |
|------|------|------|
| preflight | SCRIPT | check `veed-engine-cli` vs the latest release; offer install if stale/missing |
| footage | SCRIPT (the only step that STOPS to ask) | **only when the user supplied no footage** — generate one with VEED Fabric (`veed/generate.ts`, same VEED login as prep) → `runs/<key>/<key>.mp4`, which the PREP step then treats as ordinary footage. Three gates, in order: FOOTAGE (generate a video, take a clip they have/will record, use another model, or build from other sources instead — a non-Fabric model gets the speech caveat, since silent clips transcribe to nothing) → WORKSPACE (whose AI Playground credits; never picked for the user) → the CREDIT APPROVAL (script + presenter + voice + framing + quoted cost in ONE yes — the only gate that spends; cost tracks the finished DURATION at ~4 credits a second, and how long a script runs depends on the voice). Confirm and spend are two passes; a charged job is collected with `--resume`, never re-paid. Procedure: the FOOTAGE step of SKILL.md |
| prep | SCRIPT | transcript from the chosen provider (VEED `veed/go.ts`, local WhisperX `prep/transcribe.ts`, or the user's own via `prep/whisper.ts`; chunks = beats, real per-word timings) → `word-timings.json` (per-word absolute-ms reveal delays, synthesized by `prep/prep.ts`) + base frames; auto-detect aspect → write `meta.json` (canvas + duration + paths) |
| analyse | AGENT (vision) | **OPT-IN, skipped by default** — run only when the user asks to refine the style; then ONE nameless bg subagent reads the frames + transcript → `analysis.json` (per-beat shot/bboxes/neg-space/brightness in CANVAS px), and the DESIGN + RENDER step re-runs with it. The **only** vision pass. `analysis.json` existing IS the fast-path/refinement switch. |
| sample ONE style | SCRIPT | `pipeline/scripts/sample-style.ts` — facet-scored seeded draw of one aspect-matched ref from `refs/tags.json` (v3 RUNTIME INDEX, recipes only; `fit` = aspect SOT; transcript energy weights the draw) → `style.json` (facets, energy, coverage, alternates). Zero tokens. The CREATIVE PASS routes here too: face-1 (user brought materials) → the DESIGN + RENDER step, from-scratch; face-2 REMIX (any creative iteration) / RE-ROLL (variants asks) / patch (defects). |
Expand All @@ -17,18 +20,22 @@ is the conceptual map. One video → ONE captioned MP4. No per-shot intermediate
| preview | SCRIPT (parallel) | `preview/server.ts runs/<key>` — localhost preview opened for the user (read-only): watch and scrub the footage, follow the transcript, preview the subtitles; auto-swaps to the new `final/out.mp4` when an amend re-render lands. Launched in the background; the pipeline never waits on it |

## Where quality lives (do not let these drift)
- **Prep (1)** transcribes with whichever provider the user chose — asked once, recorded in
- **Footage** is the only step that spends the user's money, so it is the only one with gates: nothing
generates without the footage answer, no workspace is ever picked for the user, and ONE approval covers the
script, the presenter, the framing and the quoted cost. The charge is always reported, and a job already
paid for is resumed, never re-bought.
- **Prep** transcribes with whichever provider the user chose — asked once, recorded in
`.open-edit-prefs.json` at the runtime root (real per-word timings either way) — then extracts the base
frames and fixes the canvas from the source aspect; every downstream step reads `runs/<key>/meta.json`
(never re-derives dims).
- **Analysis (2)** is opt-in — the default run derives vibe from the transcript and places by safe margins.
- **Analysis** is opt-in — the default run derives vibe from the transcript and places by safe margins.
When the user asks to refine the style it becomes the sole vision pass: structured per-beat facts in CANVAS px
so the design pass composes from numbers, not pixels — the design pass never re-reads the frames.
- **Style selection (3)** is a seeded SCRIPT — same run key → same ref, `--seed/--style` to override
- **Style selection** is a seeded SCRIPT — same run key → same ref, `--seed/--style` to override
(`--style` only takes index ids — the runtime pool is recipes-only); recipe coverage grows offline. On creative face-1 the USER'S
materials are the design authority and the DIRECTION covers content/mood/placement only. Keep the
engagement seed copy **verbatim** (wording changes output).
- **Design (4)** commits ONE design system in a single pass (no aesthetic re-litigation). Recipes exist to
- **Design** commits ONE design system in a single pass (no aesthetic re-litigation). Recipes exist to
ELIMINATE per-run design analysis — and being compiled code, the per-run assembly is exact by construction
(paging, sizing, gate/fade math, verbatim `word-timings.json` delays; the arithmetic-slip failure class is
gone). Verification is engine-analytic (`--verify`) — the only fix is the mechanical ladder step-down; no
Expand Down
Loading