From 81e2e8f1134ff56570aa7d196ed95668ae886f41 Mon Sep 17 00:00:00 2001 From: ditadi Date: Mon, 12 Oct 2026 13:45:02 +0100 Subject: [PATCH] feat(appkit): offer the database plugin in databricks apps init The DatabasePlugin is no longer hidden from `databricks apps init`. The template ships config/database/schema.ts in the app schema, and app.yaml sets APPKIT_DB_MIGRATE_ON_START=true for apps that select the plugin, so the deployed app applies its committed migrations on start. The CLI manifest renders only valueFrom entries, so the literal lives in app.yaml.tmpl. drizzle-kit stays out of the template and its lockfiles: `appkit db init` installs the pinned version when the app lacks it, so apps without the plugin do not get it. `appkit db verify` warns when .env points at a branch `appkit db init` did not set up for you, such as the deploy branch the scaffolded .env names until `db init` runs. The template's server exits 1 when startup fails, so a failed migration stops the app instead of leaving it up without its routes. The owner-mismatch error names the recovery for an app recreated with a new service principal: `databricks postgres delete-role --reassign-owned-to`. The plugin docs and the template README describe the flow end to end: `appkit db init` once after scaffolding, `appkit db migrate` while you develop, migrations on deploy with their per-migration transactions and lock limits, two-step changes for tables in use, rollback, existing apps, `appkit db verify --deployed -t ` in CI, and one owner per branch. The Lakebase docs start local development on a dev branch. The template's .gitignore skips config/database/*.bak, the copy of schema.ts that `appkit db introspect` keeps. Signed-off-by: ditadi --- docs/docs/plugins/database.md | 198 +++++++++++++++++- docs/docs/plugins/lakebase.md | 25 ++- packages/appkit/src/beta.ts | 1 - .../src/database/migrations/ownership.ts | 5 +- .../migrations/tests/ownership.test.ts | 3 + .../src/database/tooling/tests/verify.test.ts | 35 +++- .../appkit/src/database/tooling/verify.ts | 19 +- .../src/plugins/beta-exports.generated.ts | 1 + .../appkit/src/plugins/database/manifest.json | 2 +- packages/shared/src/cli/commands/db/output.ts | 5 + .../src/cli/commands/db/tooling-contract.ts | 1 + .../shared/src/cli/commands/db/verify.test.ts | 16 ++ packages/shared/src/cli/index.ts | 3 +- template/README.md.tmpl | 52 ++++- template/_gitignore | 3 + template/app.yaml.tmpl | 5 + template/appkit.plugins.json | 109 ++++++++++ template/config/database/schema.ts | 5 +- template/server/server.ts | 5 +- 19 files changed, 457 insertions(+), 36 deletions(-) diff --git a/docs/docs/plugins/database.md b/docs/docs/plugins/database.md index 1467dd10b..fde484b5e 100644 --- a/docs/docs/plugins/database.md +++ b/docs/docs/plugins/database.md @@ -21,10 +21,9 @@ not support OBO and does not apply per-user or per-row authorization. Every call who can reach the generated API can perform every enabled operation on every exposed row, including delete. -Restrict access to the app and grant its service principal only the database -permissions it needs. If users need different permissions or row ownership checks, -disable the relevant generated routes and implement authorization in custom -server routes. App admission alone does not provide row-level isolation. +Restrict access to the app. If users need different permissions or row ownership +checks, disable the relevant generated routes and implement authorization in +custom server routes. App admission alone does not provide row-level isolation. ::: ## Basic usage @@ -39,9 +38,11 @@ identifier-only messages for known missing-column or missing-table errors details, and hints are omitted because they may contain row values. Apps scaffolded with the Database plugin selected include an empty -`config/database/schema.ts`, so `database()` can start without requiring sample -tables. Replace the empty declaration with your models when their PostgreSQL -tables are ready. +`config/database/schema.ts` in the Postgres schema `app`, and set +`APPKIT_DB_MIGRATE_ON_START=true` in `app.yaml` so the deployed app applies its +migrations on start. Run `npx appkit db init` once to point `.env` at a branch of +your own, declare your tables, then run `npx appkit db migrate` to create them there +(see [Development workflow](#development-workflow)). For local development, the PostgreSQL username is resolved from your Databricks credentials when `PGUSER` and `DATABRICKS_CLIENT_ID` are absent. An explicitly @@ -112,6 +113,189 @@ Migrations applied on start into `public` also need `GRANT CREATE ON SCHEMA public` to the app's service principal, because it does not own `public`. +## Development workflow + +You develop against a Lakebase branch of your own, never the deployed one. Tables +are created and changed through migrations: generated locally from `schema.ts`, +committed with your code, and applied by the deployed app when it starts. + +### Set up your branch + +Run `npx appkit db init` once after scaffolding, and again after cloning. Until +`databricks apps init` runs it for you, the scaffolded `.env` points at the branch +you picked in its prompts, usually the branch the app deploys to, and +`npx appkit db verify` warns about that. + +`db init` asks nothing: + +1. It keeps this checkout's session branch when `.env` points at one, and + otherwise uses your dev branch, `dev--`. It creates the branch when + it does not exist yet. `--session` uses a branch for this checkout that expires + after 7 days (`--ttl` changes that), and `--dev` goes back to your dev branch. +2. It makes sure the branch has an endpoint and your Postgres role, hands you the + app schema the fork copied, and points `.env` at the branch. +3. It sets up the schema. With `schema.ts` it applies the committed migrations; + without one, it writes `schema.ts` from the branch's tables (`introspect`). + `--schema clean` drops the app schema first, and `--schema none` leaves it as it + is. Without a terminal, `--schema clean` needs `--yes`. +4. When that needs drizzle-kit and the app does not have it, it installs the + version AppKit pins as a dev dependency. `migrate` and `introspect` generate + migrations with it. + +`db init` never changes the project's default branch, a protected branch or a +branch `databricks.yml` deploys to, and never edits `databricks.yml`. + +:::caution[A new branch copies the default branch] +New branches fork the project's default branch, so they start with a copy of its +rows, production data included. Your role there is a member of +`DATABRICKS_SUPERUSER`, which reads and writes every table on the branch and +bypasses row-level security. Creating a branch needs CAN MANAGE on the Lakebase +project, which also allows deleting any branch, the deploy branch included. Use a +project whose default branch holds only data every developer may see. +::: + +### Commands + +| Command | What it does | +| --- | --- | +| `npx appkit db init` | Points `.env` at your dev branch, or with `--session` at a branch for this checkout that expires, then sets up the schema. | +| `npx appkit db migrate` | Writes `config/database/migrations/NNNN_migration.sql` when `schema.ts` changed, and applies pending migrations to your branch. | +| `npx appkit db introspect` | Writes `schema.ts` from tables that already exist, and a `0000_baseline` migration (or `NNNN_introspect` when migrations already exist). | +| `npx appkit db clean` | Drops the app schema on your branch; `migrate` then rebuilds it from the migrations. `--delete-branch` deletes your session branch instead. | +| `npx appkit db seed` | Runs `config/database/seed.sql` on your branch in one transaction. | +| `npx appkit db verify` | Read-only check of your branch: missing tables or columns, pending migrations, migrations that will never run there, and migrations it ran that this checkout does not commit. `--deployed` checks a target's deployed app instead. | + +`migrate`, `clean` and `seed` change only branches `db init` created for you (your +dev branch, or this checkout's session branch). On any other branch, including the +project's default branch, a protected branch, or a branch `databricks.yml` +deploys to, `clean` and `seed` refuse and `migrate` only writes the migration files. + +A typical change: + +```bash +# edit config/database/schema.ts +npx appkit db migrate # writes and applies NNNN_migration.sql on your branch +git add config/database # commit schema.ts with its migrations +databricks apps deploy # the deployed app applies it on start +npx appkit db verify --deployed # the app runs, with every committed migration +``` + +### Changing tables in use + +While a new release migrates on start, instances of the previous release keep +serving, so every migration must work with the code that is still running. A +change that drops or renames a table or column, changes a column's type, or makes +a column required breaks them, and `db migrate` warns when it writes one. Split such a +change over two deploys. First deploy code that works with both the old and the +new schema, with a migration that only adds. Then, in a later deploy, drop what +the old code used. + +### Migrations on deploy + +When the deployed app starts with `APPKIT_DB_MIGRATE_ON_START=true` in `app.yaml`, +`database()` applies the committed migrations the deploy branch has not recorded +yet, before it checks the tables and serves requests. Apps scaffolded with the +plugin set it; add it to the `env` of an app that predates it: + +```yaml +env: + - name: APPKIT_DB_MIGRATE_ON_START + value: "true" +``` + +Migrations run as the app's service principal, one instance at a time, each in +its own transaction. If one fails, it is rolled back, the ones before it stay +applied, and the app does not start. The server log names the migration and the +SQLSTATE, or the missing migration file. An instance waits up to 5 minutes for +another one to finish migrating. A migration that waits more than 5 seconds for a +table lock, for example behind a long transaction of the running app, is rolled +back and retried twice, then fails instead of blocking that table. + +`.env` never sets the flag, so a local start (`npm run dev`, `npm start`) never +migrates: use `npx appkit db migrate`. An app pointed at a dev or session branch +does not migrate it on start either. + +A `0000_baseline` from `introspect` is recorded without running when its tables +already exist on the deploy branch and no migration was ever applied there. If +only some of them exist, the app does not start, and nothing is changed. + +### Existing apps + +- **Tables created by hand or by earlier code.** Without `schema.ts`, + `npx appkit db init` creates your dev branch from the project's default branch and + introspects it: it writes `schema.ts` from the tables there and a `0000_baseline` + migration. With a `schema.ts` already, run `npx appkit db introspect` on your + branch. Commit both files: the deployed app records the baseline without running + it, because the tables already exist on the deploy branch. +- **Tables in `public`.** Earlier releases used `public` by default. Keep them there + with `schemaName: "public"`, as [Apps with tables in `public`](#apps-with-tables-in-public) + shows. +- **No migrate-on-start flag.** Add `APPKIT_DB_MIGRATE_ON_START=true` to `app.yaml`, + as [Migrations on deploy](#migrations-on-deploy) shows. Without it, the app checks + its tables on start but creates none. + +### Rolling back + +Migrations only go forward. There are no down migrations, and the app never undoes +one. Redeploying an older commit leaves the newer tables and their recorded +migrations in place: the app logs that the branch recorded migrations the older +code does not have, applies nothing, and starts. The older code must work with the +newer schema, which [two-step changes](#changing-tables-in-use) guarantee. To undo +a schema change, commit a new migration that reverses it, and deploy it. + +### One owner per branch + +Each branch has one owner of the app schema, and only the owner can change its +tables: + +- On the deploy branch, the app's service principal creates schema `app` on its + first start and owns everything in it. The `postgres` resource's + `CAN_CONNECT_AND_CREATE` permission is all it needs; no grants are required. +- On your dev or session branch, you own it. A fork copies its parent's objects + with their owners, so `db init` hands them over to you. + +Do not apply migrations to the deploy branch yourself: you would own those objects, +and the app could no longer change them. If schema `app` on the deploy branch +already belongs to another role, the app cannot migrate it, and the server log +names both roles. Use another `schemaName` in `defineSchema()`, or have the owner +transfer the schema. + +An app that was deleted and created again has a new service principal, and the old +one still owns the schema. Someone with CAN MANAGE on the Lakebase project moves +the objects to the new one, which deletes the old role: + +```bash +# Both service principals' roles, by their client IDs +databricks postgres list-roles projects//branches/ +databricks postgres delete-role projects//branches//roles/ \ + --reassign-owned-to projects//branches//roles/ +``` + +### Verify the deployed app + +`npx appkit db verify --deployed` checks a `databricks.yml` target's app through +the Databricks API: the app exists, its `postgres` resource names the target's +branch and database with `CAN_CONNECT_AND_CREATE`, its service principal has a role +on the branch, the app is running, and its active deployment has every committed +migration. `-t, --target` picks the target; without it, verify uses the bundle's +default target. When the target's `workspace.host` is not the workspace your shell +is authenticated against, verify refuses and names the profile to set. It needs no +access to the deploy branch's database and changes nothing. A failed check names +`databricks apps logs`, `start` or `deploy`. + +### CI + +`appkit db init` sets up branches for developers, and CI does not need one. After +a deploy, check the deployed app with workspace credentials: + +```bash +databricks apps deploy -t prod +npx appkit db verify --deployed -t prod +``` + +It exits 1 when the app is not running, is bound to another branch, or lacks a +committed migration. + ## Schema discovery and overrides `database()` and `database({})` use the same defaults. During setup, the plugin diff --git a/docs/docs/plugins/lakebase.md b/docs/docs/plugins/lakebase.md index c887073da..b581b5d74 100644 --- a/docs/docs/plugins/lakebase.md +++ b/docs/docs/plugins/lakebase.md @@ -143,7 +143,30 @@ When you create the app with the Lakebase resource using the [Getting started](# ### Local development -To develop locally against a deployed Lakebase database: +Develop against a Lakebase branch of your own, so local runs never write to the +branch the app deploys to: + +```bash +npx appkit db init --schema none +``` + +It creates your dev branch from the project's default branch, gives it an endpoint +and a Postgres role for your Databricks identity, and points `.env` at it. The role +is a member of `DATABRICKS_SUPERUSER`, so you read and write every table the branch +copied. Copied objects keep their owner, usually the app's Service Principal, so +changing them (DDL) needs that owner; what you create on your branch is yours. +`--schema none` leaves the branch's schemas as they are. Apps that use the +[Database plugin](./database.md#set-up-your-branch) drop it, so `db init` also sets +up the app schema. + +:::caution +A new branch starts with a copy of the default branch's rows, production data +included. See [Set up your branch](./database.md#set-up-your-branch). +::: + +#### Against the deployed branch + +To develop locally against the branch the app deploys to: 1. **Deploy the app first.** The Service Principal creates the database schema and tables on first deploy. Apps generated from `databricks apps init` handle this automatically - they check if tables exist on startup and skip creation if they do. diff --git a/packages/appkit/src/beta.ts b/packages/appkit/src/beta.ts index 03d574653..ca75d8272 100644 --- a/packages/appkit/src/beta.ts +++ b/packages/appkit/src/beta.ts @@ -121,7 +121,6 @@ export type { SearchResult, } from "./plugins/ai-search/types"; export * from "./plugins/beta-exports.generated"; -export { database } from "./plugins/database"; export type { DatabaseApiConfig, DatabaseApiWriteOperation, diff --git a/packages/appkit/src/database/migrations/ownership.ts b/packages/appkit/src/database/migrations/ownership.ts index 59a7be8d4..6f0951da1 100644 --- a/packages/appkit/src/database/migrations/ownership.ts +++ b/packages/appkit/src/database/migrations/ownership.ts @@ -63,8 +63,9 @@ export function ownershipHint( return ( `Schema "${ownership.schemaName}" is owned by "${ownership.owner}", but the app connects as "${ownership.currentUser}". ` + "This happens when the app's database resource was removed and added again, or the app was recreated with a new service principal: " + - `Lakebase hands the objects to another role. Have "${ownership.owner}" transfer the schema and its tables to "${ownership.currentUser}" ` + - "(ALTER … OWNER TO), or set another schemaName in schema.ts. " + + `Lakebase hands the objects to another role. If "${ownership.owner}" belonged to a deleted app, someone with CAN MANAGE on the Lakebase project moves its objects with ` + + `\`databricks postgres delete-role --reassign-owned-to \` (\`databricks postgres list-roles \` names both), which also deletes its role. ` + + `Otherwise have "${ownership.owner}" transfer the schema and its tables to "${ownership.currentUser}" (ALTER … OWNER TO), or set another schemaName in schema.ts. ` + "On your dev or session branch, rerun `appkit db init`, which hands you the schema." ); } diff --git a/packages/appkit/src/database/migrations/tests/ownership.test.ts b/packages/appkit/src/database/migrations/tests/ownership.test.ts index d1dda3838..c5977aa39 100644 --- a/packages/appkit/src/database/migrations/tests/ownership.test.ts +++ b/packages/appkit/src/database/migrations/tests/ownership.test.ts @@ -25,6 +25,9 @@ describe("ownership: one owner per branch", () => { expect(hint).toContain( "the app's database resource was removed and added again, or the app was recreated", ); + expect(hint).toContain( + `If "old-app-sp" belonged to a deleted app, someone with CAN MANAGE on the Lakebase project moves its objects with \`databricks postgres delete-role --reassign-owned-to \``, + ); expect(hint).toContain("set another schemaName in schema.ts"); expect(hint).toContain( "On your dev or session branch, rerun `appkit db init`, which hands you the schema.", diff --git a/packages/appkit/src/database/tooling/tests/verify.test.ts b/packages/appkit/src/database/tooling/tests/verify.test.ts index 70d57db23..3f8cbb144 100644 --- a/packages/appkit/src/database/tooling/tests/verify.test.ts +++ b/packages/appkit/src/database/tooling/tests/verify.test.ts @@ -5,12 +5,19 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -const { withBranchPool } = vi.hoisted(() => ({ withBranchPool: vi.fn() })); +const { withBranchPool, resolveEnvBranch } = vi.hoisted(() => ({ + withBranchPool: vi.fn(), + resolveEnvBranch: vi.fn(), +})); vi.mock("../context", async (importOriginal) => ({ ...(await importOriginal()), withBranchPool, })); +vi.mock("../guard", async (importOriginal) => ({ + ...(await importOriginal()), + resolveEnvBranch, +})); import type { WorkspaceRequest } from "../../../workspace-client"; import type { DbContext } from "../context"; @@ -155,6 +162,10 @@ const WRITES = /\b(insert|update|delete|create|drop|alter|grant|truncate)\b/i; beforeEach(() => { withBranchPool.mockReset(); + resolveEnvBranch.mockReset().mockResolvedValue({ + branch: { name: BRANCH }, + kind: "dev", + }); }); afterEach(() => { @@ -169,6 +180,7 @@ describe("runVerify", () => { { tag: "0001_migration", when: 2000 }, ]; const clean = { + notYours: null, schemaProblems: [], pendingMigrations: [], strandedMigrations: [], @@ -270,6 +282,25 @@ describe("runVerify", () => { }); }); + test("a branch that is not your dev or session branch is named, and still verified", async () => { + const ctx = app(two); + branch({ + columns: { notes: ["id", "body"] }, + recorded: recordedAs(ctx, two), + }); + resolveEnvBranch.mockResolvedValue({ + branch: { name: "projects/p/branches/production" }, + kind: "default", + reason: "it is the project's default branch", + }); + + expect(await runVerify(ctx)).toMatchObject({ + branch: "projects/p/branches/production", + notYours: "it is the project's default branch", + ok: true, + }); + }); + test("no committed migrations: only the schema is checked", async () => { const ctx = app(); branch({ columns: { notes: ["id", "body"] }, recorded: "absent" }); @@ -307,6 +338,7 @@ describe("runVerify", () => { }); test("without a configured branch it points to init before connecting", async () => { + resolveEnvBranch.mockResolvedValue(null); await expect(runVerify(app([], {}))).rejects.toThrow( "Did not verify: no branch is configured in .env (PGHOST, PGDATABASE and LAKEBASE_ENDPOINT); run `appkit db init`.", ); @@ -477,6 +509,7 @@ describe("runVerify --deployed", () => { `workspace.export ${SNAPSHOT}/config/database/migrations/meta/_journal.json`, ]); expect(withBranchPool).not.toHaveBeenCalled(); + expect(resolveEnvBranch).not.toHaveBeenCalled(); expect(lakebase.calls.sort()).toEqual([ `getBranch ${PRODUCTION}`, `getDatabase ${DATABASE}`, diff --git a/packages/appkit/src/database/tooling/verify.ts b/packages/appkit/src/database/tooling/verify.ts index 3c436ff02..1885e11c0 100644 --- a/packages/appkit/src/database/tooling/verify.ts +++ b/packages/appkit/src/database/tooling/verify.ts @@ -12,17 +12,12 @@ import { readJournal, } from "../migrations/paths"; import type { DataPath } from "../runtime"; -import { - type DbContext, - hasEnvBranch, - NO_ENV_BRANCH, - withBranchPool, -} from "./context"; +import { type DbContext, NO_ENV_BRANCH, withBranchPool } from "./context"; +import { resolveEnvBranch, takesWrites } from "./guard"; import { requireSchemaFile } from "./migrate"; import { type BundleTarget, defaultBundleTarget, - parseResourceName, readBundle, resourceId, } from "./targets"; @@ -38,6 +33,8 @@ export interface VerifyOptions { interface LocalVerifyReport { readonly kind: "local"; readonly branch: string; + /** Why the branch is not your dev or session branch, e.g. "it is the project's default branch"; null when it is. */ + readonly notYours: string | null; /** Declared tables or columns the branch lacks (the plugin's startup check). */ readonly schemaProblems: readonly string[]; /** Committed migrations the branch has not run yet, newer than every one it ran. */ @@ -98,10 +95,9 @@ function rawOn(client: Pick): Pick { /** Read-only, as the developer, on the `.env` branch: one READ ONLY transaction, rolled back. */ async function verifyLocal(ctx: DbContext): Promise { requireSchemaFile(ctx.root, "verify"); - const branch = parseResourceName(ctx.env.LAKEBASE_ENDPOINT)?.branch; - if (!branch || !hasEnvBranch(ctx.env)) { - throw new Error(`Did not verify: ${NO_ENV_BRANCH}.`); - } + const env = await resolveEnvBranch(ctx); + if (!env) throw new Error(`Did not verify: ${NO_ENV_BRANCH}.`); + const branch = env.branch.name; const schema = await loadDefaultDatabaseSchema(ctx.root, { showImportError: true, }); @@ -117,6 +113,7 @@ async function verifyLocal(ctx: DbContext): Promise { return { kind: "local", branch, + notYours: takesWrites(env.kind) ? null : (env.reason ?? null), schemaProblems, pendingMigrations: drift.pending, strandedMigrations: drift.stranded, diff --git a/packages/appkit/src/plugins/beta-exports.generated.ts b/packages/appkit/src/plugins/beta-exports.generated.ts index 7e556ebd9..10982785b 100644 --- a/packages/appkit/src/plugins/beta-exports.generated.ts +++ b/packages/appkit/src/plugins/beta-exports.generated.ts @@ -7,3 +7,4 @@ export { agents } from "./agents"; export { aiSearch } from "./ai-search"; +export { database } from "./database"; diff --git a/packages/appkit/src/plugins/database/manifest.json b/packages/appkit/src/plugins/database/manifest.json index 111a7b6be..fc665fe97 100644 --- a/packages/appkit/src/plugins/database/manifest.json +++ b/packages/appkit/src/plugins/database/manifest.json @@ -4,7 +4,6 @@ "displayName": "Database (Beta)", "description": "Schema-driven typed access to Databricks Lakebase PostgreSQL", "stability": "beta", - "hidden": true, "resources": { "required": [ { @@ -17,6 +16,7 @@ "project": { "description": "Lakebase project resource name", "examples": ["projects/{project-id}"], + "bundleIgnore": true, "discovery": { "type": "kind", "resourceKind": "postgres_project", diff --git a/packages/shared/src/cli/commands/db/output.ts b/packages/shared/src/cli/commands/db/output.ts index 326091077..18594ab27 100644 --- a/packages/shared/src/cli/commands/db/output.ts +++ b/packages/shared/src/cli/commands/db/output.ts @@ -119,6 +119,11 @@ function localVerifyHint(report: LocalVerifyReport): string { function printLocalVerifyReport(report: LocalVerifyReport): void { const branch = shortName(report.branch); + if (report.notYours) { + warn( + `.env points at ${branch}, not a branch \`appkit db init\` set up for you: ${report.notYours}. \`npx appkit db init\` points it at a branch of your own.`, + ); + } if (report.ok) { ok( `${branch} has every table and column config/database/schema.ts declares, and every committed migration`, diff --git a/packages/shared/src/cli/commands/db/tooling-contract.ts b/packages/shared/src/cli/commands/db/tooling-contract.ts index 8bf01a291..80930202e 100644 --- a/packages/shared/src/cli/commands/db/tooling-contract.ts +++ b/packages/shared/src/cli/commands/db/tooling-contract.ts @@ -75,6 +75,7 @@ export interface VerifyOptions { export interface LocalVerifyReport { readonly kind: "local"; readonly branch: string; + readonly notYours: string | null; readonly schemaProblems: readonly string[]; readonly pendingMigrations: readonly string[]; readonly strandedMigrations: readonly string[]; diff --git a/packages/shared/src/cli/commands/db/verify.test.ts b/packages/shared/src/cli/commands/db/verify.test.ts index 2592175e6..3770284a0 100644 --- a/packages/shared/src/cli/commands/db/verify.test.ts +++ b/packages/shared/src/cli/commands/db/verify.test.ts @@ -32,6 +32,7 @@ function output(spy: MockInstance): string { const local = { kind: "local" as const, branch: "projects/p/branches/dev-alice-3f9a2c", + notYours: null, schemaProblems: [], pendingMigrations: [], strandedMigrations: [], @@ -87,6 +88,21 @@ describe("appkit db verify", () => { expect(process.exitCode).toBeUndefined(); }); + test("a .env branch that is not yours is named, with init as the way to your own", async () => { + runVerify.mockResolvedValue({ + ...local, + branch: "projects/p/branches/production", + notYours: "it is the project's default branch", + }); + + await dbCommand.parseAsync(["verify"], { from: "user" }); + + expect(output(log)).toContain( + ".env points at production, not a branch `appkit db init` set up for you: it is the project's default branch. `npx appkit db init` points it at a branch of your own.", + ); + expect(process.exitCode).toBeUndefined(); + }); + test("prints every schema problem and pending migration, and exits 1", async () => { runVerify.mockResolvedValue({ ...local, diff --git a/packages/shared/src/cli/index.ts b/packages/shared/src/cli/index.ts index 2fce08005..55bd50f9e 100644 --- a/packages/shared/src/cli/index.ts +++ b/packages/shared/src/cli/index.ts @@ -36,8 +36,7 @@ cmd.addCommand(docsCommand); cmd.addCommand(pluginCommand); cmd.addCommand(codemodCommand); cmd.addCommand(doctorCommand); -// Hidden from --help until the database plugin is offered by `databricks apps init`. -cmd.addCommand(dbCommand, { hidden: true }); +cmd.addCommand(dbCommand); // Registry commands are executable but hidden from --help while the feature // is still in development (registry + add work end-to-end but aren't announced). cmd.addCommand(registryCommand, { hidden: true }); diff --git a/template/README.md.tmpl b/template/README.md.tmpl index c21dc1cc0..24270e948 100644 --- a/template/README.md.tmpl +++ b/template/README.md.tmpl @@ -49,13 +49,38 @@ The Lakebase plugin requires additional environment variables for PostgreSQL con {{- if .plugins.database}} -#### Database schema +#### Database -`database()` loads `config/database/schema.ts`. The generated schema starts empty, -so initialization does not require sample tables or expose existing data. -Add your table declarations after creating the corresponding PostgreSQL tables; -AppKit does not create or migrate them. CRUD routes are enabled for declared tables -by default, and can be restricted with `api` in the plugin configuration. +`database()` loads `config/database/schema.ts`, whose tables live in the Postgres +schema `app`. CRUD routes are enabled for declared tables by default, and can be +restricted with `api` in the plugin configuration. + +Local development runs against a Lakebase branch of your own, never the deployed one. +Run `npx appkit db init` once after scaffolding, and again after cloning. It asks +nothing: it points `.env` at your branch and sets up its schema. + +```bash +npx appkit db init # your dev branch, forked from the project's default branch +npx appkit db init --session # a branch for this checkout that expires after 7 days +npx appkit db init --dev # back to your dev branch +``` + +A new branch starts with a copy of the default branch's rows, production data included. + +To change the schema, edit `config/database/schema.ts`, then: + +```bash +npx appkit db migrate # writes config/database/migrations/NNNN_migration.sql and applies it to your branch +npx appkit db verify # read-only check of your branch against schema.ts and the migrations +``` + +Commit the migrations. The deployed app applies them on start, because `app.yaml` +sets `APPKIT_DB_MIGRATE_ON_START=true`. To adopt tables that already exist, run +`npx appkit db introspect`: it writes `schema.ts` from the branch's tables, and a +`0000_baseline` migration (or `NNNN_introspect` when migrations already exist). +`npx appkit db clean` drops the app schema on your branch, and `npx appkit db seed` +runs `config/database/seed.sql` on it; both refuse any branch `db init` did not +create for you, such as the deployed one. {{- end}} ### CLI Authentication @@ -183,6 +208,14 @@ databricks apps deploy ```bash databricks apps deploy -t prod ``` +{{- if .plugins.database}} + +Then check that the deployed app runs with every committed migration: + +```bash +npx appkit db verify --deployed -t prod +``` +{{- end}} > **Restarting a stopped app:** apps stop after a period of inactivity. To start one again without redeploying, run `databricks apps start `. @@ -196,10 +229,15 @@ databricks apps deploy -t prod * server.ts # Server entry point * routes/ # Routes * shared/ # Shared types -{{- if .plugins.analytics}} +{{- if or .plugins.analytics .plugins.database}} * config/ # Configuration +{{- end}} +{{- if .plugins.analytics}} * queries/ # SQL query files {{- end}} +{{- if .plugins.database}} + * database/ # schema.ts and its migrations +{{- end}} * databricks.yml # Bundle configuration * app.yaml # App configuration * .env.example # Environment variables example diff --git a/template/_gitignore b/template/_gitignore index 29895e7c0..870cd2b9e 100644 --- a/template/_gitignore +++ b/template/_gitignore @@ -9,6 +9,9 @@ build/ test-results/ playwright-report/ +# `appkit db introspect` keeps the previous schema.ts here +config/database/*.bak + # Auto-generated types (endpoint-specific, varies per developer) shared/appkit-types/serving.d.ts diff --git a/template/app.yaml.tmpl b/template/app.yaml.tmpl index 845442fff..c0b909091 100644 --- a/template/app.yaml.tmpl +++ b/template/app.yaml.tmpl @@ -3,3 +3,8 @@ command: ['{{or .packageManager "pnpm"}}', 'run', 'start'] env: {{.appEnv}} {{- end}} +{{- if .plugins.database}} +{{- /* The plugin's postgres resource always renders LAKEBASE_ENDPOINT into .appEnv, so env: is open. */}} + - name: APPKIT_DB_MIGRATE_ON_START + value: "true" +{{- end}} diff --git a/template/appkit.plugins.json b/template/appkit.plugins.json index db53b96b7..a122753a5 100644 --- a/template/appkit.plugins.json +++ b/template/appkit.plugins.json @@ -196,6 +196,115 @@ } } }, + "database": { + "name": "database", + "displayName": "Database (Beta)", + "description": "Schema-driven typed access to Databricks Lakebase PostgreSQL", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "postgres", + "alias": "Postgres", + "resourceKey": "postgres", + "description": "Lakebase Postgres database for persistent storage", + "permission": "CAN_CONNECT_AND_CREATE", + "fields": { + "project": { + "description": "Lakebase project resource name", + "bundleIgnore": true, + "examples": [ + "projects/{project-id}" + ], + "discovery": { + "type": "kind", + "resourceKind": "postgres_project", + "select": "name" + }, + "origin": "user" + }, + "branch": { + "description": "Lakebase branch resource name", + "examples": [ + "projects/{project-id}/branches/{branch-id}" + ], + "discovery": { + "type": "kind", + "resourceKind": "postgres_branch", + "select": "name", + "dependsOn": "project" + }, + "origin": "user" + }, + "database": { + "description": "Lakebase database resource name", + "examples": [ + "projects/{project-id}/branches/{branch-id}/databases/{database-id}" + ], + "discovery": { + "type": "kind", + "resourceKind": "postgres_database", + "select": "name", + "dependsOn": "branch" + }, + "origin": "user" + }, + "host": { + "env": "PGHOST", + "description": "Postgres host", + "localOnly": true, + "resolve": "postgres:host", + "origin": "platform" + }, + "databaseName": { + "env": "PGDATABASE", + "description": "Postgres database name", + "localOnly": true, + "resolve": "postgres:databaseName", + "origin": "platform" + }, + "endpointPath": { + "env": "LAKEBASE_ENDPOINT", + "description": "Lakebase endpoint resource name", + "bundleIgnore": true, + "resolve": "postgres:endpointPath", + "origin": "cli" + }, + "port": { + "env": "PGPORT", + "description": "Postgres port", + "localOnly": true, + "value": "5432", + "origin": "platform" + }, + "sslmode": { + "env": "PGSSLMODE", + "description": "Postgres SSL mode", + "localOnly": true, + "value": "require", + "origin": "platform" + } + }, + "appOnly": true, + "binding": { + "yamlKey": "postgres", + "varFields": [ + [ + "branch", + "branch" + ], + [ + "database", + "database" + ] + ] + } + } + ], + "optional": [] + }, + "stability": "beta" + }, "files": { "name": "files", "displayName": "Files Plugin", diff --git a/template/config/database/schema.ts b/template/config/database/schema.ts index c0f6dc1e3..19163eae6 100644 --- a/template/config/database/schema.ts +++ b/template/config/database/schema.ts @@ -1,7 +1,8 @@ {{if .plugins.database -}} import { defineSchema } from '@databricks/appkit/beta'; -// database() discovers this file automatically. Start with no exposed tables. -// Add declarations here after creating the corresponding PostgreSQL tables. +// database() discovers this file automatically. Edit tables here, then run +// `npx appkit db migrate` on your dev branch. The deployed app's service principal +// owns schema "app" and applies committed migrations on start. export const schema = defineSchema(() => ({})); {{- end}} diff --git a/template/server/server.ts b/template/server/server.ts index b33bb94d8..14af3fcca 100644 --- a/template/server/server.ts +++ b/template/server/server.ts @@ -27,4 +27,7 @@ createApp({ await setupSampleLakebaseRoutes(appkit); }, {{- end}} -}).catch(console.error); +}).catch((error) => { + console.error(error); + process.exit(1); +});