From 195e6b368ff3488a84641d78869cf97849e0bf3b Mon Sep 17 00:00:00 2001 From: sdairs Date: Thu, 22 Oct 2026 11:48:13 +0100 Subject: [PATCH] Refresh Stock Reservation example to the standard README shape Restructure the README as a reference: Stack table, a standalone "Connect Bun.SQL to ClickHouse Managed Postgres" section, the standard clickhousectl provisioning steps, Try it, Test, agent prompt and Limits. Behaviour is unchanged. - Rename SQL files to sql/01_roles, 02_grants, 03_seed, 99_cleanup and the password variables to MIGRATOR_PASSWORD / APP_PASSWORD - Move the Bun CA bundle note into src/database.ts - Fold tests/README.md into the README Test section and remove the example's AGENTS.md - `bun run test` now typechecks before the unit tests - Fix the duplicated applications row in the root README Co-Authored-By: Claude Opus 5.5 --- .github/workflows/stock-reservation.yml | 2 +- README.md | 3 +- applications/stock-reservation/AGENTS.md | 13 - applications/stock-reservation/README.md | 638 ++++++------------ applications/stock-reservation/package.json | 2 +- .../sql/{bootstrap.sql => 01_roles.sql} | 6 +- .../sql/{grants.sql => 02_grants.sql} | 0 .../sql/{seed.sql => 03_seed.sql} | 0 .../sql/{cleanup.sql => 99_cleanup.sql} | 2 +- .../stock-reservation/src/database.ts | 14 +- .../stock-reservation/tests/README.md | 78 --- .../tests/integration.test.ts | 2 +- .../stock-reservation/tests/tls.test.ts | 2 +- 13 files changed, 214 insertions(+), 548 deletions(-) delete mode 100644 applications/stock-reservation/AGENTS.md rename applications/stock-reservation/sql/{bootstrap.sql => 01_roles.sql} (88%) rename applications/stock-reservation/sql/{grants.sql => 02_grants.sql} (100%) rename applications/stock-reservation/sql/{seed.sql => 03_seed.sql} (100%) rename applications/stock-reservation/sql/{cleanup.sql => 99_cleanup.sql} (86%) delete mode 100644 applications/stock-reservation/tests/README.md diff --git a/.github/workflows/stock-reservation.yml b/.github/workflows/stock-reservation.yml index d9ad2398..50e17b9d 100644 --- a/.github/workflows/stock-reservation.yml +++ b/.github/workflows/stock-reservation.yml @@ -31,4 +31,4 @@ jobs: - run: bun run test:unit # Managed Postgres integration tests are an explicit run with dedicated fixtures. -# See applications/stock-reservation/tests/README.md for the managed-service suite. +# See the "Test" section of applications/stock-reservation/README.md. diff --git a/README.md b/README.md index 194e903e..bce133f3 100644 --- a/README.md +++ b/README.md @@ -114,8 +114,7 @@ Browse [all local analytics examples](./local-analytics/README.md) for more file | Directory | What you'll find | | --- | --- | -| [applications](./applications/) | Application examples: a link shortener, feature request board, report results, and run history with ClickHouse Cloud. | -| [applications](./applications/) | Application examples: workshop bookings, a link shortener, report results, and run history with ClickHouse Cloud. | +| [applications](./applications/) | Application examples: stock reservations, workshop bookings, a feature request board, a link shortener, report results, and run history with ClickHouse Cloud. | | [ai](./ai/README.md) | AI agents, MCP integrations, and workflows using `clickhousectl`. | | [blog-examples](./blog-examples/) | Code and resources accompanying the [ClickHouse Blog](https://clickhouse.com/blog). | | [clickstack](./clickstack/) | Observability examples for LLM applications and MCP servers. | diff --git a/applications/stock-reservation/AGENTS.md b/applications/stock-reservation/AGENTS.md deleted file mode 100644 index 5ceb7b37..00000000 --- a/applications/stock-reservation/AGENTS.md +++ /dev/null @@ -1,13 +0,0 @@ -# Agent instructions - -- Read [README.md](README.md). This is a small Hono + Bun + Bun.SQL application backed by ClickHouse Managed Postgres. Preserve its four-route scope: inventory, reserve, inspect, release. -- Install dependencies and run application code only inside the isolated OrbStack machine `stock-reservation-dev`. Create it with `orb create --isolated --isolate-network ubuntu:noble stock-reservation-dev` if absent, and verify its isolation settings before use. Keep macOS file sharing, host integration, host network access, SSH-agent forwarding, and host Docker sockets disabled. Host-side source editing and read-only inspection are fine. -- Keep infrastructure commands visible in the README. Keep bootstrap, grants, ordered migrations, seed data, and cleanup in readable SQL files. Do not hide provisioning or migrations in application scripts. -- Never commit credentials, Cloud resource IDs, private endpoints, personal paths, `.deployment/`, or environment files other than the generic `.env.example`. Transfer only the credentials needed for a specific VM task; the running app receives only runtime credentials. -- Preserve the plain PostgreSQL connection URL and explicit CA and hostname verification. Do not disable TLS checks or copy driver-specific URL parameters into Bun.SQL. Separate Cloud management, database administration, migrations, and runtime access. -- Postgres must enforce stock integrity across processes. Keep inventory updates, reservation insertion, and the successful idempotency response in one transaction. Keep releases atomic and repeat-safe. Failed reservations roll back their idempotency key; successful requests replay their original response even after release. -- Derive client identity from a configured API token. Every reservation lookup and release must include the authenticated client. Do not accept an owner from a request body. -- Run `bun run typecheck` and `bun run test` in the VM. Run the documented integration suite against a disposable managed-service fixture after database changes. Recheck concurrency, rollback, ownership, and strict TLS when their implementation changes; report skipped checks explicitly. -- Record actual test commands and results in the pull request. An expected result in the README is not evidence that a check ran. -- Reuse recorded Cloud resources after interrupted setup. Inspect uncertain create/delete outcomes before retrying. Target destructive cleanup by the recorded service ID and keep receipts until deletion is confirmed. -- Keep article drafts, verification reports, and maintainer notes outside this example and its pull request. Add an article link only after publication. diff --git a/applications/stock-reservation/README.md b/applications/stock-reservation/README.md index 32055b12..459a4746 100644 --- a/applications/stock-reservation/README.md +++ b/applications/stock-reservation/README.md @@ -1,512 +1,262 @@ -# Stock Reservation API +# Stock Reservation API: reserve limited inventory with Hono, Bun.SQL, and ClickHouse Managed Postgres -Reserve and release limited inventory with **Hono**, **Bun 1.4.2**, and -**Bun.SQL**, backed by [ClickHouse Managed Postgres](https://clickhouse.com/cloud/postgres). -Competing requests cannot reserve more units than are available. Retrying a -successful request with its original idempotency key returns the original -reservation without consuming stock again. +A JSON API that reserves and releases limited stock, built with Hono on Bun 1.4 and Bun's built-in SQL client (`Bun.SQL`), backed by ClickHouse Managed Postgres and provisioned with `clickhousectl`. Competing requests can never reserve more units than are available, and retrying a request with the same `Idempotency-Key` returns the original reservation without taking stock twice. Postgres enforces both rules with a conditional `UPDATE`, a unique key and a single transaction. -Read the walkthrough: [Build a stock reservation API with Hono, Bun, and ClickHouse Managed Postgres](https://clickhouse.com/resources/engineering/build-stock-reservation-api-hono-bun-postgres). +**[Start a ClickHouse Cloud free trial with $300 in credits](https://clickhouse.com/cloud)** to run this example on ClickHouse Managed Postgres. -The example contains four authenticated routes, three application tables, seed -data, explicit SQL setup, and tests. It is an API for a small, trusted set of -clients; payments, expiring reservations, and a browser interface are outside its -scope. See [Verify changes](#verify-changes) for the test commands. +Read the walkthrough: [Build a stock reservation API with Hono, Bun, and ClickHouse Managed Postgres](https://clickhouse.com/resources/engineering/build-stock-reservation-api-hono-bun-postgres) -**[Create a ClickHouse Cloud account and start a free 30-day trial with $300 in credits](https://clickhouse.com/cloud).** +[ClickHouse Cloud](https://clickhouse.com/cloud) is a managed data platform. It runs [ClickHouse Managed Postgres](https://clickhouse.com/cloud/postgres) for transactional application data and ClickHouse for real-time analytics, managed together with one CLI, [`clickhousectl`](https://clickhouse.com/docs/concepts/features/interfaces/cli). + +## Stack + +| Layer | Choice | +| --- | --- | +| Framework | Hono 4.13.8 | +| Runtime | Bun 1.4.2 | +| Database | ClickHouse Managed Postgres (PostgreSQL 18) | +| Database access | `Bun.SQL` (built into Bun, no driver package) | +| Migrations | Numbered SQL files applied with `psql` | +| UI | None (JSON API) | +| Provisioning | `clickhousectl` 0.5.0 | ## How it works ```mermaid flowchart LR - Client[Authenticated API client] --> App[Hono on Bun] - App -->|Bun.SQL: verified TLS| PG[ClickHouse Managed Postgres] - PG --> Inventory[Inventory] - PG --> Reservations[Reservations] - PG --> Keys[Successful idempotency responses] + Client[API client] -->|Bearer token| App[Hono on Bun] -->|Bun.SQL, verified TLS, app role| PG[(ClickHouse Managed Postgres)] ``` -All stock changes happen in Postgres transactions. A conditional inventory -update rejects a reservation when too few units remain. The reservation and its -successful response commit with that update. A unique constraint coordinates -requests that use the same client identity and idempotency key, including across -different application processes. Releasing a reservation restores its quantity -once, even when release requests arrive together. - -| Route | Purpose | Success | +| Request | Handler | SQL | | --- | --- | --- | -| `GET /inventory` | List the shared inventory | `200` | -| `POST /reservations` | Reserve `{ "sku": "CAMP-MUG", "quantity": 2 }` | `201` | -| `GET /reservations/:id` | Read your reservation's current state | `200` | -| `DELETE /reservations/:id` | Release your reservation | `200` | - -Every route requires `Authorization: Bearer `. Creation also requires -`Idempotency-Key`. Client identity comes from the server's token configuration; -clients cannot nominate a reservation owner. Another client's reservation is -reported as `404` on both read and release. - -This application uses standalone Postgres. There is no analytical workload or -ClickPipe to provision. [Shortwave](../shortwave) demonstrates the larger -Postgres, ClickHouse, and ClickPipes architecture. - -## Set up the example - -The instructions use Bash, a [ClickHouse Cloud account](https://clickhouse.com/cloud) -with Managed Postgres access, Git, `curl`, `jq`, OpenSSL, and `psql` 15 or later. -Run Bun and application dependencies in a Linux development environment. -Maintainers on macOS use the -[isolated OrbStack machines](https://docs.orbstack.dev/machines/isolated); install -no Node.js or Bun packages on the host. - -### 1. Get the source and install pinned dependencies - -```sh -git clone https://github.com/ClickHouse/examples.git -cd examples/applications/stock-reservation -``` - -Inside your Linux environment, install the OS tools and Bun: - -```sh -sudo apt-get update -sudo apt-get install -y ca-certificates curl git unzip postgresql-client jq openssl -curl -fsSL https://bun.sh/install -o /tmp/bun-install.sh -less /tmp/bun-install.sh -bash /tmp/bun-install.sh bun-v1.4.2 -export PATH="$HOME/.bun/bin:$PATH" -bun --version -bun install --frozen-lockfile --ignore-scripts +| `GET /inventory` | [`src/app.ts`](src/app.ts) | `SELECT sku, name, available FROM inventory` | +| `POST /reservations` `{ "sku", "quantity" }` | `app.post("/reservations")` | One transaction: `INSERT … ON CONFLICT DO NOTHING` into `idempotency_keys`, conditional `UPDATE inventory`, `INSERT INTO reservations` | +| `GET /reservations/:id` | `app.get("/reservations/:id")` | `SELECT … WHERE id = $1 AND client_id = $2` | +| `DELETE /reservations/:id` | `app.delete("/reservations/:id")` | One transaction: `SELECT … FOR UPDATE`, `UPDATE inventory` (+quantity), `UPDATE reservations SET status = 'released'` | + +Every route needs `Authorization: Bearer `. The server maps each token to a client ID (`API_KEYS_JSON`), so a client can't choose who owns a reservation, and another client's reservation returns `404`. `POST` also needs an `Idempotency-Key` header. The schema is three tables in [`sql/migrations/001_initial.sql`](sql/migrations/001_initial.sql): `inventory`, `reservations` and `idempotency_keys`. + +**The no-overselling rule.** Reading the stock and then subtracting can oversell, because two requests can read the same number before either writes. Instead the check is part of the write ([`src/app.ts`](src/app.ts#L159)). Postgres locks the row, and a competing `UPDATE` re-checks `available >= quantity` against the committed value, so it changes no row once the stock is gone. A `CHECK (available >= 0)` constraint backs this up. + +```ts +const changed = await tx` + UPDATE stock_reservation.inventory SET available = available - ${quantity} + WHERE sku = ${sku} AND available >= ${quantity} RETURNING sku +`; +if (changed.length === 0) { + // … 404 for an unknown SKU, otherwise 409 insufficient_inventory +} ``` -The version output must be `1.4.2`. The lockfile pins the dependencies; the -application uses Bun's built-in SQL client and needs no separate Postgres driver. -The [Bun installation guide](https://bun.sh/docs/installation) explains the -versioned installer. Run subsequent application commands from this directory. - -### 2. Sign in and save the provisioning inputs - -Install [clickhousectl](https://clickhouse.com/docs/interfaces/cli) on the machine -you use to manage Cloud resources. For the isolated maintainer workflow, this is -the host; Cloud management credentials stay outside the VM. - -```sh -curl -fsSL https://clickhouse.com/cli | sh -export PATH="$HOME/.local/bin:$PATH" -clickhousectl --version -clickhousectl cloud auth login --interactive -clickhousectl cloud auth status -clickhousectl cloud org list +**The one-reservation-per-key rule.** The transaction first claims the key with `INSERT … ON CONFLICT (client_id, key) DO NOTHING` ([`src/app.ts`](src/app.ts#L138)). The primary key `(client_id, key)` makes a concurrent retry wait until the first transaction commits or rolls back. It then replays the stored response, or returns `409 idempotency_conflict` if the SKU or quantity differ. The key, the stock decrement and the reservation commit together, so a failed request leaves no key behind and can be retried. Release locks the reservation with `FOR UPDATE` and returns stock only when its status is still `active`, so a repeated `DELETE` changes nothing. + +## Connect Bun.SQL to ClickHouse Managed Postgres + +[`src/database.ts`](src/database.ts) is the only place the pool is created. This shortened version leaves out the CA bundle retry described after it. + +```ts +// ClickHouse Managed Postgres: verify the server certificate and hostname against +// the service CA from `clickhousectl cloud postgres certs get`. Keep the URL plain +// (no sslmode or channel_binding); TLS is configured here. Everything else is plain +// Postgres. +export async function createDatabase(config: DatabaseConfig): Promise { + const url = validateDatabaseUrl(config.url); // rejects query parameters + url.searchParams.set("sslmode", "verify-full"); + const ca = await Bun.file(config.caCertPath).text(); + + const sql = new SQL(url, { + max: config.maxConnections, // PG_MAX_CONNECTIONS, default 5 + idleTimeout: 30, + maxLifetime: 1800, + connectionTimeout: 10, + tls: { ca, serverName: url.hostname, rejectUnauthorized: true }, + }); + await sql`SELECT 1`; // fail at startup, not on the first request + return sql; +} ``` -Creation requires an Admin API key. Create one using the -[Cloud API instructions](https://clickhouse.com/docs/cloud/manage/openapi) and -enter it at the interactive prompt; OAuth authentication is read-only for these -operations. Keep the CLI credential store private. +- **TLS:** `tls.ca` with `rejectUnauthorized: true` and `serverName` checks both the certificate chain and the hostname, the same as `sslmode=verify-full`. `DATABASE_URL` must be a plain `postgres://user:password@host:5432/postgres` URL. Don't copy the console's connection string with `channel_binding` or `sslmode` into it. +- **CA bundles:** Bun 1.4.2 can reject a CA bundle that contains two roots with the same subject. When that happens, `createDatabase` retries each certificate on its own, still with full verification. The comment in `database.ts` explains why. +- **Pool:** five connections per process by default (`PG_MAX_CONNECTIONS`, 1–20). Multiply by the number of processes before scaling out. +- **Timeouts:** the app role has `statement_timeout = 5s`, `lock_timeout = 3s` and `idle_in_transaction_session_timeout = 10s` set in [`sql/01_roles.sql`](sql/01_roles.sql), so every pooled connection gets them. -Create a private directory for inputs, returned credentials, and command receipts: - -```sh -umask 077 -mkdir -p .deployment -``` +## Prerequisites -In an editor, create `.deployment/resources.env` with your own values: +- Bun 1.4.2, `psql` 15 or later, `jq`, `openssl`, `curl` +- A ClickHouse Cloud account and an API key with permission to create services -```dotenv -CH_ORG_ID=YOUR_ORGANIZATION_UUID -DEPLOYMENT_NAME=stock-reservation-example -CLOUD_REGION=YOUR_AVAILABLE_AWS_REGION -PG_SIZE=YOUR_AVAILABLE_POSTGRES_SIZE +```bash +bun install --frozen-lockfile ``` -Choose a supported region and instance size in your Cloud organization. Review -the [Managed Postgres pricing](https://clickhouse.com/pricing) before creation. -This example requests one Postgres 18 service with `--ha-type none`; running it -incurs database charges until you delete it. Storage, backups, network usage, and -a future application host may add costs. Stopping Bun or the VM does not stop -the database service. +## Set up ClickHouse Managed Postgres with clickhousectl -```sh -source .deployment/resources.env -``` - -### 3. Create Managed Postgres and wait for readiness +```bash +# 1. Install the CLI and sign in (an API key is required to create services) +curl -fsSL https://clickhouse.com/cli | sh +export PATH="$HOME/.local/bin:$PATH" +clickhousectl cloud auth login --api-key "$CLICKHOUSE_CLOUD_API_KEY" --api-secret "$CLICKHOUSE_CLOUD_API_SECRET" +clickhousectl cloud org list -Run creation once and preserve the private response, which contains the initial -administrator password: +# 2. Choose where the database runs +umask 077 && mkdir -p .deployment +export CH_ORG_ID="" +export PG_REGION="us-east-1" PG_SIZE="m6gd.large" -```sh +# 3. Create ClickHouse Managed Postgres (save the one-time credentials) clickhousectl cloud postgres create \ - --org-id "$CH_ORG_ID" --name "$DEPLOYMENT_NAME" \ - --provider aws --region "$CLOUD_REGION" --size "$PG_SIZE" \ - --pg-version 18 --ha-type none \ - --tag project=stock-reservation --tag environment=development --json \ - > .deployment/postgres-create.json - -PG_SERVICE_ID="$(jq -er '.id' .deployment/postgres-create.json)" -printf 'PG_SERVICE_ID=%s\n' "$PG_SERVICE_ID" >> .deployment/resources.env -``` - -Inspect the service until `state` is `running`. Repeat **get**, not **create**: - -```sh -clickhousectl cloud postgres get "$PG_SERVICE_ID" \ - --org-id "$CH_ORG_ID" --json > .deployment/postgres-status.json -jq '{id, state, hostname, username, postgresVersion, size}' \ - .deployment/postgres-status.json + --org-id "$CH_ORG_ID" --name stock-reservation \ + --provider aws --region "$PG_REGION" --size "$PG_SIZE" \ + --pg-version 18 --ha-type none --tag project=stock-reservation \ + --json > .deployment/postgres-create.json +export PG_SERVICE_ID="$(jq -er '.id' .deployment/postgres-create.json)" + +# 4. Wait until it is running, then download the CA certificate +clickhousectl cloud postgres get "$PG_SERVICE_ID" --org-id "$CH_ORG_ID" --json | jq '{state, hostname}' +clickhousectl cloud postgres certs get "$PG_SERVICE_ID" --org-id "$CH_ORG_ID" --output .deployment/postgres-ca.pem ``` -If creation failed or the terminal disconnected, first run -`clickhousectl cloud postgres list --org-id "$CH_ORG_ID" --json` and reconcile the -service against the saved receipt. A missing local response does not prove that -creation failed. Do not create another service just to recover a password. - -Download the service CA as a PEM file: - -```sh -clickhousectl cloud postgres certs get "$PG_SERVICE_ID" \ - --org-id "$CH_ORG_ID" --output .deployment/postgres-ca.pem -``` - -Use `--output` explicitly. Redirecting JSON-mode certificate output to a `.pem` -file produces an unusable certificate file. - -Record `PGHOST` and `PGADMIN` from the service's `hostname` and `username`, plus -its direct port and database, in `.deployment/resources.env`: - -```dotenv -PGHOST=YOUR_POSTGRES_HOSTNAME -PGPORT=5432 -PGDATABASE=postgres -PGADMIN=YOUR_SERVICE_ADMIN_USERNAME -``` +Repeat the `get` command until `state` is `running`. Then point `psql` at the service as the admin user: -Use the direct Postgres endpoint for this example. Match the connection details -returned for your service rather than copying a hostname from another deployment. -The recorded test setup uses the direct port `5432` and database `postgres`. -For an isolated VM, transfer only the source files, CA, and credentials needed -for the current step into its own filesystem. Keep Cloud API credentials on the -host and administrator credentials separate from the runtime configuration. - -### 4. Create roles, apply the schema, and seed inventory - -Generate passwords once. Keep this file on retries; do not regenerate a password -file for roles that already exist: - -```sh -cat > .deployment/passwords.env < .deployment/app.env < .deployment/passwords.env -```sh -source .deployment/app.env -API_TOKEN="$(printf '%s' "$API_KEYS_JSON" | jq -er '."warehouse-demo"')" -BASE_URL=http://127.0.0.1:3000 +psql -X -v ON_ERROR_STOP=1 -f sql/01_roles.sql # as postgres (admin) +export PGUSER=stock_reservation_migrator PGPASSWORD="$MIGRATOR_PASSWORD" +psql -X -v ON_ERROR_STOP=1 -f sql/migrations/001_initial.sql +psql -X -v ON_ERROR_STOP=1 -f sql/02_grants.sql +psql -X -v ON_ERROR_STOP=1 -f sql/03_seed.sql +export PGUSER=postgres PGPASSWORD="$(jq -er '.password' .deployment/postgres-create.json)" ``` -## Try the complete flow +| File | Runs as | What it does | +| --- | --- | --- | +| [`sql/01_roles.sql`](sql/01_roles.sql) | admin | Creates the schema owner `stock_reservation_owner` (no login), `stock_reservation_migrator` and `stock_reservation_app`, the `stock_reservation` schema, and the app role's timeouts. Passwords come from the environment via `\getenv`. | +| [`sql/migrations/001_initial.sql`](sql/migrations/001_initial.sql) | migrator | Creates `inventory`, `reservations` and `idempotency_keys` with their constraints. Runs once. Later changes go in new numbered files. | +| [`sql/02_grants.sql`](sql/02_grants.sql) | migrator | Gives the app role `SELECT` plus column-level `INSERT`/`UPDATE` only. No `DELETE`, `TRUNCATE` or DDL. Safe to re-run. | +| [`sql/03_seed.sql`](sql/03_seed.sql) | migrator | Adds 25 `FIELD-NOTES`, 10 `CAMP-MUG` and 5 `TRAIL-PACK`. Never resets existing stock. | -### Check the stock and reserve two mugs +The migrator runs each file with `SET LOCAL ROLE stock_reservation_owner`, so the tables belong to the owner role rather than to a login. -```sh -curl --fail-with-body -sS "$BASE_URL/inventory" \ - -H "Authorization: Bearer $API_TOKEN" | jq -``` +## Run the app -The initial response contains three products. The `CAMP-MUG` entry has -`"available": 10`. Create a reservation and save its response: - -```sh -curl --fail-with-body -sS -D .deployment/reservation-headers.txt \ - "$BASE_URL/reservations" \ - -H "Authorization: Bearer $API_TOKEN" \ - -H 'Content-Type: application/json' \ - -H 'Idempotency-Key: first-mug-order' \ - --data '{"sku":"CAMP-MUG","quantity":2}' \ - > .deployment/reservation.json -jq . .deployment/reservation.json -RESERVATION_ID="$(jq -er '.id' .deployment/reservation.json)" -``` +The app connects as the least-privilege app role, never as admin: -Expected status: `201`, with `Idempotency-Replayed: false`. The body has this -shape; your UUID and timestamps differ: - -```json -{ - "id": "8a109f49-d9e2-4be7-a3c3-d3d89e371862", - "sku": "CAMP-MUG", - "quantity": 2, - "status": "active", - "created_at": "2026-09-17T12:00:00.000Z", - "released_at": null -} +```bash +export API_TOKEN="$(openssl rand -hex 32)" +cat > .env < -H "Authorization: Bearer $API_TOKEN" | jq + ``` -Missing or invalid credentials return `401`. Creation accepts only `sku` and -`quantity`: the SKU must begin with an ASCII letter or digit and contain 1–64 -letters, digits, underscores, or hyphens; quantity must be an integer from 1 to -1,000. The idempotency key begins with a letter or digit and permits 1–128 letters, -digits, dots, underscores, colons, or hyphens. Invalid bodies are rejected before -a transaction begins. JSON requests have a 4 KiB limit; the server also rejects -bodies above its 64 KiB transport cap, which may return Bun's own `413` response. -A valid second client can see shared inventory but gets `404` when reading or -releasing the first client's reservation. +5. Race eight buyers for the five trail packs. Exactly five get `201` and three get `409`, and `TRAIL-PACK` ends at `0`. -## Verify changes + ```bash + for i in $(seq 8); do + curl -sS -o /dev/null -w '%{http_code}\n' http://127.0.0.1:3000/reservations \ + -H "Authorization: Bearer $API_TOKEN" -H 'Content-Type: application/json' \ + -H "Idempotency-Key: race-$i" --data '{"sku":"TRAIL-PACK","quantity":1}' & + done | sort | uniq -c; wait + ``` -Run these checks inside the isolated Linux environment: +## Test -```sh -bun run typecheck -bun run test:unit +```bash +bun run test # typecheck + unit tests (no database) +TEST_ADMIN_DATABASE_URL="postgres://postgres:$PGPASSWORD@$PGHOST:5432/postgres" \ + bun run test:integration # needs .env; proves no overselling and one reservation per key ``` -The integration suite exercises the live Postgres transaction path, including -competing requests, duplicate keys, rollback, release retries, two-client -ownership, and TLS failures. Use a dedicated example service. The suite creates -and deletes its own uniquely named fixtures and briefly changes a table -constraint to force a rollback; do not run it alongside real reservations. - -Copy the runtime configuration to a separate private test file: +The integration and TLS suites run against real Postgres as the app role, through two app instances with separate pools. They fail, rather than skip, when a variable is missing. They check that: -```sh -cp .deployment/app.env .deployment/test.env -``` +- 32 buyers competing for 9 units produce exactly 9 reservations, and remaining stock plus active reservations equals the starting stock; +- 24 simultaneous retries with one key create one reservation and replay the identical response; +- 20 concurrent releases restore stock once; +- a forced `INSERT` failure after the decrement rolls back both the stock and the key; +- clients can't read or release each other's reservations; +- the app role has no `CREATE`, `DELETE` or `TRUNCATE` privilege, and the connection uses TLS 1.2 or 1.3; +- an unrelated CA and a hostname not on the certificate are both rejected. -In an editor, add `TEST_ADMIN_DATABASE_URL` with the fixture administrator's plain -Postgres URL. Percent-encode the username and password components, omit query -parameters, and use the same service, port, and database as `DATABASE_URL`: +`TEST_ADMIN_DATABASE_URL` is used only to create and remove the suite's own `test-…` SKUs and to add a temporary constraint for the rollback tests. Use a dedicated service and run one copy at a time. If a run is killed mid-way, drop any leftover `test_reservation_*` constraint with the SQL in [`tests/fixtures/`](tests/fixtures). -```dotenv -TEST_ADMIN_DATABASE_URL=postgres://YOUR_ADMIN:YOUR_ENCODED_PASSWORD@YOUR_POSTGRES_HOSTNAME:5432/postgres -``` +## Build this with an AI coding agent -This privileged URL belongs only in the test file, never in `app.env`. Run the -suite explicitly: +Install the ClickHouse agent skills with `clickhousectl skills --agent claude` (or your agent's name), then prompt: -```sh -bun --env-file=.deployment/test.env run test:integration +```text +Build a stock reservation JSON API with Hono on Bun and Bun.SQL on ClickHouse Managed Postgres. +Provision the database with clickhousectl (postgres create, wait for running, certs get) and +connect with Bun.SQL tls { ca, serverName, rejectUnauthorized: true } and a plain URL. +Use an owner role, a migrator role and a least-privilege app role, with numbered SQL files run by psql. +Reserve stock with one transaction: claim the Idempotency-Key with INSERT ... ON CONFLICT DO NOTHING, +then UPDATE inventory SET available = available - $1 WHERE sku = $2 AND available >= $1, +then insert the reservation. Release with SELECT ... FOR UPDATE and restore stock once. +Add an integration test where 32 concurrent buyers compete for 9 units and exactly 9 succeed. ``` -Missing test credentials fail the run rather than skipping database checks. See -[tests/README.md](tests/README.md) for fixture permissions, scope, and cleanup. - -The important accounting check is that available stock plus active reservation -quantities remains equal to the starting stock for each SKU. HTTP responses -alone cannot establish that invariant. A successful database connection alone -does not establish concurrency correctness. - -## Scope and operating limits - -- Inventory is shared; reservations belong to one configured client. Authorization - is enforced in the application. The runtime database login is shared and is - not a separate database tenant boundary. -- Reservations remain active until explicitly released. There is no payment - confirmation, automatic expiry, restocking endpoint, or background worker. -- Successful idempotency records are retained indefinitely. A retention policy - must define how long old request keys remain safe to replay before deleting - them. A new request needs a new key. -- Each process defaults to five database connections, configurable from one to - twenty with `PG_MAX_CONNECTIONS`; multiply that limit by the number of replicas - before scaling. The runtime role has a three-second lock timeout and a - five-second statement timeout. This example is not an application capacity - benchmark. -- The default listener is local. For a hosted deployment, run Bun under a process - supervisor, inject only runtime secrets, and place it behind HTTPS with request - limits and appropriate network access. Do not expose the development HTTP - listener or put API tokens in client-side code. A hosted platform deployment is - a separate validation step from the recorded Linux development run. - -## Troubleshooting and cleanup - -For a TLS error, check that the CA file is PEM, the hostname matches this service, -and the runtime file contains the plain URL. For authentication errors, distinguish -the Cloud API key, database administrator, migrator, application database password, -and HTTP API token; each has a different purpose. For lock timeouts, retry a -reservation using its same idempotency key because a lost response may have -followed a successful commit. - -If you lost the database administrator password, reset it for the recorded -service instead of recreating the database: - -```sh -clickhousectl cloud postgres reset-password "$PG_SERVICE_ID" \ - --org-id "$CH_ORG_ID" --generate --json \ - > .deployment/postgres-password-reset.json -``` - -Store the returned password privately and use it for the admin connection. This -does not rotate the application's separately created role password. +## Limits -Stop the app with Ctrl-C. To remove only this example from a database you intend -to keep, inspect [cleanup.sql](sql/cleanup.sql), reconnect as its administrator, -and run it explicitly: +- Bearer tokens from configuration stand in for real authentication, and all clients share one database login. +- Reservations stay active until released. There is no expiry, payment or restocking endpoint. +- Successful idempotency keys are kept indefinitely. A real system needs a retention policy. +- No deployment configuration. The listener binds to `127.0.0.1` by default; put any hosted deployment behind HTTPS. -```sh -export PGPASSWORD="$(jq -er '.password' .deployment/postgres-create.json)" -psql -X -v ON_ERROR_STOP=1 -U "$PGADMIN" -f sql/cleanup.sql -unset PGPASSWORD -``` +## Clean up -Use your reset password instead if it has changed. The cleanup removes this -example's data and roles; it is destructive. It does not delete or stop the Cloud -service. - -For a service created exclusively for this example, delete the service itself -when finished. Check the ID against the receipt before deleting: - -```sh -source .deployment/resources.env -clickhousectl cloud postgres get "$PG_SERVICE_ID" --org-id "$CH_ORG_ID" --json +```bash +psql -X -v ON_ERROR_STOP=1 -f sql/99_cleanup.sql # optional if deleting the service clickhousectl cloud postgres delete "$PG_SERVICE_ID" --org-id "$CH_ORG_ID" -clickhousectl cloud postgres list --org-id "$CH_ORG_ID" --json +clickhousectl cloud postgres list --org-id "$CH_ORG_ID" --json | jq '.[].name' +rm -rf .deployment .env ``` -Deletion removes the service and its data. Confirm that the recorded ID is absent -from the service list. Preserve receipts until the outcome is clear, then remove -unneeded private credential files and the dedicated development VM. Never delete -a shared service to clean up this example. +## Learn more + +- [ClickHouse Managed Postgres docs](https://clickhouse.com/docs/products/managed-postgres/overview) +- [clickhousectl](https://clickhouse.com/docs/concepts/features/interfaces/cli) +- [Hono on Bun](https://hono.dev/docs/getting-started/bun) +- [Bun.SQL](https://bun.sh/docs/runtime/sql) +- [PostgreSQL: Read Committed isolation](https://www.postgresql.org/docs/current/transaction-iso.html#XACT-READ-COMMITTED) +- Other examples: [Feature Request Board](../feature-request-board/README.md), [Workshop Booking API](../workshop-booking/README.md), [Shortwave](../shortwave/README.md) diff --git a/applications/stock-reservation/package.json b/applications/stock-reservation/package.json index 17fc0559..77c6246d 100644 --- a/applications/stock-reservation/package.json +++ b/applications/stock-reservation/package.json @@ -9,7 +9,7 @@ "dev": "bun --watch src/server.ts", "start": "bun src/server.ts", "typecheck": "tsc --noEmit", - "test": "bun test tests/unit.test.ts", + "test": "tsc --noEmit && bun test tests/unit.test.ts", "test:unit": "bun test tests/unit.test.ts", "test:integration": "bun test --timeout 30000 tests/integration.test.ts tests/tls.test.ts" }, diff --git a/applications/stock-reservation/sql/bootstrap.sql b/applications/stock-reservation/sql/01_roles.sql similarity index 88% rename from applications/stock-reservation/sql/bootstrap.sql rename to applications/stock-reservation/sql/01_roles.sql index a9286749..6584d2a9 100644 --- a/applications/stock-reservation/sql/bootstrap.sql +++ b/applications/stock-reservation/sql/01_roles.sql @@ -1,8 +1,8 @@ -- Run ONCE as the service administrator with psql -X -v ON_ERROR_STOP=1. -- Passwords come from the environment, not process arguments or committed files. --- PG_MIGRATION_PASSWORD and PG_APP_PASSWORD must contain random generated passwords. -\getenv migration_password PG_MIGRATION_PASSWORD -\getenv app_password PG_APP_PASSWORD +-- MIGRATOR_PASSWORD and APP_PASSWORD must contain random generated passwords. +\getenv migration_password MIGRATOR_PASSWORD +\getenv app_password APP_PASSWORD BEGIN; diff --git a/applications/stock-reservation/sql/grants.sql b/applications/stock-reservation/sql/02_grants.sql similarity index 100% rename from applications/stock-reservation/sql/grants.sql rename to applications/stock-reservation/sql/02_grants.sql diff --git a/applications/stock-reservation/sql/seed.sql b/applications/stock-reservation/sql/03_seed.sql similarity index 100% rename from applications/stock-reservation/sql/seed.sql rename to applications/stock-reservation/sql/03_seed.sql diff --git a/applications/stock-reservation/sql/cleanup.sql b/applications/stock-reservation/sql/99_cleanup.sql similarity index 86% rename from applications/stock-reservation/sql/cleanup.sql rename to applications/stock-reservation/sql/99_cleanup.sql index e57ba5d6..96022ddc 100644 --- a/applications/stock-reservation/sql/cleanup.sql +++ b/applications/stock-reservation/sql/99_cleanup.sql @@ -1,5 +1,5 @@ -- DESTRUCTIVE: removes all Stock Reservation example data and its three roles. --- Stop the API first. Run as the same service administrator used for bootstrap. +-- Stop the API first. Run as the same service administrator that ran 01_roles.sql. -- This does not delete the ClickHouse Cloud service or affect other schemas. BEGIN; diff --git a/applications/stock-reservation/src/database.ts b/applications/stock-reservation/src/database.ts index 9cfd02c0..ed675777 100644 --- a/applications/stock-reservation/src/database.ts +++ b/applications/stock-reservation/src/database.ts @@ -1,6 +1,16 @@ import { SQL } from "bun"; import { validateDatabaseUrl, type DatabaseConfig } from "./config.ts"; +// ClickHouse Managed Postgres: verify the server certificate and hostname against +// the service CA from `clickhousectl cloud postgres certs get`. Keep the URL plain +// (no sslmode or channel_binding); TLS is configured here. Everything else is plain +// Postgres. +// +// Bun 1.4.2 can fail with CERT_SIGNATURE_FAILURE on a CA bundle that holds several +// roots with the same subject (as during a CA rotation). On that error only, each +// certificate in the bundle is tried on its own, still verifying certificate and +// hostname. The first anchor that works is kept for the life of the process, so +// re-download the bundle and restart after a CA rotation. export async function createDatabase(config: DatabaseConfig): Promise { const url = validateDatabaseUrl(config.url); // Only this known Bun option is added internally. It overrides ambient PGSSLMODE; @@ -37,9 +47,7 @@ export async function createDatabase(config: DatabaseConfig): Promise { return await connect(ca); } catch (originalError) { if (errorCode(originalError) !== "CERT_SIGNATURE_FAILURE" || certificates.length < 2) throw originalError; - // Bun 1.4.2 can reject a rotation bundle containing roots with the same subject. - // Retry only the supplied trust anchors, always verifying certificate AND hostname. - // The first successful pool retains that CA until the application restarts. + // See the CA bundle note above: retry each supplied anchor on its own. for (const certificate of certificates) { try { return await connect(certificate); diff --git a/applications/stock-reservation/tests/README.md b/applications/stock-reservation/tests/README.md deleted file mode 100644 index ed7f19ec..00000000 --- a/applications/stock-reservation/tests/README.md +++ /dev/null @@ -1,78 +0,0 @@ -# Verification suites - -Run Bun and dependency commands inside the isolated OrbStack development VM. -See [AGENTS.md](../AGENTS.md) for the isolation requirements. - -```sh -bun run typecheck -bun run test:unit -``` - -Unit tests check configuration, authentication, body limits (including streamed -bodies with a dishonest Content-Length), validation, and sanitized errors. They -need no database and deliberately fail if rejected HTTP input reaches a query. - -## Managed Postgres integration - -Use a disposable, migrated database created through the [README](../README.md). -The runtime connection must use `stock_reservation_app`. Fixture administration -must use a separate account that can insert/delete fixtures and alter the -reservations table, such as the service administrator. Do not point the suite at -a shared production database. Run only one copy against a database at a time. - -Store these variables in a private VM environment file; never commit it: - -```dotenv -DATABASE_URL=postgresql://stock_reservation_app:PASSWORD@HOST:5432/postgres -TEST_ADMIN_DATABASE_URL=postgresql://ADMIN:PASSWORD@HOST:5432/postgres -PG_CA_CERT_PATH=/absolute/path/to/instance-ca.pem -``` - -Both URLs must be plain PostgreSQL URLs without query parameters. Both pools use -the application factory's strict certificate validation. The test clients and -tokens are generated by the suite; it does not use your configured API tokens. - -```sh -bun --env-file=.deployment/test.env run test:integration -``` - -Missing database variables fail explicitly rather than silently skipping checks. -The suite exercises two app instances with independent connection pools: - -- 32 buyers compete for 9 units; exactly 9 reservations commit. -- 24 simultaneous retries commit one reservation and replay its original body. -- Conflicting payloads, client-scoped keys, ownership, and repeated release. -- A rejected reservation leaves its key available for a later successful retry. -- Real INSERT failure after the inventory decrement rolls back both stock and key. -- A failed release status update rolls back its stock increment and leaves the reservation active. -- Runtime role privileges, encrypted connections, wrong-CA and wrong-hostname rejection. - -The creation rollback test installs [a temporary check constraint](fixtures/rollback-guard.sql) -which rejects only generated `test-rollback-` SKUs, then removes it in `finally` -using [the explicit cleanup SQL](fixtures/remove-rollback-guard.sql). The release -rollback test similarly installs [a status constraint](fixtures/release-rollback-guard.sql) -for `test-release-rollback-` SKUs and [removes it](fixtures/remove-release-rollback-guard.sql) -after checking the unchanged stock and reservation. If the process is forcibly -killed during either test, inspect its constraint before manually running the -corresponding cleanup SQL; the next run will fail safely if it still exists. - -Fixture SKUs and authenticated client IDs contain a unique run suffix. Cleanup -deletes only those clients' idempotency records/reservations and the exact SKUs -created by the suite. It does not reset seed stock or truncate tables. A crashed -process may leave its generated fixture rows; dispose of the test database after -verification using the README cleanup steps. - -The TLS suite also requires the `openssl` command in the VM. The certificate in -`fixtures/untrusted-ca.pem` is an unrelated, public test-only -certificate. Its private key is not included. The TLS suite establishes a valid -connection first, then exercises incorrect trust and a numeric-address hostname -against the same live endpoint. The untrusted CA must fail with a certificate -verification error code. For hostname verification, OpenSSL first connects to -the exact numeric peer address and verifies the genuine DNS identity and CA; -then Bun must reject that address as the expected certificate identity, even -with an ambient `PGSSLMODE=verify-ca`. Bun 1.4.2 may emit only an empty Error with -`errno: 0` for this rejection, so the test permits that specific opaque result -alongside explicit hostname errors. The independent verified-TLS control and -successful application query before/after bound that evidence; generic timeout, -DNS, authentication, and connection-closed errors do not pass. Record actual -runtime and managed-service results in the pull request. diff --git a/applications/stock-reservation/tests/integration.test.ts b/applications/stock-reservation/tests/integration.test.ts index fcfa80ef..0df3a390 100644 --- a/applications/stock-reservation/tests/integration.test.ts +++ b/applications/stock-reservation/tests/integration.test.ts @@ -30,7 +30,7 @@ type Reservation = { function required(name: string): string { const value = process.env[name]; - if (!value) throw new Error(`Integration tests require ${name}; see tests/README.md.`); + if (!value) throw new Error(`Integration tests require ${name}; see the README Test section.`); return value; } diff --git a/applications/stock-reservation/tests/tls.test.ts b/applications/stock-reservation/tests/tls.test.ts index 6809c7d6..97bdcaa9 100644 --- a/applications/stock-reservation/tests/tls.test.ts +++ b/applications/stock-reservation/tests/tls.test.ts @@ -10,7 +10,7 @@ let trusted: SQL; beforeAll(async () => { const url = process.env.DATABASE_URL; const caCertPath = process.env.PG_CA_CERT_PATH; - if (!url || !caCertPath) throw new Error("TLS tests require DATABASE_URL and PG_CA_CERT_PATH; see tests/README.md."); + if (!url || !caCertPath) throw new Error("TLS tests require DATABASE_URL and PG_CA_CERT_PATH; see the README Test section."); configuration = { url, caCertPath, maxConnections: 1 }; trusted = await createDatabase(configuration); // Establish the positive control first. Offline/bad-credential failures must