From e7aa143ac4c9fb8db723ed1127a7146221bfca85 Mon Sep 17 00:00:00 2001 From: Renuka Fernando Date: Sun, 23 Aug 2026 21:07:34 +0530 Subject: [PATCH 1/3] feat(dbmigrate): add v1->v2 migration client and shared migrationcore Introduce a one-time, offline Platform API v1->v2 database migration tool and extract its per-row logic into a reusable migrationcore package. - cmd/dbmigrate (migrate + verify subcommands): idempotent, resumable migration of all six artifact types incl. the EventGateway websub/webbroker plugin tables; applies core + plugin DDL; file-based state (checkpoint, quarantine, flags, drops); FK-order parent gating; targeted -only-keys/-since reconcile. - migrationcore: single shared implementation of the per-row transform, idempotent v2 upsert/delete, and identity resolution -- reused by the batch and the planned live dual-write intermediate (Options, Reporter, Execer, InsertOnly, DeleteX, ResolveIdentity). - verify: read-only 6-layer decode-and-compare gate, non-zero on FAIL. - validated on a real dump: verify PASS; affordance/reconcile tests. - docs: MIGRATION_MAPPING, RUNBOOK, VALIDATION_REPORT, core README. Signed-off-by: Renuka Fernando --- .../cmd/dbmigrate/MIGRATION_MAPPING.md | 182 +++ platform-api/cmd/dbmigrate/RUNBOOK.md | 100 ++ .../cmd/dbmigrate/VALIDATION_REPORT.md | 102 ++ platform-api/cmd/dbmigrate/db.go | 152 ++ platform-api/cmd/dbmigrate/dsn.go | 89 ++ platform-api/cmd/dbmigrate/handles.go | 76 + platform-api/cmd/dbmigrate/identity.go | 124 ++ platform-api/cmd/dbmigrate/main.go | 154 ++ platform-api/cmd/dbmigrate/migrate.go | 427 ++++++ platform-api/cmd/dbmigrate/migrate_tables.go | 1359 +++++++++++++++++ platform-api/cmd/dbmigrate/reconcile.go | 181 +++ platform-api/cmd/dbmigrate/reconcile_test.go | 87 ++ platform-api/cmd/dbmigrate/state.go | 353 +++++ platform-api/cmd/dbmigrate/transform.go | 38 + platform-api/cmd/dbmigrate/verify.go | 252 +++ platform-api/cmd/dbmigrate/verify_checks.go | 738 +++++++++ platform-api/migrationcore/README.md | 65 + platform-api/migrationcore/core.go | 231 +++ platform-api/migrationcore/delete.go | 118 ++ platform-api/migrationcore/transform.go | 307 ++++ platform-api/migrationcore/upsert.go | 792 ++++++++++ platform-api/migrationcore/upsert_pg_test.go | 361 +++++ 22 files changed, 6288 insertions(+) create mode 100644 platform-api/cmd/dbmigrate/MIGRATION_MAPPING.md create mode 100644 platform-api/cmd/dbmigrate/RUNBOOK.md create mode 100644 platform-api/cmd/dbmigrate/VALIDATION_REPORT.md create mode 100644 platform-api/cmd/dbmigrate/db.go create mode 100644 platform-api/cmd/dbmigrate/dsn.go create mode 100644 platform-api/cmd/dbmigrate/handles.go create mode 100644 platform-api/cmd/dbmigrate/identity.go create mode 100644 platform-api/cmd/dbmigrate/main.go create mode 100644 platform-api/cmd/dbmigrate/migrate.go create mode 100644 platform-api/cmd/dbmigrate/migrate_tables.go create mode 100644 platform-api/cmd/dbmigrate/reconcile.go create mode 100644 platform-api/cmd/dbmigrate/reconcile_test.go create mode 100644 platform-api/cmd/dbmigrate/state.go create mode 100644 platform-api/cmd/dbmigrate/transform.go create mode 100644 platform-api/cmd/dbmigrate/verify.go create mode 100644 platform-api/cmd/dbmigrate/verify_checks.go create mode 100644 platform-api/migrationcore/README.md create mode 100644 platform-api/migrationcore/core.go create mode 100644 platform-api/migrationcore/delete.go create mode 100644 platform-api/migrationcore/transform.go create mode 100644 platform-api/migrationcore/upsert.go create mode 100644 platform-api/migrationcore/upsert_pg_test.go diff --git a/platform-api/cmd/dbmigrate/MIGRATION_MAPPING.md b/platform-api/cmd/dbmigrate/MIGRATION_MAPPING.md new file mode 100644 index 0000000000..c6b6b2fc90 --- /dev/null +++ b/platform-api/cmd/dbmigrate/MIGRATION_MAPPING.md @@ -0,0 +1,182 @@ +# Platform API v1 → v2 DB Migration — Mapping & Decisions + +**Tool:** `platform-api/cmd/dbmigrate` (one binary; subcommands `migrate` + `verify`). +**Build/deploy from the pinned v2 revision below** — the migrator compiles v2's +`internal/model` structs and column lists; struct/DDL skew silently corrupts blobs. + +| | commit | branch | schema | +|---|---|---|---| +| **v2 (target)** | `a2911a091b549ee4b4bf57f895ce0ea09932ed2c` | `migration` | `internal/database/schema.postgres.sql` + `plugins/eventgateway/schema/schema.postgres.sql` | +| **v1 (source)** | `f4556fee3df0f4f894a1699e8dbd36d6762eaff7` | `platform-api/v0.10.x` | `src/internal/database/schema.postgres.sql` | + +> **Why this tool was rebuilt (2026-08-22):** the prior run (`prod-20260818`) **dropped** +> all event APIs (84 `websub_apis` + 8 `webbroker_apis`) via the removed-feature path. Under +> the settled 2026-08-19 decision, event APIs are **first-class and always migrated** into the +> plugin tables, and the plugin DDL is always applied. This build migrates all six artifact +> types and never routes event APIs to the drop path. + +--- + +## Deliverable 0 — mechanical DDL completeness gate → **PASS** + +Parsed v1 (27 tables), v2 core (32), v2 plugin (3 → 35 v2 total). Every v2 table+column is +either **MAPPED** from v1 or explicitly **EMPTY/OPTIONAL**. Reproduce with +`scratchpad/ddl_gate.py`. Classification: + +- **MAPPED (26):** organizations, projects, applications, artifacts, rest_apis, + subscription_plans, subscription_plan_limits, subscriptions, gateways, gateway_endpoints, + artifact_gateway_mappings, gateway_custom_policies, gateway_custom_policy_usages, + gateway_tokens, deployments, deployment_status, llm_provider_templates, llm_providers, + llm_proxies, mcp_proxies, api_keys, application_api_key_mappings, + application_artifact_mappings, user_idp_references, **websub_apis**, **webbroker_apis**. +- **EMPTY, intentional (6):** gateway_states, events (runtime HA state); secrets, + secret_scopes, artifact_secret_refs (v2-new); websub_api_hmac_secrets (v2-new, §K.3). +- **OPTIONAL, empty by default (3):** artifact_subscription_plans (dead in v2 code; config + blob is source of truth — `-populate-artifact-subscription-plans` to derive rows), + user_organization_mappings (audit resolution never reads it), audit + (`-audit-marker` to emit one "migrated" marker per org). + +**v1-only tables** (dropped or renamed): `application_api_keys`→`application_api_key_mappings`, +`application_artifacts`→`application_artifact_mappings`, `association_mappings`→ +`artifact_gateway_mappings` (+dev_portal rows dropped), `devportals` (drop), +`publication_mappings` (drop). + +--- + +## Architecture (as built) + +Go, inside the v2 module so `internal/*` is importable. Stream v1 → transform → **own +parameterized `INSERT`s** into v2 (never reuse repo `Create*`, never emit a `.sql` file). +Reused verbatim (all exported, un-tagged — no `-tags experimental`): + +- `internal/model` — config structs marshaled to the BYTEA blobs (byte-identical to v2 writes). +- `internal/utils` — `GenerateHandle`, `GenerateDeterministicUUIDv7`, + `Encrypt/DecryptSubscriptionToken`, `DeriveEncryptionKey`. +- `internal/constants` — kind strings (`RestApi`/`WebSubApi`/`WebBrokerApi`/`LlmProvider`/ + `LlmProxy`/`Mcp`), throttle units (`MINUTE`/`HOUR`/`DAY`/`MONTH`), `DeletedUser`. +- `internal/database` — `NewConnection` (pgx stdlib via `database/sql`), `(*DB).InitSchemaSQL` + (applies core **and** plugin DDL), `Rebind` (`?`→`$n`), `IsDuplicateKeyError`. + +All migration state is **file-based** (checkpoint, handle map, quarantine, flags, drops, +reports) — the v2 DB contains only migrated data conforming to the v2 DDL. + +--- + +## Type conversions (applied consistently) + +| conversion | columns | +|---|---| +| JSONB → BYTEA (`json.Marshal(model.X)`, plaintext) | `*_config`/`configuration`, gateway `properties`/`manifest`, event `configuration` | +| TEXT → BYTEA | `api_keys.api_key_hashes`, llm `openapi_spec`/`model_list`, `llm_provider_templates.configuration`, `deployments.metadata`, `gateway_custom_policies.policy_definition` | +| TIMESTAMP → TIMESTAMPTZ | all `created_at`/`updated_at`/`*_at` (source TZ = `-source-tz`, default UTC; never `now()`) | +| BOOLEAN → SMALLINT (`true`→1) | `gateways.is_active`/`is_critical`, `subscription_plans.stop_on_quota_reach` | +| VARCHAR widen (safe) | `subscription_token_hash` 64→255, `gateway_custom_policies.version` 15→30 | +| VARCHAR narrow (flag if truncating) | all `handle` 255→40, `created_by` 255→200, `gateway_custom_policies.description` TEXT→1023, `api_keys.issuer`/`allowed_targets` TEXT→255 | + +## Config-blob structural reshapes + +- **RestApi / WebSub / WebBroker:** v1 `transport VARCHAR(255)` (JSON array as TEXT) is a + *column*; v2 moved it *into* the config as `transport []string`. → parse the v1 column, + `json.Unmarshal` v1 `configuration` into the v2 struct, set `cfg.Transport`, re-marshal. + In-config `name` key is **preserved** (only the outer column is renamed to `display_name`). +- **LLM Provider/Proxy, MCP:** v1 `Policies` preserved; new arrays (`GlobalPolicies`/ + `OperationPolicies`/`AdditionalProviders`) left empty. Re-marshal v1 config into v2 struct. +- **Gateway:** `properties`/`manifest` `json.Marshal(map)`; `vhost` → one `gateway_endpoints` + row (`url=vhost`); `display_name` carried from v1 `display_name`. + +### ⚠ WebSub structural reshape (discovered from real data — the "strict subset" premise was WRONG) + +The dry-run against production data disproved the assumption that the v1 WebSub config is a strict +subset of v2's (only `transport` added). Two **structural** differences exist and are handled +losslessly in `preprocessWebSubRaw` (77 of 84 websubs would otherwise fail `BLOB_UNPARSEABLE`): + +1. **`channels`: v1 ARRAY → v2 `map[string]WebSubChannel`.** v1 stores + `[{"request":{"name":"/issues","method":"SUBSCRIBE"}}]`; v2 wants a map keyed by channel + name (verified: `mapWebSubChannelsAPIToModel` keys by `name` verbatim, no slash-stripping). + → key = `request.name`, value = empty `WebSubChannel{}`. `method` is always `SUBSCRIBE` + (constant, carries no info) → dropped safely. +2. **top-level `policies` (legacy) → `allChannels`.** v2's struct has no top-level policy list, + but 6 APIs store real **auth** policies there (`basic-auth`/`api-key-auth`, with credentials). + Folded into `allChannels`: object-form `{event:[p]}` → `allChannels.event.policies`; flat + array `[p]` (whole-API auth) → `allChannels.on_subscription.policies`. None of the 6 had a + pre-existing `allChannels`, so no merge conflict. **Zero policy loss.** + +Both remaps are recorded per row as `SYNTHESIZED` flags (`structural_reshape`). WebBroker needs +no such reshape (its `channels` are already objects). This is the prompt-vs-data reconciliation +the working method calls for: **the data wins.** + +**Blob guard (three-part, §E):** (1) dry-run `Decoder.DisallowUnknownFields()` discovery lists +v1 config fields the v2 struct drops (per-field human sign-off; see §E list below); (2) reshape; +(3) FULL per-row canonical-projection round-trip in `verify` (never sampled). + +--- + +## Per-table mapping (FK order) + +| # | v1 source | v2 target(s) | key transforms | +|---|---|---|---| +| 1 | (audit identities) | **user_idp_references** | distinct `created_by` strings + `migration` actor → `uuid=GenerateDeterministicUUIDv7(idp_id, epoch)`; `ON CONFLICT(idp_id)` | +| 2 | organizations | organizations | `name`→display_name; handle carry 255→40; +`idp_organization_ref_uuid`=uuid (PLACEHOLDER_IDP); audit rewrite | +| 3 | projects | projects | **generate** handle from name; `name`→display_name | +| 4 | applications | applications | handle carry; `name`→display_name; project_uuid now nullable | +| 5 | artifacts (all 6 kinds) | artifacts | →(uuid,type,org); `kind`→`type` verbatim; carry handle/name/version/created_at/updated_at DOWN into per-type row | +| 6 | rest_apis ⋈ artifacts | rest_apis | transport col→blob; JSONB→BYTEA; lifecycle_status carried | +| 7 | llm_provider_templates | llm_provider_templates | handle 255→40; `name`→display_name; config TEXT→BYTEA; +group_id=handle, version=v1.0, managed_by=organization, is_latest=1, enabled=1, openapi_spec=NULL | +| 8 | llm_providers ⋈ artifacts | llm_providers | JSONB→BYTEA; openapi_spec/model_list TEXT→BYTEA; **status DROPPED** | +| 9 | llm_proxies ⋈ artifacts | llm_proxies | JSONB→BYTEA; openapi_spec TEXT→BYTEA; **status DROPPED** | +| 10 | mcp_proxies ⋈ artifacts | mcp_proxies | JSONB→BYTEA; **status DROPPED** | +| 11 | websub_apis ⋈ artifacts | **websub_apis (plugin)** | artifacts-split like rest_apis; transport→`WebSubAPIConfiguration.transport`; lifecycle_status carried | +| 12 | webbroker_apis ⋈ artifacts | **webbroker_apis (plugin)** | same; `WebBrokerAPIConfiguration` | +| 13 | subscription_plans | subscription_plans (+ subscription_plan_limits) | generate handle from plan_name; `plan_name`→display_name; throttle→limits row (unit CASE-CONVERT, limit_type=REQUEST_COUNT, time_amount=1, limit_count_unit=NULL); **billing_plan DROPPED** | +| 14 | subscriptions | subscriptions | `api_uuid`→`artifact_uuid`; token verbatim (+decrypt guard); hash 64→255; partial-index dup → DUP_KEY quarantine | +| 15 | gateways | gateways (+ gateway_endpoints) | generate handle from name; display_name carried; properties/manifest JSONB→BYTEA; is_active/is_critical bool→smallint; vhost→endpoints row | +| 16 | gateway_custom_policies | gateway_custom_policies | policy_definition JSONB→BYTEA; description→1023 (TRUNCATED); version 15→30; name/display_name kept | +| 17 | gateway_tokens | gateway_tokens | hashes verbatim; TS→TSTZ; +data_version/created_by/revoked_by | +| 18 | deployments | deployments | `deployment_id`→uuid; `name`→display_name; `base_deployment_id`→base_deployment_uuid; metadata TEXT→BYTEA; content verbatim | +| 19 | deployment_status | deployment_status | `deployment_id`→deployment_uuid; PK reorder; +performed_by | +| 20 | association_mappings (gateway) | artifact_gateway_mappings | dev_portal rows DROPPED; metadata=NULL | +| 21 | gateway_custom_policy_usages | gateway_custom_policy_usages | `api_uuid`→`artifact_uuid` | +| 22 | api_keys | api_keys | generate handle from name; display_name=name; api_key_hashes TEXT→BYTEA; issuer/allowed_targets→255 (TRUNCATED) | +| 23 | application_api_keys | application_api_key_mappings | rename; +created_by | +| 24 | application_artifacts | application_artifact_mappings | rename; +created_by | + +Dropped (report per row): devportals, publication_mappings, dev_portal associations, +subscription_plans.billing_plan, llm_providers/llm_proxies/mcp_proxies `status`, +gateway_states, events. + +--- + +## Decisions requiring human sign-off (defaults chosen) + +1. **`idp_organization_ref_uuid` = org's own uuid** (PLACEHOLDER_IDP flag). Backfill from IdP later. +2. **`llm_provider_templates.group_id` = template's v1 handle** (matches v2 create path). +3. **Subscription token = verbatim passthrough** + **mandatory fatal decrypt guard**. Operator + must set v2 `security.encryption_key` to the value v1 actually used + (`database.subscription_token_encryption_key`, else `auth.jwt.secret_key`). `-skip-decrypt-check` + only for dry-runs without the key. +4. **Dropped features → drop + report** (list above). Event APIs are **not** in this bucket. +5. **Audit identity (§C.1):** seed `user_idp_references` from distinct v1 audit strings + + a `migration` actor; rewrite every `created_by`/`updated_by`/`revoked_by`/`performed_by` + to the mapped UUID. **Assumption (unverifiable from DB):** v1 `idp_id` == the claim v2 + extracts from the token (`sub`/configured). If v1 stored a username but v2 auths on `sub`, + the seeded identity displays correctly but won't unify with that user's future live-login UUID. +6. **Event APIs → plugin tables**, plugin DDL applied every run. +7. **`websub_api_hmac_secrets` = empty** (v2-new; re-issue post-migration). No `APIP_CP_ENCRYPTION_KEY` needed. +8. **`artifact_subscription_plans` = empty by default** (dead in v2 code). `-populate-artifact-subscription-plans` to derive. +9. **`audit` = empty by default.** `-audit-marker` to emit one per org. +10. **Source TZ = UTC** (`-source-tz`). + +## §E dropped-field discovery (per-config-struct, filled by `migrate -dry-run`) + +> Populated from the `DisallowUnknownFields` pass in the dry-run. Any field that carries data +> (not obsolete) is a STOP-and-remap. Event configs are a strict subset of v2 → expected **zero** +> unknowns. See `migration-report--dryrun.json → dropped_config_fields`. + +## Source-data-quality lanes (fail-fast / quarantine / migrate-with-flag) + +Per the prompt's table. Quarantine (`quarantine-.jsonl`) preserves the full v1 row + +`reason_code` ∈ {ORPHAN_FK, BLOB_UNPARSEABLE, NULL_REQUIRED, HANDLE_UNRESOLVABLE, DUP_KEY}; +flags (`flags-.jsonl`) record migrate-with-flag events {DEFAULTED_NULL, TRUNCATED, +PLACEHOLDER_IDP, SYNTHESIZED, PLAINTEXT_CREDENTIAL}. `verify` gate per table: +`count(v2) + drops + quarantine(latest run) == count(v1)` AND every quarantined key resolved or +signed off (`quarantine-signoff.jsonl`). diff --git a/platform-api/cmd/dbmigrate/RUNBOOK.md b/platform-api/cmd/dbmigrate/RUNBOOK.md new file mode 100644 index 0000000000..62dd03ee81 --- /dev/null +++ b/platform-api/cmd/dbmigrate/RUNBOOK.md @@ -0,0 +1,100 @@ +# dbmigrate — Runbook + +One-time, offline migration of a Platform API **v1** PostgreSQL database into a fresh **v2** +database (core + EventGateway plugin). Build and run from the pinned v2 revision +(`a2911a091…`, see MIGRATION_MAPPING.md). + +## 0. Build + +```sh +cd /Users/renuka/git/api-platform-migration/platform-api +go build -o dbmigrate ./cmd/dbmigrate/ +``` + +## 1. Back up (non-destructive; v1 is the rollback point) + +The tool never writes to v1. Keep the v1 dump (`choreo-dev-v1-backup.sql`) as the rollback point. + +## 2. Stand up the databases + +**v1 (source)** — restore the dump into an isolated Postgres on host `:5432`: + +```sh +docker run -d --name mig-v1 -e POSTGRES_PASSWORD=admin -e POSTGRES_DB=dbv1 \ + -v "$PWD/choreo-dev-v1-backup.sql:/docker-entrypoint-initdb.d/backup.sql:ro" \ + -p 5432:5432 postgres:15 +``` + +**v2 (target)** — a fresh, empty Postgres on host `:5433`. The tool applies the core **and** +plugin DDL itself (`-init-schema`, idempotent), so no schema mounts are needed: + +```sh +docker run -d --name mig-v2 -e POSTGRES_PASSWORD=admin -e POSTGRES_DB=dbv2 \ + -p 5433:5432 postgres:15 +``` + +(Alternatively use the `platform-api-v1` / `platform-api-v2` compose stacks, which seed the v2 +schema for you; then pass `-init-schema=false`.) + +## 3. Set the subscription-token key + +v2's `security.encryption_key` must be the value v1 **actually used** for +`subscription_token` (`database.subscription_token_encryption_key`, else `auth.jwt.secret_key`). +The tool reads it from the environment (never a flag): + +```sh +export APIP_MIGRATION_ENCRYPTION_KEY="" +``` + +If v1 ran on the ephemeral fallback, those tokens are unrecoverable and must be re-issued. + +## 4. Dry run (transform + validate, NO writes) + +```sh +V1="postgres://postgres:admin@localhost:5432/dbv1?sslmode=disable" +V2="postgres://postgres:admin@localhost:5433/dbv2?sslmode=disable" +OUT=/Users/renuka/google-workspace/platform-api-migration/db-migration/migration-out + +./dbmigrate migrate -v1-dsn "$V1" -v2-dsn "$V2" -out-dir "$OUT" -run-id prod -dry-run +``` + +Review, in `$OUT`: +- `migration-report-prod-dryrun.json` → `dropped_config_fields` (§E): **any field that carries + data is a STOP** — remap before the live run. +- `quarantine-prod-dryrun.jsonl` → decide each row (fix source, or sign off as loss). +- `flags-prod-dryrun.jsonl` → every truncation / placeholder / synthesized value. + +The dry run needs no key; add `-skip-decrypt-check` if `APIP_MIGRATION_ENCRYPTION_KEY` is unset. + +## 5. Live run + +```sh +./dbmigrate migrate -v1-dsn "$V1" -v2-dsn "$V2" -out-dir "$OUT" -run-id prod +``` + +Idempotent and resumable: re-run the same command after fixing source data or an interruption +(ON CONFLICT DO NOTHING + the file checkpoint keep handles stable and rows de-duplicated). + +## 6. Verify (read-only gate; non-zero exit on FAIL) + +```sh +./dbmigrate verify -v1-dsn "$V1" -v2-dsn "$V2" -out-dir "$OUT" -run-id prod +``` + +Writes `verify-report-prod.json`. The gate passes only when every table reconciles +(`v2 + quarantine == v1`), every transform round-trips, and every quarantined key is resolved in +v2 or listed in `quarantine-signoff.jsonl` ({source_table, source_key} per line). + +No `APIP_CP_ENCRYPTION_KEY` is needed (the WebSub HMAC table stays empty, §K.3). + +## Flags of note + +| flag | default | purpose | +|---|---|---| +| `-dry-run` | off | transform+validate, no writes | +| `-init-schema` | on | apply v2 core + plugin DDL (idempotent) | +| `-source-tz` | `UTC` | timezone of naive v1 TIMESTAMP values | +| `-skip-decrypt-check` | off | skip the mandatory token decrypt guard (dry-runs only) | +| `-populate-artifact-subscription-plans` | off | derive the (redundant) artifact_subscription_plans rows | +| `-audit-marker` | off | emit one "migrated" audit row per org | +| `-migration-epoch` | `2026-01-01T00:00:00Z` | fixed epoch for deterministic synthesized UUIDs | diff --git a/platform-api/cmd/dbmigrate/VALIDATION_REPORT.md b/platform-api/cmd/dbmigrate/VALIDATION_REPORT.md new file mode 100644 index 0000000000..7c297ad5c4 --- /dev/null +++ b/platform-api/cmd/dbmigrate/VALIDATION_REPORT.md @@ -0,0 +1,102 @@ +# Validation Report — Platform API v1 → v2 migration + +**Run:** `prod` · **Date:** 2026-08-22 · **Mode:** live + verify +**Source:** choreo dev dump (`choreo-dev-v1-backup.sql`, 447 artifacts), restored to Postgres 15 on `:5432`. +**Target:** fresh empty Postgres 15 on `:5433`; core + EventGateway plugin DDL applied by the tool. +**Built from:** v2 `a2911a091` (branch `migration`); v1 `f4556fee3` (`platform-api/v0.10.x`). + +## Headline + +**Verify: PASS — 121 PASS / 1 WARN / 0 FAIL** (`verify-report-prod.json`, exit 0). All 447 artifacts +(all six types incl. **84 WebSub + 8 WebBroker**) migrated with **zero quarantine** and zero config +data loss. This corrects the prior `prod-20260818` run, which **dropped all 92 event APIs** and +quarantined 102 of their orphaned children. + +The single WARN is the sampled subscription-token *decrypt* round-trip, skipped because choreo dev's +token key is not on this machine. The token **bytes** were verified byte-identical to v1 (verbatim +passthrough), and the decrypt guard was proven to fail-fast on a wrong key (see below). + +## Coverage (processed_per_table — all reconcile v2 + quarantine(0) == v1) + +| table | rows | | table | rows | +|---|--:|---|---|--:| +| organizations | 107 | | subscriptions | 9 | +| projects | 226 | | gateways | 143 | +| applications | 20 | | gateway_endpoints | 143 | +| **artifacts** | **447** | | artifact_gateway_mappings | 108 | +| rest_apis | 193 | | gateway_custom_policies | 1 | +| llm_provider_templates | 750 | | gateway_custom_policy_usages | 3 | +| llm_providers | 93 | | gateway_tokens | 228 | +| llm_proxies | 49 | | deployments | 536 | +| mcp_proxies | 20 | | deployment_status | 194 | +| **websub_apis** | **84** | | api_keys | 139 | +| **webbroker_apis** | **8** | | application_api_key_mappings | 12 | +| subscription_plans | 60 | | application_artifact_mappings | 2 | +| subscription_plan_limits | 60 | | user_idp_references | 24 | + +`gateway_tokens` (228) and the full `deployments`/`deployment_status`/`api_keys` counts are migrated +here but were absent/short in the Aug-18 run (event-artifact children were orphaned then). + +## Verify layers (all PASS unless noted) + +- **A Coverage:** each per-type split reconciles (`websub 84`, `webbroker 8`, …); Σ per-type == v2.artifacts (447); + `subscription_plan_limits`==v1 throttled plans; `gateway_endpoints`==v1 non-empty vhosts; + `websub_api_hmac_secrets`==0; `secrets`==0. +- **B Scalar (FULL):** `display_name`==v1 `name`; `created_at` instant preserved; `kind`==`type`; + `subscriptions.api_uuid`→`artifact_uuid`; `plan_name`==`display_name` — 0 mismatch. +- **C Round-trip (FULL, decode-and-compare):** config invariants **0 mismatch** for rest_apis(193), + websub_apis(84), webbroker_apis(8), llm_providers(93), llm_proxies(49), mcp_proxies(20); + transport-column→blob **0 mismatch**; `subscription_plan_limits` values exact; `gateway_endpoints.url`==`vhost`; + `subscription_token`/`_hash` byte-verbatim. +- **D Integrity (FULL):** every v2 FK resolves (0 orphans); every `(organization_uuid, handle)` unique per + type table; `llm_provider_templates(org, group_id, version)` unique; `user_idp_references(idp_id)` unique. +- **E Generated/defaults:** carried handles == `slug(v1)`; `idp_organization_ref_uuid`==org uuid; + template `group_id`==handle, `version`=v1.0, `managed_by`=organization, `is_latest`/`enabled`=1; + `data_version`=1.0 & `origin`=control_plane everywhere; every `created_by` resolves in `user_idp_references`. +- **F Drop reconciliation:** intentional drops match v1 counts (devportals 107, dev_portal assoc 193, + billing_plan 60, llm/llm-proxy/mcp status 93/49/20); every migrated table reconciles; quarantine sign-off: 0 outstanding. + +## Intentional drops (auditable, `drops-prod.jsonl`) + +| feature | rows | scope | +|---|--:|---| +| devportals | 107 | row (feature removed in v2) | +| dev_portal_association | 193 | row (association_mappings dev_portal rows) | +| publication_mappings | 0 | row (none in source) | +| billing_plan | 60 | field (dropped column) | +| llm_provider_status / llm_proxy_status / mcp_status | 93 / 49 / 20 | field (no v2 target column) | + +Event APIs are **not** in the drop bucket. + +## Flags (migrate-with-flag audit trail, `flags-prod.jsonl`) + +| code | count | meaning | +|---|--:|---| +| SYNTHESIZED | 1421 | generated handles (projects/gateways/plans/api_keys), template defaults, seeded identities, **WebSub structural reshapes** | +| DEFAULTED_NULL | 2630 | rows with no v1 `created_by` → migration-actor UUID (§C.1) | +| PLAINTEXT_CREDENTIAL | 139 | LLM config blobs carrying an upstream apiKey in plaintext (security note) | +| PLACEHOLDER_IDP | 107 | `idp_organization_ref_uuid` = org uuid (backfill from IdP later) | +| TRUNCATED | 3 | value narrowed to a v2 VARCHAR limit (lossy but valid) | + +## Notable finding — WebSub config is NOT a strict subset of v2 + +Dry-run on real data disproved the "strict subset" premise: 77/84 v1 websubs store `channels` as an +**array** (v2 wants a map) and 6 carry top-level auth `policies` (no v2 field). Both are reshaped +losslessly (`channels[]`→map keyed by `request.name`; `policies`→`allChannels`), preserving channel +identity and auth credentials. See MIGRATION_MAPPING.md → "WebSub structural reshape". Spot-checked in v2: +`channels:{"_default":{}}`, `transport:["http","https"]`, `allChannels.on_subscription.policies:[basic-auth…]`. + +## Operational checks + +- **Idempotent/resumable:** re-running the live migration is a clean no-op (447 artifacts, 84 websub; + `gateway_endpoints` existence-checked, no duplicates); handles replay from the file checkpoint. +- **Decrypt guard is real:** with a wrong key the run **aborts before any write** — + *"decrypt guard FAILED (wrong key ⇒ all tokens garbage): … cipher: message authentication failed."* + +## Sign-off items for a production cutover + +1. Set `APIP_MIGRATION_ENCRYPTION_KEY` to the key v1 **actually used** for subscription tokens, and run + **without** `-skip-decrypt-check` (the guard was skipped here only because choreo's key is not local). +2. Confirm the §C.1 assumption: v1 `idp_id` == the claim v2 extracts from the token (`sub`/configured) — + not verifiable from the DB. +3. Accept the intentional drops and the `PLAINTEXT_CREDENTIAL` security note. diff --git a/platform-api/cmd/dbmigrate/db.go b/platform-api/cmd/dbmigrate/db.go new file mode 100644 index 0000000000..4c4cfc51f0 --- /dev/null +++ b/platform-api/cmd/dbmigrate/db.go @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "fmt" + "strings" + + "github.com/wso2/api-platform/platform-api/internal/database" +) + +// insertRow issues a single idempotent INSERT. conflict is a raw ON CONFLICT +// target such as "(uuid)" or "(organization_uuid, artifact_uuid, gateway_uuid)"; +// empty means no ON CONFLICT clause. Values are passed as bind parameters (native +// BYTEA, zero escaping). At this data scale (hundreds–thousands of rows/table) +// autocommit per-row inserts are simpler and fully resumable; batching/COPY would +// be premature optimization. +func insertRow(db *database.DB, table string, cols []string, args []any, conflict string) error { + if len(cols) != len(args) { + return fmt.Errorf("insert %s: %d cols but %d args", table, len(cols), len(args)) + } + ph := make([]string, len(cols)) + for i := range cols { + ph[i] = "?" + } + q := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", table, strings.Join(cols, ", "), strings.Join(ph, ", ")) + if conflict != "" { + q += " ON CONFLICT " + conflict + " DO NOTHING" + } + _, err := db.Exec(db.Rebind(q), args...) + if err != nil { + return fmt.Errorf("insert %s: %w", table, err) + } + return nil +} + +// rowExists reports whether the parameterized query returns at least one row. +func rowExists(db *database.DB, query string, args ...any) (bool, error) { + var one int + err := db.QueryRow(db.Rebind("SELECT 1 FROM ("+query+") _sub LIMIT 1"), args...).Scan(&one) + if err != nil { + if err.Error() == "sql: no rows in result set" { + return false, nil + } + return false, err + } + return true, nil +} + +// loadStringSet runs a single-column query and collects the results into a set. +func loadStringSet(db *database.DB, query string, args ...any) (map[string]bool, error) { + rows, err := db.Query(db.Rebind(query), args...) + if err != nil { + return nil, err + } + defer rows.Close() + set := map[string]bool{} + for rows.Next() { + var s string + if err := rows.Scan(&s); err != nil { + return nil, err + } + set[s] = true + } + return set, rows.Err() +} + +// ParentStatus is the outcome of an FK-parent lookup. +type ParentStatus int + +const ( + // ParentOK: the parent was migrated into v2. + ParentOK ParentStatus = iota + // ParentCascade: the parent exists in v1 but was NOT migrated (quarantined/ + // dropped) — the child should cascade-quarantine, not fail-fast. + ParentCascade + // ParentMissing: the parent uuid is absent from v1 entirely — genuine dangling + // reference (fail-fast if v1 enforced the FK, else quarantine). + ParentMissing +) + +// ParentSet tracks, per parent table, which uuids were seen in v1 and which were +// actually inserted into v2, so children can classify an FK reference precisely. +type ParentSet struct { + v1 map[string]bool + v2 map[string]bool +} + +func newParentSet() *ParentSet { + return &ParentSet{v1: map[string]bool{}, v2: map[string]bool{}} +} + +func (p *ParentSet) seenV1(uuid string) { p.v1[uuid] = true } +func (p *ParentSet) insertedV2(uuid string) { p.v2[uuid] = true } + +func (p *ParentSet) status(uuid string) ParentStatus { + if p.v2[uuid] { + return ParentOK + } + if p.v1[uuid] { + return ParentCascade + } + return ParentMissing +} + +// Sets bundles the in-memory FK parent sets used during a run. UUIDs are preserved +// end-to-end, so FK resolution is a set membership test against already-migrated +// parents — never a live DB round-trip, never a mid-batch Postgres FK error. +type Sets struct { + orgs *ParentSet + projects *ParentSet + artifacts *ParentSet + gateways *ParentSet + plans *ParentSet + providers *ParentSet + templates *ParentSet + applications *ParentSet + apiKeys *ParentSet + deployments *ParentSet + policies *ParentSet +} + +func newSets() *Sets { + return &Sets{ + orgs: newParentSet(), + projects: newParentSet(), + artifacts: newParentSet(), + gateways: newParentSet(), + plans: newParentSet(), + providers: newParentSet(), + templates: newParentSet(), + applications: newParentSet(), + apiKeys: newParentSet(), + deployments: newParentSet(), + policies: newParentSet(), + } +} diff --git a/platform-api/cmd/dbmigrate/dsn.go b/platform-api/cmd/dbmigrate/dsn.go new file mode 100644 index 0000000000..9aa92f74c4 --- /dev/null +++ b/platform-api/cmd/dbmigrate/dsn.go @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "fmt" + "log/slog" + "net/url" + "strconv" + "strings" + + "github.com/wso2/api-platform/platform-api/config" + "github.com/wso2/api-platform/platform-api/internal/database" +) + +// dsnToConfig parses a PostgreSQL URL DSN (postgres://user:pass@host:port/db?sslmode=...) +// into the config.Database struct that database.NewConnection expects. Only the +// PostgreSQL driver is in scope for this tool. +func dsnToConfig(dsn string) (*config.Database, error) { + u, err := url.Parse(dsn) + if err != nil { + return nil, fmt.Errorf("parse DSN: %w", err) + } + if u.Scheme != "postgres" && u.Scheme != "postgresql" { + return nil, fmt.Errorf("unsupported DSN scheme %q (only postgres:// is supported)", u.Scheme) + } + + port := 5432 + if p := u.Port(); p != "" { + port, err = strconv.Atoi(p) + if err != nil { + return nil, fmt.Errorf("invalid port %q: %w", p, err) + } + } + + name := strings.TrimPrefix(u.Path, "/") + if name == "" { + return nil, fmt.Errorf("DSN is missing a database name") + } + + sslMode := u.Query().Get("sslmode") + if sslMode == "" { + sslMode = "disable" + } + + pass, _ := u.User.Password() + return &config.Database{ + Driver: database.DriverPostgres, + Host: u.Hostname(), + Port: port, + Name: name, + User: u.User.Username(), + Password: pass, + SSLMode: sslMode, + SSLRootCert: u.Query().Get("sslrootcert"), + MaxOpenConns: 10, + MaxIdleConns: 5, + ConnMaxLifetime: 0, + }, nil +} + +// openDB opens a *database.DB for the given DSN, reusing the product's connection +// helper so we inherit its pgx wiring, Rebind, and IsDuplicateKeyError behavior. +func openDB(dsn string, logger *slog.Logger) (*database.DB, error) { + cfg, err := dsnToConfig(dsn) + if err != nil { + return nil, err + } + db, err := database.NewConnection(cfg, logger) + if err != nil { + return nil, err + } + return db, nil +} diff --git a/platform-api/cmd/dbmigrate/handles.go b/platform-api/cmd/dbmigrate/handles.go new file mode 100644 index 0000000000..f256e1c1ba --- /dev/null +++ b/platform-api/cmd/dbmigrate/handles.go @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "github.com/wso2/api-platform/platform-api/internal/utils" +) + +// handleGen produces v2 handles deterministically across runs (§B): generate once +// via the product's utils.GenerateHandle, persist to the file checkpoint, replay +// verbatim on resume. Uniqueness is scoped per (table, org); event tables get +// their own scope, so a websub handle and a rest_api handle may coincide. +type handleGen struct { + run *Run + used map[string]map[string]bool // scopeKey(table,org) -> set of used handles +} + +func newHandleGen(run *Run) *handleGen { + return &handleGen{run: run, used: map[string]map[string]bool{}} +} + +func scopeKey(table, org string) string { return table + "\x00" + org } + +func (h *handleGen) set(table, org string) map[string]bool { + k := scopeKey(table, org) + s := h.used[k] + if s == nil { + s = map[string]bool{} + h.used[k] = s + } + return s +} + +// seed records a handle already present in v2 (for the resume case) so freshly +// generated handles avoid colliding with it. +func (h *handleGen) seed(table, org, handle string) { + h.set(table, org)[handle] = true +} + +// generate returns the v2 handle for (table, org, v1uuid) built from source. +// +// - If a handle was already recorded for (table, v1uuid), reuse it VERBATIM — +// no GenerateHandle, no existsCheck (on resume the value may already be in v2; +// re-checking would falsely see "exists" and regenerate). +// - Otherwise call utils.GenerateHandle(source, existsCheck) exactly once, record +// it into the checkpoint, and mark it used in the (table, org) scope. +func (h *handleGen) generate(table, org, v1uuid, source string) (string, error) { + if hv, ok := h.run.getHandle(table, v1uuid); ok { + h.set(table, org)[hv] = true + return hv, nil + } + scope := h.set(table, org) + existsCheck := func(cand string) bool { return scope[cand] } + hv, err := utils.GenerateHandle(source, existsCheck) + if err != nil { + return "", err + } + scope[hv] = true + h.run.putHandle(table, v1uuid, hv) + return hv, nil +} diff --git a/platform-api/cmd/dbmigrate/identity.go b/platform-api/cmd/dbmigrate/identity.go new file mode 100644 index 0000000000..06a64bdbc7 --- /dev/null +++ b/platform-api/cmd/dbmigrate/identity.go @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/wso2/api-platform/platform-api/internal/database" + "github.com/wso2/api-platform/platform-api/internal/utils" +) + +// migrationActorIDPID is the synthetic IdP identity used to attribute rows whose +// v1 audit value is NULL/empty. It is seeded into user_idp_references like any +// other identity so it resolves to a real principal (not DeletedUser). +const migrationActorIDPID = "migration" + +// v1 tables that carry a raw created_by identity string (verified against the v1 +// schema). v1 has no updated_by columns. +var v1CreatedByTables = []string{ + "applications", "rest_apis", "llm_provider_templates", "llm_providers", + "llm_proxies", "mcp_proxies", "websub_apis", "webbroker_apis", "api_keys", +} + +// IdentityMap maps raw v1 audit identity strings to internal v2 user UUIDs +// (§C.1). UUIDs are deterministic (GenerateDeterministicUUIDv7), so seeding and +// rewriting are idempotent/resumable with no persisted state. +type IdentityMap struct { + epoch time.Time + actorUUID string + m map[string]string // idp_id -> uuid +} + +// resolve maps a raw v1 identity to its internal UUID, registering it on first +// sight. An empty/blank identity maps to the migration-actor UUID and reports +// defaulted=true (the caller records a DEFAULTED_NULL flag). +func (im *IdentityMap) resolve(raw string) (uuid string, defaulted bool) { + raw = strings.TrimSpace(raw) + if raw == "" { + return im.actorUUID, true + } + if u, ok := im.m[raw]; ok { + return u, false + } + u := utils.GenerateDeterministicUUIDv7(raw, im.epoch) + im.m[raw] = u + return u, false +} + +// collectAndSeedIdentities scans every v1 audit column for distinct identities, +// builds the identity→uuid map (including the migration actor), and — unless this +// is a dry-run — seeds one user_idp_references row per identity FIRST in FK order. +// Each seeded identity is recorded as SYNTHESIZED (fabricated UUID). +func collectAndSeedIdentities(v1, v2 *database.DB, run *Run, epoch time.Time) (*IdentityMap, error) { + im := &IdentityMap{epoch: epoch, m: map[string]string{}} + im.actorUUID = utils.GenerateDeterministicUUIDv7(migrationActorIDPID, epoch) + im.m[migrationActorIDPID] = im.actorUUID + + distinct := map[string]bool{} + for _, t := range v1CreatedByTables { + set, err := loadStringSet(v1, fmt.Sprintf( + "SELECT DISTINCT created_by FROM %s WHERE created_by IS NOT NULL AND created_by <> ''", t)) + if err != nil { + return nil, fmt.Errorf("collect identities from %s: %w", t, err) + } + for id := range set { + distinct[id] = true + } + } + + // Register (compute deterministic UUIDs). Sort for stable logs/flag ordering. + ids := make([]string, 0, len(distinct)) + for id := range distinct { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + im.resolve(id) + } + + run.logger.Info("collected audit identities", "distinct", len(ids), "actor", migrationActorIDPID) + + // Seed user_idp_references FIRST (dependency-free) so the map exists before any + // audited insert. created_at is pinned to the migration epoch for determinism. + seedList := append([]string{migrationActorIDPID}, ids...) + seeded := 0 + for _, idpID := range seedList { + uuid := im.m[idpID] + if len(idpID) > 200 { + run.flag("user_idp_references", uuid, FlagTruncated, + map[string]any{"len": len(idpID)}, map[string]any{"note": "raw identity > 200 chars; idp_id VARCHAR(255) holds it, but the source string is unusually long"}) + } + if !run.opts.DryRun { + if err := insertRow(v2, "user_idp_references", + []string{"uuid", "idp_id", "created_at"}, + []any{uuid, idpID, epoch}, "(idp_id)"); err != nil { + return nil, fmt.Errorf("seed user_idp_references %q: %w", idpID, err) + } + } + run.flag("user_idp_references", uuid, FlagSynthesized, + nil, map[string]any{"idp_id": idpID, "note": "fabricated internal user UUID (deterministic v7)"}) + seeded++ + } + run.addProcessed("user_idp_references", seeded) + run.markTableComplete("user_idp_references") + return im, nil +} diff --git a/platform-api/cmd/dbmigrate/main.go b/platform-api/cmd/dbmigrate/main.go new file mode 100644 index 0000000000..6403cedd5d --- /dev/null +++ b/platform-api/cmd/dbmigrate/main.go @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Command dbmigrate performs a one-time, offline migration of a Platform API v1 +// PostgreSQL database into a fresh Platform API v2 database (core + EventGateway +// plugin schema). It always migrates all six artifact types and always applies +// the plugin DDL. See MIGRATION_MAPPING.md for the full mapping and decisions. +// +// Usage: +// +// dbmigrate migrate -v1-dsn -v2-dsn -out-dir -run-id [-dry-run] [flags] +// dbmigrate verify -v1-dsn -v2-dsn -out-dir -run-id +// +// The subscription-token encryption key is read from the APIP_MIGRATION_ENCRYPTION_KEY +// environment variable or from -encryption-key-file (never a CLI flag, which would leak +// via ps/shell history). +package main + +import ( + "flag" + "fmt" + "log/slog" + "os" + "time" +) + +const defaultMigrationEpoch = "2026-01-01T00:00:00Z" + +// Options holds the parsed command-line configuration shared by the subcommands. +type Options struct { + V1DSN string + V2DSN string + OutDir string + RunID string + DryRun bool + Verbose bool + + // Schema application (migrate only). + InitSchema bool + CoreSchema string + PluginSchema string + + // Transform/decision knobs. + SourceTZ string + BatchSize int + IDPRefStrategy string + GroupIDStrategy string + MigrationEpoch time.Time + PopulateArtifactSubPlans bool + AuditMarker bool + SkipDecryptCheck bool + DecryptSampleSize int + + // Targeted reconciliation (§8.1). OnlyKeys is a file of " " lines + // to replay through the shared UpsertX/DeleteX; Since (no key list) triggers a full + // idempotent re-sync. Either one puts migrate into reconcile mode (InsertOnly:false). + OnlyKeys string + Since string + + // Encryption-key source (env var preferred; file as an alternative). Never a + // flag, which would leak the key via ps/shell history. + EncKeyFile string + + // Secrets (not flags). + EncryptionKey []byte // 32 bytes, or nil when unavailable +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + sub := os.Args[1] + args := os.Args[2:] + + var err error + switch sub { + case "migrate": + err = runMigrate(args) + case "verify": + err = runVerify(args) + case "-h", "--help", "help": + usage() + return + default: + fmt.Fprintf(os.Stderr, "unknown subcommand %q\n\n", sub) + usage() + os.Exit(2) + } + if err != nil { + fmt.Fprintf(os.Stderr, "\ndbmigrate %s: %v\n", sub, err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `dbmigrate — Platform API v1 -> v2 database migration + +Subcommands: + migrate Transform v1 data and write it into a fresh v2 database. + verify Read-only completeness/correctness gate (non-zero exit on FAIL). + +Run "dbmigrate migrate -h" or "dbmigrate verify -h" for flags. + +The subscription-token key comes from APIP_MIGRATION_ENCRYPTION_KEY or +-encryption-key-file, never a CLI flag. +`) +} + +// registerCommonFlags wires the flags shared by both subcommands into fs. +func registerCommonFlags(fs *flag.FlagSet, o *Options) { + fs.StringVar(&o.V1DSN, "v1-dsn", "", "PostgreSQL DSN of the v1 source DB (required), e.g. postgres://user:pass@host:5432/db?sslmode=disable") + fs.StringVar(&o.V2DSN, "v2-dsn", "", "PostgreSQL DSN of the v2 target DB (required)") + fs.StringVar(&o.OutDir, "out-dir", "", "Directory for state/report/quarantine/flags files (required)") + fs.StringVar(&o.RunID, "run-id", "run", "Stable identifier for this run (reused across resume)") + fs.BoolVar(&o.Verbose, "v", false, "Verbose (debug) logging") +} + +// newLogger builds a slog logger honoring -v. +func newLogger(o *Options) *slog.Logger { + level := slog.LevelInfo + if o.Verbose { + level = slog.LevelDebug + } + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})) +} + +// validateCommon checks the flags shared by both subcommands. +func validateCommon(o *Options) error { + if o.V1DSN == "" || o.V2DSN == "" { + return fmt.Errorf("-v1-dsn and -v2-dsn are required") + } + if o.OutDir == "" { + return fmt.Errorf("-out-dir is required") + } + if err := os.MkdirAll(o.OutDir, 0o755); err != nil { + return fmt.Errorf("create out-dir: %w", err) + } + return nil +} diff --git a/platform-api/cmd/dbmigrate/migrate.go b/platform-api/cmd/dbmigrate/migrate.go new file mode 100644 index 0000000000..a3c7a44016 --- /dev/null +++ b/platform-api/cmd/dbmigrate/migrate.go @@ -0,0 +1,427 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "database/sql" + "flag" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "github.com/wso2/api-platform/platform-api/internal/constants" + "github.com/wso2/api-platform/platform-api/internal/database" + "github.com/wso2/api-platform/platform-api/internal/utils" + "github.com/wso2/api-platform/platform-api/migrationcore" +) + +// migCtx bundles the shared state threaded through every table migration. +type migCtx struct { + v1, v2 *database.DB + run *Run + im *IdentityMap + sets *Sets + h *handleGen + opts *Options + loc *time.Location + encKey []byte + epoch time.Time + + // artifactPlans is collected only when -populate-artifact-subscription-plans is + // set: (artifact, org, plan handles from the config blob) for later resolution. + artifactPlans []artifactPlanRef + + // Subscription dedup sets for v2's stricter uniqueness (§G). + subAppSeen map[string]bool // org|artifact|application_id + subHashSeen map[string]bool // artifact|token_hash + + // Targeted-reconciliation state (§8.1). reconcile flips coreOpts to the live + // semantics (DO UPDATE + on-demand identity upsert); only, when non-nil, filters + // the iterators to the -only-keys work list. Both are zero/nil for the batch, so + // batch behaviour is byte-identical. + reconcile bool + only *keyFilter +} + +// deterministicUUID is a thin alias over the product's deterministic UUIDv7 helper +// so synthesized PKs are idempotent/resumable without persisted state. +func deterministicUUID(entity string, ts time.Time) string { + return utils.GenerateDeterministicUUIDv7(entity, ts) +} + +// coreOpts builds the shared-core Options. For the BATCH caller: insert-only +// (ON CONFLICT DO NOTHING) and skip per-row identity upserts (user_idp_references is +// bulk pre-seeded). For a RECONCILE run (§8.1) it uses the live dual-write semantics — +// InsertOnly:false (DO UPDATE, so changed rows are healed, not skipped) and +// SkipIdentityUpsert:false (seed the actor on demand, no bulk pre-seed). +func (mc *migCtx) coreOpts() migrationcore.Options { + return migrationcore.Options{ + EncryptionKey: mc.encKey, + SourceTZ: mc.loc, + Epoch: mc.epoch, + InsertOnly: !mc.reconcile, + SkipIdentityUpsert: !mc.reconcile, + DryRun: mc.opts.DryRun, + } +} + +// nsp / ntp convert scanned nullable columns into the pointer form V1Row uses. +func nsp(ns sql.NullString) *string { + if ns.Valid { + s := ns.String + return &s + } + return nil +} +func ntp(nt sql.NullTime) *time.Time { + if nt.Valid { + t := nt.Time + return &t + } + return nil +} + +// insert is a dry-run-guarded idempotent INSERT used only by the batch-only +// optional tables (audit markers, artifact_subscription_plans) that migrationcore +// does not own. The migrated per-type entities go through migrationcore.UpsertX. +func (mc *migCtx) insert(table string, cols []string, args []any, conflict string) error { + if mc.opts.DryRun { + return nil + } + return insertRow(mc.v2, table, cols, args, conflict) +} + +type artifactPlanRef struct { + artifactUUID string + org string + planHandles []string +} + +func runMigrate(argv []string) error { + o := &Options{} + fs := flag.NewFlagSet("migrate", flag.ContinueOnError) + registerCommonFlags(fs, o) + fs.BoolVar(&o.DryRun, "dry-run", false, "Transform and validate everything but perform no inserts") + fs.BoolVar(&o.InitSchema, "init-schema", true, "Apply the v2 core + plugin DDL before migrating (idempotent)") + fs.StringVar(&o.CoreSchema, "core-schema", + "internal/database/schema.postgres.sql", "Path to the v2 core PostgreSQL DDL") + fs.StringVar(&o.PluginSchema, "plugin-schema", + "plugins/eventgateway/schema/schema.postgres.sql", "Path to the EventGateway plugin PostgreSQL DDL") + fs.StringVar(&o.SourceTZ, "source-tz", "UTC", "Time zone of naive v1 TIMESTAMP values") + fs.IntVar(&o.BatchSize, "batch-size", 1000, "Rows per progress checkpoint") + fs.StringVar(&o.IDPRefStrategy, "idp-ref-strategy", "org-uuid", "Strategy for organizations.idp_organization_ref_uuid (org-uuid)") + fs.StringVar(&o.GroupIDStrategy, "group-id-strategy", "handle", "Strategy for llm_provider_templates.group_id (handle)") + epochStr := fs.String("migration-epoch", defaultMigrationEpoch, "Fixed epoch (RFC3339) for deterministic synthesized UUIDs") + fs.BoolVar(&o.PopulateArtifactSubPlans, "populate-artifact-subscription-plans", false, "Derive artifact_subscription_plans rows from config blobs (redundant; off by default)") + fs.BoolVar(&o.AuditMarker, "audit-marker", false, "Emit one 'migrated' audit row per organization") + fs.BoolVar(&o.SkipDecryptCheck, "skip-decrypt-check", false, "Skip the mandatory subscription-token decrypt guard (dry-runs without the key)") + fs.IntVar(&o.DecryptSampleSize, "decrypt-sample-size", 25, "Number of subscription tokens to sample for the decrypt guard") + fs.StringVar(&o.EncKeyFile, "encryption-key-file", "", "File containing the subscription-token key (alternative to APIP_MIGRATION_ENCRYPTION_KEY)") + fs.StringVar(&o.OnlyKeys, "only-keys", "", "Reconcile (§8.1): path to a file of '
' lines (op=upsert|delete) to replay through the shared UpsertX/DeleteX; enables live upsert semantics (DO UPDATE)") + fs.StringVar(&o.Since, "since", "", "Reconcile (§8.1): full idempotent re-sync of all rows (heals the enable-vs-backfill gap); RFC3339 marker, logged for the operator") + if err := fs.Parse(argv); err != nil { + return err + } + if err := validateCommon(o); err != nil { + return err + } + + epoch, err := time.Parse(time.RFC3339, *epochStr) + if err != nil { + return fmt.Errorf("invalid -migration-epoch: %w", err) + } + loc, err := time.LoadLocation(o.SourceTZ) + if err != nil { + return fmt.Errorf("invalid -source-tz %q: %w", o.SourceTZ, err) + } + if err := loadEncryptionKey(o); err != nil { + return err + } + if o.EncryptionKey == nil && !o.SkipDecryptCheck { + return fmt.Errorf("no encryption key: set APIP_MIGRATION_ENCRYPTION_KEY or -encryption-key-file for the mandatory token decrypt guard, or pass -skip-decrypt-check (dry-runs only)") + } + + logger := newLogger(o) + + // Reconcile mode (§8.1): -only-keys replays a targeted work list through the shared + // UpsertX/DeleteX; -since (no key list) re-runs every row (a full idempotent re-sync). + // Either flag flips coreOpts to the live semantics via migCtx.reconcile. + reconcile := o.OnlyKeys != "" || o.Since != "" + var only *keyFilter + if o.OnlyKeys != "" { + only, err = loadKeyFilter(o.OnlyKeys) + if err != nil { + return err + } + } + if reconcile { + logger.Info("reconcile mode: live upsert semantics (DO UPDATE + on-demand identity)", + "only_keys", o.OnlyKeys, "since", o.Since) + } + + v1, err := openDB(o.V1DSN, logger) + if err != nil { + return fmt.Errorf("open v1: %w", err) + } + defer v1.Close() + v2, err := openDB(o.V2DSN, logger) + if err != nil { + return fmt.Errorf("open v2: %w", err) + } + defer v2.Close() + + if o.InitSchema && !o.DryRun { + if err := applySchemas(v2, o, logger); err != nil { + return err + } + } + + // Preflight: fail-fast on any artifact kind outside the six migrated types. + if err := preflightKinds(v1); err != nil { + return err + } + + run, err := newRun(o, logger) + if err != nil { + return err + } + defer run.close() + + // Preflight: mandatory subscription-token decrypt guard (fail the whole run if + // any sampled token does not round-trip under the configured key). + if !o.SkipDecryptCheck { + if err := checkTokenDecryption(v1, o.EncryptionKey, o.DecryptSampleSize, logger); err != nil { + return fmt.Errorf("subscription-token decrypt guard FAILED (wrong key ⇒ all tokens garbage): %w", err) + } + logger.Info("subscription-token decrypt guard passed") + } else { + logger.Warn("skipping subscription-token decrypt guard (-skip-decrypt-check)") + } + + var im *IdentityMap + if reconcile { + // Reconcile seeds each actor on demand (SkipIdentityUpsert:false in coreOpts), so + // the bulk pre-seed is skipped. A minimal map carries only the migration actor for + // the optional tables, which reconcile does not run. + im = &IdentityMap{epoch: epoch, m: map[string]string{}} + im.actorUUID = utils.GenerateDeterministicUUIDv7(migrationActorIDPID, epoch) + im.m[migrationActorIDPID] = im.actorUUID + } else { + im, err = collectAndSeedIdentities(v1, v2, run, epoch) + if err != nil { + return err + } + } + + mc := &migCtx{ + v1: v1, v2: v2, run: run, im: im, + sets: newSets(), h: newHandleGen(run), + opts: o, loc: loc, encKey: o.EncryptionKey, epoch: epoch, + reconcile: reconcile, only: only, + } + + // Migrate in FK order. Each step checkpoints on completion. + steps := []struct { + name string + fn func(*migCtx) error + }{ + {"organizations", migrateOrganizations}, + {"projects", migrateProjects}, + {"applications", migrateApplications}, + {"rest_apis", migrateRestAPIs}, + {"llm_provider_templates", migrateLLMProviderTemplates}, + {"llm_providers", migrateLLMProviders}, + {"llm_proxies", migrateLLMProxies}, + {"mcp_proxies", migrateMCPProxies}, + {"websub_apis", migrateWebSubAPIs}, + {"webbroker_apis", migrateWebBrokerAPIs}, + {"subscription_plans", migrateSubscriptionPlans}, + {"subscriptions", migrateSubscriptions}, + {"gateways", migrateGateways}, + {"artifact_gateway_mappings", migrateArtifactGatewayMappings}, + {"gateway_custom_policies", migrateGatewayCustomPolicies}, + {"gateway_custom_policy_usages", migrateGatewayCustomPolicyUsages}, + {"gateway_tokens", migrateGatewayTokens}, + {"deployments", migrateDeployments}, + {"deployment_status", migrateDeploymentStatus}, + {"api_keys", migrateAPIKeys}, + {"application_api_key_mappings", migrateApplicationAPIKeyMappings}, + {"application_artifact_mappings", migrateApplicationArtifactMappings}, + {"artifact_subscription_plans", migrateArtifactSubscriptionPlans}, + {"audit", migrateAuditMarkers}, + {"devportals", dropDevportals}, + {"publication_mappings", dropPublicationMappings}, + } + if reconcile { + // A reconcile replay only re-runs the core per-entity upserts. The enumerate-only + // §H drop steps and the optional batch-only tables (audit markers, + // artifact_subscription_plans) are not part of a live mirror. + skip := map[string]bool{ + "artifact_subscription_plans": true, + "audit": true, + "devportals": true, + "publication_mappings": true, + } + kept := steps[:0:0] + for _, s := range steps { + if !skip[s.name] { + kept = append(kept, s) + } + } + steps = kept + } + + for _, s := range steps { + logger.Info("migrating", "table", s.name, "mode", run.mode) + if err := s.fn(mc); err != nil { + _ = run.saveCheckpoint() + _ = run.writeReport() + return fmt.Errorf("migrate %s: %w", s.name, err) + } + run.markTableComplete(s.name) + if err := run.saveCheckpoint(); err != nil { + return fmt.Errorf("checkpoint after %s: %w", s.name, err) + } + } + + // Replay tombstoned deletes last: an upsert-reconcile can never heal a delete (the v1 + // row is gone), so DeleteX is applied directly against v2 (§8.1, delete-reconcile gap). + if reconcile { + if err := mc.reconcileDeletes(); err != nil { + _ = run.writeReport() + return err + } + } + + if err := run.writeReport(); err != nil { + return err + } + logger.Info("migration complete", "mode", run.mode, "run_id", o.RunID, "out_dir", o.OutDir) + printReportSummary(run.report) + return nil +} + +// loadEncryptionKey resolves the 32-byte key from env or file (never a flag). +func loadEncryptionKey(o *Options) error { + raw := os.Getenv("APIP_MIGRATION_ENCRYPTION_KEY") + if raw == "" && o.EncKeyFile != "" { + b, err := os.ReadFile(o.EncKeyFile) + if err != nil { + return fmt.Errorf("read encryption-key-file: %w", err) + } + raw = strings.TrimSpace(string(b)) + } + if raw == "" { + return nil + } + key, err := utils.DeriveEncryptionKey(raw) + if err != nil { + return fmt.Errorf("encryption key: %w", err) + } + o.EncryptionKey = key + return nil +} + +// applySchemas applies the v2 core DDL then the EventGateway plugin DDL. Both are +// idempotent (CREATE TABLE IF NOT EXISTS). The plugin DDL is NOT auto-applied for +// Postgres by the product, so the tool applies it every run (§K.2). +func applySchemas(v2 *database.DB, o *Options, logger *slog.Logger) error { + core, err := os.ReadFile(o.CoreSchema) + if err != nil { + return fmt.Errorf("read core schema %s: %w", o.CoreSchema, err) + } + if err := v2.InitSchemaSQL(string(core), logger); err != nil { + return fmt.Errorf("apply core schema: %w", err) + } + plugin, err := os.ReadFile(o.PluginSchema) + if err != nil { + return fmt.Errorf("read plugin schema %s: %w", o.PluginSchema, err) + } + if err := v2.InitSchemaSQL(string(plugin), logger); err != nil { + return fmt.Errorf("apply plugin schema: %w", err) + } + logger.Info("applied v2 core + plugin DDL") + return nil +} + +// preflightKinds fails fast if v1 holds any artifact kind outside the six migrated +// types (unknown destination ⇒ mapping incomplete). +func preflightKinds(v1 *database.DB) error { + set, err := loadStringSet(v1, "SELECT DISTINCT kind FROM artifacts") + if err != nil { + return fmt.Errorf("preflight kinds: %w", err) + } + valid := map[string]bool{ + constants.RestApi: true, constants.LLMProvider: true, constants.LLMProxy: true, + constants.MCPProxy: true, constants.WebSubApi: true, constants.WebBrokerApi: true, + } + for k := range set { + if !valid[k] { + return fmt.Errorf("fail-fast: v1 artifacts contains unmapped kind %q (expected one of RestApi/LlmProvider/LlmProxy/Mcp/WebSubApi/WebBrokerApi)", k) + } + } + return nil +} + +// checkTokenDecryption samples subscription tokens and fails if any does not +// round-trip under the configured key. +func checkTokenDecryption(v1 *database.DB, key []byte, sample int, logger *slog.Logger) error { + if sample <= 0 { + sample = 25 + } + rows, err := v1.Query(v1.Rebind( + fmt.Sprintf("SELECT uuid, subscription_token FROM subscriptions WHERE subscription_token <> '' ORDER BY uuid %s", v1.FetchFirstClause(sample)))) + if err != nil { + return err + } + defer rows.Close() + checked := 0 + for rows.Next() { + var uuid, token string + if err := rows.Scan(&uuid, &token); err != nil { + return err + } + if _, err := utils.DecryptSubscriptionToken(key, token); err != nil { + return fmt.Errorf("token for subscription %s did not decrypt: %w", uuid, err) + } + checked++ + } + if err := rows.Err(); err != nil { + return err + } + logger.Info("decrypt guard sampled tokens", "checked", checked) + return nil +} + +// carriedHandle generates+persists the handle for a table that CARRIES the v1 +// handle (source = v1 handle), flagging TRUNCATED when the 255→40 narrowing bites. +func (mc *migCtx) carriedHandle(table, org, v1uuid, v1handle string) (string, error) { + h, err := mc.h.generate(table, org, v1uuid, v1handle) + if err != nil { + return "", err + } + if _, cut := truncateStr(v1handle, 40); cut || slug(v1handle) != h { + mc.run.flag(table, v1uuid, FlagTruncated, + map[string]any{"handle": v1handle, "len": len(v1handle)}, + map[string]any{"handle": h}) + } + return h, nil +} + diff --git a/platform-api/cmd/dbmigrate/migrate_tables.go b/platform-api/cmd/dbmigrate/migrate_tables.go new file mode 100644 index 0000000000..4329606343 --- /dev/null +++ b/platform-api/cmd/dbmigrate/migrate_tables.go @@ -0,0 +1,1359 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +// This file is the BATCH-ONLY orchestration: table iteration in FK order, FK-parent +// gating, handle generation (persist-and-replay), subscription dedup, and drop +// enumeration. The per-row transform + v2 write is delegated to the shared +// migrationcore.UpsertX — the ONE implementation the live dual-write path also uses. + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/wso2/api-platform/platform-api/migrationcore" +) + +// blobSkipped reports whether err is the core's "config blob unparseable" sentinel +// (already quarantined by the core) — the batch just skips the row. +func blobSkipped(err error) bool { return errors.Is(err, migrationcore.ErrBlobUnparseable) } + +// parentOK classifies a child's FK reference against an in-memory parent set and +// routes violations: cascade (parent seen in v1 but not migrated) → quarantine; +// missing (absent from v1) → fail-fast if v1 enforced the FK, else quarantine. +func (mc *migCtx) parentOK(ps *ParentSet, parentUUID, childTable, childKey, fk string, enforced bool, fullRow any) (bool, error) { + // In a targeted reconcile the in-memory parent sets only hold the filtered rows, so + // the usual FK gating would wrongly cascade-quarantine children whose parents already + // exist in v2 from the backfill. Skip gating and let v2's FKs reject a genuinely + // missing parent (surfaced as the upsert error). No-op for the batch (reconcile=false). + if mc.reconcile { + return true, nil + } + switch ps.status(parentUUID) { + case ParentOK: + return true, nil + case ParentCascade: + mc.run.quarantine(childTable, childKey, ReasonOrphanFK, + fmt.Sprintf("%s parent %s exists in v1 but was not migrated", fk, parentUUID), fullRow) + return false, nil + default: // ParentMissing + if enforced { + return false, fmt.Errorf("fail-fast: %s.%s=%s is absent from v1 (v1-enforced FK ⇒ source corruption); key=%s", + childTable, fk, parentUUID, childKey) + } + mc.run.quarantine(childTable, childKey, ReasonOrphanFK, + fmt.Sprintf("%s references %s=%s not present in v1", childTable, fk, parentUUID), fullRow) + return false, nil + } +} + +// ---- organizations ---- + +func migrateOrganizations(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT uuid, handle, name, region, created_at, updated_at FROM organizations`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var uuid, handle, name, region string + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &handle, &name, ®ion, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.orgs.seenV1(uuid) + h, err := mc.carriedHandle("organizations", "", uuid, handle) + if err != nil { + mc.run.quarantine("organizations", uuid, ReasonHandleUnresolvable, err.Error(), + map[string]any{"uuid": uuid, "handle": handle, "name": name}) + continue + } + if !mc.want("organizations", uuid) { + continue + } + if err := migrationcore.UpsertOrganization(mc.v2, migrationcore.OrganizationRow{ + UUID: uuid, Handle: h, DisplayName: name, Region: region, + CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + mc.sets.orgs.insertedV2(uuid) + n++ + } + mc.run.addProcessed("organizations", n) + return rows.Err() +} + +// ---- projects ---- + +func migrateProjects(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT uuid, name, organization_uuid, description, created_at, updated_at FROM projects`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var uuid, name, org string + var description sql.NullString + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &name, &org, &description, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.projects.seenV1(uuid) + full := map[string]any{"uuid": uuid, "name": name, "organization_uuid": org} + if ok, err := mc.parentOK(mc.sets.orgs, org, "projects", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + h, err := mc.h.generate("projects", org, uuid, name) + if err != nil { + mc.run.quarantine("projects", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + mc.run.flag("projects", uuid, FlagSynthesized, nil, map[string]any{"handle": h, "note": "generated from name"}) + if !mc.want("projects", uuid) { + continue + } + if err := migrationcore.UpsertProject(mc.v2, migrationcore.ProjectRow{ + UUID: uuid, Handle: h, DisplayName: name, Org: org, Description: nsp(description), + CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + mc.sets.projects.insertedV2(uuid) + n++ + } + mc.run.addProcessed("projects", n) + return rows.Err() +} + +// ---- applications ---- + +func migrateApplications(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT uuid, handle, project_uuid, organization_uuid, created_by, name, description, type, created_at, updated_at FROM applications`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var uuid, handle, projectUUID, org, name, typ string + var createdByRaw, description sql.NullString + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &handle, &projectUUID, &org, &createdByRaw, &name, &description, &typ, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.applications.seenV1(uuid) + full := map[string]any{"uuid": uuid, "handle": handle, "organization_uuid": org, "project_uuid": projectUUID, "name": name} + if ok, err := mc.parentOK(mc.sets.orgs, org, "applications", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.projects, projectUUID, "applications", uuid, "project_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + h, err := mc.carriedHandle("applications", org, uuid, handle) + if err != nil { + mc.run.quarantine("applications", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + if !mc.want("applications", uuid) { + continue + } + if err := migrationcore.UpsertApplication(mc.v2, migrationcore.ApplicationRow{ + UUID: uuid, Handle: h, ProjectUUID: projectUUID, Org: org, DisplayName: name, Type: typ, + Description: nsp(description), CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), CreatedBy: strv(createdByRaw), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + mc.sets.applications.insertedV2(uuid) + n++ + } + mc.run.addProcessed("applications", n) + return rows.Err() +} + +// ---- rest_apis (RestApi) ---- + +func migrateRestAPIs(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT t.uuid, t.description, t.created_by, t.project_uuid, t.lifecycle_status, t.transport, t.configuration, + a.handle, a.name, a.version, a.organization_uuid, a.created_at, a.updated_at + FROM rest_apis t INNER JOIN artifacts a ON t.uuid = a.uuid`)) + if err != nil { + return err + } + defer rows.Close() + na, nt := 0, 0 + for rows.Next() { + var uuid, projectUUID, handle, name, version, org string + var description, createdByRaw, lifecycle, transport sql.NullString + var config []byte + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &description, &createdByRaw, &projectUUID, &lifecycle, &transport, &config, + &handle, &name, &version, &org, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.artifacts.seenV1(uuid) + full := map[string]any{"uuid": uuid, "handle": handle, "name": name, "organization_uuid": org} + if ok, err := mc.parentOK(mc.sets.orgs, org, "rest_apis", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.projects, projectUUID, "rest_apis", uuid, "project_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + h, err := mc.carriedHandle("rest_apis", org, uuid, handle) + if err != nil { + mc.run.quarantine("rest_apis", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + if !mc.want("rest_apis", uuid) { + continue + } + if err := migrationcore.UpsertRestAPI(mc.v2, migrationcore.RestAPIRow{ + UUID: uuid, Handle: h, DisplayName: name, Version: version, Org: org, ProjectUUID: projectUUID, + Description: nsp(description), Lifecycle: nsp(lifecycle), Transport: nsp(transport), Configuration: config, + CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), CreatedBy: strv(createdByRaw), + }, mc.coreOpts(), mc.run); err != nil { + if blobSkipped(err) { + continue + } + return err + } + mc.sets.artifacts.insertedV2(uuid) + mc.collectArtifactPlans(uuid, org, config) + na++ + nt++ + } + mc.run.addProcessed("artifacts", na) + mc.run.addProcessed("rest_apis", nt) + return rows.Err() +} + +// ---- llm_provider_templates ---- + +func migrateLLMProviderTemplates(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT uuid, organization_uuid, handle, name, description, created_by, configuration, created_at, updated_at + FROM llm_provider_templates`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var uuid, org, handle, name, config string + var description, createdByRaw sql.NullString + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &org, &handle, &name, &description, &createdByRaw, &config, &createdAt, &updatedAt); err != nil { + return err + } + full := map[string]any{"uuid": uuid, "handle": handle, "name": name, "organization_uuid": org} + if ok, err := mc.parentOK(mc.sets.orgs, org, "llm_provider_templates", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + h, err := mc.carriedHandle("llm_provider_templates", org, uuid, handle) + if err != nil { + mc.run.quarantine("llm_provider_templates", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + if !mc.want("llm_provider_templates", uuid) { + continue + } + if err := migrationcore.UpsertLLMProviderTemplate(mc.v2, migrationcore.LLMTemplateRow{ + UUID: uuid, Org: org, Handle: h, DisplayName: name, Description: nsp(description), + Configuration: []byte(config), CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), CreatedBy: strv(createdByRaw), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + mc.sets.templates.insertedV2(uuid) + n++ + } + mc.run.addProcessed("llm_provider_templates", n) + return rows.Err() +} + +// ---- llm_providers (LlmProvider) ---- + +func migrateLLMProviders(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT t.uuid, t.description, t.created_by, t.template_uuid, t.openapi_spec, t.model_list, t.status, t.configuration, + a.handle, a.name, a.version, a.organization_uuid, a.created_at, a.updated_at + FROM llm_providers t INNER JOIN artifacts a ON t.uuid = a.uuid`)) + if err != nil { + return err + } + defer rows.Close() + na, nt := 0, 0 + for rows.Next() { + var uuid, templateUUID, handle, name, version, org string + var description, createdByRaw, openapiSpec, modelList, status sql.NullString + var config []byte + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &description, &createdByRaw, &templateUUID, &openapiSpec, &modelList, &status, &config, + &handle, &name, &version, &org, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.artifacts.seenV1(uuid) + full := map[string]any{"uuid": uuid, "handle": handle, "organization_uuid": org, "template_uuid": templateUUID} + if ok, err := mc.parentOK(mc.sets.orgs, org, "llm_providers", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.templates, templateUUID, "llm_providers", uuid, "template_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + h, err := mc.carriedHandle("llm_providers", org, uuid, handle) + if err != nil { + mc.run.quarantine("llm_providers", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + if !mc.want("llm_providers", uuid) { + continue + } + if err := migrationcore.UpsertLLMProvider(mc.v2, migrationcore.LLMProviderRow{ + UUID: uuid, Handle: h, DisplayName: name, Version: version, Org: org, TemplateUUID: templateUUID, + Description: nsp(description), OpenAPISpec: nsp(openapiSpec), ModelList: nsp(modelList), Status: nsp(status), + Configuration: config, CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), CreatedBy: strv(createdByRaw), + }, mc.coreOpts(), mc.run); err != nil { + if blobSkipped(err) { + continue + } + return err + } + mc.sets.artifacts.insertedV2(uuid) + mc.sets.providers.insertedV2(uuid) + na++ + nt++ + } + mc.run.addProcessed("artifacts", na) + mc.run.addProcessed("llm_providers", nt) + return rows.Err() +} + +// ---- llm_proxies (LlmProxy) ---- + +func migrateLLMProxies(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT t.uuid, t.project_uuid, t.description, t.created_by, t.provider_uuid, t.openapi_spec, t.status, t.configuration, + a.handle, a.name, a.version, a.organization_uuid, a.created_at, a.updated_at + FROM llm_proxies t INNER JOIN artifacts a ON t.uuid = a.uuid`)) + if err != nil { + return err + } + defer rows.Close() + na, nt := 0, 0 + for rows.Next() { + var uuid, projectUUID, providerUUID, handle, name, version, org string + var description, createdByRaw, openapiSpec, status sql.NullString + var config []byte + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &projectUUID, &description, &createdByRaw, &providerUUID, &openapiSpec, &status, &config, + &handle, &name, &version, &org, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.artifacts.seenV1(uuid) + full := map[string]any{"uuid": uuid, "handle": handle, "organization_uuid": org, "provider_uuid": providerUUID} + if ok, err := mc.parentOK(mc.sets.orgs, org, "llm_proxies", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.projects, projectUUID, "llm_proxies", uuid, "project_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.providers, providerUUID, "llm_proxies", uuid, "provider_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + h, err := mc.carriedHandle("llm_proxies", org, uuid, handle) + if err != nil { + mc.run.quarantine("llm_proxies", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + if !mc.want("llm_proxies", uuid) { + continue + } + if err := migrationcore.UpsertLLMProxy(mc.v2, migrationcore.LLMProxyRow{ + UUID: uuid, Handle: h, DisplayName: name, Version: version, ProjectUUID: projectUUID, Org: org, ProviderUUID: providerUUID, + Description: nsp(description), OpenAPISpec: nsp(openapiSpec), Status: nsp(status), + Configuration: config, CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), CreatedBy: strv(createdByRaw), + }, mc.coreOpts(), mc.run); err != nil { + if blobSkipped(err) { + continue + } + return err + } + mc.sets.artifacts.insertedV2(uuid) + na++ + nt++ + } + mc.run.addProcessed("artifacts", na) + mc.run.addProcessed("llm_proxies", nt) + return rows.Err() +} + +// ---- mcp_proxies (Mcp) ---- + +func migrateMCPProxies(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT t.uuid, t.project_uuid, t.description, t.created_by, t.status, t.configuration, + a.handle, a.name, a.version, a.organization_uuid, a.created_at, a.updated_at + FROM mcp_proxies t INNER JOIN artifacts a ON t.uuid = a.uuid`)) + if err != nil { + return err + } + defer rows.Close() + na, nt := 0, 0 + for rows.Next() { + var uuid, handle, name, version, org string + var projectUUID, description, createdByRaw, status sql.NullString + var config []byte + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &projectUUID, &description, &createdByRaw, &status, &config, + &handle, &name, &version, &org, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.artifacts.seenV1(uuid) + full := map[string]any{"uuid": uuid, "handle": handle, "organization_uuid": org} + if ok, err := mc.parentOK(mc.sets.orgs, org, "mcp_proxies", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if projectUUID.Valid && projectUUID.String != "" { + if ok, err := mc.parentOK(mc.sets.projects, projectUUID.String, "mcp_proxies", uuid, "project_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + } + h, err := mc.carriedHandle("mcp_proxies", org, uuid, handle) + if err != nil { + mc.run.quarantine("mcp_proxies", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + if !mc.want("mcp_proxies", uuid) { + continue + } + if err := migrationcore.UpsertMCPProxy(mc.v2, migrationcore.MCPProxyRow{ + UUID: uuid, Handle: h, DisplayName: name, Version: version, Org: org, + ProjectUUID: nsp(projectUUID), Description: nsp(description), Status: nsp(status), + Configuration: config, CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), CreatedBy: strv(createdByRaw), + }, mc.coreOpts(), mc.run); err != nil { + if blobSkipped(err) { + continue + } + return err + } + mc.sets.artifacts.insertedV2(uuid) + na++ + nt++ + } + mc.run.addProcessed("artifacts", na) + mc.run.addProcessed("mcp_proxies", nt) + return rows.Err() +} + +// ---- websub_apis (WebSubApi, plugin table) ---- + +func migrateWebSubAPIs(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT t.uuid, t.project_uuid, t.description, t.created_by, t.lifecycle_status, t.transport, t.configuration, + a.handle, a.name, a.version, a.organization_uuid, a.created_at, a.updated_at + FROM websub_apis t INNER JOIN artifacts a ON t.uuid = a.uuid`)) + if err != nil { + return err + } + defer rows.Close() + na, nt := 0, 0 + for rows.Next() { + var uuid, projectUUID, handle, name, version, org string + var description, createdByRaw, lifecycle, transport sql.NullString + var config []byte + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &projectUUID, &description, &createdByRaw, &lifecycle, &transport, &config, + &handle, &name, &version, &org, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.artifacts.seenV1(uuid) + full := map[string]any{"uuid": uuid, "handle": handle, "organization_uuid": org} + if ok, err := mc.parentOK(mc.sets.orgs, org, "websub_apis", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.projects, projectUUID, "websub_apis", uuid, "project_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + h, err := mc.carriedHandle("websub_apis", org, uuid, handle) + if err != nil { + mc.run.quarantine("websub_apis", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + if !mc.want("websub_apis", uuid) { + continue + } + if err := migrationcore.UpsertWebSubAPI(mc.v2, migrationcore.WebSubRow{ + UUID: uuid, Handle: h, DisplayName: name, Version: version, Org: org, ProjectUUID: projectUUID, + Description: nsp(description), Lifecycle: nsp(lifecycle), Transport: nsp(transport), Configuration: config, + CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), CreatedBy: strv(createdByRaw), + }, mc.coreOpts(), mc.run); err != nil { + if blobSkipped(err) { + continue + } + return err + } + mc.sets.artifacts.insertedV2(uuid) + mc.collectArtifactPlans(uuid, org, config) + na++ + nt++ + } + mc.run.addProcessed("artifacts", na) + mc.run.addProcessed("websub_apis", nt) + return rows.Err() +} + +// ---- webbroker_apis (WebBrokerApi, plugin table) ---- + +func migrateWebBrokerAPIs(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT t.uuid, t.project_uuid, t.description, t.created_by, t.lifecycle_status, t.transport, t.configuration, + a.handle, a.name, a.version, a.organization_uuid, a.created_at, a.updated_at + FROM webbroker_apis t INNER JOIN artifacts a ON t.uuid = a.uuid`)) + if err != nil { + return err + } + defer rows.Close() + na, nt := 0, 0 + for rows.Next() { + var uuid, projectUUID, handle, name, version, org string + var description, createdByRaw, lifecycle, transport sql.NullString + var config []byte + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &projectUUID, &description, &createdByRaw, &lifecycle, &transport, &config, + &handle, &name, &version, &org, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.artifacts.seenV1(uuid) + full := map[string]any{"uuid": uuid, "handle": handle, "organization_uuid": org} + if ok, err := mc.parentOK(mc.sets.orgs, org, "webbroker_apis", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.projects, projectUUID, "webbroker_apis", uuid, "project_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + h, err := mc.carriedHandle("webbroker_apis", org, uuid, handle) + if err != nil { + mc.run.quarantine("webbroker_apis", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + if !mc.want("webbroker_apis", uuid) { + continue + } + if err := migrationcore.UpsertWebBrokerAPI(mc.v2, migrationcore.WebBrokerRow{ + UUID: uuid, Handle: h, DisplayName: name, Version: version, Org: org, ProjectUUID: projectUUID, + Description: nsp(description), Lifecycle: nsp(lifecycle), Transport: nsp(transport), Configuration: config, + CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), CreatedBy: strv(createdByRaw), + }, mc.coreOpts(), mc.run); err != nil { + if blobSkipped(err) { + continue + } + return err + } + mc.sets.artifacts.insertedV2(uuid) + mc.collectArtifactPlans(uuid, org, config) + na++ + nt++ + } + mc.run.addProcessed("artifacts", na) + mc.run.addProcessed("webbroker_apis", nt) + return rows.Err() +} + +// ---- subscription_plans (+ subscription_plan_limits) ---- + +func migrateSubscriptionPlans(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT uuid, plan_name, billing_plan, stop_on_quota_reach, throttle_limit_count, throttle_limit_unit, + expiry_time, organization_uuid, status, created_at, updated_at + FROM subscription_plans`)) + if err != nil { + return err + } + defer rows.Close() + np, nl := 0, 0 + for rows.Next() { + var uuid, planName, org, status string + var billingPlan, throttleUnit sql.NullString + var stopOnQuota sql.NullBool + var throttleCount sql.NullInt64 + var expiry, createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &planName, &billingPlan, &stopOnQuota, &throttleCount, &throttleUnit, + &expiry, &org, &status, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.plans.seenV1(uuid) + full := map[string]any{"uuid": uuid, "plan_name": planName, "organization_uuid": org} + if ok, err := mc.parentOK(mc.sets.orgs, org, "subscription_plans", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + h, err := mc.h.generate("subscription_plans", org, uuid, planName) + if err != nil { + mc.run.quarantine("subscription_plans", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + mc.run.flag("subscription_plans", uuid, FlagSynthesized, nil, map[string]any{"handle": h, "note": "generated from plan_name"}) + row := migrationcore.SubscriptionPlanRow{ + UUID: uuid, Handle: h, DisplayName: planName, Org: org, Status: status, + BillingPlan: nsp(billingPlan), ThrottleUnit: nsp(throttleUnit), + ExpiryTime: ntp(expiry), CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), + } + if stopOnQuota.Valid { + b := stopOnQuota.Bool + row.StopOnQuota = &b + } + if throttleCount.Valid { + c := throttleCount.Int64 + row.ThrottleCount = &c + } + if !mc.want("subscription_plans", uuid) { + continue + } + if err := migrationcore.UpsertSubscriptionPlan(mc.v2, row, mc.coreOpts(), mc.run); err != nil { + var unmapped migrationcore.ErrUnmappedThrottleUnit + if errors.As(err, &unmapped) { + return fmt.Errorf("fail-fast: subscription_plan %s has %w", uuid, err) + } + return err + } + mc.sets.plans.insertedV2(uuid) + np++ + if throttleCount.Valid && throttleUnit.Valid && throttleUnit.String != "" { + nl++ + } + } + mc.run.addProcessed("subscription_plans", np) + mc.run.addProcessed("subscription_plan_limits", nl) + return rows.Err() +} + +// ---- subscriptions ---- + +func migrateSubscriptions(mc *migCtx) error { + if mc.subAppSeen == nil { + mc.subAppSeen = map[string]bool{} + mc.subHashSeen = map[string]bool{} + } + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT uuid, api_uuid, subscriber_id, application_id, subscription_token, subscription_token_hash, + subscription_plan_uuid, organization_uuid, status, created_at, updated_at + FROM subscriptions`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var uuid, apiUUID, subscriberID, token, hash, org, status string + var applicationID, planUUID sql.NullString + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &apiUUID, &subscriberID, &applicationID, &token, &hash, + &planUUID, &org, &status, &createdAt, &updatedAt); err != nil { + return err + } + full := map[string]any{"uuid": uuid, "api_uuid": apiUUID, "organization_uuid": org, "subscriber_id": subscriberID} + if ok, err := mc.parentOK(mc.sets.artifacts, apiUUID, "subscriptions", uuid, "api_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if planUUID.Valid && planUUID.String != "" { + if ok, err := mc.parentOK(mc.sets.plans, planUUID.String, "subscriptions", uuid, "subscription_plan_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + } + hkey := apiUUID + "|" + hash + if mc.subHashSeen[hkey] { + mc.run.quarantine("subscriptions", uuid, ReasonDupKey, "duplicate (artifact_uuid, subscription_token_hash)", full) + continue + } + if applicationID.Valid && applicationID.String != "" { + akey := org + "|" + apiUUID + "|" + applicationID.String + if mc.subAppSeen[akey] { + mc.run.quarantine("subscriptions", uuid, ReasonDupKey, + "duplicate (organization_uuid, artifact_uuid, application_id) — v1 enforced subscriber_id, not application_id", full) + continue + } + mc.subAppSeen[akey] = true + } + mc.subHashSeen[hkey] = true + if !mc.want("subscriptions", uuid) { + continue + } + if err := migrationcore.UpsertSubscription(mc.v2, migrationcore.SubscriptionRow{ + UUID: uuid, ArtifactUUID: apiUUID, SubscriberID: subscriberID, Token: token, Hash: hash, Org: org, Status: status, + ApplicationID: nsp(applicationID), PlanUUID: nsp(planUUID), CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + n++ + } + mc.run.addProcessed("subscriptions", n) + return rows.Err() +} + +// ---- gateways (+ gateway_endpoints) ---- + +func migrateGateways(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT uuid, organization_uuid, name, version, display_name, description, properties, vhost, + is_critical, gateway_functionality_type, is_active, manifest, created_at, updated_at + FROM gateways`)) + if err != nil { + return err + } + defer rows.Close() + ng, ne := 0, 0 + for rows.Next() { + var uuid, org, name, version, displayName, funcType, vhost string + var description sql.NullString + var properties, manifest []byte + var isCritical, isActive sql.NullBool + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &org, &name, &version, &displayName, &description, &properties, &vhost, + &isCritical, &funcType, &isActive, &manifest, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.gateways.seenV1(uuid) + full := map[string]any{"uuid": uuid, "name": name, "organization_uuid": org, "vhost": vhost} + if ok, err := mc.parentOK(mc.sets.orgs, org, "gateways", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + h, err := mc.h.generate("gateways", org, uuid, name) + if err != nil { + mc.run.quarantine("gateways", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + mc.run.flag("gateways", uuid, FlagSynthesized, nil, map[string]any{"handle": h, "note": "generated from name"}) + if !mc.want("gateways", uuid) { + continue + } + if err := migrationcore.UpsertGateway(mc.v2, migrationcore.GatewayRow{ + UUID: uuid, Org: org, Handle: h, DisplayName: displayName, Version: version, FuncType: funcType, Vhost: vhost, + Description: nsp(description), Properties: properties, Manifest: manifest, + IsCritical: isCritical.Bool, IsActive: isActive.Bool, + CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + mc.sets.gateways.insertedV2(uuid) + ng++ + if strings.TrimSpace(vhost) != "" { + ne++ + } + } + mc.run.addProcessed("gateways", ng) + mc.run.addProcessed("gateway_endpoints", ne) + return rows.Err() +} + +// ---- artifact_gateway_mappings (from association_mappings gateway rows) ---- + +func migrateArtifactGatewayMappings(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT id, artifact_uuid, organization_uuid, resource_uuid, association_type, created_at, updated_at + FROM association_mappings`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var id int64 + var artifactUUID, org, resourceUUID, assocType string + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&id, &artifactUUID, &org, &resourceUUID, &assocType, &createdAt, &updatedAt); err != nil { + return err + } + key := fmt.Sprintf("assoc:%d", id) + full := map[string]any{"id": id, "artifact_uuid": artifactUUID, "organization_uuid": org, "resource_uuid": resourceUUID, "association_type": assocType} + switch assocType { + case "gateway": + // migrate below + case "dev_portal": + mc.run.dropRow("association_mappings", key, DropDevPortalAssoc, fmt.Sprintf("artifact %s → resource %s", artifactUUID, resourceUUID)) + continue + default: + return fmt.Errorf("fail-fast: association_mappings id=%d has unmapped association_type %q", id, assocType) + } + if ok, err := mc.parentOK(mc.sets.artifacts, artifactUUID, "artifact_gateway_mappings", key, "artifact_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.gateways, resourceUUID, "artifact_gateway_mappings", key, "gateway_uuid", false, full); err != nil { + return err + } else if !ok { + continue + } + if !mc.want("artifact_gateway_mappings", org+"|"+artifactUUID+"|"+resourceUUID) { + continue + } + if err := migrationcore.UpsertArtifactGatewayMapping(mc.v2, migrationcore.ArtifactGatewayMappingRow{ + ArtifactUUID: artifactUUID, Org: org, GatewayUUID: resourceUUID, CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + n++ + } + mc.run.addProcessed("artifact_gateway_mappings", n) + return rows.Err() +} + +// ---- gateway_custom_policies ---- + +func migrateGatewayCustomPolicies(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT uuid, organization_uuid, name, display_name, version, description, policy_definition, created_at, updated_at + FROM gateway_custom_policies`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var uuid, org, name, version string + var displayName, description sql.NullString + var policyDef []byte + var createdAt, updatedAt sql.NullTime + if err := rows.Scan(&uuid, &org, &name, &displayName, &version, &description, &policyDef, &createdAt, &updatedAt); err != nil { + return err + } + mc.sets.policies.seenV1(uuid) + full := map[string]any{"uuid": uuid, "name": name, "organization_uuid": org} + if ok, err := mc.parentOK(mc.sets.orgs, org, "gateway_custom_policies", uuid, "organization_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if !mc.want("gateway_custom_policies", uuid) { + continue + } + if err := migrationcore.UpsertGatewayCustomPolicy(mc.v2, migrationcore.GatewayCustomPolicyRow{ + UUID: uuid, Org: org, Name: name, Version: version, DisplayName: nsp(displayName), Description: nsp(description), + PolicyDefinition: policyDef, CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + mc.sets.policies.insertedV2(uuid) + n++ + } + mc.run.addProcessed("gateway_custom_policies", n) + return rows.Err() +} + +// ---- gateway_custom_policy_usages ---- + +func migrateGatewayCustomPolicyUsages(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind(`SELECT policy_uuid, api_uuid FROM gateway_custom_policy_usages`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var policyUUID, apiUUID string + if err := rows.Scan(&policyUUID, &apiUUID); err != nil { + return err + } + key := policyUUID + "|" + apiUUID + full := map[string]any{"policy_uuid": policyUUID, "api_uuid": apiUUID} + if ok, err := mc.parentOK(mc.sets.policies, policyUUID, "gateway_custom_policy_usages", key, "policy_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.artifacts, apiUUID, "gateway_custom_policy_usages", key, "artifact_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if !mc.want("gateway_custom_policy_usages", policyUUID+"|"+apiUUID) { + continue + } + if err := migrationcore.UpsertPolicyUsage(mc.v2, migrationcore.PolicyUsageRow{PolicyUUID: policyUUID, ArtifactUUID: apiUUID}, mc.coreOpts(), mc.run); err != nil { + return err + } + n++ + } + mc.run.addProcessed("gateway_custom_policy_usages", n) + return rows.Err() +} + +// ---- gateway_tokens ---- + +func migrateGatewayTokens(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT uuid, gateway_uuid, token_hash, salt, status, created_at, revoked_at FROM gateway_tokens`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var uuid, gatewayUUID, tokenHash, salt, status string + var createdAt, revokedAt sql.NullTime + if err := rows.Scan(&uuid, &gatewayUUID, &tokenHash, &salt, &status, &createdAt, &revokedAt); err != nil { + return err + } + full := map[string]any{"uuid": uuid, "gateway_uuid": gatewayUUID, "status": status} + if ok, err := mc.parentOK(mc.sets.gateways, gatewayUUID, "gateway_tokens", uuid, "gateway_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if !mc.want("gateway_tokens", uuid) { + continue + } + if err := migrationcore.UpsertGatewayToken(mc.v2, migrationcore.GatewayTokenRow{ + UUID: uuid, GatewayUUID: gatewayUUID, TokenHash: tokenHash, Salt: salt, Status: status, + CreatedAt: ntp(createdAt), RevokedAt: ntp(revokedAt), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + n++ + } + mc.run.addProcessed("gateway_tokens", n) + return rows.Err() +} + +// ---- deployments (ordered by created_at so base_deployment predecessors exist first) ---- + +func migrateDeployments(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT deployment_id, name, artifact_uuid, organization_uuid, gateway_uuid, base_deployment_id, + content, metadata, created_at + FROM deployments ORDER BY created_at ASC NULLS FIRST`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var uuid, name, artifactUUID, org, gatewayUUID string + var baseDeployment, metadata sql.NullString + var content []byte + var createdAt sql.NullTime + if err := rows.Scan(&uuid, &name, &artifactUUID, &org, &gatewayUUID, &baseDeployment, &content, &metadata, &createdAt); err != nil { + return err + } + mc.sets.deployments.seenV1(uuid) + full := map[string]any{"deployment_id": uuid, "artifact_uuid": artifactUUID, "gateway_uuid": gatewayUUID, "organization_uuid": org} + if ok, err := mc.parentOK(mc.sets.artifacts, artifactUUID, "deployments", uuid, "artifact_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.gateways, gatewayUUID, "deployments", uuid, "gateway_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + var base *string + if baseDeployment.Valid && baseDeployment.String != "" && mc.sets.deployments.status(baseDeployment.String) == ParentOK { + s := baseDeployment.String + base = &s + } + if !mc.want("deployments", uuid) { + continue + } + if err := migrationcore.UpsertDeployment(mc.v2, migrationcore.DeploymentRow{ + UUID: uuid, DisplayName: name, ArtifactUUID: artifactUUID, Org: org, GatewayUUID: gatewayUUID, + BaseDeploymentUUID: base, Metadata: nsp(metadata), Content: content, CreatedAt: ntp(createdAt), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + mc.sets.deployments.insertedV2(uuid) + n++ + } + mc.run.addProcessed("deployments", n) + return rows.Err() +} + +// ---- deployment_status ---- + +func migrateDeploymentStatus(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT artifact_uuid, organization_uuid, gateway_uuid, deployment_id, status, status_desired, + performed_at, status_reason, updated_at + FROM deployment_status`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var artifactUUID, org, gatewayUUID, deploymentUUID, status string + var statusDesired, statusReason sql.NullString + var performedAt, updatedAt sql.NullTime + if err := rows.Scan(&artifactUUID, &org, &gatewayUUID, &deploymentUUID, &status, &statusDesired, &performedAt, &statusReason, &updatedAt); err != nil { + return err + } + key := artifactUUID + "|" + org + "|" + gatewayUUID + full := map[string]any{"artifact_uuid": artifactUUID, "organization_uuid": org, "gateway_uuid": gatewayUUID, "deployment_id": deploymentUUID} + if ok, err := mc.parentOK(mc.sets.artifacts, artifactUUID, "deployment_status", key, "artifact_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.gateways, gatewayUUID, "deployment_status", key, "gateway_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.deployments, deploymentUUID, "deployment_status", key, "deployment_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if !mc.want("deployment_status", org+"|"+artifactUUID+"|"+gatewayUUID) { + continue + } + if err := migrationcore.UpsertDeploymentStatus(mc.v2, migrationcore.DeploymentStatusRow{ + ArtifactUUID: artifactUUID, Org: org, GatewayUUID: gatewayUUID, DeploymentUUID: deploymentUUID, Status: status, + StatusDesired: nsp(statusDesired), StatusReason: nsp(statusReason), PerformedAt: ntp(performedAt), UpdatedAt: ntp(updatedAt), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + n++ + } + mc.run.addProcessed("deployment_status", n) + return rows.Err() +} + +// ---- api_keys ---- + +func migrateAPIKeys(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT uuid, artifact_uuid, name, masked_api_key, api_key_hashes, status, created_at, created_by, + updated_at, expires_at, issuer, allowed_targets + FROM api_keys`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var uuid, artifactUUID, name, maskedKey, apiKeyHashes, status, allowedTargets string + var createdByRaw, issuer sql.NullString + var createdAt, updatedAt, expiresAt sql.NullTime + if err := rows.Scan(&uuid, &artifactUUID, &name, &maskedKey, &apiKeyHashes, &status, &createdAt, &createdByRaw, + &updatedAt, &expiresAt, &issuer, &allowedTargets); err != nil { + return err + } + mc.sets.apiKeys.seenV1(uuid) + full := map[string]any{"uuid": uuid, "artifact_uuid": artifactUUID, "name": name} + if ok, err := mc.parentOK(mc.sets.artifacts, artifactUUID, "api_keys", uuid, "artifact_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + h, err := mc.h.generate("api_keys", artifactUUID, uuid, name) + if err != nil { + mc.run.quarantine("api_keys", uuid, ReasonHandleUnresolvable, err.Error(), full) + continue + } + mc.run.flag("api_keys", uuid, FlagSynthesized, nil, map[string]any{"handle": h, "note": "generated from name"}) + if !mc.want("api_keys", uuid) { + continue + } + if err := migrationcore.UpsertAPIKey(mc.v2, migrationcore.APIKeyRow{ + UUID: uuid, ArtifactUUID: artifactUUID, Handle: h, DisplayName: name, MaskedKey: maskedKey, + APIKeyHashes: apiKeyHashes, Status: status, AllowedTargets: allowedTargets, Issuer: nsp(issuer), + CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), ExpiresAt: ntp(expiresAt), CreatedBy: strv(createdByRaw), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + mc.sets.apiKeys.insertedV2(uuid) + n++ + } + mc.run.addProcessed("api_keys", n) + return rows.Err() +} + +// ---- application_api_key_mappings ---- + +func migrateApplicationAPIKeyMappings(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT application_uuid, api_key_id, created_at FROM application_api_keys`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var appUUID, apiKeyID string + var createdAt sql.NullTime + if err := rows.Scan(&appUUID, &apiKeyID, &createdAt); err != nil { + return err + } + key := appUUID + "|" + apiKeyID + full := map[string]any{"application_uuid": appUUID, "api_key_id": apiKeyID} + if ok, err := mc.parentOK(mc.sets.applications, appUUID, "application_api_key_mappings", key, "application_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.apiKeys, apiKeyID, "application_api_key_mappings", key, "api_key_id", true, full); err != nil { + return err + } else if !ok { + continue + } + if !mc.want("application_api_key_mappings", appUUID+"|"+apiKeyID) { + continue + } + if err := migrationcore.UpsertApplicationAPIKeyMapping(mc.v2, migrationcore.ApplicationAPIKeyMappingRow{ + ApplicationUUID: appUUID, APIKeyID: apiKeyID, CreatedAt: ntp(createdAt), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + n++ + } + mc.run.addProcessed("application_api_key_mappings", n) + return rows.Err() +} + +// ---- application_artifact_mappings ---- + +func migrateApplicationArtifactMappings(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind( + `SELECT application_uuid, artifact_uuid, created_at FROM application_artifacts`)) + if err != nil { + return err + } + defer rows.Close() + n := 0 + for rows.Next() { + var appUUID, artifactUUID string + var createdAt sql.NullTime + if err := rows.Scan(&appUUID, &artifactUUID, &createdAt); err != nil { + return err + } + key := appUUID + "|" + artifactUUID + full := map[string]any{"application_uuid": appUUID, "artifact_uuid": artifactUUID} + if ok, err := mc.parentOK(mc.sets.applications, appUUID, "application_artifact_mappings", key, "application_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if ok, err := mc.parentOK(mc.sets.artifacts, artifactUUID, "application_artifact_mappings", key, "artifact_uuid", true, full); err != nil { + return err + } else if !ok { + continue + } + if !mc.want("application_artifact_mappings", appUUID+"|"+artifactUUID) { + continue + } + if err := migrationcore.UpsertApplicationArtifactMapping(mc.v2, migrationcore.ApplicationArtifactMappingRow{ + ApplicationUUID: appUUID, ArtifactUUID: artifactUUID, CreatedAt: ntp(createdAt), + }, mc.coreOpts(), mc.run); err != nil { + return err + } + n++ + } + mc.run.addProcessed("application_artifact_mappings", n) + return rows.Err() +} + +// ---- artifact_subscription_plans (optional; dead in v2 code) ---- + +func migrateArtifactSubscriptionPlans(mc *migCtx) error { + if !mc.opts.PopulateArtifactSubPlans { + mc.run.logger.Info("skipping artifact_subscription_plans (redundant; -populate-artifact-subscription-plans to enable)") + return nil + } + planByHandle := map[string]string{} + rows, err := mc.v2.Query(mc.v2.Rebind(`SELECT organization_uuid, handle, uuid FROM subscription_plans`)) + if err != nil { + return err + } + for rows.Next() { + var org, handle, uuid string + if err := rows.Scan(&org, &handle, &uuid); err != nil { + rows.Close() + return err + } + planByHandle[org+"|"+handle] = uuid + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + n := 0 + for _, ap := range mc.artifactPlans { + for _, ph := range ap.planHandles { + planUUID, ok := planByHandle[ap.org+"|"+ph] + if !ok { + mc.run.flag("artifact_subscription_plans", ap.artifactUUID, FlagSynthesized, + map[string]any{"plan_handle": ph}, map[string]any{"note": "config plan handle did not resolve to a migrated plan; skipped"}) + continue + } + if err := mc.insert("artifact_subscription_plans", + []string{"artifact_uuid", "subscription_plan_uuid", "created_by", "created_at"}, + []any{ap.artifactUUID, planUUID, mc.im.actorUUID, mc.epoch}, + "(artifact_uuid, subscription_plan_uuid)"); err != nil { + return err + } + n++ + } + } + mc.run.addProcessed("artifact_subscription_plans", n) + return nil +} + +// collectArtifactPlans records an artifact's config subscription-plan handles for +// the optional artifact_subscription_plans derivation (parsed from the v1 config). +func (mc *migCtx) collectArtifactPlans(artifactUUID, org string, rawConfig []byte) { + if !mc.opts.PopulateArtifactSubPlans { + return + } + var m struct { + SubscriptionPlans []string `json:"subscriptionPlans"` + } + if json.Unmarshal(rawConfig, &m) == nil && len(m.SubscriptionPlans) > 0 { + mc.artifactPlans = append(mc.artifactPlans, artifactPlanRef{artifactUUID: artifactUUID, org: org, planHandles: m.SubscriptionPlans}) + } +} + +// ---- audit markers (optional) ---- + +func migrateAuditMarkers(mc *migCtx) error { + if !mc.opts.AuditMarker { + mc.run.logger.Info("skipping audit markers (-audit-marker to enable)") + return nil + } + n := 0 + for org := range mc.sets.orgs.v2 { + uuid := deterministicUUID("audit-migrated|"+org, mc.epoch) + if err := mc.insert("audit", + []string{"uuid", "action", "resource_uuid", "resource_type", "organization_uuid", "performed_by", "performed_at"}, + []any{uuid, "migrated", org, "organization", org, mc.im.actorUUID, mc.epoch}, + "(uuid)"); err != nil { + return err + } + n++ + } + mc.run.addProcessed("audit", n) + return nil +} + +// ---- intentional §H row drops (enumerate + report, never migrate) ---- + +func dropDevportals(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind(`SELECT uuid, name FROM devportals`)) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var uuid, name string + if err := rows.Scan(&uuid, &name); err != nil { + return err + } + mc.run.dropRow("devportals", uuid, DropDevportals, name) + } + return rows.Err() +} + +func dropPublicationMappings(mc *migCtx) error { + rows, err := mc.v1.Query(mc.v1.Rebind(`SELECT api_uuid, devportal_uuid FROM publication_mappings`)) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var apiUUID, devportalUUID string + if err := rows.Scan(&apiUUID, &devportalUUID); err != nil { + return err + } + mc.run.dropRow("publication_mappings", apiUUID+"|"+devportalUUID, DropPublicationMappings, + fmt.Sprintf("api %s → devportal %s", apiUUID, devportalUUID)) + } + return rows.Err() +} + +// strv returns a NullString's value or "" (raw actor for the core's audit resolution). +func strv(ns sql.NullString) string { + if ns.Valid { + return ns.String + } + return "" +} + +// printReportSummary prints a compact human summary of a completed run. +func printReportSummary(r *Report) { + fmt.Printf("\n=== migration summary (%s, run %s) ===\n", r.Mode, r.RunID) + fmt.Printf("processed_per_table: %v\n", r.ProcessedPerTable) + if len(r.QuarantineByCode) > 0 { + fmt.Printf("quarantine_by_code: %v\n", r.QuarantineByCode) + } + if len(r.DropsByFeature) > 0 { + fmt.Printf("drops_by_feature: %v\n", r.DropsByFeature) + } + fmt.Printf("flags_by_code: %v\n", r.FlagsByCode) + if len(r.DroppedConfigFields) > 0 { + fmt.Printf("dropped_config_fields (REVIEW): %v\n", r.DroppedConfigFields) + } +} diff --git a/platform-api/cmd/dbmigrate/reconcile.go b/platform-api/cmd/dbmigrate/reconcile.go new file mode 100644 index 0000000000..751ad97af8 --- /dev/null +++ b/platform-api/cmd/dbmigrate/reconcile.go @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +// Targeted reconciliation (§8.1). `dbmigrate migrate -only-keys ` re-runs a +// specific set of rows through the SAME per-row migrationcore.UpsertX/DeleteX the batch +// uses — no new transform logic. It is the live dual-write path's reconcile step: an +// operator dumps the unreconciled `v2_dual_write_failures` rows to a keys file and +// replays them here. `-since` (no key list) triggers a full idempotent re-sync, which +// heals the enable-vs-backfill gap. +// +// UPSERT replays reuse the existing per-table iterators, filtered in-loop to the key +// set (see mc.want). DELETE replays go through migrationcore.DeleteX directly, because +// an upsert-reconcile can NEVER heal a delete: the v1 row is already gone, so the +// iterator would never see it and the orphaned v2 row would survive. + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/wso2/api-platform/platform-api/internal/database" + "github.com/wso2/api-platform/platform-api/migrationcore" +) + +// keyFilter is the reconciliation work list parsed from -only-keys. +type keyFilter struct { + upserts map[string]map[string]bool // v2 table -> key -> present + deletes []deleteKey // ordered delete replays +} + +type deleteKey struct{ table, key string } + +// loadKeyFilter parses a -only-keys file. Each non-blank, non-comment ('#') line is +// "
" where op is "upsert" or "delete". key is the row's natural key; +// composite keys are "|"-joined in the DeleteX argument order (documented per table in +// dispatchDelete) — e.g. deployment_status = "||". +func loadKeyFilter(path string) (*keyFilter, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open -only-keys file: %w", err) + } + defer f.Close() + + kf := &keyFilter{upserts: map[string]map[string]bool{}} + sc := bufio.NewScanner(f) + lineNo := 0 + for sc.Scan() { + lineNo++ + text := strings.TrimSpace(sc.Text()) + if text == "" || strings.HasPrefix(text, "#") { + continue + } + fields := strings.Fields(text) + if len(fields) != 3 { + return nil, fmt.Errorf("-only-keys line %d: want '
', got %q", lineNo, text) + } + op, table, key := fields[0], fields[1], fields[2] + switch op { + case "upsert": + if kf.upserts[table] == nil { + kf.upserts[table] = map[string]bool{} + } + kf.upserts[table][key] = true + case "delete": + kf.deletes = append(kf.deletes, deleteKey{table: table, key: key}) + default: + return nil, fmt.Errorf("-only-keys line %d: unknown op %q (want upsert|delete)", lineNo, op) + } + } + if err := sc.Err(); err != nil { + return nil, err + } + return kf, nil +} + +// want reports whether (table, key) is in the upsert work list. In batch mode +// (mc.only == nil) it always returns true, so the per-iterator guard is a no-op and +// batch output stays byte-identical. In a -since full re-sync mc.only is also nil, so +// every row is re-run. +func (mc *migCtx) want(table, key string) bool { + if mc.only == nil { + return true + } + return mc.only.upserts[table][key] +} + +// reconcileDeletes replays each delete from the work list through migrationcore.DeleteX. +func (mc *migCtx) reconcileDeletes() error { + if mc.only == nil { + return nil + } + n := 0 + for _, d := range mc.only.deletes { + if err := dispatchDelete(mc.v2, d.table, d.key); err != nil { + return fmt.Errorf("reconcile delete %s %q: %w", d.table, d.key, err) + } + n++ + } + if n > 0 { + mc.run.logger.Info("reconciled deletes", "count", n) + } + return nil +} + +// dispatchDelete routes a (table, key) to the matching migrationcore.DeleteX. The +// artifact type tables (rest_apis/llm_*/mcp_proxies/websub_apis/webbroker_apis) delete +// via DeleteArtifact so the v2 FK cascade removes the type row + children, mirroring the +// live path. Composite keys are "|"-joined in DeleteX argument order. +func dispatchDelete(v2 *database.DB, table, key string) error { + p := strings.Split(key, "|") + switch table { + case "artifacts", "rest_apis", "llm_providers", "llm_proxies", "mcp_proxies", "websub_apis", "webbroker_apis": + return migrationcore.DeleteArtifact(v2, p[0]) + case "organizations": + return migrationcore.DeleteOrganization(v2, p[0]) + case "projects": + return migrationcore.DeleteProject(v2, p[0]) + case "applications": + return migrationcore.DeleteApplication(v2, p[0]) + case "subscription_plans": + return migrationcore.DeleteSubscriptionPlan(v2, p[0]) + case "subscriptions": + return migrationcore.DeleteSubscription(v2, p[0]) + case "gateways": + return migrationcore.DeleteGateway(v2, p[0]) + case "gateway_tokens": + return migrationcore.DeleteGatewayToken(v2, p[0]) + case "gateway_custom_policies": + return migrationcore.DeleteGatewayCustomPolicy(v2, p[0]) + case "api_keys": + return migrationcore.DeleteAPIKey(v2, p[0]) + case "deployments": + return migrationcore.DeleteDeployment(v2, p[0]) + case "llm_provider_templates": + return migrationcore.DeleteLLMProviderTemplate(v2, p[0]) + case "artifact_gateway_mappings": // || + if len(p) != 3 { + return fmt.Errorf("artifact_gateway_mappings key %q: want ||", key) + } + return migrationcore.DeleteArtifactGatewayMapping(v2, p[0], p[1], p[2]) + case "application_api_key_mappings": // | + if len(p) != 2 { + return fmt.Errorf("application_api_key_mappings key %q: want |", key) + } + return migrationcore.DeleteApplicationAPIKeyMapping(v2, p[0], p[1]) + case "application_artifact_mappings": // | + if len(p) != 2 { + return fmt.Errorf("application_artifact_mappings key %q: want |", key) + } + return migrationcore.DeleteApplicationArtifactMapping(v2, p[0], p[1]) + case "gateway_custom_policy_usages": // | + if len(p) != 2 { + return fmt.Errorf("gateway_custom_policy_usages key %q: want |", key) + } + return migrationcore.DeletePolicyUsage(v2, p[0], p[1]) + case "deployment_status": // || + if len(p) != 3 { + return fmt.Errorf("deployment_status key %q: want ||", key) + } + return migrationcore.DeleteDeploymentStatus(v2, p[0], p[1], p[2]) + default: + return fmt.Errorf("unknown delete table %q", table) + } +} diff --git a/platform-api/cmd/dbmigrate/reconcile_test.go b/platform-api/cmd/dbmigrate/reconcile_test.go new file mode 100644 index 0000000000..d83b17c24b --- /dev/null +++ b/platform-api/cmd/dbmigrate/reconcile_test.go @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadKeyFilter(t *testing.T) { + path := filepath.Join(t.TempDir(), "keys.txt") + content := "# a comment\n" + + "upsert rest_apis uuid-1\n" + + "upsert gateways uuid-2\n" + + "delete artifacts uuid-3\n" + + "delete deployment_status org|art|gw\n" + + "\n" + + "upsert rest_apis uuid-4\n" + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + kf, err := loadKeyFilter(path) + if err != nil { + t.Fatalf("loadKeyFilter: %v", err) + } + if !kf.upserts["rest_apis"]["uuid-1"] || !kf.upserts["rest_apis"]["uuid-4"] { + t.Errorf("rest_apis upsert keys missing: %v", kf.upserts["rest_apis"]) + } + if !kf.upserts["gateways"]["uuid-2"] { + t.Errorf("gateways upsert key missing: %v", kf.upserts["gateways"]) + } + if len(kf.deletes) != 2 { + t.Fatalf("deletes = %d, want 2 (%v)", len(kf.deletes), kf.deletes) + } + if kf.deletes[0].table != "artifacts" || kf.deletes[0].key != "uuid-3" { + t.Errorf("deletes[0] = %+v, want {artifacts uuid-3}", kf.deletes[0]) + } + if kf.deletes[1].table != "deployment_status" || kf.deletes[1].key != "org|art|gw" { + t.Errorf("deletes[1] = %+v, want {deployment_status org|art|gw}", kf.deletes[1]) + } + + // want() matches only listed upsert keys; anything else (and the delete keys) is skipped. + mc := &migCtx{only: kf} + if !mc.want("rest_apis", "uuid-1") { + t.Error("want(rest_apis, uuid-1) should be true") + } + if mc.want("rest_apis", "uuid-3") { + t.Error("want(rest_apis, uuid-3) should be false (it is a delete, not an upsert)") + } + // Batch mode (only == nil) always processes. + if !(&migCtx{}).want("anything", "anykey") { + t.Error("want() must return true in batch mode (only == nil)") + } +} + +func TestLoadKeyFilterRejectsBadLines(t *testing.T) { + write := func(content string) string { + p := filepath.Join(t.TempDir(), "k.txt") + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p + } + if _, err := loadKeyFilter(write("upsert rest_apis\n")); err == nil { + t.Error("expected an error for a 2-field line") + } + if _, err := loadKeyFilter(write("frob rest_apis uuid\n")); err == nil { + t.Error("expected an error for an unknown op") + } +} diff --git a/platform-api/cmd/dbmigrate/state.go b/platform-api/cmd/dbmigrate/state.go new file mode 100644 index 0000000000..3e93bd6c51 --- /dev/null +++ b/platform-api/cmd/dbmigrate/state.go @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "sync" + "time" +) + +// Quarantine reason codes (closed enum). +const ( + ReasonOrphanFK = "ORPHAN_FK" + ReasonBlobUnparseable = "BLOB_UNPARSEABLE" + ReasonNullRequired = "NULL_REQUIRED" + ReasonHandleUnresolvable = "HANDLE_UNRESOLVABLE" + ReasonDupKey = "DUP_KEY" +) + +// Migrate-with-flag codes. +const ( + FlagDefaultedNull = "DEFAULTED_NULL" + FlagTruncated = "TRUNCATED" + FlagPlaceholderIDP = "PLACEHOLDER_IDP" + FlagSynthesized = "SYNTHESIZED" + FlagPlaintextCredential = "PLAINTEXT_CREDENTIAL" +) + +// Drop feature labels (intentional §H drops). +const ( + DropDevportals = "devportals" + DropDevPortalAssoc = "dev_portal_association" + DropBillingPlan = "billing_plan" + DropLLMProviderStatus = "llm_provider_status" + DropLLMProxyStatus = "llm_proxy_status" + DropMCPStatus = "mcp_status" + DropPublicationMappings = "publication_mappings" +) + +// Report is the machine-readable summary written at the end of a run. +type Report struct { + RunID string `json:"run_id"` + Mode string `json:"mode"` + SourceTZ string `json:"source_tz"` + IDPRefStrategy string `json:"idp_ref_strategy"` + GroupIDStrategy string `json:"group_id_strategy"` + PopulateArtSubplan bool `json:"populate_art_subplan"` + ProcessedPerTable map[string]int `json:"processed_per_table"` + QuarantineByCode map[string]int `json:"quarantine_by_code"` + QuarantineByTable map[string]int `json:"quarantine_by_table"` + DropsByFeature map[string]int `json:"drops_by_feature"` + DropsByTable map[string]int `json:"drops_by_table"` + FlagsByCode map[string]int `json:"flags_by_code"` + DroppedConfigFields map[string][]string `json:"dropped_config_fields,omitempty"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` +} + +// Checkpoint persists resume state: the generated-handle map (persist-and-replay, +// §B) and which tables have completed. It lives in a file, never the v2 DB. +type Checkpoint struct { + RunID string `json:"run_id"` + CompletedTables map[string]bool `json:"completed_tables"` + Handles map[string]string `json:"handles"` // "table|v1uuid" -> final handle +} + +// Run bundles all file-based migration state and the append-only output writers. +type Run struct { + opts *Options + logger *slog.Logger + mode string // "live" | "dry-run" + + mu sync.Mutex + qFile *os.File + fFile *os.File + dFile *os.File + report *Report + ckpt *Checkpoint + + ckptPath string +} + +func suffixed(base, runID string, dryRun bool, ext string) string { + if dryRun { + return fmt.Sprintf("%s-%s-dryrun.%s", base, runID, ext) + } + return fmt.Sprintf("%s-%s.%s", base, runID, ext) +} + +// newRun opens the output writers, loads any existing checkpoint (for the handle +// map), and initializes the report. The JSONL outputs are truncated per invocation: +// a resume re-processes all tables idempotently (ON CONFLICT DO NOTHING), so the +// outputs are always a clean, non-duplicated picture of the latest attempt. +func newRun(opts *Options, logger *slog.Logger) (*Run, error) { + dir := opts.OutDir + open := func(base string) (*os.File, error) { + p := filepath.Join(dir, suffixed(base, opts.RunID, opts.DryRun, "jsonl")) + return os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + } + qf, err := open("quarantine") + if err != nil { + return nil, err + } + ff, err := open("flags") + if err != nil { + return nil, err + } + df, err := open("drops") + if err != nil { + return nil, err + } + + mode := "live" + if opts.DryRun { + mode = "dry-run" + } + + r := &Run{ + opts: opts, + logger: logger, + mode: mode, + qFile: qf, + fFile: ff, + dFile: df, + report: &Report{ + RunID: opts.RunID, + Mode: mode, + SourceTZ: opts.SourceTZ, + IDPRefStrategy: opts.IDPRefStrategy, + GroupIDStrategy: opts.GroupIDStrategy, + PopulateArtSubplan: opts.PopulateArtifactSubPlans, + ProcessedPerTable: map[string]int{}, + QuarantineByCode: map[string]int{}, + QuarantineByTable: map[string]int{}, + DropsByFeature: map[string]int{}, + DropsByTable: map[string]int{}, + FlagsByCode: map[string]int{}, + StartedAt: nowRFC3339(), + }, + } + + // The checkpoint (handle map) is only meaningful for live runs; dry-runs never + // insert, so they neither read nor write it. + if !opts.DryRun { + r.ckptPath = filepath.Join(dir, suffixed("migration-state", opts.RunID, false, "json")) + r.ckpt = loadCheckpoint(r.ckptPath, opts.RunID, logger) + } else { + r.ckpt = &Checkpoint{RunID: opts.RunID, CompletedTables: map[string]bool{}, Handles: map[string]string{}} + } + return r, nil +} + +func loadCheckpoint(path, runID string, logger *slog.Logger) *Checkpoint { + c := &Checkpoint{RunID: runID, CompletedTables: map[string]bool{}, Handles: map[string]string{}} + data, err := os.ReadFile(path) + if err != nil { + return c + } + var loaded Checkpoint + if err := json.Unmarshal(data, &loaded); err != nil { + logger.Warn("checkpoint unreadable, starting fresh", "path", path, "error", err) + return c + } + if loaded.CompletedTables == nil { + loaded.CompletedTables = map[string]bool{} + } + if loaded.Handles == nil { + loaded.Handles = map[string]string{} + } + logger.Info("resuming from checkpoint", "handles", len(loaded.Handles), "completed_tables", len(loaded.CompletedTables)) + return &loaded +} + +func nowRFC3339() string { return time.Now().UTC().Format(time.RFC3339Nano) } + +func (r *Run) writeJSONL(f *os.File, obj any) { + b, err := json.Marshal(obj) + if err != nil { + r.logger.Error("marshal jsonl record", "error", err) + return + } + if _, err := f.Write(append(b, '\n')); err != nil { + r.logger.Error("write jsonl record", "error", err) + } +} + +// quarantine records a non-migrated row (full v1 row preserved for re-injection). +func (r *Run) quarantine(sourceTable, sourceKey, reasonCode, detail string, fullRow any) { + r.mu.Lock() + defer r.mu.Unlock() + r.writeJSONL(r.qFile, map[string]any{ + "run_id": r.opts.RunID, + "source_table": sourceTable, + "source_key": sourceKey, + "reason_code": reasonCode, + "detail": detail, + "full_row": fullRow, + "quarantined_at": nowRFC3339(), + }) + r.report.QuarantineByCode[reasonCode]++ + r.report.QuarantineByTable[sourceTable]++ +} + +// flag records a migrate-with-flag event (the row IS in v2; this is the audit trail). +func (r *Run) flag(sourceTable, sourceKey, flagCode string, oldV, newV any) { + r.mu.Lock() + defer r.mu.Unlock() + r.writeJSONL(r.fFile, map[string]any{ + "run_id": r.opts.RunID, + "source_table": sourceTable, + "source_key": sourceKey, + "flag_code": flagCode, + "old": oldV, + "new": newV, + }) + r.report.FlagsByCode[flagCode]++ +} + +// drop records an intentional §H drop (id + human label) so loss is auditable. +// scope is "row" (the whole v1 row is discarded — devportals, dev_portal assoc, +// publication_mappings) or "field" (the row IS migrated but one column is dropped — +// billing_plan, llm/mcp status). verify reconciles row-drops only. +func (r *Run) drop(scope, sourceTable, sourceKey, feature, label string) { + r.mu.Lock() + defer r.mu.Unlock() + r.writeJSONL(r.dFile, map[string]any{ + "run_id": r.opts.RunID, + "source_table": sourceTable, + "source_key": sourceKey, + "scope": scope, + "feature": feature, + "label": label, + }) + r.report.DropsByFeature[feature]++ + r.report.DropsByTable[sourceTable]++ +} + +// dropRow / dropField are thin, self-documenting wrappers over drop. +func (r *Run) dropRow(sourceTable, sourceKey, feature, label string) { + r.drop("row", sourceTable, sourceKey, feature, label) +} +func (r *Run) dropField(sourceTable, sourceKey, feature, label string) { + r.drop("field", sourceTable, sourceKey, feature, label) +} + +// --- migrationcore.Reporter implementation (exported method set) --- +// The batch writes the existing JSONL files; the live dual-write path implements +// the same interface with logs + the v2_dual_write_failures table. + +func (r *Run) Flag(table, key, code string, oldV, newV any) { r.flag(table, key, code, oldV, newV) } +func (r *Run) Quarantine(table, key, code, detail string, row any) { r.quarantine(table, key, code, detail, row) } +func (r *Run) Dropped(scope, table, key, feature, label string) { r.drop(scope, table, key, feature, label) } +func (r *Run) DroppedFields(structName string, fields []string) { r.recordDroppedFields(structName, fields) } + +// addProcessed increments the migrated-row count for a v2 table. +func (r *Run) addProcessed(table string, n int) { + r.mu.Lock() + defer r.mu.Unlock() + r.report.ProcessedPerTable[table] += n +} + +// recordDroppedFields stores the §E DisallowUnknownFields discovery for a config struct. +func (r *Run) recordDroppedFields(structName string, fields []string) { + r.mu.Lock() + defer r.mu.Unlock() + if len(fields) == 0 { + return + } + if r.report.DroppedConfigFields == nil { + r.report.DroppedConfigFields = map[string][]string{} + } + r.report.DroppedConfigFields[structName] = fields +} + +// getHandle returns a previously generated handle for (table, v1uuid), if any. +func (r *Run) getHandle(table, v1uuid string) (string, bool) { + r.mu.Lock() + defer r.mu.Unlock() + h, ok := r.ckpt.Handles[table+"|"+v1uuid] + return h, ok +} + +// putHandle records a generated handle for (table, v1uuid). +func (r *Run) putHandle(table, v1uuid, handle string) { + r.mu.Lock() + defer r.mu.Unlock() + r.ckpt.Handles[table+"|"+v1uuid] = handle +} + +// saveCheckpoint atomically flushes the handle map + completion state to disk. +func (r *Run) saveCheckpoint() error { + if r.opts.DryRun { + return nil + } + r.mu.Lock() + defer r.mu.Unlock() + b, err := json.MarshalIndent(r.ckpt, "", " ") + if err != nil { + return err + } + tmp := r.ckptPath + ".tmp" + if err := os.WriteFile(tmp, b, 0o644); err != nil { + return err + } + return os.Rename(tmp, r.ckptPath) +} + +func (r *Run) markTableComplete(table string) { + r.mu.Lock() + r.ckpt.CompletedTables[table] = true + r.mu.Unlock() +} + +// writeReport writes the migration-report JSON and syncs the JSONL files. +func (r *Run) writeReport() error { + r.mu.Lock() + defer r.mu.Unlock() + r.report.FinishedAt = nowRFC3339() + b, err := json.MarshalIndent(r.report, "", " ") + if err != nil { + return err + } + p := filepath.Join(r.opts.OutDir, suffixed("migration-report", r.opts.RunID, r.opts.DryRun, "json")) + return os.WriteFile(p, b, 0o644) +} + +func (r *Run) close() { + for _, f := range []*os.File{r.qFile, r.fFile, r.dFile} { + if f != nil { + _ = f.Sync() + _ = f.Close() + } + } +} diff --git a/platform-api/cmd/dbmigrate/transform.go b/platform-api/cmd/dbmigrate/transform.go new file mode 100644 index 0000000000..7ff1dcd6ae --- /dev/null +++ b/platform-api/cmd/dbmigrate/transform.go @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +// The per-row transforms now live in the shared migrationcore package (the ONE +// implementation reused by the live dual-write path). These file-local aliases keep +// the batch call sites unchanged. +import "github.com/wso2/api-platform/platform-api/migrationcore" + +var ( + reinterpretTZ = migrationcore.ReinterpretTZ + boolToSmallint = migrationcore.BoolToSmallint + truncateStr = migrationcore.TruncateStr + slug = migrationcore.Slug + parseTransport = migrationcore.ParseTransport + topLevelUnknownFields = migrationcore.TopLevelUnknownFields + reshapeRestAPIConfig = migrationcore.ReshapeRestAPIConfig + reshapeWebSubConfig = migrationcore.ReshapeWebSubConfig + preprocessWebSubRaw = migrationcore.PreprocessWebSubRaw + reshapeWebBrokerConfig = migrationcore.ReshapeWebBrokerConfig + remarshalConfig = migrationcore.RemarshalConfig + caseConvertThrottleUnit = migrationcore.CaseConvertThrottleUnit +) diff --git a/platform-api/cmd/dbmigrate/verify.go b/platform-api/cmd/dbmigrate/verify.go new file mode 100644 index 0000000000..e0ea487eb5 --- /dev/null +++ b/platform-api/cmd/dbmigrate/verify.go @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "time" + + "github.com/wso2/api-platform/platform-api/internal/database" +) + +// Check is one verification assertion. v1/v2/expected are optional counts. +type Check struct { + Layer string `json:"layer"` + Name string `json:"name"` + Status string `json:"status"` // PASS | FAIL | WARN + V1 *int64 `json:"v1,omitempty"` + V2 *int64 `json:"v2,omitempty"` + Expected *int64 `json:"expected,omitempty"` + Detail string `json:"detail,omitempty"` +} + +// VerifyReport is the structured output of the verify subcommand. +type VerifyReport struct { + RunID string `json:"run_id"` + Checks []Check `json:"checks"` + Overall string `json:"overall"` + Summary map[string]int `json:"summary"` +} + +// verifier holds both read-only connections plus loaded reconciliation state. +type verifier struct { + v1, v2 *database.DB + opts *Options + loc *time.Location + epoch time.Time + encKey []byte + + quarByTable map[string]int64 + quarKeys map[string]bool // "table|key" + signoff map[string]bool // "table|key" + + checks []Check +} + +const ( + statusPass = "PASS" + statusFail = "FAIL" + statusWarn = "WARN" +) + +func runVerify(argv []string) error { + o := &Options{} + fs := flag.NewFlagSet("verify", flag.ContinueOnError) + registerCommonFlags(fs, o) + fs.StringVar(&o.SourceTZ, "source-tz", "UTC", "Time zone of naive v1 TIMESTAMP values (must match the migrate run)") + epochStr := fs.String("migration-epoch", defaultMigrationEpoch, "Migration epoch used at migrate time (for audit-identity checks)") + fs.IntVar(&o.DecryptSampleSize, "decrypt-sample-size", 25, "Subscription tokens to sample for the decrypt round-trip") + fs.StringVar(&o.EncKeyFile, "encryption-key-file", "", "File with the subscription-token key (else APIP_MIGRATION_ENCRYPTION_KEY); enables the token round-trip check") + if err := fs.Parse(argv); err != nil { + return err + } + if err := validateCommon(o); err != nil { + return err + } + loc, err := time.LoadLocation(o.SourceTZ) + if err != nil { + return fmt.Errorf("invalid -source-tz: %w", err) + } + epoch, err := time.Parse(time.RFC3339, *epochStr) + if err != nil { + return fmt.Errorf("invalid -migration-epoch: %w", err) + } + _ = loadEncryptionKey(o) // optional for verify + + logger := newLogger(o) + v1, err := openDB(o.V1DSN, logger) + if err != nil { + return fmt.Errorf("open v1: %w", err) + } + defer v1.Close() + v2, err := openDB(o.V2DSN, logger) + if err != nil { + return fmt.Errorf("open v2: %w", err) + } + defer v2.Close() + + vr := &verifier{ + v1: v1, v2: v2, opts: o, loc: loc, epoch: epoch, encKey: o.EncryptionKey, + quarByTable: map[string]int64{}, quarKeys: map[string]bool{}, signoff: map[string]bool{}, + } + vr.loadReconciliationState() + + // Run all layers. Individual check errors are recorded as FAIL, not returned, + // so one broken check never masks the rest. + vr.layerA_counts() + vr.layerB_scalars() + vr.layerC_transforms() + vr.layerD_integrity() + vr.layerE_generated() + vr.layerF_dropReconcile() + + report := vr.finish(o.RunID) + if err := writeVerifyReport(o, report); err != nil { + return err + } + printVerify(report) + if report.Summary["FAIL"] > 0 { + return fmt.Errorf("verify FAILED: %d check(s) failed", report.Summary["FAIL"]) + } + return nil +} + +// loadReconciliationState reads the latest quarantine JSONL and the sign-off file. +func (vr *verifier) loadReconciliationState() { + qPath := filepath.Join(vr.opts.OutDir, suffixed("quarantine", vr.opts.RunID, false, "jsonl")) + forEachJSONL(qPath, func(m map[string]any) { + t, _ := m["source_table"].(string) + k, _ := m["source_key"].(string) + vr.quarByTable[t]++ + vr.quarKeys[t+"|"+k] = true + }) + sPath := filepath.Join(vr.opts.OutDir, "quarantine-signoff.jsonl") + forEachJSONL(sPath, func(m map[string]any) { + t, _ := m["source_table"].(string) + k, _ := m["source_key"].(string) + vr.signoff[t+"|"+k] = true + }) +} + +func forEachJSONL(path string, fn func(map[string]any)) { + f, err := os.Open(path) + if err != nil { + return + } + defer f.Close() + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1024*1024), 16*1024*1024) + for sc.Scan() { + line := sc.Bytes() + if len(line) == 0 { + continue + } + var m map[string]any + if json.Unmarshal(line, &m) == nil { + fn(m) + } + } +} + +// ---- check recording helpers ---- + +func i64(v int64) *int64 { return &v } + +func (vr *verifier) add(c Check) { vr.checks = append(vr.checks, c) } + +func (vr *verifier) pass(layer, name, detail string) { + vr.add(Check{Layer: layer, Name: name, Status: statusPass, Detail: detail}) +} +func (vr *verifier) fail(layer, name, detail string) { + vr.add(Check{Layer: layer, Name: name, Status: statusFail, Detail: detail}) +} +func (vr *verifier) warn(layer, name, detail string) { + vr.add(Check{Layer: layer, Name: name, Status: statusWarn, Detail: detail}) +} + +// eq records a PASS/FAIL count comparison. +func (vr *verifier) eq(layer, name string, got, want int64, detail string) { + st := statusPass + if got != want { + st = statusFail + } + vr.add(Check{Layer: layer, Name: name, Status: st, V2: i64(got), Expected: i64(want), Detail: detail}) +} + +func (vr *verifier) finish(runID string) *VerifyReport { + summary := map[string]int{"PASS": 0, "FAIL": 0, "WARN": 0, "total": len(vr.checks)} + for _, c := range vr.checks { + summary[c.Status]++ + } + overall := statusPass + if summary["FAIL"] > 0 { + overall = statusFail + } + return &VerifyReport{RunID: runID, Checks: vr.checks, Overall: overall, Summary: summary} +} + +func writeVerifyReport(o *Options, r *VerifyReport) error { + b, err := json.MarshalIndent(r, "", " ") + if err != nil { + return err + } + p := filepath.Join(o.OutDir, suffixed("verify-report", o.RunID, false, "json")) + return os.WriteFile(p, b, 0o644) +} + +func printVerify(r *VerifyReport) { + // Sort for stable output: FAILs first, then WARNs, then PASS. + order := map[string]int{statusFail: 0, statusWarn: 1, statusPass: 2} + sorted := make([]Check, len(r.Checks)) + copy(sorted, r.Checks) + sort.SliceStable(sorted, func(i, j int) bool { return order[sorted[i].Status] < order[sorted[j].Status] }) + fmt.Printf("\n=== verify report (run %s) — overall %s ===\n", r.RunID, r.Overall) + for _, c := range sorted { + if c.Status == statusPass { + continue // keep the console focused on problems; full detail is in JSON + } + fmt.Printf(" [%s] %s/%s %s\n", c.Status, c.Layer, c.Name, c.Detail) + } + fmt.Printf("summary: PASS=%d WARN=%d FAIL=%d total=%d\n", + r.Summary["PASS"], r.Summary["WARN"], r.Summary["FAIL"], r.Summary["total"]) +} + +// count returns the row count of a table (0 on error, recorded as a FAIL check). +func (vr *verifier) count(db *database.DB, table string) int64 { + var n int64 + if err := db.QueryRow("SELECT count(*) FROM " + table).Scan(&n); err != nil { + vr.fail("A", "count "+table, err.Error()) + return 0 + } + return n +} + +func (vr *verifier) countWhere(db *database.DB, table, where string) int64 { + var n int64 + if err := db.QueryRow("SELECT count(*) FROM " + table + " WHERE " + where).Scan(&n); err != nil { + vr.fail("A", "count "+table+" where "+where, err.Error()) + return 0 + } + return n +} diff --git a/platform-api/cmd/dbmigrate/verify_checks.go b/platform-api/cmd/dbmigrate/verify_checks.go new file mode 100644 index 0000000000..5640293440 --- /dev/null +++ b/platform-api/cmd/dbmigrate/verify_checks.go @@ -0,0 +1,738 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "sort" + + "github.com/wso2/api-platform/platform-api/internal/constants" + "github.com/wso2/api-platform/platform-api/internal/database" + "github.com/wso2/api-platform/platform-api/internal/model" + "github.com/wso2/api-platform/platform-api/internal/utils" +) + +// perTypeKinds maps each v2 per-type table to its v1 artifacts.kind. +var perTypeKinds = []struct{ table, kind string }{ + {"rest_apis", constants.RestApi}, + {"llm_providers", constants.LLMProvider}, + {"llm_proxies", constants.LLMProxy}, + {"mcp_proxies", constants.MCPProxy}, + {"websub_apis", constants.WebSubApi}, + {"webbroker_apis", constants.WebBrokerApi}, +} + +// ---- Layer A: coverage counts ---- + +func (vr *verifier) layerA_counts() { + // Split: each per-type count reconciles with v1 artifacts of that kind. + var sumPerType int64 + for _, pt := range perTypeKinds { + v1kind := vr.countWhere(vr.v1, "artifacts", "kind = '"+pt.kind+"'") + v2t := vr.count(vr.v2, pt.table) + sumPerType += v2t + quar := vr.quarByTable[pt.table] + vr.add(Check{Layer: "A", Name: "split artifacts[" + pt.kind + "] -> " + pt.table, + Status: passIf(v2t+quar == v1kind), V1: i64(v1kind), V2: i64(v2t), Expected: i64(v1kind), + Detail: fmt.Sprintf("v2(%d)+quarantine(%d)==v1(%d)", v2t, quar, v1kind)}) + } + // Σ per-type == v2.artifacts. + v2art := vr.count(vr.v2, "artifacts") + vr.eq("A", "v2 artifacts == Σ per-type", v2art, sumPerType, "") + + // Derived tables. + v1throttled := vr.countWhere(vr.v1, "subscription_plans", "throttle_limit_count IS NOT NULL AND throttle_limit_unit IS NOT NULL") + vr.eq("A", "subscription_plan_limits == v1 throttled plans", vr.count(vr.v2, "subscription_plan_limits"), v1throttled, "") + + v1vhost := vr.countWhere(vr.v1, "gateways", "vhost IS NOT NULL AND vhost <> ''") + vr.eq("A", "gateway_endpoints == v1 gateways with vhost", vr.count(vr.v2, "gateway_endpoints"), v1vhost, "") + + v1gwAssoc := vr.countWhere(vr.v1, "association_mappings", "association_type = 'gateway'") + agm := vr.count(vr.v2, "artifact_gateway_mappings") + vr.add(Check{Layer: "A", Name: "artifact_gateway_mappings == v1 gateway associations", + Status: passIf(agm+vr.quarByTable["artifact_gateway_mappings"] == v1gwAssoc), + V1: i64(v1gwAssoc), V2: i64(agm), Expected: i64(v1gwAssoc), + Detail: fmt.Sprintf("v2(%d)+quar(%d)==v1(%d)", agm, vr.quarByTable["artifact_gateway_mappings"], v1gwAssoc)}) + + // New v2 feature tables that must stay empty. + vr.eq("A", "websub_api_hmac_secrets == 0", vr.count(vr.v2, "websub_api_hmac_secrets"), 0, "new v2 feature, no v1 source") + vr.eq("A", "secrets == 0", vr.count(vr.v2, "secrets"), 0, "") +} + +// ---- Layer B: scalar equivalence (uuid join, FULL) ---- + +type v1Art struct { + name, version, kind, org string + createdAt, updatedAt sql.NullTime +} + +func (vr *verifier) loadV1Artifacts() map[string]v1Art { + m := map[string]v1Art{} + rows, err := vr.v1.Query(`SELECT uuid, name, version, kind, organization_uuid, created_at, updated_at FROM artifacts`) + if err != nil { + vr.fail("B", "load v1 artifacts", err.Error()) + return m + } + defer rows.Close() + for rows.Next() { + var uuid string + var a v1Art + if err := rows.Scan(&uuid, &a.name, &a.version, &a.kind, &a.org, &a.createdAt, &a.updatedAt); err != nil { + vr.fail("B", "scan v1 artifacts", err.Error()) + return m + } + m[uuid] = a + } + return m +} + +func (vr *verifier) layerB_scalars() { + arts := vr.loadV1Artifacts() + + // display_name == v1 artifacts.name, created_at instant preserved, kind == type. + nameMismatch, tsMismatch, typeMismatch, orphan := 0, 0, 0, 0 + for _, pt := range perTypeKinds { + rows, err := vr.v2.Query("SELECT uuid, display_name, created_at FROM " + pt.table) + if err != nil { + vr.fail("B", "read "+pt.table, err.Error()) + continue + } + for rows.Next() { + var uuid, displayName string + var createdAt sql.NullTime + if err := rows.Scan(&uuid, &displayName, &createdAt); err != nil { + vr.fail("B", "scan "+pt.table, err.Error()) + break + } + a, ok := arts[uuid] + if !ok { + orphan++ + continue + } + if a.name != displayName { + nameMismatch++ + } + if a.createdAt.Valid && createdAt.Valid && !reinterpretTZ(a.createdAt.Time, vr.loc).Equal(createdAt.Time.UTC()) { + tsMismatch++ + } + } + rows.Close() + } + // kind == type via v2 artifacts. + rows, err := vr.v2.Query("SELECT uuid, type FROM artifacts") + if err == nil { + for rows.Next() { + var uuid, typ string + _ = rows.Scan(&uuid, &typ) + if a, ok := arts[uuid]; ok && a.kind != typ { + typeMismatch++ + } + } + rows.Close() + } + vr.add(Check{Layer: "B", Name: "display_name == v1 artifacts.name", Status: passIf(nameMismatch == 0), Detail: fmt.Sprintf("%d mismatch", nameMismatch)}) + vr.add(Check{Layer: "B", Name: "created_at instant preserved", Status: passIf(tsMismatch == 0), Detail: fmt.Sprintf("%d mismatch", tsMismatch)}) + vr.add(Check{Layer: "B", Name: "v1 kind == v2 artifacts.type", Status: passIf(typeMismatch == 0), Detail: fmt.Sprintf("%d mismatch", typeMismatch)}) + vr.add(Check{Layer: "B", Name: "no v2 per-type row missing its v1 artifact", Status: passIf(orphan == 0), Detail: fmt.Sprintf("%d orphan", orphan)}) + + // subscriptions.artifact_uuid == v1.api_uuid. + v1sub := loadKV(vr.v1, "SELECT uuid, api_uuid FROM subscriptions") + subMismatch := 0 + rs, err := vr.v2.Query("SELECT uuid, artifact_uuid FROM subscriptions") + if err == nil { + for rs.Next() { + var uuid, art string + _ = rs.Scan(&uuid, &art) + if v, ok := v1sub[uuid]; ok && v != art { + subMismatch++ + } + } + rs.Close() + } + vr.add(Check{Layer: "B", Name: "subscriptions api_uuid -> artifact_uuid", Status: passIf(subMismatch == 0), Detail: fmt.Sprintf("%d mismatch", subMismatch)}) + + // subscription_plans.display_name == v1.plan_name. + v1plan := loadKV(vr.v1, "SELECT uuid, plan_name FROM subscription_plans") + planMismatch := 0 + rp, err := vr.v2.Query("SELECT uuid, display_name FROM subscription_plans") + if err == nil { + for rp.Next() { + var uuid, dn string + _ = rp.Scan(&uuid, &dn) + if v, ok := v1plan[uuid]; ok && v != dn { + planMismatch++ + } + } + rp.Close() + } + vr.add(Check{Layer: "B", Name: "subscription_plans plan_name -> display_name", Status: passIf(planMismatch == 0), Detail: fmt.Sprintf("%d mismatch", planMismatch)}) +} + +// ---- Layer C: transform round-trip (decode-and-compare, FULL) ---- + +type cfgRow struct { + config []byte + transport string +} + +func (vr *verifier) layerC_transforms() { + websubPrep := func(b []byte) []byte { out, _, _ := preprocessWebSubRaw(b); return out } + + // Transport-bearing configs (v1 column moved into the blob). + compareConfigs(vr, "rest_apis", + vr.loadV1Cfg("rest_apis", true), vr.loadV2Cfg("rest_apis"), + func(c *model.RestAPIConfig, tr string) { + if p := parseTransport(tr); len(p) > 0 { + c.Transport = p + } + }, + func(c *model.RestAPIConfig) []string { return c.Transport }, nil) + compareConfigs(vr, "websub_apis", + vr.loadV1Cfg("websub_apis", true), vr.loadV2Cfg("websub_apis"), + func(c *model.WebSubAPIConfiguration, tr string) { + if p := parseTransport(tr); len(p) > 0 { + c.Transport = p + } + }, + func(c *model.WebSubAPIConfiguration) []string { return c.Transport }, websubPrep) + compareConfigs(vr, "webbroker_apis", + vr.loadV1Cfg("webbroker_apis", true), vr.loadV2Cfg("webbroker_apis"), + func(c *model.WebBrokerAPIConfiguration, tr string) { + if p := parseTransport(tr); len(p) > 0 { + c.Transport = p + } + }, + func(c *model.WebBrokerAPIConfiguration) []string { return c.Transport }, nil) + + // Non-transport configs (re-marshal equality only). + compareConfigs[model.LLMProviderConfig](vr, "llm_providers", + vr.loadV1Cfg("llm_providers", false), vr.loadV2Cfg("llm_providers"), nil, nil, nil) + compareConfigs[model.LLMProxyConfig](vr, "llm_proxies", + vr.loadV1Cfg("llm_proxies", false), vr.loadV2Cfg("llm_proxies"), nil, nil, nil) + compareConfigs[model.MCPProxyConfiguration](vr, "mcp_proxies", + vr.loadV1Cfg("mcp_proxies", false), vr.loadV2Cfg("mcp_proxies"), nil, nil, nil) + + vr.checkPlanLimits() + vr.checkGatewayEndpoints() + vr.checkTokens() +} + +func (vr *verifier) loadV1Cfg(table string, withTransport bool) map[string]cfgRow { + m := map[string]cfgRow{} + q := "SELECT uuid, configuration" + if withTransport { + q += ", transport" + } + q += " FROM " + table + rows, err := vr.v1.Query(q) + if err != nil { + vr.fail("C", "load v1 cfg "+table, err.Error()) + return m + } + defer rows.Close() + for rows.Next() { + var uuid string + var cfg []byte + var tr sql.NullString + var scanErr error + if withTransport { + scanErr = rows.Scan(&uuid, &cfg, &tr) + } else { + scanErr = rows.Scan(&uuid, &cfg) + } + if scanErr != nil { + vr.fail("C", "scan v1 cfg "+table, scanErr.Error()) + return m + } + m[uuid] = cfgRow{config: cfg, transport: tr.String} + } + return m +} + +func (vr *verifier) loadV2Cfg(table string) map[string][]byte { + m := map[string][]byte{} + rows, err := vr.v2.Query("SELECT uuid, configuration FROM " + table) + if err != nil { + vr.fail("C", "load v2 cfg "+table, err.Error()) + return m + } + defer rows.Close() + for rows.Next() { + var uuid string + var cfg []byte + if err := rows.Scan(&uuid, &cfg); err != nil { + vr.fail("C", "scan v2 cfg "+table, err.Error()) + return m + } + m[uuid] = cfg + } + return m +} + +// compareConfigs decodes each v2 config blob back into the v2 model, applies the +// documented forward transform to the v1 source, and asserts equality — the +// round-trip correctness gate. When getTransport != nil it additionally asserts +// the moved transport column landed inside the blob. +func compareConfigs[T any](vr *verifier, table string, v1 map[string]cfgRow, v2 map[string][]byte, + inject func(*T, string), getTransport func(*T) []string, prep func([]byte) []byte) { + total, decodeErr, mismatch, transportBad, missing := 0, 0, 0, 0, 0 + for uuid, v2b := range v2 { + total++ + var t2 T + if err := json.Unmarshal(v2b, &t2); err != nil { + decodeErr++ + continue + } + v1r, ok := v1[uuid] + if !ok { + missing++ + continue + } + src := v1r.config + if prep != nil { + src = prep(src) + } + var t1 T + if err := json.Unmarshal(src, &t1); err != nil { + mismatch++ + continue + } + if inject != nil { + inject(&t1, v1r.transport) + } + b1, _ := json.Marshal(t1) + b2, _ := json.Marshal(t2) + if string(b1) != string(b2) { + mismatch++ + } + if getTransport != nil { + if !stringsEqual(getTransport(&t2), parseTransport(v1r.transport)) { + transportBad++ + } + } + } + vr.add(Check{Layer: "C", Name: "blob decodes into v2 model " + table, Status: passIf(decodeErr == 0), + V2: i64(int64(total)), Detail: fmt.Sprintf("%d decode error(s) of %d", decodeErr, total)}) + vr.add(Check{Layer: "C", Name: "config round-trip invariants " + table, Status: passIf(mismatch == 0 && missing == 0), + Detail: fmt.Sprintf("%d mismatch, %d missing-in-v1 of %d", mismatch, missing, total)}) + if getTransport != nil { + vr.add(Check{Layer: "C", Name: "transport column -> blob " + table, Status: passIf(transportBad == 0), + Detail: fmt.Sprintf("%d transport mismatch of %d", transportBad, total)}) + } +} + +func (vr *verifier) checkPlanLimits() { + // v1 plan -> throttle; v2 limits by plan. + type thr struct { + count sql.NullInt64 + unit sql.NullString + stop sql.NullBool + } + v1 := map[string]thr{} + rows, err := vr.v1.Query(`SELECT uuid, throttle_limit_count, throttle_limit_unit, stop_on_quota_reach FROM subscription_plans`) + if err != nil { + vr.fail("C", "load v1 plans", err.Error()) + return + } + for rows.Next() { + var uuid string + var t thr + _ = rows.Scan(&uuid, &t.count, &t.unit, &t.stop) + v1[uuid] = t + } + rows.Close() + + type lim struct { + count int64 + unit, ltype string + amount, stop int64 + } + v2 := map[string]lim{} + lr, err := vr.v2.Query(`SELECT subscription_plan_uuid, limit_count, time_unit, limit_type, time_amount, stop_on_quota_reach FROM subscription_plan_limits`) + if err != nil { + vr.fail("C", "load v2 limits", err.Error()) + return + } + for lr.Next() { + var plan string + var l lim + _ = lr.Scan(&plan, &l.count, &l.unit, &l.ltype, &l.amount, &l.stop) + v2[plan] = l + } + lr.Close() + + bad, missing, spurious := 0, 0, 0 + for uuid, t := range v1 { + hasThrottle := t.count.Valid && t.unit.Valid && t.unit.String != "" + l, present := v2[uuid] + if hasThrottle { + if !present { + missing++ + continue + } + wantUnit, _ := caseConvertThrottleUnit(t.unit.String) + wantStop := int64(1) + if t.stop.Valid { + wantStop = int64(boolToSmallint(t.stop.Bool)) + } + if l.count != t.count.Int64 || l.unit != wantUnit || l.ltype != constants.LimitTypeRequestCount || l.amount != 1 || l.stop != wantStop { + bad++ + } + } else if present { + spurious++ + } + } + vr.add(Check{Layer: "C", Name: "subscription_plan_limits values", Status: passIf(bad == 0 && missing == 0 && spurious == 0), + Detail: fmt.Sprintf("%d wrong, %d missing, %d spurious", bad, missing, spurious)}) +} + +func (vr *verifier) checkGatewayEndpoints() { + v1 := loadKV(vr.v1, "SELECT uuid, vhost FROM gateways WHERE vhost IS NOT NULL AND vhost <> ''") + rows, err := vr.v2.Query("SELECT gateway_uuid, url FROM gateway_endpoints") + if err != nil { + vr.fail("C", "load gateway_endpoints", err.Error()) + return + } + defer rows.Close() + bad := 0 + seen := map[string]bool{} + for rows.Next() { + var gw, url string + _ = rows.Scan(&gw, &url) + seen[gw] = true + if v, ok := v1[gw]; !ok || v != url { + bad++ + } + } + missing := 0 + for gw := range v1 { + if !seen[gw] { + missing++ + } + } + vr.add(Check{Layer: "C", Name: "gateway_endpoints url == v1 vhost", Status: passIf(bad == 0 && missing == 0), + Detail: fmt.Sprintf("%d wrong url, %d gateways missing endpoint", bad, missing)}) +} + +func (vr *verifier) checkTokens() { + v1 := loadKV(vr.v1, "SELECT uuid, subscription_token FROM subscriptions") + v1hash := loadKV(vr.v1, "SELECT uuid, subscription_token_hash FROM subscriptions") + rows, err := vr.v2.Query("SELECT uuid, subscription_token, subscription_token_hash FROM subscriptions") + if err != nil { + vr.fail("C", "load v2 subscriptions", err.Error()) + return + } + defer rows.Close() + tokBad, hashBad := 0, 0 + var sampleTok []string + for rows.Next() { + var uuid, tok, hash string + _ = rows.Scan(&uuid, &tok, &hash) + if v, ok := v1[uuid]; ok && v != tok { + tokBad++ + } + if v, ok := v1hash[uuid]; ok && v != hash { + hashBad++ + } + if len(sampleTok) < vr.opts.DecryptSampleSize { + sampleTok = append(sampleTok, tok) + } + } + vr.add(Check{Layer: "C", Name: "subscription_token bytes == v1 (verbatim)", Status: passIf(tokBad == 0), Detail: fmt.Sprintf("%d mismatch", tokBad)}) + vr.add(Check{Layer: "C", Name: "subscription_token_hash == v1", Status: passIf(hashBad == 0), Detail: fmt.Sprintf("%d mismatch", hashBad)}) + + if vr.encKey != nil { + fail := 0 + for _, tok := range sampleTok { + if _, err := utils.DecryptSubscriptionToken(vr.encKey, tok); err != nil { + fail++ + } + } + vr.add(Check{Layer: "C", Name: "sampled token decrypt round-trip (v2 key)", Status: passIf(fail == 0), + Detail: fmt.Sprintf("%d/%d failed", fail, len(sampleTok))}) + } else { + vr.warn("C", "sampled token decrypt round-trip", "skipped (no key provided)") + } +} + +// ---- Layer D: v2 referential integrity + uniqueness (FULL) ---- + +func (vr *verifier) layerD_integrity() { + fks := []struct{ child, col, parent, pcol string }{ + {"rest_apis", "uuid", "artifacts", "uuid"}, + {"llm_providers", "uuid", "artifacts", "uuid"}, + {"llm_providers", "template_uuid", "llm_provider_templates", "uuid"}, + {"llm_proxies", "provider_uuid", "llm_providers", "uuid"}, + {"mcp_proxies", "uuid", "artifacts", "uuid"}, + {"websub_apis", "uuid", "artifacts", "uuid"}, + {"webbroker_apis", "uuid", "artifacts", "uuid"}, + {"subscriptions", "artifact_uuid", "artifacts", "uuid"}, + {"subscription_plan_limits", "subscription_plan_uuid", "subscription_plans", "uuid"}, + {"gateway_endpoints", "gateway_uuid", "gateways", "uuid"}, + {"artifact_gateway_mappings", "artifact_uuid", "artifacts", "uuid"}, + {"artifact_gateway_mappings", "gateway_uuid", "gateways", "uuid"}, + {"deployments", "artifact_uuid", "artifacts", "uuid"}, + {"deployment_status", "deployment_uuid", "deployments", "uuid"}, + {"gateway_custom_policy_usages", "artifact_uuid", "artifacts", "uuid"}, + {"api_keys", "artifact_uuid", "artifacts", "uuid"}, + {"application_api_key_mappings", "api_key_id", "api_keys", "uuid"}, + {"application_artifact_mappings", "artifact_uuid", "artifacts", "uuid"}, + } + for _, fk := range fks { + q := fmt.Sprintf("SELECT count(*) FROM %s c LEFT JOIN %s p ON c.%s = p.%s WHERE c.%s IS NOT NULL AND p.%s IS NULL", + fk.child, fk.parent, fk.col, fk.pcol, fk.col, fk.pcol) + var orphans int64 + if err := vr.v2.QueryRow(q).Scan(&orphans); err != nil { + vr.fail("D", "FK "+fk.child+"."+fk.col, err.Error()) + continue + } + vr.add(Check{Layer: "D", Name: "FK " + fk.child + "." + fk.col + " -> " + fk.parent, Status: passIf(orphans == 0), + Detail: fmt.Sprintf("%d orphan(s)", orphans)}) + } + + // Uniqueness: (organization_uuid, handle) per type table + others. + uniq := []struct{ table, cols string }{ + {"organizations", "handle"}, + {"projects", "organization_uuid, handle"}, + {"applications", "organization_uuid, handle"}, + {"rest_apis", "organization_uuid, handle"}, + {"llm_providers", "organization_uuid, handle"}, + {"llm_proxies", "organization_uuid, handle"}, + {"mcp_proxies", "organization_uuid, handle"}, + {"websub_apis", "organization_uuid, handle"}, + {"webbroker_apis", "organization_uuid, handle"}, + {"subscription_plans", "organization_uuid, handle"}, + {"gateways", "organization_uuid, handle"}, + {"llm_provider_templates", "organization_uuid, group_id, version"}, + {"llm_provider_templates", "organization_uuid, handle"}, + {"api_keys", "artifact_uuid, handle"}, + {"user_idp_references", "idp_id"}, + } + for _, u := range uniq { + q := fmt.Sprintf("SELECT count(*) FROM (SELECT %s FROM %s GROUP BY %s HAVING count(*) > 1) d", u.cols, u.table, u.cols) + var dups int64 + if err := vr.v2.QueryRow(q).Scan(&dups); err != nil { + vr.fail("D", "unique "+u.table+"("+u.cols+")", err.Error()) + continue + } + vr.add(Check{Layer: "D", Name: "unique " + u.table + "(" + u.cols + ")", Status: passIf(dups == 0), + Detail: fmt.Sprintf("%d duplicate group(s)", dups)}) + } +} + +// ---- Layer E: generated / placeholder / default correctness ---- + +func (vr *verifier) layerE_generated() { + // Carried handles: v2.handle == slug(v1 handle) for the common (non-suffixed) case. + vr.checkCarriedHandles("organizations", "organizations") + vr.checkCarriedHandles("applications", "applications") + vr.checkCarriedHandles("llm_provider_templates", "llm_provider_templates") + + // idp_organization_ref_uuid == uuid. + vr.eq("E", "idp_organization_ref_uuid == org uuid", + vr.countWhere(vr.v2, "organizations", "idp_organization_ref_uuid <> uuid"), 0, "must be 0") + + // Template defaults. + vr.eq("E", "template group_id == handle", vr.countWhere(vr.v2, "llm_provider_templates", "group_id <> handle"), 0, "") + vr.eq("E", "template version == v1.0", vr.countWhere(vr.v2, "llm_provider_templates", "version <> 'v1.0'"), 0, "") + vr.eq("E", "template managed_by == organization", vr.countWhere(vr.v2, "llm_provider_templates", "managed_by <> 'organization'"), 0, "") + vr.eq("E", "template is_latest == 1", vr.countWhere(vr.v2, "llm_provider_templates", "is_latest <> 1"), 0, "") + vr.eq("E", "template enabled == 1", vr.countWhere(vr.v2, "llm_provider_templates", "enabled <> 1"), 0, "") + + // data_version / origin defaults where the columns exist. + for _, t := range []string{"organizations", "projects", "applications", "rest_apis", "llm_providers", "llm_proxies", "mcp_proxies", "websub_apis", "webbroker_apis", "subscription_plans", "gateways", "llm_provider_templates"} { + vr.eq("E", "data_version=1.0 in "+t, vr.countWhere(vr.v2, t, "data_version <> '1.0'"), 0, "") + } + for _, t := range []string{"rest_apis", "llm_providers", "llm_proxies", "mcp_proxies", "websub_apis", "webbroker_apis", "llm_provider_templates"} { + vr.eq("E", "origin=control_plane in "+t, vr.countWhere(vr.v2, t, "origin <> 'control_plane'"), 0, "") + } + + // Audit identity: every non-null created_by/updated_by resolves in user_idp_references. + auditTables := []string{"organizations", "projects", "applications", "rest_apis", "llm_providers", + "llm_proxies", "mcp_proxies", "websub_apis", "webbroker_apis", "subscription_plans", + "gateways", "gateway_custom_policies", "gateway_tokens", "deployments", "api_keys"} + unresolved := int64(0) + for _, t := range auditTables { + unresolved += vr.countWhere(vr.v2, t, "created_by IS NOT NULL AND created_by NOT IN (SELECT uuid FROM user_idp_references)") + } + vr.eq("E", "all created_by resolve in user_idp_references", unresolved, 0, "across audited tables") +} + +func (vr *verifier) checkCarriedHandles(v1table, v2table string) { + v1 := loadKV(vr.v1, "SELECT uuid, handle FROM "+v1table) + rows, err := vr.v2.Query("SELECT uuid, handle FROM " + v2table) + if err != nil { + vr.fail("E", "carried handle "+v2table, err.Error()) + return + } + defer rows.Close() + mismatch, suffixed := 0, 0 + for rows.Next() { + var uuid, h string + _ = rows.Scan(&uuid, &h) + src, ok := v1[uuid] + if !ok { + continue + } + if h != slug(src) { + // A collision suffix or degenerate-short pad is expected/allowed; only a + // wholesale divergence from the slug prefix is a real problem. + if len(h) >= 5 && len(slug(src)) >= 3 && h[:min2(len(slug(src)), len(h))] != slug(src)[:min2(len(slug(src)), len(h))] { + mismatch++ + } else { + suffixed++ + } + } + } + st := statusPass + if mismatch > 0 { + st = statusFail + } + vr.add(Check{Layer: "E", Name: "carried handle == slug(v1) in " + v2table, Status: st, + Detail: fmt.Sprintf("%d divergent, %d collision/degenerate-suffixed (tolerated)", mismatch, suffixed)}) +} + +// ---- Layer F: drop reconciliation + quarantine sign-off gate ---- + +func (vr *verifier) layerF_dropReconcile() { + drops := vr.loadDropsByFeature() + + // Intentional row-drops reconcile with v1 source counts. + vr.eq("F", "drop devportals == v1 devportals", drops["devportals"], vr.count(vr.v1, "devportals"), "") + vr.eq("F", "drop publication_mappings == v1 publication_mappings", drops["publication_mappings"], vr.count(vr.v1, "publication_mappings"), "") + vr.eq("F", "drop dev_portal assoc == v1", drops["dev_portal_association"], vr.countWhere(vr.v1, "association_mappings", "association_type = 'dev_portal'"), "") + // Field-drops reconcile with v1 source counts. + vr.eq("F", "drop billing_plan == v1 plans with billing_plan", drops["billing_plan"], vr.countWhere(vr.v1, "subscription_plans", "billing_plan IS NOT NULL AND billing_plan <> ''"), "") + vr.eq("F", "drop llm_provider_status == v1", drops["llm_provider_status"], vr.countWhere(vr.v1, "llm_providers", "status IS NOT NULL AND status <> ''"), "") + vr.eq("F", "drop llm_proxy_status == v1", drops["llm_proxy_status"], vr.countWhere(vr.v1, "llm_proxies", "status IS NOT NULL AND status <> ''"), "") + vr.eq("F", "drop mcp_status == v1", drops["mcp_status"], vr.countWhere(vr.v1, "mcp_proxies", "status IS NOT NULL AND status <> ''"), "") + + // 1:1 / renamed table reconciliation: v2 + quarantine == v1. + recon := []struct{ v1t, v2t string }{ + {"organizations", "organizations"}, + {"projects", "projects"}, + {"applications", "applications"}, + {"subscription_plans", "subscription_plans"}, + {"subscriptions", "subscriptions"}, + {"gateways", "gateways"}, + {"gateway_custom_policies", "gateway_custom_policies"}, + {"gateway_custom_policy_usages", "gateway_custom_policy_usages"}, + {"gateway_tokens", "gateway_tokens"}, + {"deployments", "deployments"}, + {"deployment_status", "deployment_status"}, + {"api_keys", "api_keys"}, + {"application_api_keys", "application_api_key_mappings"}, + {"application_artifacts", "application_artifact_mappings"}, + } + for _, r := range recon { + v1c := vr.count(vr.v1, r.v1t) + v2c := vr.count(vr.v2, r.v2t) + quar := vr.quarByTable[r.v2t] + vr.add(Check{Layer: "F", Name: "reconcile " + r.v1t, Status: passIf(v2c+quar == v1c), + V1: i64(v1c), V2: i64(v2c), Expected: i64(v1c), + Detail: fmt.Sprintf("v2(%d)+quarantine(%d)==v1(%d)", v2c, quar, v1c)}) + } + + // Quarantine sign-off gate: every quarantined key resolved in v2 or signed off. + unresolved := 0 + var sample []string + for key := range vr.quarKeys { + if vr.signoff[key] { + continue + } + // "resolved in v2" = the key's uuid is now present. Best-effort: the key's + // first path component is the source table; we cannot cheaply re-check every + // composite key, so a signed-off OR present-in-v2 decision is required. + unresolved++ + if len(sample) < 20 { + sample = append(sample, key) + } + } + st := statusPass + detail := fmt.Sprintf("all %d quarantined key(s) signed off or resolved", len(vr.quarKeys)) + if unresolved > 0 { + st = statusFail + detail = fmt.Sprintf("%d quarantined key(s) NOT signed off (e.g. %v)", unresolved, sample) + } + vr.add(Check{Layer: "F", Name: "quarantine sign-off", Status: st, Detail: detail}) +} + +func (vr *verifier) loadDropsByFeature() map[string]int64 { + m := map[string]int64{} + path := filepathJoin(vr.opts.OutDir, suffixed("drops", vr.opts.RunID, false, "jsonl")) + forEachJSONL(path, func(rec map[string]any) { + if f, ok := rec["feature"].(string); ok { + m[f]++ + } + }) + return m +} + +// ---- small helpers ---- + +func passIf(ok bool) string { + if ok { + return statusPass + } + return statusFail +} + +func loadKV(db *database.DB, query string) map[string]string { + m := map[string]string{} + rows, err := db.Query(query) + if err != nil { + return m + } + defer rows.Close() + for rows.Next() { + var k string + var v sql.NullString + if rows.Scan(&k, &v) == nil { + m[k] = v.String + } + } + return m +} + +func stringsEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + sa := append([]string(nil), a...) + sb := append([]string(nil), b...) + sort.Strings(sa) + sort.Strings(sb) + for i := range sa { + if sa[i] != sb[i] { + return false + } + } + return true +} + +func min2(a, b int) int { + if a < b { + return a + } + return b +} + +// filepathJoin builds an output-file path (out-dir is always a plain local dir). +func filepathJoin(dir, name string) string { return dir + "/" + name } diff --git a/platform-api/migrationcore/README.md b/platform-api/migrationcore/README.md new file mode 100644 index 0000000000..46037f765e --- /dev/null +++ b/platform-api/migrationcore/README.md @@ -0,0 +1,65 @@ +# migrationcore + +The **one implementation** of the Platform API v1→v2 per-row transform + idempotent +v2 write, shared by two callers: + +- **Batch backfill** — `cmd/dbmigrate` (the one-time migrator). +- **Live dual-write intermediate** — the v1 build that mirrors each v1 mutation into + the v2 DB after the v1 repo write (spec: `../../smooth-migration'/agent-prompt.md`). + +Both call the SAME `UpsertX`/`DeleteX`/`ResolveIdentity` so config-blob reshapes, +type conversions, handle/identity rules and the v2 column layout can never drift +between the backfill and the live path. + +## What's here + +| file | contents | +|---|---| +| `transform.go` | pure config-blob reshapes (`ReshapeRestAPIConfig`, `ReshapeWebSubConfig`+`PreprocessWebSubRaw`, `ReshapeWebBrokerConfig`, `RemarshalConfig`) and type conversions (`ReinterpretTZ`, `BoolToSmallint`, `TruncateStr`, `Slug`, `ParseTransport`) | +| `core.go` | `Options`, `Reporter`, `Execer`, the idempotent `upsert`/`deleteWhere` engine, `ResolveIdentity`, `DeterministicUUID` | +| `upsert.go` | per-entity `V1Row` structs + `UpsertX` (each writes an entity's FULL v2 footprint — `artifacts` + type row + child rows) | +| `delete.go` | per-entity `DeleteX` (live-only; reproduce v2's cascade shape) | +| `upsert_pg_test.go` | Postgres affordance tests (upsert/update, InsertOnly, delete, incremental identity, dry-run) | + +## Design contract + +- **Caller owns the transaction.** `UpsertX(ex Execer, row V1Row, opts Options, rep Reporter)` + takes a `*sql.Tx | *sql.DB`, so one entity's artifact + type + child rows upsert atomically. +- **Parameterized, no globals.** Everything comes via `Options{ EncryptionKey, SourceTZ, + Epoch, InsertOnly, SkipIdentityUpsert, DryRun }`. The core reads no env/flags. + **The live path MUST use the same `EncryptionKey` + `Epoch` as the batch backfill**, or + synthesized UUIDs / token passthrough diverge. +- **Idempotent.** `InsertOnly` → `ON CONFLICT DO NOTHING` (batch); otherwise `DO UPDATE` + (live UPDATE mirroring). `DeleteX` mirrors deletes. +- **Pluggable reporting.** The core emits flags/quarantine/drops through `Reporter`; the + batch writes the JSONL files, the live path writes logs + a `v2_dual_write_failures` table. +- **Batch-only concerns stay in `cmd/dbmigrate`:** table iteration + FK order, FK-parent + gating, handle persist-and-replay, checkpoint/resume, dropped-field discovery, DDL init. + +Caller wiring: + +``` +batch (cmd/dbmigrate): Options{InsertOnly:true, SkipIdentityUpsert:true, DryRun:cfg.DryRun} + Reporter = the JSONL Run +live (v1 dual-write): Options{InsertOnly:false, SkipIdentityUpsert:false} + Reporter = logs + v2_dual_write_failures +``` + +## Validated (2026-08-23) + +Extracted from the already-validated `cmd/dbmigrate` with a regression gate: + +1. **Byte-identical batch behavior** — `dbmigrate migrate -dry-run` on the real choreo dump: + report aggregates, drops, and quarantine identical to pre-extraction; flags identical + except the `artifact_gateway_mappings` audit `source_key` was reformatted from the + batch-only `assoc:` to the natural `artifact|gateway` key (same 108 rows — + the live path produces that key too). +2. **Live + verify** — refactored batch → fresh v2 → `dbmigrate verify` **PASS (121/1/0)**. +3. **Affordance tests** (`go test ./migrationcore` with `MIGRATIONCORE_TEST_DSN`) — upsert + INSERT, `ON CONFLICT DO UPDATE`, `InsertOnly` DO-NOTHING, `DeleteX`, incremental + `ResolveIdentity` (deterministic + idempotent), and `DryRun` (no write) all pass. + +## Pin + +Built against v2 `internal/model` at commit `a2911a091…`. Deploy the batch, the live build, +and this package from the same revision — struct/DDL skew silently corrupts blobs. diff --git a/platform-api/migrationcore/core.go b/platform-api/migrationcore/core.go new file mode 100644 index 0000000000..e658311475 --- /dev/null +++ b/platform-api/migrationcore/core.go @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package migrationcore + +import ( + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "github.com/wso2/api-platform/platform-api/internal/utils" +) + +// Data-version / origin constants written on every migrated row. +const ( + DataVersion = "1.0" + OriginCP = "control_plane" +) + +// MigrationActorIDPID is the synthetic IdP identity attributed to rows whose v1 +// audit value is NULL/empty. Seeded into user_idp_references like any identity so +// it resolves to a real principal (not DeletedUser). +const MigrationActorIDPID = "migration" + +// ErrBlobUnparseable signals that a config blob would not unmarshal into the v2 +// model — the caller should quarantine (batch) or fail the mutation (live) rather +// than write a half-artifact. +var ErrBlobUnparseable = errors.New("config blob does not unmarshal into the v2 model") + +// Options parameterizes the shared core. NO global env/flags are read inside the +// core — the batch main builds this from flags/env, the live path from v1 config. +// The EncryptionKey + Epoch used live MUST equal the batch backfill's, or +// synthesized UUIDs / token passthrough diverge. +type Options struct { + EncryptionKey []byte // subscription-token key (verbatim passthrough; used only by the decrypt guard) + SourceTZ *time.Location // timezone of naive v1 TIMESTAMP values (default UTC) + Epoch time.Time // fixed epoch for deterministic synthesized UUIDs + // InsertOnly makes upserts ON CONFLICT DO NOTHING (batch backfill semantics). + // The live dual-write path sets this false so UPDATEs mirror via DO UPDATE. + InsertOnly bool + // SkipIdentityUpsert: when true (batch, which bulk-pre-seeds user_idp_references) + // ResolveIdentity computes the deterministic UUID without a per-row write. The + // live path leaves it false so each mutation upserts its actor on demand. + SkipIdentityUpsert bool + // DryRun runs every transform + Reporter event but performs NO database write + // (the batch's --dry-run transform-and-validate pass). The live path never sets it. + DryRun bool +} + +func (o Options) loc() *time.Location { + if o.SourceTZ == nil { + return time.UTC + } + return o.SourceTZ +} + +// Reporter receives per-row migrate-with-flag / quarantine / drop events. The batch +// implements it with the JSONL files (output unchanged); the live path implements +// it with structured logs + the v2_dual_write_failures table. +type Reporter interface { + Flag(table, key, code string, oldV, newV any) + Quarantine(table, key, code, detail string, row any) + Dropped(scope, table, key, feature, label string) + DroppedFields(structName string, fields []string) +} + +// NopReporter discards all events (for callers that don't need reporting). +type NopReporter struct{} + +func (NopReporter) Flag(string, string, string, any, any) {} +func (NopReporter) Quarantine(string, string, string, string, any) {} +func (NopReporter) Dropped(string, string, string, string, string) {} +func (NopReporter) DroppedFields(string, []string) {} + +// Execer is satisfied by *sql.DB and *sql.Tx (and *database.DB, which embeds +// *sql.DB). The caller owns the transaction: pass a *sql.Tx to make one entity's +// full v2 footprint (artifact + type row + child rows) upsert atomically. +type Execer interface { + Exec(query string, args ...any) (sql.Result, error) + QueryRow(query string, args ...any) *sql.Row +} + +// immutableOnUpdate lists creation-audit columns that a live ON CONFLICT DO UPDATE +// must never overwrite (§8.2): once a row exists, its created_at/created_by are fixed, +// so a later mirroring UPDATE keeps the original creation audit rather than stamping it +// with the updater's time/identity. The batch backfill is InsertOnly, so its INSERT … +// DO NOTHING path never consults this set — batch output is byte-identical. +var immutableOnUpdate = map[string]bool{ + "created_at": true, + "created_by": true, +} + +// upsert executes an idempotent INSERT … ON CONFLICT with $n placeholders (postgres). +// InsertOnly → DO NOTHING (batch); otherwise DO UPDATE SET every non-conflict, +// non-immutable column = excluded.(live UPDATE mirroring; created_at/created_by +// are held immutable per §8.2). Empty conflictCols → plain INSERT. +func upsert(ex Execer, opts Options, table string, cols []string, args []any, conflictCols []string) error { + if len(cols) != len(args) { + return fmt.Errorf("upsert %s: %d cols but %d args", table, len(cols), len(args)) + } + ph := make([]string, len(cols)) + for i := range cols { + ph[i] = fmt.Sprintf("$%d", i+1) + } + q := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", table, strings.Join(cols, ", "), strings.Join(ph, ", ")) + if len(conflictCols) > 0 { + conflictSet := map[string]bool{} + for _, c := range conflictCols { + conflictSet[c] = true + } + set := make([]string, 0, len(cols)) + if !opts.InsertOnly { + for _, c := range cols { + if !conflictSet[c] && !immutableOnUpdate[c] { + set = append(set, c+" = excluded."+c) + } + } + } + if len(set) == 0 { + q += " ON CONFLICT (" + strings.Join(conflictCols, ", ") + ") DO NOTHING" + } else { + q += " ON CONFLICT (" + strings.Join(conflictCols, ", ") + ") DO UPDATE SET " + strings.Join(set, ", ") + } + } + if opts.DryRun { + return nil + } + if _, err := ex.Exec(q, args...); err != nil { + return fmt.Errorf("upsert %s: %w", table, err) + } + return nil +} + +// deleteWhere executes DELETE FROM table WHERE . Used only by the +// live path's DeleteX (batch never deletes). +func deleteWhere(ex Execer, table string, whereCols []string, whereArgs []any) error { + conds := make([]string, len(whereCols)) + for i, c := range whereCols { + conds[i] = fmt.Sprintf("%s = $%d", c, i+1) + } + q := fmt.Sprintf("DELETE FROM %s WHERE %s", table, strings.Join(conds, " AND ")) + if _, err := ex.Exec(q, whereArgs...); err != nil { + return fmt.Errorf("delete %s: %w", table, err) + } + return nil +} + +// DeterministicUUID is a thin alias over the product's deterministic UUIDv7 helper +// so synthesized PKs are idempotent/resumable without persisted state. +func DeterministicUUID(entity string, ts time.Time) string { + return utils.GenerateDeterministicUUIDv7(entity, ts) +} + +// ResolveIdentity maps a raw v1 actor to its internal v2 user UUID (§C.1). Empty → +// the migration actor (defaulted=true). The UUID is deterministic; when +// opts.SkipIdentityUpsert is false and ex != nil, it also upserts the +// user_idp_references row on demand (ON CONFLICT (idp_id) DO NOTHING) — the live +// path's incremental seeding. The batch pre-seeds and sets SkipIdentityUpsert. +func ResolveIdentity(ex Execer, rawActor string, opts Options) (uuid string, defaulted bool, err error) { + idpID := strings.TrimSpace(rawActor) + if idpID == "" { + idpID = MigrationActorIDPID + defaulted = true + } + uuid = utils.GenerateDeterministicUUIDv7(idpID, opts.Epoch) + if !opts.SkipIdentityUpsert && !opts.DryRun && ex != nil { + if _, err = ex.Exec( + `INSERT INTO user_idp_references (uuid, idp_id, created_at) VALUES ($1, $2, $3) ON CONFLICT (idp_id) DO NOTHING`, + uuid, idpID, opts.Epoch); err != nil { + return "", defaulted, fmt.Errorf("resolve identity %q: %w", idpID, err) + } + } + return uuid, defaulted, nil +} + +// audit resolves the created_by actor into the v2 audit UUID and, when the source +// was NULL/empty, emits a DEFAULTED_NULL flag (§C.1). updated_by == created_by. +func audit(ex Execer, opts Options, rep Reporter, table, key, rawActor string) (string, error) { + uuid, defaulted, err := ResolveIdentity(ex, rawActor, opts) + if err != nil { + return "", err + } + if defaulted { + rep.Flag(table, key, FlagDefaultedNull, + map[string]any{"created_by": nil}, + map[string]any{"created_by": uuid, "actor": MigrationActorIDPID}) + } + return uuid, nil +} + +// tsArg converts a nullable v1 TIMESTAMP into a TIMESTAMPTZ insert arg (UTC +// instant) or nil. Never substitutes now(). +func tsArg(t *time.Time, opts Options) any { + if t == nil { + return nil + } + return ReinterpretTZ(*t, opts.loc()) +} + +// Flag / quarantine / drop codes (shared vocabulary; the batch's JSONL and the +// live path's failure table use the same strings). +const ( + FlagDefaultedNull = "DEFAULTED_NULL" + FlagTruncated = "TRUNCATED" + FlagPlaceholderIDP = "PLACEHOLDER_IDP" + FlagSynthesized = "SYNTHESIZED" + FlagPlaintextCredential = "PLAINTEXT_CREDENTIAL" + + ReasonBlobUnparseable = "BLOB_UNPARSEABLE" + + DropBillingPlan = "billing_plan" + DropLLMProviderStatus = "llm_provider_status" + DropLLMProxyStatus = "llm_proxy_status" + DropMCPStatus = "mcp_status" +) diff --git a/platform-api/migrationcore/delete.go b/platform-api/migrationcore/delete.go new file mode 100644 index 0000000000..9f6d98c9fc --- /dev/null +++ b/platform-api/migrationcore/delete.go @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package migrationcore + +// Per-entity deletes for the LIVE dual-write path (the batch backfill never calls +// these). They reproduce v2's cascade shape: deleting the parent row lets the v2 +// FK ON DELETE CASCADE remove the type row and artifact_uuid/gateway_uuid children; +// tables without a cascading FK are deleted explicitly by their composite key. + +// DeleteArtifact removes an artifact (any of the six types). The v2 FKs +// (rest_apis/llm_*/mcp/websub/webbroker .uuid→artifacts, and api_keys/deployments/ +// deployment_status/subscriptions/mappings .artifact_uuid→artifacts, all ON DELETE +// CASCADE) remove the type row and every artifact-keyed child. +func DeleteArtifact(ex Execer, uuid string) error { + return deleteWhere(ex, "artifacts", []string{"uuid"}, []any{uuid}) +} + +// DeleteOrganization removes an org; org-scoped rows cascade via their FKs. +func DeleteOrganization(ex Execer, uuid string) error { + return deleteWhere(ex, "organizations", []string{"uuid"}, []any{uuid}) +} + +// DeleteProject removes a project (project-scoped children cascade). +func DeleteProject(ex Execer, uuid string) error { + return deleteWhere(ex, "projects", []string{"uuid"}, []any{uuid}) +} + +// DeleteApplication removes an application (its mappings cascade). +func DeleteApplication(ex Execer, uuid string) error { + return deleteWhere(ex, "applications", []string{"uuid"}, []any{uuid}) +} + +// DeleteLLMProviderTemplate removes a template (blocked by FK if providers reference it). +func DeleteLLMProviderTemplate(ex Execer, uuid string) error { + return deleteWhere(ex, "llm_provider_templates", []string{"uuid"}, []any{uuid}) +} + +// DeleteSubscriptionPlan removes a plan; subscription_plan_limits cascade. +func DeleteSubscriptionPlan(ex Execer, uuid string) error { + return deleteWhere(ex, "subscription_plans", []string{"uuid"}, []any{uuid}) +} + +// DeleteSubscription removes a single subscription. +func DeleteSubscription(ex Execer, uuid string) error { + return deleteWhere(ex, "subscriptions", []string{"uuid"}, []any{uuid}) +} + +// DeleteGateway removes a gateway; gateway_endpoints and gateway_tokens cascade. +func DeleteGateway(ex Execer, uuid string) error { + return deleteWhere(ex, "gateways", []string{"uuid"}, []any{uuid}) +} + +// DeleteGatewayToken removes a single gateway token. +func DeleteGatewayToken(ex Execer, uuid string) error { + return deleteWhere(ex, "gateway_tokens", []string{"uuid"}, []any{uuid}) +} + +// DeleteGatewayCustomPolicy removes a policy; gateway_custom_policy_usages cascade. +func DeleteGatewayCustomPolicy(ex Execer, uuid string) error { + return deleteWhere(ex, "gateway_custom_policies", []string{"uuid"}, []any{uuid}) +} + +// DeleteAPIKey removes an api key; application_api_key_mappings cascade. +func DeleteAPIKey(ex Execer, uuid string) error { + return deleteWhere(ex, "api_keys", []string{"uuid"}, []any{uuid}) +} + +// DeleteDeployment removes a single deployment (deployment_status cascades). +func DeleteDeployment(ex Execer, uuid string) error { + return deleteWhere(ex, "deployments", []string{"uuid"}, []any{uuid}) +} + +// ---- composite-key children (deleted directly when only the mapping changes) ---- + +// DeleteArtifactGatewayMapping removes one artifact↔gateway association. +func DeleteArtifactGatewayMapping(ex Execer, org, artifactUUID, gatewayUUID string) error { + return deleteWhere(ex, "artifact_gateway_mappings", + []string{"organization_uuid", "artifact_uuid", "gateway_uuid"}, []any{org, artifactUUID, gatewayUUID}) +} + +// DeleteApplicationAPIKeyMapping removes one application↔api-key mapping. +func DeleteApplicationAPIKeyMapping(ex Execer, applicationUUID, apiKeyID string) error { + return deleteWhere(ex, "application_api_key_mappings", + []string{"application_uuid", "api_key_id"}, []any{applicationUUID, apiKeyID}) +} + +// DeleteApplicationArtifactMapping removes one application↔artifact mapping. +func DeleteApplicationArtifactMapping(ex Execer, applicationUUID, artifactUUID string) error { + return deleteWhere(ex, "application_artifact_mappings", + []string{"application_uuid", "artifact_uuid"}, []any{applicationUUID, artifactUUID}) +} + +// DeletePolicyUsage removes one policy-usage row. +func DeletePolicyUsage(ex Execer, policyUUID, artifactUUID string) error { + return deleteWhere(ex, "gateway_custom_policy_usages", + []string{"policy_uuid", "artifact_uuid"}, []any{policyUUID, artifactUUID}) +} + +// DeleteDeploymentStatus removes the current-state row for an artifact+gateway. +func DeleteDeploymentStatus(ex Execer, org, artifactUUID, gatewayUUID string) error { + return deleteWhere(ex, "deployment_status", + []string{"organization_uuid", "artifact_uuid", "gateway_uuid"}, []any{org, artifactUUID, gatewayUUID}) +} diff --git a/platform-api/migrationcore/transform.go b/platform-api/migrationcore/transform.go new file mode 100644 index 0000000000..4c09f87f7e --- /dev/null +++ b/platform-api/migrationcore/transform.go @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package migrationcore holds the ONE implementation of the Platform API v1→v2 +// per-row transform + idempotent v2 write, shared by the batch migrator +// (cmd/dbmigrate) and the live dual-write intermediate v1 build. It contains only +// pure per-row logic and idempotent upserts — no table iteration, quarantine +// bookkeeping, checkpointing, or DDL init (those stay in the batch caller). +package migrationcore + +import ( + "encoding/json" + "reflect" + "regexp" + "strings" + "time" + + "github.com/wso2/api-platform/platform-api/internal/model" +) + +// ReinterpretTZ treats t's wall-clock as being in loc and returns the equivalent +// UTC instant. v1 stores naive TIMESTAMP; pgx hands them back with a UTC location, +// so for source-tz=UTC this is identity. Used identically by migrate and verify. +func ReinterpretTZ(t time.Time, loc *time.Location) time.Time { + if loc == nil || loc == time.UTC { + return t.UTC() + } + y, mo, d := t.Date() + h, mi, s := t.Clock() + return time.Date(y, mo, d, h, mi, s, t.Nanosecond(), loc).UTC() +} + +// BoolToSmallint maps a SQL boolean to v2's SMALLINT convention. +func BoolToSmallint(b bool) int { + if b { + return 1 + } + return 0 +} + +// TruncateStr truncates s to max runes, reporting whether truncation occurred. +func TruncateStr(s string, max int) (string, bool) { + if len(s) <= max { + return s, false + } + r := []rune(s) + if len(r) <= max { + return s, false + } + return string(r[:max]), true +} + +var ( + slugInvalidChars = regexp.MustCompile(`[^a-z0-9\-_ ]`) + slugMultiHyphen = regexp.MustCompile(`-+`) +) + +// Slug reproduces the deterministic part of internal/utils.sanitizeToHandle (no +// random padding). Keep in sync with internal/utils/handle.go. +func Slug(s string) string { + h := strings.ToLower(s) + h = strings.ReplaceAll(h, " ", "-") + h = strings.ReplaceAll(h, "_", "-") + h = slugInvalidChars.ReplaceAllString(h, "") + h = slugMultiHyphen.ReplaceAllString(h, "-") + h = strings.Trim(h, "-") + if len(h) > 40 { + h = h[:40] + h = strings.TrimRight(h, "-") + } + return h +} + +// ParseTransport parses the v1 `transport` column (a JSON array stored as TEXT, +// e.g. `["https","wss"]`). Empty/invalid input yields a nil slice. +func ParseTransport(s string) []string { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + var out []string + if err := json.Unmarshal([]byte(s), &out); err != nil { + return nil + } + return out +} + +// jsonTagSet returns the set of top-level json field names on v (a struct or +// pointer to one), ignoring "-" and honoring embedded structs. +func jsonTagSet(v any) map[string]bool { + set := map[string]bool{} + t := reflect.TypeOf(v) + for t != nil && t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t == nil || t.Kind() != reflect.Struct { + return set + } + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.Anonymous { + for k := range jsonTagSet(reflect.New(f.Type).Interface()) { + set[k] = true + } + continue + } + tag := f.Tag.Get("json") + if tag == "" { + set[f.Name] = true + continue + } + name := strings.Split(tag, ",")[0] + if name == "-" { + continue + } + if name == "" { + name = f.Name + } + set[name] = true + } + return set +} + +// TopLevelUnknownFields returns the top-level keys present in the v1 config JSON +// that the v2 struct target does NOT have — fields the permissive unmarshal would +// silently drop (§E discovery). Catches struct-level unknowns only. +func TopLevelUnknownFields(raw []byte, target any) []string { + var top map[string]json.RawMessage + if err := json.Unmarshal(raw, &top); err != nil { + return nil + } + known := jsonTagSet(target) + var unknown []string + for k := range top { + if !known[k] { + unknown = append(unknown, k) + } + } + return unknown +} + +// ReshapeRestAPIConfig unmarshals a v1 rest_apis.configuration blob into the v2 +// RestAPIConfig, injects the moved `transport` column, and re-marshals to BYTEA. +func ReshapeRestAPIConfig(raw []byte, transportCol string) (out []byte, cfg model.RestAPIConfig, unknown []string, err error) { + unknown = TopLevelUnknownFields(raw, &cfg) + if err = json.Unmarshal(raw, &cfg); err != nil { + return nil, cfg, unknown, err + } + if tr := ParseTransport(transportCol); len(tr) > 0 { + cfg.Transport = tr + } + out, err = json.Marshal(&cfg) + return out, cfg, unknown, err +} + +// ReshapeWebSubConfig reshapes a v1 WebSub config into the v2 struct. Beyond the +// transport injection, two STRUCTURAL v1→v2 differences (discovered from real data, +// contradicting the "strict subset" premise) are handled losslessly: +// +// - channels: v1 stores an ARRAY [{request:{name,method}}]; v2 wants a +// map[string]WebSubChannel keyed by channel name. → key = request.name, +// value = empty WebSubChannel (method is always SUBSCRIBE, carries no info). +// - policies (top-level, legacy): folded into allChannels — object-form +// {event:[p]} → allChannels.event.policies; flat array [p] (whole-API auth) → +// allChannels.on_subscription.policies. +// +// notes describes any structural remap applied (recorded as a flag by the caller). +func ReshapeWebSubConfig(raw []byte, transportCol string) (out []byte, cfg model.WebSubAPIConfiguration, unknown, notes []string, err error) { + fixed, notes, err := PreprocessWebSubRaw(raw) + if err != nil { + return nil, cfg, nil, notes, err + } + unknown = TopLevelUnknownFields(fixed, &cfg) + if err = json.Unmarshal(fixed, &cfg); err != nil { + return nil, cfg, unknown, notes, err + } + if tr := ParseTransport(transportCol); len(tr) > 0 { + cfg.Transport = tr + } + out, err = json.Marshal(&cfg) + return out, cfg, unknown, notes, err +} + +// jsonKind returns the first non-whitespace byte of a JSON value ('{', '[', ...), +// or 0 for empty/whitespace-only input. +func jsonKind(raw []byte) byte { + for _, b := range raw { + switch b { + case ' ', '\t', '\n', '\r': + continue + default: + return b + } + } + return 0 +} + +// PreprocessWebSubRaw applies the two structural fixups (channels array→map, +// top-level policies→allChannels) at the JSON level, before struct unmarshal. +func PreprocessWebSubRaw(raw []byte) ([]byte, []string, error) { + var top map[string]json.RawMessage + if err := json.Unmarshal(raw, &top); err != nil { + return raw, nil, err + } + var notes []string + + // channels: array → map keyed by request.name. + if ch, ok := top["channels"]; ok && jsonKind(ch) == '[' { + var arr []struct { + Request struct { + Name string `json:"name"` + Method string `json:"method"` + } `json:"request"` + } + if err := json.Unmarshal(ch, &arr); err == nil { + m := map[string]json.RawMessage{} + for _, e := range arr { + if e.Request.Name != "" { + m[e.Request.Name] = json.RawMessage("{}") + } + } + b, _ := json.Marshal(m) + top["channels"] = b + notes = append(notes, "channels[] -> map (keyed by request.name)") + } + } + + // policies (top-level legacy) → allChannels. + if pol, ok := top["policies"]; ok && jsonKind(pol) != 0 && string(pol) != "null" { + allch := map[string]json.RawMessage{} + if ac, ok := top["allChannels"]; ok && jsonKind(ac) == '{' { + _ = json.Unmarshal(ac, &allch) + } + wrap := func(policyArray json.RawMessage) json.RawMessage { + b, _ := json.Marshal(map[string]json.RawMessage{"policies": policyArray}) + return b + } + switch jsonKind(pol) { + case '{': // object-form: {event: [policy...]} + var byEvent map[string]json.RawMessage + if json.Unmarshal(pol, &byEvent) == nil { + for event, arr := range byEvent { + if _, exists := allch[event]; !exists { + allch[event] = wrap(arr) + } + } + notes = append(notes, "policies{event:[...]} -> allChannels") + } + case '[': // flat array of whole-API policies (auth) → on_subscription + var arr []json.RawMessage + if json.Unmarshal(pol, &arr) == nil && len(arr) > 0 { + if _, exists := allch["on_subscription"]; !exists { + allch["on_subscription"] = wrap(pol) + } + notes = append(notes, "policies[] (whole-API) -> allChannels.on_subscription") + } + } + delete(top, "policies") + if len(allch) > 0 { + b, _ := json.Marshal(allch) + top["allChannels"] = b + } + } + + out, err := json.Marshal(top) + return out, notes, err +} + +// ReshapeWebBrokerConfig mirrors ReshapeRestAPIConfig for WebBroker configs (a +// strict subset of v2 + transport). +func ReshapeWebBrokerConfig(raw []byte, transportCol string) (out []byte, cfg model.WebBrokerAPIConfiguration, unknown []string, err error) { + unknown = TopLevelUnknownFields(raw, &cfg) + if err = json.Unmarshal(raw, &cfg); err != nil { + return nil, cfg, unknown, err + } + if tr := ParseTransport(transportCol); len(tr) > 0 { + cfg.Transport = tr + } + out, err = json.Marshal(&cfg) + return out, cfg, unknown, err +} + +// RemarshalConfig unmarshals a v1 config blob into the given v2 config struct +// (target must be a pointer) and re-marshals it to BYTEA, returning any dropped +// top-level fields. Used for LLM provider/proxy and MCP configs (no field moves). +func RemarshalConfig(raw []byte, target any) (out []byte, unknown []string, err error) { + unknown = TopLevelUnknownFields(raw, target) + if err = json.Unmarshal(raw, target); err != nil { + return nil, unknown, err + } + out, err = json.Marshal(target) + return out, unknown, err +} diff --git a/platform-api/migrationcore/upsert.go b/platform-api/migrationcore/upsert.go new file mode 100644 index 0000000000..9b7d67b98b --- /dev/null +++ b/platform-api/migrationcore/upsert.go @@ -0,0 +1,792 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package migrationcore + +import ( + "fmt" + "strings" + "time" + + "github.com/wso2/api-platform/platform-api/internal/constants" + "github.com/wso2/api-platform/platform-api/internal/model" +) + +// sarg returns a nullable-string insert arg (value or nil for NULL). +func sarg(p *string) any { + if p == nil { + return nil + } + return *p +} + +// insertArtifact writes the parent artifacts(uuid, type, organization_uuid) row. +func insertArtifact(ex Execer, opts Options, uuid, kind, org string) error { + return upsert(ex, opts, "artifacts", []string{"uuid", "type", "organization_uuid"}, + []any{uuid, kind, org}, []string{"uuid"}) +} + +// ---- organizations ---- + +// OrganizationRow mirrors the v1 organizations columns needed for the v2 write. +// Handle is pre-resolved by the caller (batch: persist-and-replay; live: slug+uuid). +type OrganizationRow struct { + UUID, Handle, DisplayName, Region string + CreatedAt, UpdatedAt *time.Time + CreatedBy string // raw v1 actor ("" ⇒ migration actor) +} + +func UpsertOrganization(ex Execer, r OrganizationRow, opts Options, rep Reporter) error { + // idp_organization_ref_uuid = org uuid (deterministic placeholder). + rep.Flag("organizations", r.UUID, FlagPlaceholderIDP, nil, + map[string]any{"idp_organization_ref_uuid": r.UUID}) + createdBy, err := audit(ex, opts, rep, "organizations", r.UUID, r.CreatedBy) + if err != nil { + return err + } + return upsert(ex, opts, "organizations", + []string{"uuid", "handle", "display_name", "region", "idp_organization_ref_uuid", + "data_version", "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.UUID, r.Handle, r.DisplayName, r.Region, r.UUID, DataVersion, createdBy, + tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"uuid"}) +} + +// ---- projects ---- + +type ProjectRow struct { + UUID, Handle, DisplayName, Org string + Description *string + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertProject(ex Execer, r ProjectRow, opts Options, rep Reporter) error { + createdBy, err := audit(ex, opts, rep, "projects", r.UUID, r.CreatedBy) + if err != nil { + return err + } + return upsert(ex, opts, "projects", + []string{"uuid", "handle", "display_name", "organization_uuid", "description", + "data_version", "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.UUID, r.Handle, r.DisplayName, r.Org, sarg(r.Description), DataVersion, createdBy, + tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"uuid"}) +} + +// ---- applications ---- + +type ApplicationRow struct { + UUID, Handle, ProjectUUID, Org, DisplayName, Type string + Description *string + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertApplication(ex Execer, r ApplicationRow, opts Options, rep Reporter) error { + createdBy, err := audit(ex, opts, rep, "applications", r.UUID, r.CreatedBy) + if err != nil { + return err + } + return upsert(ex, opts, "applications", + []string{"uuid", "handle", "project_uuid", "organization_uuid", "display_name", "description", + "type", "data_version", "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.UUID, r.Handle, r.ProjectUUID, r.Org, r.DisplayName, sarg(r.Description), r.Type, + DataVersion, createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"uuid"}) +} + +// ---- rest_apis (RestApi) ---- + +type RestAPIRow struct { + UUID, Handle, DisplayName, Version, Org, ProjectUUID string + Description, Lifecycle, Transport *string + Configuration []byte // v1 JSONB + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertRestAPI(ex Execer, r RestAPIRow, opts Options, rep Reporter) error { + blob, _, unknown, err := ReshapeRestAPIConfig(r.Configuration, deref(r.Transport)) + if err != nil { + rep.Quarantine("rest_apis", r.UUID, ReasonBlobUnparseable, err.Error(), + map[string]any{"uuid": r.UUID}) + return ErrBlobUnparseable + } + rep.DroppedFields("RestAPIConfig", unknown) + lc := lifecycleOr(r.Lifecycle) + createdBy, err := audit(ex, opts, rep, "rest_apis", r.UUID, r.CreatedBy) + if err != nil { + return err + } + if err := insertArtifact(ex, opts, r.UUID, constants.RestApi, r.Org); err != nil { + return err + } + return upsert(ex, opts, "rest_apis", + []string{"uuid", "organization_uuid", "handle", "display_name", "version", "project_uuid", + "description", "lifecycle_status", "configuration", "data_version", "origin", + "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.UUID, r.Org, r.Handle, r.DisplayName, r.Version, r.ProjectUUID, sarg(r.Description), lc, blob, + DataVersion, OriginCP, createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"uuid"}) +} + +// ---- llm_provider_templates ---- + +type LLMTemplateRow struct { + UUID, Org, Handle, DisplayName string + Description *string + Configuration []byte // v1 TEXT + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertLLMProviderTemplate(ex Execer, r LLMTemplateRow, opts Options, rep Reporter) error { + groupID := r.Handle // group_id = the template's v1 handle (matches v2 create path) + rep.Flag("llm_provider_templates", r.UUID, FlagSynthesized, nil, map[string]any{ + "group_id": groupID, "version": "v1.0", "managed_by": constants.TemplateManagedByOrganization, + "is_latest": 1, "enabled": 1, "openapi_spec": nil}) + createdBy, err := audit(ex, opts, rep, "llm_provider_templates", r.UUID, r.CreatedBy) + if err != nil { + return err + } + return upsert(ex, opts, "llm_provider_templates", + []string{"uuid", "organization_uuid", "handle", "group_id", "display_name", "managed_by", + "version", "description", "configuration", "openapi_spec", "is_latest", "enabled", + "data_version", "origin", "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.UUID, r.Org, r.Handle, groupID, r.DisplayName, constants.TemplateManagedByOrganization, + "v1.0", sarg(r.Description), r.Configuration, nil, 1, 1, + DataVersion, OriginCP, createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"uuid"}) +} + +// ---- llm_providers (LlmProvider) ---- + +type LLMProviderRow struct { + UUID, Handle, DisplayName, Version, Org, TemplateUUID string + Description, OpenAPISpec, ModelList, Status *string + Configuration []byte + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertLLMProvider(ex Execer, r LLMProviderRow, opts Options, rep Reporter) error { + var cfg model.LLMProviderConfig + blob, unknown, err := RemarshalConfig(r.Configuration, &cfg) + if err != nil { + rep.Quarantine("llm_providers", r.UUID, ReasonBlobUnparseable, err.Error(), map[string]any{"uuid": r.UUID}) + return ErrBlobUnparseable + } + rep.DroppedFields("LLMProviderConfig", unknown) + flagPlaintextCredential(rep, "llm_providers", r.UUID, cfg.Security) + if r.Status != nil && *r.Status != "" { + rep.Dropped("field", "llm_providers", r.UUID, DropLLMProviderStatus, *r.Status) + } + createdBy, err := audit(ex, opts, rep, "llm_providers", r.UUID, r.CreatedBy) + if err != nil { + return err + } + if err := insertArtifact(ex, opts, r.UUID, constants.LLMProvider, r.Org); err != nil { + return err + } + return upsert(ex, opts, "llm_providers", + []string{"uuid", "handle", "display_name", "version", "description", "template_uuid", + "openapi_spec", "model_list", "configuration", "data_version", "origin", + "created_by", "created_at", "updated_by", "updated_at", "organization_uuid"}, + []any{r.UUID, r.Handle, r.DisplayName, r.Version, sarg(r.Description), r.TemplateUUID, + bytesOrNilP(r.OpenAPISpec), bytesOrNilP(r.ModelList), blob, DataVersion, OriginCP, + createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts), r.Org}, + []string{"uuid"}) +} + +// ---- llm_proxies (LlmProxy) ---- + +type LLMProxyRow struct { + UUID, Handle, DisplayName, Version, ProjectUUID, Org, ProviderUUID string + Description, OpenAPISpec, Status *string + Configuration []byte + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertLLMProxy(ex Execer, r LLMProxyRow, opts Options, rep Reporter) error { + var cfg model.LLMProxyConfig + blob, unknown, err := RemarshalConfig(r.Configuration, &cfg) + if err != nil { + rep.Quarantine("llm_proxies", r.UUID, ReasonBlobUnparseable, err.Error(), map[string]any{"uuid": r.UUID}) + return ErrBlobUnparseable + } + rep.DroppedFields("LLMProxyConfig", unknown) + flagPlaintextCredential(rep, "llm_proxies", r.UUID, cfg.Security) + if r.Status != nil && *r.Status != "" { + rep.Dropped("field", "llm_proxies", r.UUID, DropLLMProxyStatus, *r.Status) + } + createdBy, err := audit(ex, opts, rep, "llm_proxies", r.UUID, r.CreatedBy) + if err != nil { + return err + } + if err := insertArtifact(ex, opts, r.UUID, constants.LLMProxy, r.Org); err != nil { + return err + } + return upsert(ex, opts, "llm_proxies", + []string{"uuid", "handle", "display_name", "version", "project_uuid", "description", "provider_uuid", + "openapi_spec", "configuration", "data_version", "origin", + "created_by", "created_at", "updated_by", "updated_at", "organization_uuid"}, + []any{r.UUID, r.Handle, r.DisplayName, r.Version, r.ProjectUUID, sarg(r.Description), r.ProviderUUID, + bytesOrNilP(r.OpenAPISpec), blob, DataVersion, OriginCP, + createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts), r.Org}, + []string{"uuid"}) +} + +// ---- mcp_proxies (Mcp) ---- + +type MCPProxyRow struct { + UUID, Handle, DisplayName, Version, Org string + ProjectUUID, Description, Status *string + Configuration []byte + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertMCPProxy(ex Execer, r MCPProxyRow, opts Options, rep Reporter) error { + var cfg model.MCPProxyConfiguration + blob, unknown, err := RemarshalConfig(r.Configuration, &cfg) + if err != nil { + rep.Quarantine("mcp_proxies", r.UUID, ReasonBlobUnparseable, err.Error(), map[string]any{"uuid": r.UUID}) + return ErrBlobUnparseable + } + rep.DroppedFields("MCPProxyConfiguration", unknown) + if r.Status != nil && *r.Status != "" { + rep.Dropped("field", "mcp_proxies", r.UUID, DropMCPStatus, *r.Status) + } + createdBy, err := audit(ex, opts, rep, "mcp_proxies", r.UUID, r.CreatedBy) + if err != nil { + return err + } + if err := insertArtifact(ex, opts, r.UUID, constants.MCPProxy, r.Org); err != nil { + return err + } + return upsert(ex, opts, "mcp_proxies", + []string{"uuid", "handle", "display_name", "version", "project_uuid", "description", + "configuration", "data_version", "origin", + "created_by", "created_at", "updated_by", "updated_at", "organization_uuid"}, + []any{r.UUID, r.Handle, r.DisplayName, r.Version, sarg(r.ProjectUUID), sarg(r.Description), + blob, DataVersion, OriginCP, + createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts), r.Org}, + []string{"uuid"}) +} + +// ---- websub_apis (WebSubApi, plugin table) ---- + +type WebSubRow struct { + UUID, Handle, DisplayName, Version, Org, ProjectUUID string + Description, Lifecycle, Transport *string + Configuration []byte + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertWebSubAPI(ex Execer, r WebSubRow, opts Options, rep Reporter) error { + blob, _, unknown, notes, err := ReshapeWebSubConfig(r.Configuration, deref(r.Transport)) + if err != nil { + rep.Quarantine("websub_apis", r.UUID, ReasonBlobUnparseable, err.Error(), map[string]any{"uuid": r.UUID}) + return ErrBlobUnparseable + } + rep.DroppedFields("WebSubAPIConfiguration", unknown) + if len(notes) > 0 { + rep.Flag("websub_apis", r.UUID, FlagSynthesized, nil, map[string]any{"structural_reshape": notes}) + } + lc := lifecycleOr(r.Lifecycle) + createdBy, err := audit(ex, opts, rep, "websub_apis", r.UUID, r.CreatedBy) + if err != nil { + return err + } + if err := insertArtifact(ex, opts, r.UUID, constants.WebSubApi, r.Org); err != nil { + return err + } + return upsert(ex, opts, "websub_apis", + []string{"uuid", "organization_uuid", "handle", "display_name", "version", "project_uuid", + "description", "lifecycle_status", "configuration", "data_version", "origin", + "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.UUID, r.Org, r.Handle, r.DisplayName, r.Version, r.ProjectUUID, sarg(r.Description), lc, blob, + DataVersion, OriginCP, createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"uuid"}) +} + +// ---- webbroker_apis (WebBrokerApi, plugin table) ---- + +type WebBrokerRow struct { + UUID, Handle, DisplayName, Version, Org, ProjectUUID string + Description, Lifecycle, Transport *string + Configuration []byte + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertWebBrokerAPI(ex Execer, r WebBrokerRow, opts Options, rep Reporter) error { + blob, _, unknown, err := ReshapeWebBrokerConfig(r.Configuration, deref(r.Transport)) + if err != nil { + rep.Quarantine("webbroker_apis", r.UUID, ReasonBlobUnparseable, err.Error(), map[string]any{"uuid": r.UUID}) + return ErrBlobUnparseable + } + rep.DroppedFields("WebBrokerAPIConfiguration", unknown) + lc := lifecycleOr(r.Lifecycle) + createdBy, err := audit(ex, opts, rep, "webbroker_apis", r.UUID, r.CreatedBy) + if err != nil { + return err + } + if err := insertArtifact(ex, opts, r.UUID, constants.WebBrokerApi, r.Org); err != nil { + return err + } + return upsert(ex, opts, "webbroker_apis", + []string{"uuid", "organization_uuid", "handle", "display_name", "version", "project_uuid", + "description", "lifecycle_status", "configuration", "data_version", "origin", + "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.UUID, r.Org, r.Handle, r.DisplayName, r.Version, r.ProjectUUID, sarg(r.Description), lc, blob, + DataVersion, OriginCP, createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"uuid"}) +} + +// ---- subscription_plans (+ subscription_plan_limits) ---- + +type SubscriptionPlanRow struct { + UUID, Handle, DisplayName, Org, Status string + BillingPlan, ThrottleUnit *string + StopOnQuota *bool + ThrottleCount *int64 + ExpiryTime, CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +// ErrUnmappedThrottleUnit is returned when a v1 throttle unit has no v2 mapping. +type ErrUnmappedThrottleUnit struct{ Unit string } + +func (e ErrUnmappedThrottleUnit) Error() string { + return "unmapped throttle_limit_unit " + e.Unit +} + +func UpsertSubscriptionPlan(ex Execer, r SubscriptionPlanRow, opts Options, rep Reporter) error { + if r.BillingPlan != nil && *r.BillingPlan != "" { + rep.Dropped("field", "subscription_plans", r.UUID, DropBillingPlan, *r.BillingPlan) + } + createdBy, err := audit(ex, opts, rep, "subscription_plans", r.UUID, r.CreatedBy) + if err != nil { + return err + } + if err := upsert(ex, opts, "subscription_plans", + []string{"uuid", "handle", "display_name", "expiry_time", "organization_uuid", "status", + "data_version", "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.UUID, r.Handle, r.DisplayName, tsArg(r.ExpiryTime, opts), r.Org, r.Status, DataVersion, + createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"uuid"}); err != nil { + return err + } + + // Throttle → exactly one subscription_plan_limits row. On a live UPDATE (§8.3) the + // throttle unit may have changed (which shifts the row's natural key and would leave + // the old limit behind) or the throttle may have been cleared: upsert the desired + // limit, then delete any superseded limit rows so the v2 child state matches the v1 + // plan. Batch is InsertOnly, so the replacement deletes are skipped → byte-identical. + if r.ThrottleCount != nil && r.ThrottleUnit != nil && *r.ThrottleUnit != "" { + timeUnit, ok := CaseConvertThrottleUnit(*r.ThrottleUnit) + if !ok { + return ErrUnmappedThrottleUnit{Unit: *r.ThrottleUnit} + } + limitTS := opts.Epoch + if r.CreatedAt != nil { + limitTS = ReinterpretTZ(*r.CreatedAt, opts.loc()) + } + limitUUID := DeterministicUUID(r.UUID+"|"+constants.LimitTypeRequestCount+"|"+timeUnit, limitTS) + stop := 1 + if r.StopOnQuota != nil { + stop = BoolToSmallint(*r.StopOnQuota) + } + if err := upsert(ex, opts, "subscription_plan_limits", + []string{"uuid", "subscription_plan_uuid", "limit_type", "time_unit", "time_amount", + "limit_count", "limit_count_unit", "stop_on_quota_reach"}, + []any{limitUUID, r.UUID, constants.LimitTypeRequestCount, timeUnit, 1, *r.ThrottleCount, nil, stop}, + []string{"subscription_plan_uuid", "limit_type", "time_amount", "time_unit"}); err != nil { + return err + } + if !opts.InsertOnly && !opts.DryRun { + if _, err := ex.Exec("DELETE FROM subscription_plan_limits WHERE subscription_plan_uuid = $1 AND uuid <> $2", r.UUID, limitUUID); err != nil { + return fmt.Errorf("replace subscription_plan_limits for %s: %w", r.UUID, err) + } + } + } else if !opts.InsertOnly && !opts.DryRun { + // Throttle cleared on a live UPDATE → drop any stale limit rows. + if _, err := ex.Exec("DELETE FROM subscription_plan_limits WHERE subscription_plan_uuid = $1", r.UUID); err != nil { + return fmt.Errorf("clear subscription_plan_limits for %s: %w", r.UUID, err) + } + } + return nil +} + +// CaseConvertThrottleUnit maps a v1 PascalCase throttle unit to v2's UPPERCASE. +func CaseConvertThrottleUnit(u string) (string, bool) { + switch strings.ToLower(strings.TrimSpace(u)) { + case "min", "minute": + return constants.ThrottleLimitUnitMinute, true + case "hour": + return constants.ThrottleLimitUnitHour, true + case "day": + return constants.ThrottleLimitUnitDay, true + case "month": + return constants.ThrottleLimitUnitMonth, true + default: + return "", false + } +} + +// ---- subscriptions ---- + +type SubscriptionRow struct { + UUID, ArtifactUUID, SubscriberID, Token, Hash, Org, Status string + ApplicationID, PlanUUID *string + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertSubscription(ex Execer, r SubscriptionRow, opts Options, rep Reporter) error { + createdBy, err := audit(ex, opts, rep, "subscriptions", r.UUID, r.CreatedBy) + if err != nil { + return err + } + return upsert(ex, opts, "subscriptions", + []string{"uuid", "artifact_uuid", "subscriber_id", "application_id", "subscription_token", + "subscription_token_hash", "subscription_plan_uuid", "organization_uuid", "status", + "data_version", "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.UUID, r.ArtifactUUID, r.SubscriberID, sarg(r.ApplicationID), r.Token, r.Hash, sarg(r.PlanUUID), r.Org, r.Status, + DataVersion, createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"uuid"}) +} + +// ---- gateways (+ gateway_endpoints) ---- + +type GatewayRow struct { + UUID, Org, Handle, DisplayName, Version, FuncType, Vhost string + Description *string + Properties, Manifest []byte + IsCritical, IsActive bool + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertGateway(ex Execer, r GatewayRow, opts Options, rep Reporter) error { + ver, cut := TruncateStr(r.Version, 30) + if cut { + rep.Flag("gateways", r.UUID, FlagTruncated, map[string]any{"version": r.Version}, map[string]any{"version": ver}) + } + createdBy, err := audit(ex, opts, rep, "gateways", r.UUID, r.CreatedBy) + if err != nil { + return err + } + if err := upsert(ex, opts, "gateways", + []string{"uuid", "organization_uuid", "handle", "display_name", "description", "version", + "gateway_functionality_type", "properties", "manifest", "is_active", "is_critical", + "data_version", "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.UUID, r.Org, r.Handle, r.DisplayName, sarg(r.Description), ver, r.FuncType, r.Properties, r.Manifest, + BoolToSmallint(r.IsActive), BoolToSmallint(r.IsCritical), + DataVersion, createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"uuid"}); err != nil { + return err + } + // vhost → exactly one gateway_endpoints row. On a live UPDATE (§8.3) the vhost may + // have changed (gateway_endpoints has a SERIAL id and no natural key, so the changed + // vhost inserts a second row) or been cleared: insert the desired endpoint, then + // delete any superseded ones so the v2 child state matches the v1 gateway. Batch is + // InsertOnly, so the replacement deletes are skipped → batch output is byte-identical. + if strings.TrimSpace(r.Vhost) != "" { + if err := UpsertGatewayEndpoint(ex, r.UUID, r.Vhost, opts); err != nil { + return err + } + if !opts.InsertOnly && !opts.DryRun { + if _, err := ex.Exec("DELETE FROM gateway_endpoints WHERE gateway_uuid = $1 AND url <> $2", r.UUID, r.Vhost); err != nil { + return fmt.Errorf("replace gateway_endpoints for %s: %w", r.UUID, err) + } + } + return nil + } + // vhost cleared on a live UPDATE → drop any stale endpoint rows. + if !opts.InsertOnly && !opts.DryRun { + if _, err := ex.Exec("DELETE FROM gateway_endpoints WHERE gateway_uuid = $1", r.UUID); err != nil { + return fmt.Errorf("clear gateway_endpoints for %s: %w", r.UUID, err) + } + } + return nil +} + +// UpsertGatewayEndpoint inserts (gateway_uuid, url) once; gateway_endpoints has a +// SERIAL id and no natural unique key, so idempotency is an explicit existence check. +// Returns whether a row was inserted. +func UpsertGatewayEndpoint(ex Execer, gatewayUUID, url string, opts Options) error { + if opts.DryRun { + return nil + } + var one int + err := ex.QueryRow("SELECT 1 FROM gateway_endpoints WHERE gateway_uuid = $1 AND url = $2", gatewayUUID, url).Scan(&one) + if err == nil { + return nil // already present + } + if err.Error() != "sql: no rows in result set" { + return err + } + return upsert(ex, opts, "gateway_endpoints", []string{"gateway_uuid", "url"}, []any{gatewayUUID, url}, nil) +} + +// ---- artifact_gateway_mappings ---- + +type ArtifactGatewayMappingRow struct { + ArtifactUUID, Org, GatewayUUID string + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertArtifactGatewayMapping(ex Execer, r ArtifactGatewayMappingRow, opts Options, rep Reporter) error { + key := r.ArtifactUUID + "|" + r.GatewayUUID + createdBy, err := audit(ex, opts, rep, "artifact_gateway_mappings", key, r.CreatedBy) + if err != nil { + return err + } + return upsert(ex, opts, "artifact_gateway_mappings", + []string{"artifact_uuid", "organization_uuid", "gateway_uuid", "metadata", + "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.ArtifactUUID, r.Org, r.GatewayUUID, nil, createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"organization_uuid", "artifact_uuid", "gateway_uuid"}) +} + +// ---- gateway_custom_policies ---- + +type GatewayCustomPolicyRow struct { + UUID, Org, Name, Version string + DisplayName, Description *string + PolicyDefinition []byte + CreatedAt, UpdatedAt *time.Time + CreatedBy string +} + +func UpsertGatewayCustomPolicy(ex Execer, r GatewayCustomPolicyRow, opts Options, rep Reporter) error { + var descVal any + if r.Description != nil { + d, cut := TruncateStr(*r.Description, 1023) + if cut { + rep.Flag("gateway_custom_policies", r.UUID, FlagTruncated, + map[string]any{"description_len": len(*r.Description)}, map[string]any{"description_len": len(d)}) + } + descVal = d + } + createdBy, err := audit(ex, opts, rep, "gateway_custom_policies", r.UUID, r.CreatedBy) + if err != nil { + return err + } + return upsert(ex, opts, "gateway_custom_policies", + []string{"uuid", "organization_uuid", "name", "display_name", "version", "description", + "policy_definition", "data_version", "created_by", "created_at", "updated_by", "updated_at"}, + []any{r.UUID, r.Org, r.Name, sarg(r.DisplayName), r.Version, descVal, r.PolicyDefinition, DataVersion, + createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts)}, + []string{"uuid"}) +} + +// ---- gateway_custom_policy_usages ---- + +type PolicyUsageRow struct{ PolicyUUID, ArtifactUUID string } + +func UpsertPolicyUsage(ex Execer, r PolicyUsageRow, opts Options, rep Reporter) error { + return upsert(ex, opts, "gateway_custom_policy_usages", + []string{"policy_uuid", "artifact_uuid"}, []any{r.PolicyUUID, r.ArtifactUUID}, + []string{"policy_uuid", "artifact_uuid"}) +} + +// ---- gateway_tokens ---- + +type GatewayTokenRow struct { + UUID, GatewayUUID, TokenHash, Salt, Status string + CreatedAt, RevokedAt *time.Time + CreatedBy string +} + +func UpsertGatewayToken(ex Execer, r GatewayTokenRow, opts Options, rep Reporter) error { + createdBy, err := audit(ex, opts, rep, "gateway_tokens", r.UUID, r.CreatedBy) + if err != nil { + return err + } + var revokedBy any + if r.RevokedAt != nil { + actor, _, e := ResolveIdentity(ex, MigrationActorIDPID, opts) + if e != nil { + return e + } + revokedBy = actor + } + return upsert(ex, opts, "gateway_tokens", + []string{"uuid", "gateway_uuid", "token_hash", "salt", "status", "data_version", + "created_by", "created_at", "revoked_by", "revoked_at"}, + []any{r.UUID, r.GatewayUUID, r.TokenHash, r.Salt, r.Status, DataVersion, + createdBy, tsArg(r.CreatedAt, opts), revokedBy, tsArg(r.RevokedAt, opts)}, + []string{"uuid"}) +} + +// ---- deployments ---- + +type DeploymentRow struct { + UUID, DisplayName, ArtifactUUID, Org, GatewayUUID string + BaseDeploymentUUID, Metadata *string + Content []byte + CreatedAt *time.Time + CreatedBy string +} + +func UpsertDeployment(ex Execer, r DeploymentRow, opts Options, rep Reporter) error { + createdBy, err := audit(ex, opts, rep, "deployments", r.UUID, r.CreatedBy) + if err != nil { + return err + } + return upsert(ex, opts, "deployments", + []string{"uuid", "display_name", "artifact_uuid", "organization_uuid", "gateway_uuid", + "base_deployment_uuid", "content", "metadata", "data_version", "created_by", "created_at"}, + []any{r.UUID, r.DisplayName, r.ArtifactUUID, r.Org, r.GatewayUUID, sarg(r.BaseDeploymentUUID), + r.Content, bytesOrNilP(r.Metadata), DataVersion, createdBy, tsArg(r.CreatedAt, opts)}, + []string{"uuid"}) +} + +// ---- deployment_status ---- + +type DeploymentStatusRow struct { + ArtifactUUID, Org, GatewayUUID, DeploymentUUID, Status string + StatusDesired, StatusReason *string + PerformedAt, UpdatedAt *time.Time + PerformedBy string +} + +func UpsertDeploymentStatus(ex Execer, r DeploymentStatusRow, opts Options, rep Reporter) error { + key := r.ArtifactUUID + "|" + r.Org + "|" + r.GatewayUUID + performedBy, err := audit(ex, opts, rep, "deployment_status", key, r.PerformedBy) + if err != nil { + return err + } + return upsert(ex, opts, "deployment_status", + []string{"artifact_uuid", "organization_uuid", "gateway_uuid", "deployment_uuid", "status", + "status_desired", "performed_at", "performed_by", "status_reason", "updated_at"}, + []any{r.ArtifactUUID, r.Org, r.GatewayUUID, r.DeploymentUUID, r.Status, sarg(r.StatusDesired), + tsArg(r.PerformedAt, opts), performedBy, sarg(r.StatusReason), tsArg(r.UpdatedAt, opts)}, + []string{"organization_uuid", "artifact_uuid", "gateway_uuid"}) +} + +// ---- api_keys ---- + +type APIKeyRow struct { + UUID, ArtifactUUID, Handle, DisplayName, MaskedKey, APIKeyHashes, Status, AllowedTargets string + Issuer *string + CreatedAt, UpdatedAt, ExpiresAt *time.Time + CreatedBy string +} + +func UpsertAPIKey(ex Execer, r APIKeyRow, opts Options, rep Reporter) error { + var issuerVal any + if r.Issuer != nil { + iv, cut := TruncateStr(*r.Issuer, 255) + if cut { + rep.Flag("api_keys", r.UUID, FlagTruncated, map[string]any{"issuer_len": len(*r.Issuer)}, map[string]any{"issuer_len": len(iv)}) + } + issuerVal = iv + } + at, cut := TruncateStr(r.AllowedTargets, 255) + if cut { + rep.Flag("api_keys", r.UUID, FlagTruncated, map[string]any{"allowed_targets_len": len(r.AllowedTargets)}, map[string]any{"allowed_targets_len": len(at)}) + } + createdBy, err := audit(ex, opts, rep, "api_keys", r.UUID, r.CreatedBy) + if err != nil { + return err + } + return upsert(ex, opts, "api_keys", + []string{"uuid", "artifact_uuid", "handle", "display_name", "masked_api_key", "api_key_hashes", + "status", "data_version", "created_by", "created_at", "updated_by", "updated_at", + "expires_at", "issuer", "allowed_targets"}, + []any{r.UUID, r.ArtifactUUID, r.Handle, r.DisplayName, r.MaskedKey, []byte(r.APIKeyHashes), r.Status, DataVersion, + createdBy, tsArg(r.CreatedAt, opts), createdBy, tsArg(r.UpdatedAt, opts), tsArg(r.ExpiresAt, opts), issuerVal, at}, + []string{"uuid"}) +} + +// ---- application_api_key_mappings ---- + +type ApplicationAPIKeyMappingRow struct { + ApplicationUUID, APIKeyID string + CreatedAt *time.Time + CreatedBy string +} + +func UpsertApplicationAPIKeyMapping(ex Execer, r ApplicationAPIKeyMappingRow, opts Options, rep Reporter) error { + key := r.ApplicationUUID + "|" + r.APIKeyID + createdBy, err := audit(ex, opts, rep, "application_api_key_mappings", key, r.CreatedBy) + if err != nil { + return err + } + return upsert(ex, opts, "application_api_key_mappings", + []string{"application_uuid", "api_key_id", "created_by", "created_at"}, + []any{r.ApplicationUUID, r.APIKeyID, createdBy, tsArg(r.CreatedAt, opts)}, + []string{"application_uuid", "api_key_id"}) +} + +// ---- application_artifact_mappings ---- + +type ApplicationArtifactMappingRow struct { + ApplicationUUID, ArtifactUUID string + CreatedAt *time.Time + CreatedBy string +} + +func UpsertApplicationArtifactMapping(ex Execer, r ApplicationArtifactMappingRow, opts Options, rep Reporter) error { + key := r.ApplicationUUID + "|" + r.ArtifactUUID + createdBy, err := audit(ex, opts, rep, "application_artifact_mappings", key, r.CreatedBy) + if err != nil { + return err + } + return upsert(ex, opts, "application_artifact_mappings", + []string{"application_uuid", "artifact_uuid", "created_by", "created_at"}, + []any{r.ApplicationUUID, r.ArtifactUUID, createdBy, tsArg(r.CreatedAt, opts)}, + []string{"application_uuid", "artifact_uuid"}) +} + +// ---- shared helpers ---- + +func deref(p *string) string { + if p == nil { + return "" + } + return *p +} + +func lifecycleOr(p *string) string { + if p != nil && *p != "" { + return *p + } + return "CREATED" +} + +func bytesOrNilP(p *string) any { + if p == nil { + return nil + } + return []byte(*p) +} + +func flagPlaintextCredential(rep Reporter, table, key string, sec *model.SecurityConfig) { + if sec != nil && sec.APIKey != nil && sec.APIKey.Key != "" { + rep.Flag(table, key, FlagPlaintextCredential, nil, + map[string]any{"note": "upstream apiKey.key stored in plaintext in the config blob"}) + } +} diff --git a/platform-api/migrationcore/upsert_pg_test.go b/platform-api/migrationcore/upsert_pg_test.go new file mode 100644 index 0000000000..461ecce59c --- /dev/null +++ b/platform-api/migrationcore/upsert_pg_test.go @@ -0,0 +1,361 @@ +/* + * Copyright (c) 2026, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package migrationcore + +import ( + "database/sql" + "os" + "testing" + "time" + + _ "github.com/jackc/pgx/v5/stdlib" + "github.com/wso2/api-platform/platform-api/internal/constants" +) + +// These integration tests exercise the affordances the batch backfill never uses +// but the live dual-write path needs: ON CONFLICT DO UPDATE (upsert), InsertOnly +// (DO NOTHING), per-entity delete, and on-demand identity resolution. They require +// a PostgreSQL DB with the v2 core schema applied; set MIGRATIONCORE_TEST_DSN +// (e.g. postgres://postgres:admin@localhost:5433/dbtest?sslmode=disable). +func openTestDB(t *testing.T) *sql.DB { + t.Helper() + dsn := os.Getenv("MIGRATIONCORE_TEST_DSN") + if dsn == "" { + t.Skip("MIGRATIONCORE_TEST_DSN not set; skipping migrationcore Postgres integration tests") + } + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatalf("open: %v", err) + } + if err := db.Ping(); err != nil { + t.Fatalf("ping: %v", err) + } + return db +} + +var testEpoch = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + +func baseOpts() Options { + return Options{SourceTZ: time.UTC, Epoch: testEpoch, SkipIdentityUpsert: true} +} + +func TestUpsertUpdateVsInsertOnly(t *testing.T) { + db := openTestDB(t) + defer db.Close() + const uuid = "test-org-affordance-0001" + _, _ = db.Exec("DELETE FROM organizations WHERE uuid = $1", uuid) + + getName := func() string { + var n string + if err := db.QueryRow("SELECT display_name FROM organizations WHERE uuid = $1", uuid).Scan(&n); err != nil { + t.Fatalf("select: %v", err) + } + return n + } + + row := OrganizationRow{UUID: uuid, Handle: "affordance-org", DisplayName: "Acme", Region: "us"} + + // (1) upsert INSERT. + if err := UpsertOrganization(db, row, baseOpts(), NopReporter{}); err != nil { + t.Fatalf("insert: %v", err) + } + if got := getName(); got != "Acme" { + t.Fatalf("after insert display_name=%q want Acme", got) + } + + // (2) upsert UPDATE (InsertOnly=false → ON CONFLICT DO UPDATE). + row.DisplayName = "AcmeUpdated" + opts := baseOpts() + opts.InsertOnly = false + if err := UpsertOrganization(db, row, opts, NopReporter{}); err != nil { + t.Fatalf("update: %v", err) + } + if got := getName(); got != "AcmeUpdated" { + t.Fatalf("after upsert-update display_name=%q want AcmeUpdated", got) + } + + // (3) InsertOnly=true → DO NOTHING (no change). + row.DisplayName = "ShouldNotStick" + opts.InsertOnly = true + if err := UpsertOrganization(db, row, opts, NopReporter{}); err != nil { + t.Fatalf("insert-only: %v", err) + } + if got := getName(); got != "AcmeUpdated" { + t.Fatalf("after InsertOnly display_name=%q want AcmeUpdated (unchanged)", got) + } + + // (4) delete. + if err := DeleteOrganization(db, uuid); err != nil { + t.Fatalf("delete: %v", err) + } + var cnt int + if err := db.QueryRow("SELECT count(*) FROM organizations WHERE uuid = $1", uuid).Scan(&cnt); err != nil { + t.Fatalf("count: %v", err) + } + if cnt != 0 { + t.Fatalf("after delete count=%d want 0", cnt) + } +} + +func TestResolveIdentityIncremental(t *testing.T) { + db := openTestDB(t) + defer db.Close() + const idp = "affordance-user@example.com" + _, _ = db.Exec("DELETE FROM user_idp_references WHERE idp_id = $1", idp) + + opts := baseOpts() + opts.SkipIdentityUpsert = false // live path: upsert on demand + + uuid1, defaulted, err := ResolveIdentity(db, idp, opts) + if err != nil { + t.Fatalf("resolve: %v", err) + } + if defaulted { + t.Fatalf("non-empty actor should not be defaulted") + } + if want := DeterministicUUID(idp, testEpoch); uuid1 != want { + t.Fatalf("uuid=%s want deterministic %s", uuid1, want) + } + // The row must now exist. + var stored string + if err := db.QueryRow("SELECT uuid FROM user_idp_references WHERE idp_id = $1", idp).Scan(&stored); err != nil { + t.Fatalf("select idp: %v", err) + } + if stored != uuid1 { + t.Fatalf("stored uuid=%s want %s", stored, uuid1) + } + // Idempotent: a second resolve neither errors nor duplicates. + uuid2, _, err := ResolveIdentity(db, idp, opts) + if err != nil || uuid2 != uuid1 { + t.Fatalf("second resolve uuid=%s err=%v want %s", uuid2, err, uuid1) + } + var cnt int + if err := db.QueryRow("SELECT count(*) FROM user_idp_references WHERE idp_id = $1", idp).Scan(&cnt); err != nil { + t.Fatalf("count: %v", err) + } + if cnt != 1 { + t.Fatalf("idp rows=%d want 1", cnt) + } + + // Empty actor → the migration actor, flagged defaulted. + _, def, err := ResolveIdentity(db, "", opts) + if err != nil || !def { + t.Fatalf("empty actor should default: def=%v err=%v", def, err) + } + _, _ = db.Exec("DELETE FROM user_idp_references WHERE idp_id IN ($1, $2)", idp, MigrationActorIDPID) +} + +// TestDryRunNoWrite proves DryRun runs the transform but writes nothing. +func TestDryRunNoWrite(t *testing.T) { + db := openTestDB(t) + defer db.Close() + const uuid = "test-org-dryrun-0002" + _, _ = db.Exec("DELETE FROM organizations WHERE uuid = $1", uuid) + + opts := baseOpts() + opts.DryRun = true + if err := UpsertOrganization(db, OrganizationRow{UUID: uuid, Handle: "dry", DisplayName: "Dry", Region: "us"}, opts, NopReporter{}); err != nil { + t.Fatalf("dry-run upsert: %v", err) + } + var cnt int + if err := db.QueryRow("SELECT count(*) FROM organizations WHERE uuid = $1", uuid).Scan(&cnt); err != nil { + t.Fatalf("count: %v", err) + } + if cnt != 0 { + t.Fatalf("dry-run wrote a row (count=%d); want 0", cnt) + } +} + +// TestUpsertPreservesCreationAuditOnUpdate proves §8.2: a live ON CONFLICT DO UPDATE +// leaves created_at/created_by untouched (creation audit is immutable) while payload +// and updated_at/updated_by move to the updater's values. +func TestUpsertPreservesCreationAuditOnUpdate(t *testing.T) { + db := openTestDB(t) + defer db.Close() + const uuid = "test-org-audit-8002" + _, _ = db.Exec("DELETE FROM organizations WHERE uuid = $1", uuid) + defer db.Exec("DELETE FROM organizations WHERE uuid = $1", uuid) + + t1 := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC) + t2 := time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC) + opts := baseOpts() + opts.InsertOnly = false + + // (1) create with actor "alice" at t1. + row := OrganizationRow{UUID: uuid, Handle: "audit-org-8002", DisplayName: "Acme", Region: "us", + CreatedAt: &t1, UpdatedAt: &t1, CreatedBy: "alice"} + if err := UpsertOrganization(db, row, opts, NopReporter{}); err != nil { + t.Fatalf("create: %v", err) + } + wantCreatedBy := DeterministicUUID("alice", testEpoch) + + // (2) update with actor "bob" at t2 — created_at/created_by must NOT move (§8.2). + row.DisplayName = "AcmeUpdated" + row.CreatedAt = &t2 + row.UpdatedAt = &t2 + row.CreatedBy = "bob" + if err := UpsertOrganization(db, row, opts, NopReporter{}); err != nil { + t.Fatalf("update: %v", err) + } + + var displayName, createdBy, updatedBy string + var createdAt, updatedAt time.Time + if err := db.QueryRow( + "SELECT display_name, created_by, created_at, updated_by, updated_at FROM organizations WHERE uuid = $1", uuid, + ).Scan(&displayName, &createdBy, &createdAt, &updatedBy, &updatedAt); err != nil { + t.Fatalf("select: %v", err) + } + if displayName != "AcmeUpdated" { + t.Errorf("display_name=%q want AcmeUpdated (payload should update)", displayName) + } + if createdBy != wantCreatedBy { + t.Errorf("created_by=%q want %q (creation audit must be immutable on update)", createdBy, wantCreatedBy) + } + if !createdAt.UTC().Equal(t1) { + t.Errorf("created_at=%s want %s (creation audit must be immutable on update)", createdAt.UTC(), t1) + } + if want := DeterministicUUID("bob", testEpoch); updatedBy != want { + t.Errorf("updated_by=%q want %q (updated audit should move to the updater)", updatedBy, want) + } + if !updatedAt.UTC().Equal(t2) { + t.Errorf("updated_at=%s want %s (updated audit should move)", updatedAt.UTC(), t2) + } +} + +// TestUpsertGatewayReplacesEndpointOnVhostChange proves §8.3: a live UpsertGateway +// UPDATE with a changed vhost leaves exactly one gateway_endpoints row (the stale one +// is removed), rather than accumulating a second SERIAL-keyed row. +func TestUpsertGatewayReplacesEndpointOnVhostChange(t *testing.T) { + db := openTestDB(t) + defer db.Close() + const org = "test-org-gw-8003" + const gw = "test-gw-8003" + _, _ = db.Exec("DELETE FROM gateways WHERE uuid = $1", gw) + _, _ = db.Exec("DELETE FROM organizations WHERE uuid = $1", org) + defer func() { + db.Exec("DELETE FROM gateways WHERE uuid = $1", gw) + db.Exec("DELETE FROM organizations WHERE uuid = $1", org) + }() + + opts := baseOpts() + opts.InsertOnly = false + if err := UpsertOrganization(db, OrganizationRow{UUID: org, Handle: "gw-org-8003", DisplayName: "GwOrg", Region: "us"}, opts, NopReporter{}); err != nil { + t.Fatalf("seed org: %v", err) + } + + countEndpoints := func() int { + var n int + if err := db.QueryRow("SELECT count(*) FROM gateway_endpoints WHERE gateway_uuid = $1", gw).Scan(&n); err != nil { + t.Fatalf("count endpoints: %v", err) + } + return n + } + getEndpoint := func() string { + var u string + if err := db.QueryRow("SELECT url FROM gateway_endpoints WHERE gateway_uuid = $1", gw).Scan(&u); err != nil { + t.Fatalf("get endpoint: %v", err) + } + return u + } + + base := GatewayRow{UUID: gw, Org: org, Handle: "gw-8003", DisplayName: "GW", Version: "1.0", + FuncType: "regular", Vhost: "v1.example.com", Properties: []byte("{}"), IsActive: true} + if err := UpsertGateway(db, base, opts, NopReporter{}); err != nil { + t.Fatalf("create gateway: %v", err) + } + if n := countEndpoints(); n != 1 || getEndpoint() != "v1.example.com" { + t.Fatalf("after create: endpoints=%d url=%q want 1 / v1.example.com", n, getEndpoint()) + } + + // vhost change → the old endpoint must be replaced, not accumulated (§8.3). + base.Vhost = "v2.example.com" + if err := UpsertGateway(db, base, opts, NopReporter{}); err != nil { + t.Fatalf("update gateway: %v", err) + } + if n := countEndpoints(); n != 1 || getEndpoint() != "v2.example.com" { + t.Fatalf("after vhost change: endpoints=%d url=%q want 1 / v2.example.com (stale endpoint must be removed)", n, getEndpoint()) + } +} + +// TestUpsertSubscriptionPlanReplacesLimitOnThrottleChange proves §8.3: a live +// UpsertSubscriptionPlan UPDATE that changes the throttle unit (which shifts the +// limit's natural key) leaves exactly one subscription_plan_limits row. +func TestUpsertSubscriptionPlanReplacesLimitOnThrottleChange(t *testing.T) { + db := openTestDB(t) + defer db.Close() + const org = "test-org-plan-8003" + const plan = "test-plan-8003" + _, _ = db.Exec("DELETE FROM subscription_plans WHERE uuid = $1", plan) + _, _ = db.Exec("DELETE FROM organizations WHERE uuid = $1", org) + defer func() { + db.Exec("DELETE FROM subscription_plans WHERE uuid = $1", plan) + db.Exec("DELETE FROM organizations WHERE uuid = $1", org) + }() + + opts := baseOpts() + opts.InsertOnly = false + if err := UpsertOrganization(db, OrganizationRow{UUID: org, Handle: "plan-org-8003", DisplayName: "PlanOrg", Region: "us"}, opts, NopReporter{}); err != nil { + t.Fatalf("seed org: %v", err) + } + + countLimits := func() int { + var n int + if err := db.QueryRow("SELECT count(*) FROM subscription_plan_limits WHERE subscription_plan_uuid = $1", plan).Scan(&n); err != nil { + t.Fatalf("count limits: %v", err) + } + return n + } + getLimit := func() (string, int64) { + var unit string + var count int64 + if err := db.QueryRow("SELECT time_unit, limit_count FROM subscription_plan_limits WHERE subscription_plan_uuid = $1", plan).Scan(&unit, &count); err != nil { + t.Fatalf("get limit: %v", err) + } + return unit, count + } + + minUnit := "min" + cnt100 := int64(100) + base := SubscriptionPlanRow{UUID: plan, Handle: "plan-8003", DisplayName: "Plan", Org: org, Status: "ACTIVE", + ThrottleUnit: &minUnit, ThrottleCount: &cnt100} + if err := UpsertSubscriptionPlan(db, base, opts, NopReporter{}); err != nil { + t.Fatalf("create plan: %v", err) + } + if n := countLimits(); n != 1 { + t.Fatalf("after create: limits=%d want 1", n) + } + if u, c := getLimit(); u != constants.ThrottleLimitUnitMinute || c != 100 { + t.Fatalf("after create: limit=%s/%d want %s/100", u, c, constants.ThrottleLimitUnitMinute) + } + + // throttle unit change (min→hour, which moves the natural key) → the old MINUTE + // limit must be replaced, not left behind (§8.3). + hourUnit := "hour" + cnt200 := int64(200) + base.ThrottleUnit = &hourUnit + base.ThrottleCount = &cnt200 + if err := UpsertSubscriptionPlan(db, base, opts, NopReporter{}); err != nil { + t.Fatalf("update plan: %v", err) + } + if n := countLimits(); n != 1 { + t.Fatalf("after throttle change: limits=%d want 1 (stale MINUTE limit must be removed)", n) + } + if u, c := getLimit(); u != constants.ThrottleLimitUnitHour || c != 200 { + t.Fatalf("after throttle change: limit=%s/%d want %s/200", u, c, constants.ThrottleLimitUnitHour) + } +} From cf6f559b651fecc61be668f9b4b3fc7bdb01f7bb Mon Sep 17 00:00:00 2001 From: Renuka Fernando Date: Mon, 24 Aug 2026 15:26:47 +0530 Subject: [PATCH 2/3] fix(dbmigrate): address PR review (security, cascade, dry-run deletes) Address the CodeRabbit review on #3290. Security: - restrict -skip-decrypt-check to --dry-run (a live run must validate tokens); - stop leaking the DSN password via wrapped url.Parse errors; warn on sslmode=disable; write run artifacts with 0600; report the verify key error. Correctness: - record providers/templates in the v1 parent-set so a quarantined parent cascades to children instead of failing fast; - use errors.Is(sql.ErrNoRows); check rows.Err()/scan errors in verify loops; - honor Options.DryRun in DeleteX/deleteWhere and reconcile delete replays; - apply the -only-keys filter before claiming subscription dedup keys; - record the dropped base_deployment link; keep an unrecognized websub top-level `policies` shape for unknown-field discovery instead of dropping it; - validate single-component reconcile delete keys. Docs: fix the migrationcore README spec path + fence language, genericize the RUNBOOK paths, and reword the DDL-gate reproduction note. Regression: migrate -dry-run output is byte-identical to pre-fix; the migrationcore affordance tests pass. Signed-off-by: Renuka Fernando --- .../cmd/dbmigrate/MIGRATION_MAPPING.md | 5 +- platform-api/cmd/dbmigrate/RUNBOOK.md | 4 +- platform-api/cmd/dbmigrate/dsn.go | 7 +- platform-api/cmd/dbmigrate/migrate.go | 5 ++ platform-api/cmd/dbmigrate/migrate_tables.go | 28 ++++-- platform-api/cmd/dbmigrate/reconcile.go | 85 +++++++++++++------ platform-api/cmd/dbmigrate/reconcile_test.go | 22 +++++ platform-api/cmd/dbmigrate/state.go | 6 +- platform-api/cmd/dbmigrate/verify.go | 9 +- platform-api/cmd/dbmigrate/verify_checks.go | 37 ++++++-- platform-api/migrationcore/README.md | 4 +- platform-api/migrationcore/core.go | 7 +- platform-api/migrationcore/delete.go | 71 ++++++++-------- platform-api/migrationcore/transform.go | 20 +++-- platform-api/migrationcore/upsert.go | 4 +- platform-api/migrationcore/upsert_pg_test.go | 2 +- 16 files changed, 218 insertions(+), 98 deletions(-) diff --git a/platform-api/cmd/dbmigrate/MIGRATION_MAPPING.md b/platform-api/cmd/dbmigrate/MIGRATION_MAPPING.md index c6b6b2fc90..f69cdba339 100644 --- a/platform-api/cmd/dbmigrate/MIGRATION_MAPPING.md +++ b/platform-api/cmd/dbmigrate/MIGRATION_MAPPING.md @@ -20,8 +20,9 @@ ## Deliverable 0 — mechanical DDL completeness gate → **PASS** Parsed v1 (27 tables), v2 core (32), v2 plugin (3 → 35 v2 total). Every v2 table+column is -either **MAPPED** from v1 or explicitly **EMPTY/OPTIONAL**. Reproduce with -`scratchpad/ddl_gate.py`. Classification: +either **MAPPED** from v1 or explicitly **EMPTY/OPTIONAL**. Reproduce by extracting the +`CREATE TABLE` table+column sets from the three schema files and diffing them against the +classification below (any v2 object not listed is a blocker). Classification: - **MAPPED (26):** organizations, projects, applications, artifacts, rest_apis, subscription_plans, subscription_plan_limits, subscriptions, gateways, gateway_endpoints, diff --git a/platform-api/cmd/dbmigrate/RUNBOOK.md b/platform-api/cmd/dbmigrate/RUNBOOK.md index 62dd03ee81..ce7dde1d35 100644 --- a/platform-api/cmd/dbmigrate/RUNBOOK.md +++ b/platform-api/cmd/dbmigrate/RUNBOOK.md @@ -7,7 +7,7 @@ database (core + EventGateway plugin). Build and run from the pinned v2 revision ## 0. Build ```sh -cd /Users/renuka/git/api-platform-migration/platform-api +cd /platform-api # the v2 repo root go build -o dbmigrate ./cmd/dbmigrate/ ``` @@ -53,7 +53,7 @@ If v1 ran on the ephemeral fallback, those tokens are unrecoverable and must be ```sh V1="postgres://postgres:admin@localhost:5432/dbv1?sslmode=disable" V2="postgres://postgres:admin@localhost:5433/dbv2?sslmode=disable" -OUT=/Users/renuka/google-workspace/platform-api-migration/db-migration/migration-out +OUT=./migration-out # any writable dir for run artifacts (report/quarantine/flags) ./dbmigrate migrate -v1-dsn "$V1" -v2-dsn "$V2" -out-dir "$OUT" -run-id prod -dry-run ``` diff --git a/platform-api/cmd/dbmigrate/dsn.go b/platform-api/cmd/dbmigrate/dsn.go index 9aa92f74c4..0219c4e87a 100644 --- a/platform-api/cmd/dbmigrate/dsn.go +++ b/platform-api/cmd/dbmigrate/dsn.go @@ -34,7 +34,8 @@ import ( func dsnToConfig(dsn string) (*config.Database, error) { u, err := url.Parse(dsn) if err != nil { - return nil, fmt.Errorf("parse DSN: %w", err) + // Do NOT wrap err — url.Parse echoes the raw input, which includes the password. + return nil, fmt.Errorf("could not parse DSN (expected postgres://user:pass@host:port/db?sslmode=...)") } if u.Scheme != "postgres" && u.Scheme != "postgresql" { return nil, fmt.Errorf("unsupported DSN scheme %q (only postgres:// is supported)", u.Scheme) @@ -81,6 +82,10 @@ func openDB(dsn string, logger *slog.Logger) (*database.DB, error) { if err != nil { return nil, err } + if cfg.SSLMode == "" || cfg.SSLMode == "disable" { + logger.Warn("database TLS is disabled (sslmode=disable) — credentials and data traverse the network in plaintext; use sslmode=require or verify-full for a remote database", + "host", cfg.Host) + } db, err := database.NewConnection(cfg, logger) if err != nil { return nil, err diff --git a/platform-api/cmd/dbmigrate/migrate.go b/platform-api/cmd/dbmigrate/migrate.go index a3c7a44016..fec34a0df7 100644 --- a/platform-api/cmd/dbmigrate/migrate.go +++ b/platform-api/cmd/dbmigrate/migrate.go @@ -151,6 +151,11 @@ func runMigrate(argv []string) error { if err != nil { return fmt.Errorf("invalid -source-tz %q: %w", o.SourceTZ, err) } + // The subscription-token decrypt guard is mandatory for a live run; only a + // --dry-run may skip it (it validates tokens against the real key before writes). + if o.SkipDecryptCheck && !o.DryRun { + return fmt.Errorf("-skip-decrypt-check is only allowed with -dry-run; a live run must validate subscription tokens against the real key") + } if err := loadEncryptionKey(o); err != nil { return err } diff --git a/platform-api/cmd/dbmigrate/migrate_tables.go b/platform-api/cmd/dbmigrate/migrate_tables.go index 4329606343..886d456ae5 100644 --- a/platform-api/cmd/dbmigrate/migrate_tables.go +++ b/platform-api/cmd/dbmigrate/migrate_tables.go @@ -279,6 +279,9 @@ func migrateLLMProviderTemplates(mc *migCtx) error { if err := rows.Scan(&uuid, &org, &handle, &name, &description, &createdByRaw, &config, &createdAt, &updatedAt); err != nil { return err } + // Record v1 presence so a quarantined template cascades (not fail-fast) for + // providers that reference it via template_uuid. + mc.sets.templates.seenV1(uuid) full := map[string]any{"uuid": uuid, "handle": handle, "name": name, "organization_uuid": org} if ok, err := mc.parentOK(mc.sets.orgs, org, "llm_provider_templates", uuid, "organization_uuid", true, full); err != nil { return err @@ -328,6 +331,9 @@ func migrateLLMProviders(mc *migCtx) error { return err } mc.sets.artifacts.seenV1(uuid) + // Record v1 presence so a quarantined provider cascades (not fail-fast) for + // proxies that reference it via provider_uuid. + mc.sets.providers.seenV1(uuid) full := map[string]any{"uuid": uuid, "handle": handle, "organization_uuid": org, "template_uuid": templateUUID} if ok, err := mc.parentOK(mc.sets.orgs, org, "llm_providers", uuid, "organization_uuid", true, full); err != nil { return err @@ -722,6 +728,12 @@ func migrateSubscriptions(mc *migCtx) error { continue } } + // Apply the reconcile filter BEFORE claiming dedup keys, so a row skipped by + // -only-keys never quarantines a wanted row that shares its key. In batch mode + // want() is always true, so ordering is a no-op and output stays identical. + if !mc.want("subscriptions", uuid) { + continue + } hkey := apiUUID + "|" + hash if mc.subHashSeen[hkey] { mc.run.quarantine("subscriptions", uuid, ReasonDupKey, "duplicate (artifact_uuid, subscription_token_hash)", full) @@ -737,9 +749,6 @@ func migrateSubscriptions(mc *migCtx) error { mc.subAppSeen[akey] = true } mc.subHashSeen[hkey] = true - if !mc.want("subscriptions", uuid) { - continue - } if err := migrationcore.UpsertSubscription(mc.v2, migrationcore.SubscriptionRow{ UUID: uuid, ArtifactUUID: apiUUID, SubscriberID: subscriberID, Token: token, Hash: hash, Org: org, Status: status, ApplicationID: nsp(applicationID), PlanUUID: nsp(planUUID), CreatedAt: ntp(createdAt), UpdatedAt: ntp(updatedAt), @@ -1012,9 +1021,16 @@ func migrateDeployments(mc *migCtx) error { continue } var base *string - if baseDeployment.Valid && baseDeployment.String != "" && mc.sets.deployments.status(baseDeployment.String) == ParentOK { - s := baseDeployment.String - base = &s + if baseDeployment.Valid && baseDeployment.String != "" { + if mc.sets.deployments.status(baseDeployment.String) == ParentOK { + s := baseDeployment.String + base = &s + } else { + // Predecessor not migrated → drop the (nullable) base link, but record it. + mc.run.flag("deployments", uuid, FlagDefaultedNull, + map[string]any{"base_deployment_uuid": baseDeployment.String}, + map[string]any{"base_deployment_uuid": nil, "note": "base deployment not migrated; link dropped"}) + } } if !mc.want("deployments", uuid) { continue diff --git a/platform-api/cmd/dbmigrate/reconcile.go b/platform-api/cmd/dbmigrate/reconcile.go index 751ad97af8..94d9602eb7 100644 --- a/platform-api/cmd/dbmigrate/reconcile.go +++ b/platform-api/cmd/dbmigrate/reconcile.go @@ -31,6 +31,7 @@ package main import ( "bufio" + "encoding/json" "fmt" "os" "strings" @@ -47,10 +48,11 @@ type keyFilter struct { type deleteKey struct{ table, key string } -// loadKeyFilter parses a -only-keys file. Each non-blank, non-comment ('#') line is -// "
" where op is "upsert" or "delete". key is the row's natural key; -// composite keys are "|"-joined in the DeleteX argument order (documented per table in -// dispatchDelete) — e.g. deployment_status = "||". +// loadKeyFilter parses a -only-keys file. Each non-blank, non-comment ('#') line is EITHER +// "
" OR a JSON object with op/table/key fields — so the live path's JSONL +// failure log (dual_write.failure_log) can be fed in verbatim. op is "upsert" or "delete"; +// key is the row's natural key; composite keys are "|"-joined in the DeleteX argument order +// (documented per table in dispatchDelete) — e.g. deployment_status = "||". func loadKeyFilter(path string) (*keyFilter, error) { f, err := os.Open(path) if err != nil { @@ -60,6 +62,7 @@ func loadKeyFilter(path string) (*keyFilter, error) { kf := &keyFilter{upserts: map[string]map[string]bool{}} sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) // JSONL failure lines can be long (error text) lineNo := 0 for sc.Scan() { lineNo++ @@ -67,11 +70,27 @@ func loadKeyFilter(path string) (*keyFilter, error) { if text == "" || strings.HasPrefix(text, "#") { continue } - fields := strings.Fields(text) - if len(fields) != 3 { - return nil, fmt.Errorf("-only-keys line %d: want '
', got %q", lineNo, text) + var op, table, key string + if strings.HasPrefix(text, "{") { + var rec struct { + Op string `json:"op"` + Table string `json:"table"` + Key string `json:"key"` + } + if err := json.Unmarshal([]byte(text), &rec); err != nil { + return nil, fmt.Errorf("-only-keys line %d: invalid JSON: %w", lineNo, err) + } + op, table, key = rec.Op, rec.Table, rec.Key + if op == "" || table == "" || key == "" { + return nil, fmt.Errorf("-only-keys line %d: JSON must have non-empty op/table/key", lineNo) + } + } else { + fields := strings.Fields(text) + if len(fields) != 3 { + return nil, fmt.Errorf("-only-keys line %d: want '
' or a JSON object, got %q", lineNo, text) + } + op, table, key = fields[0], fields[1], fields[2] } - op, table, key := fields[0], fields[1], fields[2] switch op { case "upsert": if kf.upserts[table] == nil { @@ -108,7 +127,7 @@ func (mc *migCtx) reconcileDeletes() error { } n := 0 for _, d := range mc.only.deletes { - if err := dispatchDelete(mc.v2, d.table, d.key); err != nil { + if err := dispatchDelete(mc.v2, mc.coreOpts(), d.table, d.key); err != nil { return fmt.Errorf("reconcile delete %s %q: %w", d.table, d.key, err) } n++ @@ -123,58 +142,70 @@ func (mc *migCtx) reconcileDeletes() error { // artifact type tables (rest_apis/llm_*/mcp_proxies/websub_apis/webbroker_apis) delete // via DeleteArtifact so the v2 FK cascade removes the type row + children, mirroring the // live path. Composite keys are "|"-joined in DeleteX argument order. -func dispatchDelete(v2 *database.DB, table, key string) error { +func dispatchDelete(v2 *database.DB, opts migrationcore.Options, table, key string) error { p := strings.Split(key, "|") + // Single-uuid tables must carry exactly one key component — reject a stray + // composite key rather than silently deleting on p[0]. + singleKey := map[string]bool{ + "artifacts": true, "rest_apis": true, "llm_providers": true, "llm_proxies": true, + "mcp_proxies": true, "websub_apis": true, "webbroker_apis": true, "organizations": true, + "projects": true, "applications": true, "subscription_plans": true, "subscriptions": true, + "gateways": true, "gateway_tokens": true, "gateway_custom_policies": true, "api_keys": true, + "deployments": true, "llm_provider_templates": true, + } + if singleKey[table] && (len(p) != 1 || p[0] == "") { + return fmt.Errorf("%s delete key %q: want a single uuid, got %d component(s)", table, key, len(p)) + } switch table { case "artifacts", "rest_apis", "llm_providers", "llm_proxies", "mcp_proxies", "websub_apis", "webbroker_apis": - return migrationcore.DeleteArtifact(v2, p[0]) + return migrationcore.DeleteArtifact(v2, opts, p[0]) case "organizations": - return migrationcore.DeleteOrganization(v2, p[0]) + return migrationcore.DeleteOrganization(v2, opts, p[0]) case "projects": - return migrationcore.DeleteProject(v2, p[0]) + return migrationcore.DeleteProject(v2, opts, p[0]) case "applications": - return migrationcore.DeleteApplication(v2, p[0]) + return migrationcore.DeleteApplication(v2, opts, p[0]) case "subscription_plans": - return migrationcore.DeleteSubscriptionPlan(v2, p[0]) + return migrationcore.DeleteSubscriptionPlan(v2, opts, p[0]) case "subscriptions": - return migrationcore.DeleteSubscription(v2, p[0]) + return migrationcore.DeleteSubscription(v2, opts, p[0]) case "gateways": - return migrationcore.DeleteGateway(v2, p[0]) + return migrationcore.DeleteGateway(v2, opts, p[0]) case "gateway_tokens": - return migrationcore.DeleteGatewayToken(v2, p[0]) + return migrationcore.DeleteGatewayToken(v2, opts, p[0]) case "gateway_custom_policies": - return migrationcore.DeleteGatewayCustomPolicy(v2, p[0]) + return migrationcore.DeleteGatewayCustomPolicy(v2, opts, p[0]) case "api_keys": - return migrationcore.DeleteAPIKey(v2, p[0]) + return migrationcore.DeleteAPIKey(v2, opts, p[0]) case "deployments": - return migrationcore.DeleteDeployment(v2, p[0]) + return migrationcore.DeleteDeployment(v2, opts, p[0]) case "llm_provider_templates": - return migrationcore.DeleteLLMProviderTemplate(v2, p[0]) + return migrationcore.DeleteLLMProviderTemplate(v2, opts, p[0]) case "artifact_gateway_mappings": // || if len(p) != 3 { return fmt.Errorf("artifact_gateway_mappings key %q: want ||", key) } - return migrationcore.DeleteArtifactGatewayMapping(v2, p[0], p[1], p[2]) + return migrationcore.DeleteArtifactGatewayMapping(v2, opts, p[0], p[1], p[2]) case "application_api_key_mappings": // | if len(p) != 2 { return fmt.Errorf("application_api_key_mappings key %q: want |", key) } - return migrationcore.DeleteApplicationAPIKeyMapping(v2, p[0], p[1]) + return migrationcore.DeleteApplicationAPIKeyMapping(v2, opts, p[0], p[1]) case "application_artifact_mappings": // | if len(p) != 2 { return fmt.Errorf("application_artifact_mappings key %q: want |", key) } - return migrationcore.DeleteApplicationArtifactMapping(v2, p[0], p[1]) + return migrationcore.DeleteApplicationArtifactMapping(v2, opts, p[0], p[1]) case "gateway_custom_policy_usages": // | if len(p) != 2 { return fmt.Errorf("gateway_custom_policy_usages key %q: want |", key) } - return migrationcore.DeletePolicyUsage(v2, p[0], p[1]) + return migrationcore.DeletePolicyUsage(v2, opts, p[0], p[1]) case "deployment_status": // || if len(p) != 3 { return fmt.Errorf("deployment_status key %q: want ||", key) } - return migrationcore.DeleteDeploymentStatus(v2, p[0], p[1], p[2]) + return migrationcore.DeleteDeploymentStatus(v2, opts, p[0], p[1], p[2]) default: return fmt.Errorf("unknown delete table %q", table) } diff --git a/platform-api/cmd/dbmigrate/reconcile_test.go b/platform-api/cmd/dbmigrate/reconcile_test.go index d83b17c24b..712c5c12cb 100644 --- a/platform-api/cmd/dbmigrate/reconcile_test.go +++ b/platform-api/cmd/dbmigrate/reconcile_test.go @@ -70,6 +70,28 @@ func TestLoadKeyFilter(t *testing.T) { } } +// TestLoadKeyFilterAcceptsJSONL proves the live path's JSONL failure log can be fed to +// -only-keys verbatim (each line is a JSON object with op/table/key). +func TestLoadKeyFilterAcceptsJSONL(t *testing.T) { + path := filepath.Join(t.TempDir(), "failures.jsonl") + content := `{"code":"V2_DUAL_WRITE_FAILURE","entity":"rest_api","op":"upsert","table":"rest_apis","key":"uuid-1","occurred_at":"2026-01-01T00:00:00Z"} +{"code":"V2_DUAL_WRITE_FAILURE","entity":"gateway","op":"delete","table":"gateways","key":"gw-1"} +` + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + kf, err := loadKeyFilter(path) + if err != nil { + t.Fatalf("loadKeyFilter: %v", err) + } + if !kf.upserts["rest_apis"]["uuid-1"] { + t.Error("JSONL upsert line not parsed") + } + if len(kf.deletes) != 1 || kf.deletes[0].table != "gateways" || kf.deletes[0].key != "gw-1" { + t.Errorf("JSONL delete line not parsed: %v", kf.deletes) + } +} + func TestLoadKeyFilterRejectsBadLines(t *testing.T) { write := func(content string) string { p := filepath.Join(t.TempDir(), "k.txt") diff --git a/platform-api/cmd/dbmigrate/state.go b/platform-api/cmd/dbmigrate/state.go index 3e93bd6c51..a62f03850f 100644 --- a/platform-api/cmd/dbmigrate/state.go +++ b/platform-api/cmd/dbmigrate/state.go @@ -114,7 +114,7 @@ func newRun(opts *Options, logger *slog.Logger) (*Run, error) { dir := opts.OutDir open := func(base string) (*os.File, error) { p := filepath.Join(dir, suffixed(base, opts.RunID, opts.DryRun, "jsonl")) - return os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + return os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) } qf, err := open("quarantine") if err != nil { @@ -318,7 +318,7 @@ func (r *Run) saveCheckpoint() error { return err } tmp := r.ckptPath + ".tmp" - if err := os.WriteFile(tmp, b, 0o644); err != nil { + if err := os.WriteFile(tmp, b, 0o600); err != nil { return err } return os.Rename(tmp, r.ckptPath) @@ -340,7 +340,7 @@ func (r *Run) writeReport() error { return err } p := filepath.Join(r.opts.OutDir, suffixed("migration-report", r.opts.RunID, r.opts.DryRun, "json")) - return os.WriteFile(p, b, 0o644) + return os.WriteFile(p, b, 0o600) } func (r *Run) close() { diff --git a/platform-api/cmd/dbmigrate/verify.go b/platform-api/cmd/dbmigrate/verify.go index e0ea487eb5..a49da464ea 100644 --- a/platform-api/cmd/dbmigrate/verify.go +++ b/platform-api/cmd/dbmigrate/verify.go @@ -92,9 +92,12 @@ func runVerify(argv []string) error { if err != nil { return fmt.Errorf("invalid -migration-epoch: %w", err) } - _ = loadEncryptionKey(o) // optional for verify - logger := newLogger(o) + // The key is optional for verify (it only enables the token decrypt round-trip); + // report a load failure rather than silently swallowing it. + if err := loadEncryptionKey(o); err != nil { + logger.Warn("could not load subscription-token key; the decrypt round-trip check will be skipped", "error", err) + } v1, err := openDB(o.V1DSN, logger) if err != nil { return fmt.Errorf("open v1: %w", err) @@ -212,7 +215,7 @@ func writeVerifyReport(o *Options, r *VerifyReport) error { return err } p := filepath.Join(o.OutDir, suffixed("verify-report", o.RunID, false, "json")) - return os.WriteFile(p, b, 0o644) + return os.WriteFile(p, b, 0o600) } func printVerify(r *VerifyReport) { diff --git a/platform-api/cmd/dbmigrate/verify_checks.go b/platform-api/cmd/dbmigrate/verify_checks.go index 5640293440..2c51a3e19e 100644 --- a/platform-api/cmd/dbmigrate/verify_checks.go +++ b/platform-api/cmd/dbmigrate/verify_checks.go @@ -100,6 +100,9 @@ func (vr *verifier) loadV1Artifacts() map[string]v1Art { } m[uuid] = a } + if err := rows.Err(); err != nil { + vr.fail("B", "iterate v1 artifacts", err.Error()) + } return m } @@ -140,11 +143,17 @@ func (vr *verifier) layerB_scalars() { if err == nil { for rows.Next() { var uuid, typ string - _ = rows.Scan(&uuid, &typ) + if err := rows.Scan(&uuid, &typ); err != nil { + vr.fail("B", "scan artifacts", err.Error()) + break + } if a, ok := arts[uuid]; ok && a.kind != typ { typeMismatch++ } } + if err := rows.Err(); err != nil { + vr.fail("B", "iterate artifacts", err.Error()) + } rows.Close() } vr.add(Check{Layer: "B", Name: "display_name == v1 artifacts.name", Status: passIf(nameMismatch == 0), Detail: fmt.Sprintf("%d mismatch", nameMismatch)}) @@ -159,11 +168,17 @@ func (vr *verifier) layerB_scalars() { if err == nil { for rs.Next() { var uuid, art string - _ = rs.Scan(&uuid, &art) + if err := rs.Scan(&uuid, &art); err != nil { + vr.fail("B", "scan subscriptions", err.Error()) + break + } if v, ok := v1sub[uuid]; ok && v != art { subMismatch++ } } + if err := rs.Err(); err != nil { + vr.fail("B", "iterate subscriptions", err.Error()) + } rs.Close() } vr.add(Check{Layer: "B", Name: "subscriptions api_uuid -> artifact_uuid", Status: passIf(subMismatch == 0), Detail: fmt.Sprintf("%d mismatch", subMismatch)}) @@ -175,11 +190,17 @@ func (vr *verifier) layerB_scalars() { if err == nil { for rp.Next() { var uuid, dn string - _ = rp.Scan(&uuid, &dn) + if err := rp.Scan(&uuid, &dn); err != nil { + vr.fail("B", "scan subscription_plans", err.Error()) + break + } if v, ok := v1plan[uuid]; ok && v != dn { planMismatch++ } } + if err := rp.Err(); err != nil { + vr.fail("B", "iterate subscription_plans", err.Error()) + } rp.Close() } vr.add(Check{Layer: "B", Name: "subscription_plans plan_name -> display_name", Status: passIf(planMismatch == 0), Detail: fmt.Sprintf("%d mismatch", planMismatch)}) @@ -650,23 +671,23 @@ func (vr *verifier) layerF_dropReconcile() { Detail: fmt.Sprintf("v2(%d)+quarantine(%d)==v1(%d)", v2c, quar, v1c)}) } - // Quarantine sign-off gate: every quarantined key resolved in v2 or signed off. + // Quarantine sign-off gate: every quarantined key must be signed off + // (quarantine-signoff.jsonl). "Resolved by a re-run" is handled implicitly — a + // key fixed at source no longer appears in the latest run's quarantine file, so + // it never reaches this gate; what remains here must be explicitly accepted as loss. unresolved := 0 var sample []string for key := range vr.quarKeys { if vr.signoff[key] { continue } - // "resolved in v2" = the key's uuid is now present. Best-effort: the key's - // first path component is the source table; we cannot cheaply re-check every - // composite key, so a signed-off OR present-in-v2 decision is required. unresolved++ if len(sample) < 20 { sample = append(sample, key) } } st := statusPass - detail := fmt.Sprintf("all %d quarantined key(s) signed off or resolved", len(vr.quarKeys)) + detail := fmt.Sprintf("all %d quarantined key(s) signed off", len(vr.quarKeys)) if unresolved > 0 { st = statusFail detail = fmt.Sprintf("%d quarantined key(s) NOT signed off (e.g. %v)", unresolved, sample) diff --git a/platform-api/migrationcore/README.md b/platform-api/migrationcore/README.md index 46037f765e..60ed263093 100644 --- a/platform-api/migrationcore/README.md +++ b/platform-api/migrationcore/README.md @@ -5,7 +5,7 @@ v2 write, shared by two callers: - **Batch backfill** — `cmd/dbmigrate` (the one-time migrator). - **Live dual-write intermediate** — the v1 build that mirrors each v1 mutation into - the v2 DB after the v1 repo write (spec: `../../smooth-migration'/agent-prompt.md`). + the v2 DB after the v1 repo write (a separate design; its spec lives outside this repo). Both call the SAME `UpsertX`/`DeleteX`/`ResolveIdentity` so config-blob reshapes, type conversions, handle/identity rules and the v2 column layout can never drift @@ -38,7 +38,7 @@ between the backfill and the live path. Caller wiring: -``` +```text batch (cmd/dbmigrate): Options{InsertOnly:true, SkipIdentityUpsert:true, DryRun:cfg.DryRun} Reporter = the JSONL Run live (v1 dual-write): Options{InsertOnly:false, SkipIdentityUpsert:false} diff --git a/platform-api/migrationcore/core.go b/platform-api/migrationcore/core.go index e658311475..6bad3082b4 100644 --- a/platform-api/migrationcore/core.go +++ b/platform-api/migrationcore/core.go @@ -148,13 +148,16 @@ func upsert(ex Execer, opts Options, table string, cols []string, args []any, co } // deleteWhere executes DELETE FROM table WHERE . Used only by the -// live path's DeleteX (batch never deletes). -func deleteWhere(ex Execer, table string, whereCols []string, whereArgs []any) error { +// live path's DeleteX (batch never deletes). Honors opts.DryRun (no write). +func deleteWhere(ex Execer, opts Options, table string, whereCols []string, whereArgs []any) error { conds := make([]string, len(whereCols)) for i, c := range whereCols { conds[i] = fmt.Sprintf("%s = $%d", c, i+1) } q := fmt.Sprintf("DELETE FROM %s WHERE %s", table, strings.Join(conds, " AND ")) + if opts.DryRun { + return nil + } if _, err := ex.Exec(q, whereArgs...); err != nil { return fmt.Errorf("delete %s: %w", table, err) } diff --git a/platform-api/migrationcore/delete.go b/platform-api/migrationcore/delete.go index 9f6d98c9fc..b598a46146 100644 --- a/platform-api/migrationcore/delete.go +++ b/platform-api/migrationcore/delete.go @@ -20,99 +20,100 @@ package migrationcore // Per-entity deletes for the LIVE dual-write path (the batch backfill never calls // these). They reproduce v2's cascade shape: deleting the parent row lets the v2 // FK ON DELETE CASCADE remove the type row and artifact_uuid/gateway_uuid children; -// tables without a cascading FK are deleted explicitly by their composite key. +// tables without a cascading FK are deleted explicitly by their composite key. All +// honor opts.DryRun (via deleteWhere), symmetric with UpsertX. // DeleteArtifact removes an artifact (any of the six types). The v2 FKs // (rest_apis/llm_*/mcp/websub/webbroker .uuid→artifacts, and api_keys/deployments/ // deployment_status/subscriptions/mappings .artifact_uuid→artifacts, all ON DELETE // CASCADE) remove the type row and every artifact-keyed child. -func DeleteArtifact(ex Execer, uuid string) error { - return deleteWhere(ex, "artifacts", []string{"uuid"}, []any{uuid}) +func DeleteArtifact(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "artifacts", []string{"uuid"}, []any{uuid}) } // DeleteOrganization removes an org; org-scoped rows cascade via their FKs. -func DeleteOrganization(ex Execer, uuid string) error { - return deleteWhere(ex, "organizations", []string{"uuid"}, []any{uuid}) +func DeleteOrganization(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "organizations", []string{"uuid"}, []any{uuid}) } // DeleteProject removes a project (project-scoped children cascade). -func DeleteProject(ex Execer, uuid string) error { - return deleteWhere(ex, "projects", []string{"uuid"}, []any{uuid}) +func DeleteProject(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "projects", []string{"uuid"}, []any{uuid}) } // DeleteApplication removes an application (its mappings cascade). -func DeleteApplication(ex Execer, uuid string) error { - return deleteWhere(ex, "applications", []string{"uuid"}, []any{uuid}) +func DeleteApplication(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "applications", []string{"uuid"}, []any{uuid}) } // DeleteLLMProviderTemplate removes a template (blocked by FK if providers reference it). -func DeleteLLMProviderTemplate(ex Execer, uuid string) error { - return deleteWhere(ex, "llm_provider_templates", []string{"uuid"}, []any{uuid}) +func DeleteLLMProviderTemplate(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "llm_provider_templates", []string{"uuid"}, []any{uuid}) } // DeleteSubscriptionPlan removes a plan; subscription_plan_limits cascade. -func DeleteSubscriptionPlan(ex Execer, uuid string) error { - return deleteWhere(ex, "subscription_plans", []string{"uuid"}, []any{uuid}) +func DeleteSubscriptionPlan(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "subscription_plans", []string{"uuid"}, []any{uuid}) } // DeleteSubscription removes a single subscription. -func DeleteSubscription(ex Execer, uuid string) error { - return deleteWhere(ex, "subscriptions", []string{"uuid"}, []any{uuid}) +func DeleteSubscription(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "subscriptions", []string{"uuid"}, []any{uuid}) } // DeleteGateway removes a gateway; gateway_endpoints and gateway_tokens cascade. -func DeleteGateway(ex Execer, uuid string) error { - return deleteWhere(ex, "gateways", []string{"uuid"}, []any{uuid}) +func DeleteGateway(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "gateways", []string{"uuid"}, []any{uuid}) } // DeleteGatewayToken removes a single gateway token. -func DeleteGatewayToken(ex Execer, uuid string) error { - return deleteWhere(ex, "gateway_tokens", []string{"uuid"}, []any{uuid}) +func DeleteGatewayToken(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "gateway_tokens", []string{"uuid"}, []any{uuid}) } // DeleteGatewayCustomPolicy removes a policy; gateway_custom_policy_usages cascade. -func DeleteGatewayCustomPolicy(ex Execer, uuid string) error { - return deleteWhere(ex, "gateway_custom_policies", []string{"uuid"}, []any{uuid}) +func DeleteGatewayCustomPolicy(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "gateway_custom_policies", []string{"uuid"}, []any{uuid}) } // DeleteAPIKey removes an api key; application_api_key_mappings cascade. -func DeleteAPIKey(ex Execer, uuid string) error { - return deleteWhere(ex, "api_keys", []string{"uuid"}, []any{uuid}) +func DeleteAPIKey(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "api_keys", []string{"uuid"}, []any{uuid}) } // DeleteDeployment removes a single deployment (deployment_status cascades). -func DeleteDeployment(ex Execer, uuid string) error { - return deleteWhere(ex, "deployments", []string{"uuid"}, []any{uuid}) +func DeleteDeployment(ex Execer, opts Options, uuid string) error { + return deleteWhere(ex, opts, "deployments", []string{"uuid"}, []any{uuid}) } // ---- composite-key children (deleted directly when only the mapping changes) ---- // DeleteArtifactGatewayMapping removes one artifact↔gateway association. -func DeleteArtifactGatewayMapping(ex Execer, org, artifactUUID, gatewayUUID string) error { - return deleteWhere(ex, "artifact_gateway_mappings", +func DeleteArtifactGatewayMapping(ex Execer, opts Options, org, artifactUUID, gatewayUUID string) error { + return deleteWhere(ex, opts, "artifact_gateway_mappings", []string{"organization_uuid", "artifact_uuid", "gateway_uuid"}, []any{org, artifactUUID, gatewayUUID}) } // DeleteApplicationAPIKeyMapping removes one application↔api-key mapping. -func DeleteApplicationAPIKeyMapping(ex Execer, applicationUUID, apiKeyID string) error { - return deleteWhere(ex, "application_api_key_mappings", +func DeleteApplicationAPIKeyMapping(ex Execer, opts Options, applicationUUID, apiKeyID string) error { + return deleteWhere(ex, opts, "application_api_key_mappings", []string{"application_uuid", "api_key_id"}, []any{applicationUUID, apiKeyID}) } // DeleteApplicationArtifactMapping removes one application↔artifact mapping. -func DeleteApplicationArtifactMapping(ex Execer, applicationUUID, artifactUUID string) error { - return deleteWhere(ex, "application_artifact_mappings", +func DeleteApplicationArtifactMapping(ex Execer, opts Options, applicationUUID, artifactUUID string) error { + return deleteWhere(ex, opts, "application_artifact_mappings", []string{"application_uuid", "artifact_uuid"}, []any{applicationUUID, artifactUUID}) } // DeletePolicyUsage removes one policy-usage row. -func DeletePolicyUsage(ex Execer, policyUUID, artifactUUID string) error { - return deleteWhere(ex, "gateway_custom_policy_usages", +func DeletePolicyUsage(ex Execer, opts Options, policyUUID, artifactUUID string) error { + return deleteWhere(ex, opts, "gateway_custom_policy_usages", []string{"policy_uuid", "artifact_uuid"}, []any{policyUUID, artifactUUID}) } // DeleteDeploymentStatus removes the current-state row for an artifact+gateway. -func DeleteDeploymentStatus(ex Execer, org, artifactUUID, gatewayUUID string) error { - return deleteWhere(ex, "deployment_status", +func DeleteDeploymentStatus(ex Execer, opts Options, org, artifactUUID, gatewayUUID string) error { + return deleteWhere(ex, opts, "deployment_status", []string{"organization_uuid", "artifact_uuid", "gateway_uuid"}, []any{org, artifactUUID, gatewayUUID}) } diff --git a/platform-api/migrationcore/transform.go b/platform-api/migrationcore/transform.go index 4c09f87f7e..f6a251fa25 100644 --- a/platform-api/migrationcore/transform.go +++ b/platform-api/migrationcore/transform.go @@ -249,16 +249,18 @@ func PreprocessWebSubRaw(raw []byte) ([]byte, []string, error) { b, _ := json.Marshal(map[string]json.RawMessage{"policies": policyArray}) return b } + folded := false switch jsonKind(pol) { case '{': // object-form: {event: [policy...]} var byEvent map[string]json.RawMessage - if json.Unmarshal(pol, &byEvent) == nil { + if json.Unmarshal(pol, &byEvent) == nil && len(byEvent) > 0 { for event, arr := range byEvent { if _, exists := allch[event]; !exists { allch[event] = wrap(arr) } } notes = append(notes, "policies{event:[...]} -> allChannels") + folded = true } case '[': // flat array of whole-API policies (auth) → on_subscription var arr []json.RawMessage @@ -267,12 +269,20 @@ func PreprocessWebSubRaw(raw []byte) ([]byte, []string, error) { allch["on_subscription"] = wrap(pol) } notes = append(notes, "policies[] (whole-API) -> allChannels.on_subscription") + folded = true } } - delete(top, "policies") - if len(allch) > 0 { - b, _ := json.Marshal(allch) - top["allChannels"] = b + if folded { + delete(top, "policies") + if len(allch) > 0 { + b, _ := json.Marshal(allch) + top["allChannels"] = b + } + } else { + // Unrecognized shape (not object/array, or empty): do NOT silently drop + // it — leave it in place so the DisallowUnknownFields pass surfaces it for + // human review rather than losing it. + notes = append(notes, "policies present but in an unhandled shape; left for unknown-field discovery") } } diff --git a/platform-api/migrationcore/upsert.go b/platform-api/migrationcore/upsert.go index 9b7d67b98b..943da4e8df 100644 --- a/platform-api/migrationcore/upsert.go +++ b/platform-api/migrationcore/upsert.go @@ -18,6 +18,8 @@ package migrationcore import ( + "database/sql" + "errors" "fmt" "strings" "time" @@ -542,7 +544,7 @@ func UpsertGatewayEndpoint(ex Execer, gatewayUUID, url string, opts Options) err if err == nil { return nil // already present } - if err.Error() != "sql: no rows in result set" { + if !errors.Is(err, sql.ErrNoRows) { return err } return upsert(ex, opts, "gateway_endpoints", []string{"gateway_uuid", "url"}, []any{gatewayUUID, url}, nil) diff --git a/platform-api/migrationcore/upsert_pg_test.go b/platform-api/migrationcore/upsert_pg_test.go index 461ecce59c..6a4d960419 100644 --- a/platform-api/migrationcore/upsert_pg_test.go +++ b/platform-api/migrationcore/upsert_pg_test.go @@ -100,7 +100,7 @@ func TestUpsertUpdateVsInsertOnly(t *testing.T) { } // (4) delete. - if err := DeleteOrganization(db, uuid); err != nil { + if err := DeleteOrganization(db, baseOpts(), uuid); err != nil { t.Fatalf("delete: %v", err) } var cnt int From c6b66be04e6f83dfb2f2c14eaaf277ec786b95aa Mon Sep 17 00:00:00 2001 From: Renuka Fernando Date: Mon, 24 Aug 2026 18:59:04 +0530 Subject: [PATCH 3/3] fix(dbmigrate): address 2nd-round PR review follow-ups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - reconcile: preserve the base_deployment link during a targeted -only-keys reconcile (the in-memory parent set is filtered then, but the predecessor exists in v2 from the backfill — let v2's FK arbitrate); - reconcile: bound the total -only-keys input size (per-line was capped, total was not) to avoid unbounded memory growth on a malformed file; - state/verify: chmod run artifacts to 0600 — O_CREATE and WriteFile only set the mode on creation, so a pre-existing world-readable file stayed loose; - verify: do not log the key-load error verbatim (it may echo key/DSN material). Signed-off-by: Renuka Fernando --- platform-api/cmd/dbmigrate/migrate_tables.go | 8 ++++++-- platform-api/cmd/dbmigrate/reconcile.go | 9 ++++++++- platform-api/cmd/dbmigrate/state.go | 20 ++++++++++++++++++-- platform-api/cmd/dbmigrate/verify.go | 9 +++++++-- 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/platform-api/cmd/dbmigrate/migrate_tables.go b/platform-api/cmd/dbmigrate/migrate_tables.go index 886d456ae5..3f6139a938 100644 --- a/platform-api/cmd/dbmigrate/migrate_tables.go +++ b/platform-api/cmd/dbmigrate/migrate_tables.go @@ -1022,11 +1022,15 @@ func migrateDeployments(mc *migCtx) error { } var base *string if baseDeployment.Valid && baseDeployment.String != "" { - if mc.sets.deployments.status(baseDeployment.String) == ParentOK { + // In a targeted reconcile the in-memory deployment set holds only the + // filtered rows, so the predecessor is (correctly) absent — but it exists + // in v2 from the backfill. Keep the link and let v2's FK arbitrate. + if mc.reconcile || mc.sets.deployments.status(baseDeployment.String) == ParentOK { s := baseDeployment.String base = &s } else { - // Predecessor not migrated → drop the (nullable) base link, but record it. + // Full backfill and the predecessor was not migrated → drop the + // (nullable) base link, but record it. mc.run.flag("deployments", uuid, FlagDefaultedNull, map[string]any{"base_deployment_uuid": baseDeployment.String}, map[string]any{"base_deployment_uuid": nil, "note": "base deployment not migrated; link dropped"}) diff --git a/platform-api/cmd/dbmigrate/reconcile.go b/platform-api/cmd/dbmigrate/reconcile.go index 94d9602eb7..c19fa3b33a 100644 --- a/platform-api/cmd/dbmigrate/reconcile.go +++ b/platform-api/cmd/dbmigrate/reconcile.go @@ -48,6 +48,10 @@ type keyFilter struct { type deleteKey struct{ table, key string } +// maxOnlyKeysEntries bounds a -only-keys file so a malformed or unbounded input +// cannot exhaust memory building the in-memory work list. +const maxOnlyKeysEntries = 5_000_000 + // loadKeyFilter parses a -only-keys file. Each non-blank, non-comment ('#') line is EITHER // "
" OR a JSON object with op/table/key fields — so the live path's JSONL // failure log (dual_write.failure_log) can be fed in verbatim. op is "upsert" or "delete"; @@ -63,13 +67,16 @@ func loadKeyFilter(path string) (*keyFilter, error) { kf := &keyFilter{upserts: map[string]map[string]bool{}} sc := bufio.NewScanner(f) sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) // JSONL failure lines can be long (error text) - lineNo := 0 + lineNo, entries := 0, 0 for sc.Scan() { lineNo++ text := strings.TrimSpace(sc.Text()) if text == "" || strings.HasPrefix(text, "#") { continue } + if entries++; entries > maxOnlyKeysEntries { + return nil, fmt.Errorf("-only-keys file has more than %d entries; split it into batches", maxOnlyKeysEntries) + } var op, table, key string if strings.HasPrefix(text, "{") { var rec struct { diff --git a/platform-api/cmd/dbmigrate/state.go b/platform-api/cmd/dbmigrate/state.go index a62f03850f..05b9e86948 100644 --- a/platform-api/cmd/dbmigrate/state.go +++ b/platform-api/cmd/dbmigrate/state.go @@ -114,7 +114,17 @@ func newRun(opts *Options, logger *slog.Logger) (*Run, error) { dir := opts.OutDir open := func(base string) (*os.File, error) { p := filepath.Join(dir, suffixed(base, opts.RunID, opts.DryRun, "jsonl")) - return os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + f, err := os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return nil, err + } + // O_CREATE sets the mode only on creation; enforce 0600 on a pre-existing + // (possibly world-readable) file too — these artifacts hold full v1 rows. + if err := f.Chmod(0o600); err != nil { + _ = f.Close() + return nil, err + } + return f, nil } qf, err := open("quarantine") if err != nil { @@ -321,6 +331,9 @@ func (r *Run) saveCheckpoint() error { if err := os.WriteFile(tmp, b, 0o600); err != nil { return err } + if err := os.Chmod(tmp, 0o600); err != nil { // enforce on a pre-existing stale tmp + return err + } return os.Rename(tmp, r.ckptPath) } @@ -340,7 +353,10 @@ func (r *Run) writeReport() error { return err } p := filepath.Join(r.opts.OutDir, suffixed("migration-report", r.opts.RunID, r.opts.DryRun, "json")) - return os.WriteFile(p, b, 0o600) + if err := os.WriteFile(p, b, 0o600); err != nil { + return err + } + return os.Chmod(p, 0o600) // enforce 0600 on a pre-existing report file } func (r *Run) close() { diff --git a/platform-api/cmd/dbmigrate/verify.go b/platform-api/cmd/dbmigrate/verify.go index a49da464ea..735978a2c5 100644 --- a/platform-api/cmd/dbmigrate/verify.go +++ b/platform-api/cmd/dbmigrate/verify.go @@ -96,7 +96,9 @@ func runVerify(argv []string) error { // The key is optional for verify (it only enables the token decrypt round-trip); // report a load failure rather than silently swallowing it. if err := loadEncryptionKey(o); err != nil { - logger.Warn("could not load subscription-token key; the decrypt round-trip check will be skipped", "error", err) + // Do not log err verbatim — it may echo key material or the DSN. A generic + // warning is enough; the token round-trip check simply won't run. + logger.Warn("could not load the subscription-token key; the decrypt round-trip check will be skipped") } v1, err := openDB(o.V1DSN, logger) if err != nil { @@ -215,7 +217,10 @@ func writeVerifyReport(o *Options, r *VerifyReport) error { return err } p := filepath.Join(o.OutDir, suffixed("verify-report", o.RunID, false, "json")) - return os.WriteFile(p, b, 0o600) + if err := os.WriteFile(p, b, 0o600); err != nil { + return err + } + return os.Chmod(p, 0o600) // enforce 0600 on a pre-existing report file } func printVerify(r *VerifyReport) {