Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .changeset/legacy-bigint-storage-class.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
---
"executor": patch
---

**Saved integrations come back after an upgrade left an OAuth expiry in the old number format**

Some installs lost every saved integration from the MCP gateway at once. The credentials were never deleted — the gateway simply could not read the table they live in, so it served an empty tool list and restarting did not help.

The `connection.expires_at` column records when an OAuth access token expires. It used to be a plain number; it now holds the value's digits, because a millisecond timestamp is larger than a 32-bit integer. SQLite does not rewrite rows when a column's type changes, so a connection saved by an older build still held the old form. Reading one back failed, and because the failure happened while mapping the row, it failed the whole query rather than that one field — one stale row was enough to hide every integration.

A boot-time migration now converts those values to the current form. It runs before anything reads the table, so the integrations are back on the first restart after upgrading. It only touches values still in the old numeric form: rows already written by a current build are left exactly as they are, and it runs once.
11 changes: 10 additions & 1 deletion apps/host-selfhost/src/db/data-migrations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@
// renamed.
// ---------------------------------------------------------------------------

import { sqliteDataMigration, type SqliteDataMigration } from "@executor-js/sdk";
import {
bigintStorageClassSqliteMigration,
sqliteDataMigration,
type SqliteDataMigration,
} from "@executor-js/sdk";
import { runSqliteAuthConfigMigration } from "@executor-js/sdk/http-auth";
import {
openApiNdjsonOutputDataMigration,
Expand All @@ -20,6 +24,11 @@ import { encryptedSecretsRepartitionDataMigration } from "@executor-js/plugin-en
import { authConfigTransforms } from "./auth-config-migration";

export const selfHostDataMigrations: readonly SqliteDataMigration[] = [
// FIRST, because it un-bricks reads every later migration and the whole app
// depend on: `bigint` columns an older build left in SQLite's INTEGER storage
// class cannot be read by the bigint row mapper, so a single legacy
// `connection.expires_at` failed every catalog read (issue #1771).
bigintStorageClassSqliteMigration,
// Rewrite pre-canonical integration auth configs into the shared
// placements model.
sqliteDataMigration("2026-06-05-auth-config-placements", (client) =>
Expand Down
6 changes: 6 additions & 0 deletions apps/local/src/db/data-migrations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import {
Effect,
bigintStorageClassSqliteMigration,
oauthClientGcSqliteMigration,
sqliteDataMigration,
type SqliteDataMigration,
Expand All @@ -30,6 +31,11 @@ export const localDataMigrations: readonly SqliteDataMigration[] = [
// stamped atomically inside the staged v2 build; fresh/pre-v2-native DBs get
// the same stamp here so future boots skip legacy shape probing.
{ name: LOCAL_V1_V2_LEDGER_NAME, run: () => Effect.void },
// FIRST, because it un-bricks reads every later migration and the whole app
// depend on: `bigint` columns an older build left in SQLite's INTEGER storage
// class cannot be read by the bigint row mapper, so a single legacy
// `connection.expires_at` failed every catalog read (issue #1771).
bigintStorageClassSqliteMigration,
// Rewrite pre-canonical integration auth configs (incl. v1→v2 outputs)
// into the shared placements model.
sqliteDataMigration("2026-06-05-auth-config-placements", (client) =>
Expand Down
120 changes: 120 additions & 0 deletions apps/local/src/db/legacy-bigint-boot.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
// ---------------------------------------------------------------------------
// Boot-level proof for issue #1771: a local database holding a legacy INTEGER
// `connection.expires_at` is unreadable, and the local boot sequence heals it.
//
// The migration body is unit-tested in the SDK. What this pins is the WIRING —
// that `localDataMigrations` actually carries the entry, early enough that the
// catalog reads which follow the ledger run see repaired rows. That wiring is
// the part that recovers a user's install; a correct migration nobody runs
// would leave the gateway just as empty.
// ---------------------------------------------------------------------------

import { afterEach, beforeEach, describe, expect, it } from "@effect/vitest";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { Effect } from "effect";
import { withQueryContext } from "@executor-js/fumadb/query";

import { collectTables } from "@executor-js/api/server";
import { runSqliteDataMigrations } from "@executor-js/sdk";

import { localDataMigrations } from "./data-migrations";
import { createSqliteFumaDb } from "./sqlite-fumadb";

const TENANT = "executor-workspace-1771";
const SUBJECT = "local";
// Epoch millis, the shape an OAuth token expiry takes.
const LEGACY_EXPIRES_AT = 1787321623456;

let workDir: string;

beforeEach(() => {
workDir = mkdtempSync(join(tmpdir(), "executor-legacy-bigint-"));
});

afterEach(() => {
rmSync(workDir, { recursive: true, force: true });
});

const openDb = (dbPath: string) =>
createSqliteFumaDb({
tables: collectTables(),
namespace: "executor_local",
path: dbPath,
});

/** Write the row a pre-`bigint` build left behind: `expires_at` as a bare
* integer literal, which SQLite keeps in the INTEGER storage class. */
const seedLegacyConnection = async (dbPath: string): Promise<void> => {
const sqlite = await openDb(dbPath);
await sqlite.client.execute({
sql: `INSERT INTO connection
(row_id, tenant, owner, subject, integration, name, template, provider, item_ids,
expires_at, created_at, updated_at)
VALUES ('c1', ?, 'user', ?, 'acme', 'default', 'oauth2', 'file', ?,
${LEGACY_EXPIRES_AT}, ?, ?)`,
args: [
TENANT,
SUBJECT,
JSON.stringify({ token: "item_1" }),
Math.floor(Date.now() / 1000),
Math.floor(Date.now() / 1000),
],
});
await sqlite.close();
};

describe("local boot over a legacy bigint database", () => {
it("cannot read the connection table before the migrations run", async () => {
const dbPath = join(workDir, "data.db");
await seedLegacyConnection(dbPath);

const sqlite = await openDb(dbPath);
const scoped = withQueryContext(sqlite.db, { tenant: TENANT, subject: SUBJECT });
// The reported symptom: not a wrong value, a throw — so the gateway lost
// every saved integration at once.
await expect(scoped.findMany("connection", {})).rejects.toThrow(/type number/);
await sqlite.close();
});

it("heals it through the local data-migration registry", async () => {
const dbPath = join(workDir, "data.db");
await seedLegacyConnection(dbPath);

const sqlite = await openDb(dbPath);
const applied = await Effect.runPromise(
runSqliteDataMigrations(sqlite.client, localDataMigrations),
);
expect(applied).toContain("2026-08-28-bigint-storage-class");

const scoped = withQueryContext(sqlite.db, { tenant: TENANT, subject: SUBJECT });
const rows = await scoped.findMany("connection", {});
expect(rows.map((row) => [row.name, Number(row.expires_at)])).toEqual([
["default", LEGACY_EXPIRES_AT],
]);
await sqlite.close();
});

it("stays readable across a reboot", async () => {
const dbPath = join(workDir, "data.db");
await seedLegacyConnection(dbPath);

const first = await openDb(dbPath);
await Effect.runPromise(runSqliteDataMigrations(first.client, localDataMigrations));
await first.close();

// Second boot: the entry is stamped, so it is skipped — the data must
// already be in the shape the mapper reads.
const second = await openDb(dbPath);
const applied = await Effect.runPromise(
runSqliteDataMigrations(second.client, localDataMigrations),
);
expect(applied).not.toContain("2026-08-28-bigint-storage-class");

const scoped = withQueryContext(second.db, { tenant: TENANT, subject: SUBJECT });
const rows = await scoped.findMany("connection", {});
expect(rows.map((row) => Number(row.expires_at))).toEqual([LEGACY_EXPIRES_AT]);
await second.close();
});
});
8 changes: 8 additions & 0 deletions packages/core/sdk/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,14 @@ export {
oauthClientGcSqliteMigration,
runSqliteOAuthClientGcMigration,
} from "./sqlite-oauth-client-gc-migration";
// Rewrite `bigint` columns an earlier build left in SQLite's INTEGER storage
// class, which the bigint row mapper cannot read (issue #1771).
export {
bigintStorageClassSqliteMigration,
runSqliteBigintStorageClassMigration,
LEGACY_BIGINT_STORAGE_CLASS_COLUMNS,
type BigintStorageClassColumn,
} from "./sqlite-bigint-storage-class-migration";
export {
authToolFailure,
isUnauthorizedToolFailure,
Expand Down
202 changes: 202 additions & 0 deletions packages/core/sdk/src/sqlite-bigint-storage-class-migration.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
import { describe, expect, it } from "@effect/vitest";
import { Effect } from "effect";
import { withQueryContext } from "@executor-js/fumadb/query";

import { collectTables } from "./executor";
import { createSqliteTestFumaDb, type SqliteTestFumaDb } from "./sqlite-test-db";
import {
LEGACY_BIGINT_STORAGE_CLASS_COLUMNS,
bigintStorageClassSqliteMigration,
runSqliteBigintStorageClassMigration,
} from "./sqlite-bigint-storage-class-migration";

// A `bigint` column is stored on SQLite as a blob holding the decimal digits
// (drizzle's `blob({ mode: "bigint" })`). Before the columns below carried the
// `bigint` flag they were plain numbers, so SQLite kept them in the INTEGER
// storage class — and an install written by that build still holds integers
// today. Reading one back through the bigint mapper reaches
// `Buffer.from(<number>)`, which throws `ERR_INVALID_ARG_TYPE`, and because the
// throw is on the ROW mapper it takes down the whole `findMany`, not one field.
//
// That is issue #1771: `connection.expires_at` held an epoch-millis integer, so
// every catalog read threw and the MCP gateway served an empty tool list even
// though the integrations were still saved.

const TENANT = "t1";
const SUBJECT = "user_a";
const LEGACY_EXPIRES_AT = 1787321623456;
const HEALTHY_EXPIRES_AT = 1787321699999;

const withDb = <A>(body: (db: SqliteTestFumaDb) => Promise<A>): Promise<A> =>
Effect.runPromise(
Effect.acquireUseRelease(
Effect.promise(() => createSqliteTestFumaDb({ tables: collectTables() })),
(db) => Effect.promise(() => body(db)),
(db) => Effect.promise(() => db.close()),
),
);

const seconds = (ms: number) => Math.floor(ms / 1000);

/** Insert a connection row whose `expires_at` is set by a raw SQL expression,
* so the test controls its SQLite storage class exactly: an integer literal
* lands as INTEGER — what a build that declared the column `integer` wrote —
* while `CAST(... AS BLOB)` lands as BLOB, what the ORM writes today. A bound
* JS number would land as REAL under the column's current BLOB affinity, which
* is a different (also broken) shape. */
const insertConnection = (
db: SqliteTestFumaDb,
row: { readonly rowId: string; readonly name: string; readonly expiresAtSql: string },
): Promise<unknown> =>
db.client.execute({
sql: `INSERT INTO connection
(row_id, tenant, owner, subject, integration, name, template, provider, item_ids,
expires_at, created_at, updated_at)
VALUES (?, ?, 'user', ?, 'acme', ?, 'oauth2', 'file', ?, ${row.expiresAtSql}, ?, ?)`,
args: [
row.rowId,
TENANT,
SUBJECT,
row.name,
JSON.stringify({ token: "item_1" }),
seconds(Date.now()),
seconds(Date.now()),
],
});

/** The legacy shape: a bare integer literal, stored in the INTEGER class. */
const legacyInteger = String(LEGACY_EXPIRES_AT);
/** The current shape: the decimal digits as bytes. */
const currentBlob = `CAST('${HEALTHY_EXPIRES_AT}' AS BLOB)`;

const storageClassOf = async (db: SqliteTestFumaDb, rowId: string): Promise<string> => {
const result = await db.client.execute({
sql: "SELECT typeof(expires_at) AS kind FROM connection WHERE row_id = ?",
args: [rowId],
});
return String(result.rows[0]?.["kind"]);
};

describe("legacy bigint storage class migration", () => {
it.effect("reproduces the catalog read failure on a legacy integer row", () =>
Effect.promise(() =>
withDb(async (db) => {
await insertConnection(db, {
rowId: "c_legacy",
name: "legacy",
expiresAtSql: legacyInteger,
});
expect(await storageClassOf(db, "c_legacy")).toBe("integer");

const scoped = withQueryContext(db.db, { tenant: TENANT, subject: SUBJECT });
// Not "returns a bad value" — the read THROWS, which is why the gateway
// lost every saved integration rather than one field of one row.
await expect(scoped.findMany("connection", {})).rejects.toThrow(/type number/);
}),
),
);

it.effect("converts legacy integers so the catalog reads again", () =>
Effect.promise(() =>
withDb(async (db) => {
await insertConnection(db, {
rowId: "c_legacy",
name: "legacy",
expiresAtSql: legacyInteger,
});
await insertConnection(db, {
rowId: "c_healthy",
name: "healthy",
expiresAtSql: currentBlob,
});
await insertConnection(db, { rowId: "c_null", name: "null", expiresAtSql: "NULL" });

const converted = await Effect.runPromise(runSqliteBigintStorageClassMigration(db.client));
expect(converted).toBe(1);

expect(await storageClassOf(db, "c_legacy")).toBe("blob");
expect(await storageClassOf(db, "c_healthy")).toBe("blob");
expect(await storageClassOf(db, "c_null")).toBe("null");

const scoped = withQueryContext(db.db, { tenant: TENANT, subject: SUBJECT });
const rows = await scoped.findMany("connection", {});
expect(
rows.map((row) => [row.name, row.expires_at == null ? null : Number(row.expires_at)]),
).toEqual([
["healthy", HEALTHY_EXPIRES_AT],
["legacy", LEGACY_EXPIRES_AT],
["null", null],
]);
}),
),
);

it.effect("is idempotent", () =>
Effect.promise(() =>
withDb(async (db) => {
await insertConnection(db, {
rowId: "c_legacy",
name: "legacy",
expiresAtSql: legacyInteger,
});

expect(await Effect.runPromise(runSqliteBigintStorageClassMigration(db.client))).toBe(1);
expect(await Effect.runPromise(runSqliteBigintStorageClassMigration(db.client))).toBe(0);

const scoped = withQueryContext(db.db, { tenant: TENANT, subject: SUBJECT });
const rows = await scoped.findMany("connection", {});
expect(rows.map((row) => Number(row.expires_at))).toEqual([LEGACY_EXPIRES_AT]);
}),
),
);

it.effect("converts every bigint column that predates the flag", () =>
Effect.promise(() =>
withDb(async (db) => {
// `oauth_session.expires_at` was re-typed in the same change, and
// `connection.tools_synced_at` / `integration.config_revised_at` /
// `subject.last_seen_at` share the representation.
await db.client.execute({
sql: `INSERT INTO integration
(row_id, tenant, slug, plugin_id, description, config_revised_at, created_at, updated_at)
VALUES ('i1', ?, 'acme', 'openapi', '', ${legacyInteger}, ?, ?)`,
args: [TENANT, seconds(Date.now()), seconds(Date.now())],
});
await insertConnection(db, { rowId: "c1", name: "c1", expiresAtSql: "NULL" });
await db.client.execute(
`UPDATE connection SET tools_synced_at = ${legacyInteger} WHERE row_id = 'c1'`,
);

expect(await Effect.runPromise(runSqliteBigintStorageClassMigration(db.client))).toBe(2);

const scoped = withQueryContext(db.db, { tenant: TENANT, subject: SUBJECT });
const integrations = await scoped.findMany("integration", {});
expect(integrations.map((row) => Number(row.config_revised_at))).toEqual([
LEGACY_EXPIRES_AT,
]);
const connections = await scoped.findMany("connection", {});
expect(connections.map((row) => Number(row.tools_synced_at))).toEqual([LEGACY_EXPIRES_AT]);
}),
),
);

it("covers every bigint column the core schema declares", () => {
const tables = collectTables() as Record<string, { readonly columns: Record<string, unknown> }>;
const declared: string[] = [];
for (const [tableName, table] of Object.entries(tables)) {
for (const [columnName, column] of Object.entries(table.columns)) {
if ((column as { readonly type?: string }).type === "bigint") {
declared.push(`${tableName}.${columnName}`);
}
}
}
const covered = LEGACY_BIGINT_STORAGE_CLASS_COLUMNS.map(
(entry) => `${entry.table}.${entry.column}`,
);
expect(covered.slice().sort()).toEqual(declared.sort());
});

it("is registered under a stable, date-prefixed name", () => {
expect(bigintStorageClassSqliteMigration.name).toBe("2026-08-28-bigint-storage-class");
});
});
Loading
Loading