From 1c42e8cf1fc62001def6bcf1e026c94cbfa76178 Mon Sep 17 00:00:00 2001 From: NewtTheWolf Date: Tue, 18 Aug 2026 20:41:47 +0200 Subject: [PATCH 1/3] feat(releases): capture the README per release and serve it per version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README was only ever read on the git-ref manifest path. The asset-first path — the default, and the one every documented release takes — returned readme: null, and manifestPatch wrote that null over the plugin row. So the README appeared after a manual refresh and was wiped again by the next release, which is what authors saw as 'it will not populate consistently'. Two changes behind that symptom: - The asset path now reads the README at the release's tag. A tag is immutable, so that is the README the release actually shipped with. The lookup order (readmes locale map, readme path, then the conventional root filenames) is shared with the git-ref path instead of duplicated. - manifestPatch omits when a pass resolved none, rather than writing null. Mirrors the manifestSha256/manifestRaw guard that persistRelease already had for the same reason. The README is also stored on the release row, so browsing an older version shows the docs that shipped with it instead of whatever the plugin's README says today. Served by GET /api/plugins/:slug/releases/:version/readme and loaded lazily when a release row is expanded, so the detail payload does not grow a full README per version. Releases predating the column report captured:false instead of falling back to the current README, which would quietly misrepresent the old version. For that history, POST /api/admin/plugins/:id/backfill-readmes reads each release's README at its tag and fills it in — tags are still on the forge, so nothing is lost. Migration added for all three dialects; the app applies them at boot. --- .../migration.sql | 2 + .../migration.sql | 2 + .../migration.sql | 5 + apps/api/src/db/schema.mysql.ts | 5 + apps/api/src/db/schema.pg.ts | 5 + apps/api/src/db/schema.ts | 5 + apps/api/src/lib/manifest-apply.ts | 19 ++- apps/api/src/lib/manifest.ts | 110 +++++++++----- apps/api/src/lib/readme.ts | 26 ++++ apps/api/src/lib/release-ingest.ts | 91 ++++++++++-- .../admin/plugins/[id]/backfill-readmes.ts | 66 +++++++++ .../api/admin/plugins/[id]/replay-webhook.ts | 6 +- .../src/routes/api/plugins/[slug]/index.ts | 25 +--- .../[slug]/releases/[version]/readme.ts | 79 ++++++++++ apps/api/src/routes/api/submit/oauth.ts | 8 +- apps/api/src/routes/api/webhooks/release.ts | 6 +- apps/api/tests/lib/release-readme.test.ts | 135 ++++++++++++++++++ apps/frontend/messages/de.json | 2 + apps/frontend/messages/en.json | 2 + apps/frontend/messages/es.json | 2 + apps/frontend/messages/fr.json | 2 + apps/frontend/messages/it.json | 2 + apps/frontend/messages/zh-CN.json | 2 + .../src/routes/plugins/[slug]/+page.svelte | 46 +++++- 24 files changed, 575 insertions(+), 78 deletions(-) create mode 100644 apps/api/src/db/migrations.mysql/20260818200000_release_readme/migration.sql create mode 100644 apps/api/src/db/migrations.pg/20260818200000_release_readme/migration.sql create mode 100644 apps/api/src/db/migrations/20260818200000_release_readme/migration.sql create mode 100644 apps/api/src/lib/readme.ts create mode 100644 apps/api/src/routes/api/admin/plugins/[id]/backfill-readmes.ts create mode 100644 apps/api/src/routes/api/plugins/[slug]/releases/[version]/readme.ts create mode 100644 apps/api/tests/lib/release-readme.test.ts diff --git a/apps/api/src/db/migrations.mysql/20260818200000_release_readme/migration.sql b/apps/api/src/db/migrations.mysql/20260818200000_release_readme/migration.sql new file mode 100644 index 0000000..869f83b --- /dev/null +++ b/apps/api/src/db/migrations.mysql/20260818200000_release_readme/migration.sql @@ -0,0 +1,2 @@ +-- Per-release README markdown. See sqlite migration for rationale. +ALTER TABLE `releases` ADD `readme` text; diff --git a/apps/api/src/db/migrations.pg/20260818200000_release_readme/migration.sql b/apps/api/src/db/migrations.pg/20260818200000_release_readme/migration.sql new file mode 100644 index 0000000..748e176 --- /dev/null +++ b/apps/api/src/db/migrations.pg/20260818200000_release_readme/migration.sql @@ -0,0 +1,2 @@ +-- Per-release README markdown. See sqlite migration for rationale. +ALTER TABLE "releases" ADD COLUMN "readme" text; diff --git a/apps/api/src/db/migrations/20260818200000_release_readme/migration.sql b/apps/api/src/db/migrations/20260818200000_release_readme/migration.sql new file mode 100644 index 0000000..338bcb2 --- /dev/null +++ b/apps/api/src/db/migrations/20260818200000_release_readme/migration.sql @@ -0,0 +1,5 @@ +-- README markdown captured at each release's tag, so an older version can be +-- read exactly as it shipped instead of showing whatever the plugin's current +-- README happens to be. Raw markdown, or a JSON locale map when the manifest +-- declares `readmes`. NULL on releases ingested before this column existed. +ALTER TABLE `releases` ADD `readme` text; diff --git a/apps/api/src/db/schema.mysql.ts b/apps/api/src/db/schema.mysql.ts index 712179c..987d0c3 100644 --- a/apps/api/src/db/schema.mysql.ts +++ b/apps/api/src/db/schema.mysql.ts @@ -125,6 +125,11 @@ export const releases = mysqlTable( manifestSha256: varchar('manifest_sha256', { length: 64 }), // Canonical raw .tabularium bytes — see schema.ts for rationale. manifestRaw: text('manifest_raw'), + // README markdown captured at this release's tag. Raw markdown, or a + // JSON locale map when the manifest declares `readmes` — same shape as + // plugins.readme, so pickReadme() handles both. NULL on releases + // ingested before this column existed. + readme: text('readme'), createdAt: ts('created_at').notNull().$defaultFn(now), }, (t) => ({ diff --git a/apps/api/src/db/schema.pg.ts b/apps/api/src/db/schema.pg.ts index d19ff8b..3b1d256 100644 --- a/apps/api/src/db/schema.pg.ts +++ b/apps/api/src/db/schema.pg.ts @@ -125,6 +125,11 @@ export const releases = pgTable( manifestSha256: text('manifest_sha256'), // Canonical raw .tabularium bytes — see schema.ts for rationale. manifestRaw: text('manifest_raw'), + // README markdown captured at this release's tag. Raw markdown, or a + // JSON locale map when the manifest declares `readmes` — same shape as + // plugins.readme, so pickReadme() handles both. NULL on releases + // ingested before this column existed. + readme: text('readme'), createdAt: ts('created_at').notNull().$defaultFn(now), }, (t) => ({ diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 8e214f7..8cb2957 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -134,6 +134,11 @@ export const releases = sqliteTable( // registry can serve the manifest itself — clients verify // sha256(manifest_raw) === manifestSha256 (JWS-signed) without the forge. manifestRaw: text('manifest_raw'), + // README markdown captured at this release's tag. Raw markdown, or a + // JSON locale map when the manifest declares `readmes` — same shape as + // plugins.readme, so pickReadme() handles both. NULL on releases + // ingested before this column existed. + readme: text('readme'), createdAt: integer('created_at').notNull().$defaultFn(now), }, (t) => ({ diff --git a/apps/api/src/lib/manifest-apply.ts b/apps/api/src/lib/manifest-apply.ts index 21562d5..774e27e 100644 --- a/apps/api/src/lib/manifest-apply.ts +++ b/apps/api/src/lib/manifest-apply.ts @@ -46,7 +46,11 @@ export type PluginManifestUpdate = { license: string | null iconUrl: string | null screenshots: string | null - readme: string | null + // Optional on purpose: a pass that resolved no README leaves the stored one + // alone instead of nulling it. Mirrors the manifestSha256/manifestRaw guard + // in persistRelease — a blank overwrite used to wipe the README on every + // asset-resolved release. + readme?: string | null documentationUrl: string | null supportEmail: string | null issuesUrl: string | null @@ -63,12 +67,19 @@ import { jsonArrayOrNull as jsonArray } from './util' * Convert a parsed manifest into a column patch for the `plugins` row. * `repoBase` is used to resolve relative icon/screenshot paths to absolute URLs. */ +// Storage shape for a resolved README: a JSON locale map when the manifest +// declares `readmes`, otherwise plain markdown. Same encoding for the plugin +// column and the per-release column, so pickReadme() reads either one. +export function readmePayloadOf(m: ResolvedManifest): string | null { + return m.readmeLocales ? JSON.stringify(m.readmeLocales) : (m.readmeMarkdown ?? null) +} + export function manifestPatch( m: ResolvedManifest, opts: { repoBase: string; version: string | null }, ): PluginManifestUpdate { - const { parsed, readmeMarkdown, readmeLocales } = m - const readmePayload = readmeLocales ? JSON.stringify(readmeLocales) : (readmeMarkdown ?? null) + const { parsed } = m + const readmePayload = readmePayloadOf(m) const iconUrl = parsed.icon ? resolveAbsolute(opts.repoBase, parsed.icon) : null const screenshots = @@ -91,7 +102,6 @@ export function manifestPatch( license: parsed.license ?? null, iconUrl, screenshots: jsonArray(screenshots), - readme: readmePayload, documentationUrl: parsed.documentation_url ?? null, supportEmail: parsed.support?.email ?? null, issuesUrl: parsed.support?.issues_url ?? null, @@ -100,6 +110,7 @@ export function manifestPatch( manifestVersion: opts.version, updatedAt: Date.now(), } + if (readmePayload !== null) patch.readme = readmePayload if (parsed.name) patch.name = parsed.name if (parsed.description) patch.description = parsed.description if (parsed.homepage) patch.homepage = parsed.homepage diff --git a/apps/api/src/lib/manifest.ts b/apps/api/src/lib/manifest.ts index 100b62a..6a36be9 100644 --- a/apps/api/src/lib/manifest.ts +++ b/apps/api/src/lib/manifest.ts @@ -238,13 +238,79 @@ async function fetchAssetContent(url: string, accessToken: string): Promise<{ co return { content: text, bytes: new TextEncoder().encode(text).length } } +// Resolves the README a manifest points at: the localized `readmes` map first, +// then a single `readme` path, then the conventional root filenames. Shared by +// both manifest paths so an asset-resolved release captures the same README a +// git-ref-resolved one would. +async function resolveReadme( + fetch: FileFetcher, + parsed: Manifest, +): Promise<{ readmeMarkdown: string | null; readmeLocales: ReadmeMap | null }> { + let readmeMarkdown: string | null = null + let readmeLocales: ReadmeMap | null = null + + if (parsed.readmes && Object.keys(parsed.readmes).length > 0) { + readmeLocales = {} + for (const [locale, path] of Object.entries(parsed.readmes)) { + try { + const r = await fetch(path) + if (r) readmeLocales[locale] = r.content + } catch (err) { + log.warn({ err, locale, path }, 'localized readme fetch failed') + } + } + if (Object.keys(readmeLocales).length === 0) readmeLocales = null + } + if (!readmeLocales && parsed.readme) { + try { + const r = await fetch(parsed.readme) + if (r) readmeMarkdown = r.content + } catch (err) { + log.warn({ err, readme: parsed.readme }, 'readme path fetch failed — manifest still applied') + } + } + if (!readmeLocales && !readmeMarkdown) { + for (const fallback of ['README.md', 'readme.md', 'README.markdown']) { + try { + const r = await fetch(fallback) + if (r) { + readmeMarkdown = r.content + break + } + } catch { + // try next + } + } + } + return { readmeMarkdown, readmeLocales } +} + +// Reads the README for an already-ingested release, given the manifest that +// shipped with it. Used by the backfill: releases stored before the per-release +// README column existed still have their tag on the forge, and a tag is +// immutable, so this recovers exactly what that version shipped with. +export async function fetchReadmeAtTag( + accessToken: string, + ref: RepoRef, + tag: string, + parsed: Manifest, +): Promise<{ readmeMarkdown: string | null; readmeLocales: ReadmeMap | null }> { + return resolveReadme(fetcherFor(accessToken, ref, tag), parsed) +} + // Asset-first manifest resolution: scan the release's published assets for // any filename in the configured candidate list and ingest that. Avoids the // race + auth flakiness of the git-ref fetch path because release assets are // immutable per release and served by the forge's CDN. +// +// The README is not a release asset, so it is read from the repo at the +// release's tag when `readmeAt` is supplied. A tag is immutable, so that is +// the same content the release shipped with — without it the plugin's README +// would be blank for every asset-resolved release. export async function resolveManifestFromReleaseAssets( accessToken: string, assets: ReleaseAsset[], + readmeAt?: { ref: RepoRef; tag: string }, ): Promise { if (assets.length === 0) return null const byName = new Map(assets.map((a) => [a.name, a])) @@ -264,7 +330,12 @@ export async function resolveManifestFromReleaseAssets( continue } const parsed = parseManifestText(got.content) - return { raw: got.content, parsed, readmeMarkdown: null, readmeLocales: null } + if (!readmeAt) return { raw: got.content, parsed, readmeMarkdown: null, readmeLocales: null } + const { readmeMarkdown, readmeLocales } = await resolveReadme( + fetcherFor(accessToken, readmeAt.ref, readmeAt.tag), + parsed, + ) + return { raw: got.content, parsed, readmeMarkdown, readmeLocales } } catch (err) { if (err instanceof UpstreamUnauthorizedError) throw err if (err instanceof ManifestValidationError) { @@ -297,42 +368,7 @@ export async function resolveManifest( continue } const parsed = parseManifestText(got.content) - let readmeMarkdown: string | null = null - let readmeLocales: ReadmeMap | null = null - - if (parsed.readmes && Object.keys(parsed.readmes).length > 0) { - readmeLocales = {} - for (const [locale, path] of Object.entries(parsed.readmes)) { - try { - const r = await fetch(path) - if (r) readmeLocales[locale] = r.content - } catch (err) { - log.warn({ err, locale, path }, 'localized readme fetch failed') - } - } - if (Object.keys(readmeLocales).length === 0) readmeLocales = null - } - if (!readmeLocales && parsed.readme) { - try { - const r = await fetch(parsed.readme) - if (r) readmeMarkdown = r.content - } catch (err) { - log.warn({ err, readme: parsed.readme }, 'readme path fetch failed — manifest still applied') - } - } - if (!readmeLocales && !readmeMarkdown) { - for (const fallback of ['README.md', 'readme.md', 'README.markdown']) { - try { - const r = await fetch(fallback) - if (r) { - readmeMarkdown = r.content - break - } - } catch { - // try next - } - } - } + const { readmeMarkdown, readmeLocales } = await resolveReadme(fetch, parsed) return { raw: got.content, parsed, readmeMarkdown, readmeLocales } } catch (err) { if (err instanceof UpstreamUnauthorizedError) throw err diff --git a/apps/api/src/lib/readme.ts b/apps/api/src/lib/readme.ts new file mode 100644 index 0000000..d57fd72 --- /dev/null +++ b/apps/api/src/lib/readme.ts @@ -0,0 +1,26 @@ +// README storage is either plain markdown or a JSON locale map (written when +// the manifest declares `readmes`). Both the plugin column and the per-release +// column use that shape, so one reader serves both. +export function pickReadme( + raw: string | null, + preferredLocale: string | undefined, +): { markdown: string | null; locale: string | null; available: string[] } { + if (!raw) return { markdown: null, locale: null, available: [] } + if (!raw.startsWith('{')) return { markdown: raw, locale: null, available: [] } + try { + const map = JSON.parse(raw) as Record + const available = Object.keys(map).filter((k) => typeof map[k] === 'string') + if (available.length === 0) return { markdown: null, locale: null, available: [] } + const pick = (locale: string | undefined): string | null => { + if (locale && typeof map[locale] === 'string') return locale + return null + } + const baseLocale = preferredLocale?.split('-')[0] + const chosen = pick(preferredLocale) ?? pick(baseLocale) ?? pick('en') ?? available[0] + return { markdown: map[chosen] as string, locale: chosen, available } + } catch { + return { markdown: raw, locale: null, available: [] } + } +} + +export const README_TTL = 600 diff --git a/apps/api/src/lib/release-ingest.ts b/apps/api/src/lib/release-ingest.ts index f4a7bec..08bcbf9 100644 --- a/apps/api/src/lib/release-ingest.ts +++ b/apps/api/src/lib/release-ingest.ts @@ -12,9 +12,18 @@ import { recordAudit } from './audit' import { fetchAttestation } from './attestation' import { logger } from './logger' import { compareSemver, tagToVersion } from './semver' -import { resolveManifest, resolveManifestFromReleaseAssets, fetchReleaseAssetList, rawContentBase } from './manifest' +import { + resolveManifest, + resolveManifestFromReleaseAssets, + fetchReleaseAssetList, + fetchReadmeAtTag, + parseManifestText, + rawContentBase, + type Manifest, +} from './manifest' import { manifestPatch, + readmePayloadOf, applyManifestToPlugin, assertManifestVersionMatches, ManifestVersionMismatchError, @@ -31,7 +40,7 @@ const log = logger.child({ module: 'release-ingest' }) export async function persistRelease( plugin: { id: string; latestVersion: string | null }, normalized: NormalizedRelease, - opts: { manifestSha256?: string | null; manifestRaw?: string | null } = {}, + opts: { manifestSha256?: string | null; manifestRaw?: string | null; readme?: string | null } = {}, ): Promise<{ version: string; assetMap: AssetMap }> { // Strict semver gate: rejecting here keeps lax tags ("v1.2", "release-foo") // out of the releases table entirely. Manifest-side validation already @@ -49,11 +58,17 @@ export async function persistRelease( // manifest this pass. Asset-only re-ingests (rehash, asset backfill) leave // a previously-signed manifestSha256/manifestRaw intact instead of nulling // it — a wiped hash would make the integrity endpoint silently 404. - const set: { assets: string; manifestSha256?: string | null; manifestRaw?: string | null } = { + const set: { + assets: string + manifestSha256?: string | null + manifestRaw?: string | null + readme?: string | null + } = { assets: serializeAssets(assetMap), } if (opts.manifestSha256 !== undefined) set.manifestSha256 = opts.manifestSha256 if (opts.manifestRaw !== undefined) set.manifestRaw = opts.manifestRaw + if (opts.readme !== undefined) set.readme = opts.readme await db .insert(releases) .values({ @@ -63,6 +78,7 @@ export async function persistRelease( assets: serializeAssets(assetMap), manifestSha256: opts.manifestSha256 ?? null, manifestRaw: opts.manifestRaw ?? null, + readme: opts.readme ?? null, }) .onConflictDoUpdate({ target: [releases.pluginId, releases.version], set }) @@ -198,7 +214,7 @@ export async function refreshManifestAtRelease( tag: string, version: string, assets: Array<{ name: string; url: string }> = [], -): Promise<{ sha: string; raw: string } | null> { +): Promise<{ sha: string; raw: string; readme: string | null } | null> { const ref = parseRepoUrl(plugin.repoUrl) if (!ref) return null const ownerIdentity = await db.query.identities.findFirst({ @@ -234,7 +250,7 @@ export async function refreshManifestAtRelease( // only when the operator allows it via the require_release_asset setting. let manifest: Awaited> = null try { - manifest = await resolveManifestFromReleaseAssets(token, assets) + manifest = await resolveManifestFromReleaseAssets(token, assets, { ref, tag }) } catch (err) { log.warn({ err, slug: plugin.id, tag }, 'asset-based manifest resolution errored') } @@ -311,7 +327,7 @@ export async function refreshManifestAtRelease( await cache().del(latestCacheKey(plugin.id)) const sha = manifestSha256(manifest.raw) log.info({ slug: plugin.id, version }, 'manifest refreshed at release') - return { sha, raw: manifest.raw } + return { sha, raw: manifest.raw, readme: readmePayloadOf(manifest) } } catch (err) { const reason = err instanceof Error ? err.message : String(err) log.warn({ err, slug: plugin.id }, 'manifest apply failed after fetch succeeded') @@ -372,7 +388,7 @@ async function recheckAssetsOnce(plugin: PluginRef, tag: string, version: string } let manifest: Awaited> = null try { - manifest = await resolveManifestFromReleaseAssets(token, assets) + manifest = await resolveManifestFromReleaseAssets(token, assets, { ref, tag }) } catch (err) { log.warn({ err, slug: plugin.id, tag, attempt }, 'asset recheck resolution errored') } @@ -408,7 +424,11 @@ async function recheckAssetsOnce(plugin: PluginRef, tag: string, version: string // the publish webhook, so the original persistRelease saw no manifest). await db .update(releases) - .set({ manifestSha256: manifestSha256(manifest.raw), manifestRaw: manifest.raw }) + .set({ + manifestSha256: manifestSha256(manifest.raw), + manifestRaw: manifest.raw, + readme: readmePayloadOf(manifest), + }) .where(and(eq(releases.pluginId, plugin.id), eq(releases.version, expectedVersion))) // Back-fill the binary asset URL map. The original webhook race left @@ -465,3 +485,58 @@ async function recheckAssetsOnce(plugin: PluginRef, tag: string, version: string log.warn({ err, slug: plugin.id, tag }, 'asset recheck apply failed') } } + +// One-shot repair for releases ingested before READMEs were captured per +// release. Walks the plugin's releases that have none, reads the README at +// each release's tag, and stores it. Tags are immutable, so this recovers the +// exact docs each version shipped with. Releases whose tag or README is gone +// upstream are skipped, not failed — a partial backfill is still an +// improvement over a blank history. +export async function backfillReleaseReadmes(plugin: { + id: string + ownerId: string + repoUrl: string +}): Promise<{ scanned: number; filled: number; skipped: number }> { + const ref = parseRepoUrl(plugin.repoUrl) + if (!ref) return { scanned: 0, filled: 0, skipped: 0 } + const ownerIdentity = await db.query.identities.findFirst({ + where: { userId: plugin.ownerId, providerInstanceId: ref.instance.id }, + }) + if (!ownerIdentity?.accessToken) throw new Error('owner has no stored access token') + const token = await getValidAccessToken(ownerIdentity, ref.instance) + + const rows = await db.query.releases.findMany({ + where: { pluginId: plugin.id }, + columns: { id: true, version: true, readme: true, manifestRaw: true }, + }) + const pending = rows.filter((r) => !r.readme) + + let filled = 0 + let skipped = 0 + for (const row of pending) { + // The stored manifest tells us where that version kept its README; without + // one, resolveReadme still falls back to the conventional root filenames. + let parsed: Manifest + try { + parsed = row.manifestRaw ? parseManifestText(row.manifestRaw) : ({ readme: null } as unknown as Manifest) + } catch { + parsed = { readme: null } as unknown as Manifest + } + // Releases are stored without the "v", tags commonly carry it. + for (const tag of [`v${row.version}`, row.version]) { + try { + const got = await fetchReadmeAtTag(token, ref, tag, parsed) + const payload = readmePayloadOf({ raw: '', parsed, ...got }) + if (!payload) continue + await db.update(releases).set({ readme: payload }).where(eq(releases.id, row.id)) + filled++ + break + } catch (err) { + log.warn({ err, slug: plugin.id, version: row.version, tag }, 'readme backfill failed for tag') + } + } + } + skipped = pending.length - filled + log.info({ slug: plugin.id, scanned: pending.length, filled, skipped }, 'release readme backfill done') + return { scanned: pending.length, filled, skipped } +} diff --git a/apps/api/src/routes/api/admin/plugins/[id]/backfill-readmes.ts b/apps/api/src/routes/api/admin/plugins/[id]/backfill-readmes.ts new file mode 100644 index 0000000..8587d22 --- /dev/null +++ b/apps/api/src/routes/api/admin/plugins/[id]/backfill-readmes.ts @@ -0,0 +1,66 @@ +import { Elysia, t } from 'elysia' +import { adminMiddleware } from '$middleware/admin' +import { db } from '$db' +import { backfillReleaseReadmes } from '$lib/release-ingest' +import { OAuthExpiredError, UpstreamUnauthorizedError, reauthErrorBody } from '$lib/oauth-tokens' +import { recordAudit, actorFromAdmin } from '$lib/audit' +import { logger } from '$lib/logger' + +const log = logger.child({ module: 'readme-backfill' }) + +export default new Elysia().use(adminMiddleware).post( + '/', + async ({ params, set, admin, request }) => { + const plugin = await db.query.plugins.findFirst({ where: { id: params.id } }) + if (!plugin) { + set.status = 404 + return { error: 'Plugin not found' } + } + + let result: Awaited> + try { + result = await backfillReleaseReadmes(plugin) + } catch (e) { + if (e instanceof OAuthExpiredError || e instanceof UpstreamUnauthorizedError) { + set.status = 401 + return reauthErrorBody(e) + } + log.warn({ err: e, slug: plugin.id }, 'readme backfill failed') + set.status = 422 + return { error: e instanceof Error ? e.message : 'Backfill failed' } + } + + await recordAudit({ + ...actorFromAdmin(admin, request), + action: 'plugin.backfill_readmes', + target: `plugin:${plugin.id}`, + meta: result, + }) + + return { ok: true, ...result } + }, + { + detail: { + tags: ['Admin'], + summary: 'Backfill per-release READMEs from each release tag', + description: + 'Releases ingested before READMEs were captured per release have none stored. This reads the README at ' + + "each such release's tag using the owner's stored OAuth token and fills it in. Tags are immutable, so the " + + 'recovered README is what that version actually shipped with. Already-populated releases are left alone.', + operationId: 'backfillReleaseReadmes', + security: [{ bearerAuth: [] }, { cookieAuth: [] }], + }, + params: t.Object({ id: t.String() }), + response: { + 200: t.Object({ + ok: t.Boolean(), + scanned: t.Number(), + filled: t.Number(), + skipped: t.Number(), + }), + 401: t.Object({ error: t.String(), reauthFor: t.String() }), + 404: t.Object({ error: t.String() }), + 422: t.Object({ error: t.String() }), + }, + }, +) diff --git a/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts b/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts index 460e541..4213d12 100644 --- a/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts +++ b/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts @@ -80,7 +80,11 @@ export default new Elysia().use(adminMiddleware).post( queueMicrotask(async () => { const manifest = await refreshManifestAtRelease(plugin, normalized.tag, version) if (manifest) - await persistRelease(plugin, normalized, { manifestSha256: manifest.sha, manifestRaw: manifest.raw }) + await persistRelease(plugin, normalized, { + manifestSha256: manifest.sha, + manifestRaw: manifest.raw, + readme: manifest.readme, + }) }) queueMicrotask(() => { diff --git a/apps/api/src/routes/api/plugins/[slug]/index.ts b/apps/api/src/routes/api/plugins/[slug]/index.ts index d77dc7d..b18f703 100644 --- a/apps/api/src/routes/api/plugins/[slug]/index.ts +++ b/apps/api/src/routes/api/plugins/[slug]/index.ts @@ -6,6 +6,7 @@ import { authMiddleware } from '$middleware/auth' import { projectPluginDetail } from '$lib/plugin-projection' import { renderMarkdown } from '$lib/markdown' import { cache, isString } from '$lib/cache' +import { pickReadme, README_TTL } from '$lib/readme' import { buildIntegrity } from '$lib/release-integrity' const screenshotSchema = t.Object({ @@ -77,32 +78,8 @@ const pluginDetailSchema = t.Object({ releases: t.Array(releaseSchema), }) -function pickReadme( - raw: string | null, - preferredLocale: string | undefined, -): { markdown: string | null; locale: string | null; available: string[] } { - if (!raw) return { markdown: null, locale: null, available: [] } - if (!raw.startsWith('{')) return { markdown: raw, locale: null, available: [] } - try { - const map = JSON.parse(raw) as Record - const available = Object.keys(map).filter((k) => typeof map[k] === 'string') - if (available.length === 0) return { markdown: null, locale: null, available: [] } - const pick = (locale: string | undefined): string | null => { - if (locale && typeof map[locale] === 'string') return locale - return null - } - const baseLocale = preferredLocale?.split('-')[0] - const chosen = pick(preferredLocale) ?? pick(baseLocale) ?? pick('en') ?? available[0] - return { markdown: map[chosen] as string, locale: chosen, available } - } catch { - return { markdown: raw, locale: null, available: [] } - } -} - const errorSchema = t.Object({ error: t.String() }) -const README_TTL = 600 - export default new Elysia() .get( '/', diff --git a/apps/api/src/routes/api/plugins/[slug]/releases/[version]/readme.ts b/apps/api/src/routes/api/plugins/[slug]/releases/[version]/readme.ts new file mode 100644 index 0000000..8baad0d --- /dev/null +++ b/apps/api/src/routes/api/plugins/[slug]/releases/[version]/readme.ts @@ -0,0 +1,79 @@ +import { Elysia, t } from 'elysia' +import { db } from '$db' +import { renderMarkdown } from '$lib/markdown' +import { cache, isString } from '$lib/cache' +import { pickReadme, README_TTL } from '$lib/readme' + +// The README as it stood at a specific release. Captured at ingest from the +// release's tag, so browsing an older version shows the docs that shipped with +// it instead of the plugin's current README. Releases ingested before the +// column existed have none — the response says so rather than falling back to +// the current one, which would quietly misrepresent the old version. +export default new Elysia().get( + '/', + async ({ params, query, set }) => { + const plugin = await db.query.plugins.findFirst({ + where: { id: params.slug }, + columns: { id: true, status: true }, + }) + if (!plugin || plugin.status !== 'approved') { + set.status = 404 + return { error: 'Plugin not found' } + } + + const release = await db.query.releases.findFirst({ + where: { pluginId: plugin.id, version: params.version }, + columns: { id: true, version: true, readme: true }, + }) + if (!release) { + set.status = 404 + return { error: 'Release not found' } + } + + const picked = pickReadme(release.readme ?? null, query.locale) + if (!picked.markdown) { + return { + version: release.version, + readmeHtml: null, + readmeLocale: null, + readmeAvailableLocales: [], + captured: false, + } + } + + const cacheKey = `release:readme:${plugin.id}:${release.id}:${picked.locale ?? 'default'}` + let readmeHtml = await cache().get(cacheKey, isString) + if (!readmeHtml) { + readmeHtml = await renderMarkdown(picked.markdown) + await cache().set(cacheKey, readmeHtml, README_TTL) + } + + return { + version: release.version, + readmeHtml, + readmeLocale: picked.locale, + readmeAvailableLocales: picked.available, + captured: true, + } + }, + { + detail: { + tags: ['Plugins'], + summary: 'README captured at a specific release', + operationId: 'getReleaseReadme', + }, + params: t.Object({ slug: t.String(), version: t.String() }), + query: t.Object({ locale: t.Optional(t.String({ maxLength: 16 })) }), + response: { + 200: t.Object({ + version: t.String(), + readmeHtml: t.Nullable(t.String()), + readmeLocale: t.Nullable(t.String()), + readmeAvailableLocales: t.Array(t.String()), + // false when this release predates per-release README capture + captured: t.Boolean(), + }), + 404: t.Object({ error: t.String() }), + }, + }, +) diff --git a/apps/api/src/routes/api/submit/oauth.ts b/apps/api/src/routes/api/submit/oauth.ts index 919d0f8..a6fe5e6 100644 --- a/apps/api/src/routes/api/submit/oauth.ts +++ b/apps/api/src/routes/api/submit/oauth.ts @@ -11,7 +11,7 @@ import { getValidAccessToken, OAuthExpiredError } from '$lib/oauth-tokens' import { env } from '$lib/env' import { getSetting } from '$lib/settings' import { resolveManifest, rawContentBase, ManifestValidationError } from '$lib/manifest' -import { manifestPatch, applyManifestToPlugin } from '$lib/manifest-apply' +import { manifestPatch, readmePayloadOf, applyManifestToPlugin } from '$lib/manifest-apply' import { fetchLatestRelease } from '$lib/release-fetch' import { persistRelease, hashReleaseAssetsAsync, manifestSha256 } from '$lib/release-ingest' import { getFeatures } from '$lib/features' @@ -154,7 +154,11 @@ export default new Elysia() // factored out so both call sites stay in sync. if (latestRelease.published) { const manifestOpts = manifest - ? { manifestSha256: manifestSha256(manifest.raw), manifestRaw: manifest.raw } + ? { + manifestSha256: manifestSha256(manifest.raw), + manifestRaw: manifest.raw, + readme: readmePayloadOf(manifest), + } : {} const { version, assetMap } = await persistRelease( { id: slug, latestVersion: null }, diff --git a/apps/api/src/routes/api/webhooks/release.ts b/apps/api/src/routes/api/webhooks/release.ts index 5c476b4..a8eea51 100644 --- a/apps/api/src/routes/api/webhooks/release.ts +++ b/apps/api/src/routes/api/webhooks/release.ts @@ -107,7 +107,11 @@ export default new Elysia().use(rateLimit({ bucket: 'webhook-release', limit: 60 if (manifest) { // Re-persist the row with the now-known manifest sha + raw bytes // (persistRelease ran without them because the fetch is async). - await persistRelease(plugin, normalized, { manifestSha256: manifest.sha, manifestRaw: manifest.raw }) + await persistRelease(plugin, normalized, { + manifestSha256: manifest.sha, + manifestRaw: manifest.raw, + readme: manifest.readme, + }) } }) diff --git a/apps/api/tests/lib/release-readme.test.ts b/apps/api/tests/lib/release-readme.test.ts new file mode 100644 index 0000000..1ae4f66 --- /dev/null +++ b/apps/api/tests/lib/release-readme.test.ts @@ -0,0 +1,135 @@ +import { describe, it, expect, spyOn, afterEach, beforeEach } from 'bun:test' +import { clearDb } from '../helpers' +import { resolveManifestFromReleaseAssets } from '../../src/lib/manifest' +import { manifestPatch, readmePayloadOf } from '../../src/lib/manifest-apply' +import type { RepoRef } from '../../src/lib/providers' + +const MANIFEST = JSON.stringify({ name: 'alpha', version: '1.0.0', description: 'A test plugin.' }) + +const ref: RepoRef = { + instance: { + id: 'github', + kind: 'github', + displayName: 'GitHub', + baseUrl: 'https://github.com', + clientId: '', + clientSecret: '', + logoUrl: null, + enabled: true, + }, + owner: 'alice', + repo: 'my-plugin', + fullName: 'alice/my-plugin', +} + +let spy: ReturnType | null = null +let requested: string[] = [] + +function mockFetch(byUrlContains: Array<[string, string]>) { + requested = [] + spy = spyOn(global, 'fetch').mockImplementation((async (url: string | URL | Request) => { + const key = String(typeof url === 'string' ? url : url instanceof URL ? url.toString() : url.url) + requested.push(key) + const hit = byUrlContains.find(([needle]) => key.includes(needle)) + if (!hit) return new Response('not found', { status: 404 }) + return new Response(hit[1], { + status: 200, + headers: { 'content-length': String(new TextEncoder().encode(hit[1]).length) }, + }) + }) as unknown as typeof fetch) +} + +afterEach(() => { + spy?.mockRestore() + spy = null +}) + +describe('resolveManifestFromReleaseAssets — README capture', () => { + beforeEach(clearDb) + + // Regression: the asset path used to return readme: null unconditionally, so + // every asset-resolved release stored a blank README — and the patch then + // wrote that null over whatever the plugin already had. + it('reads the README at the release tag when told where to look', async () => { + mockFetch([ + ['/default.tabularium', MANIFEST], + ['contents/README.md', '# Alpha\n\nDocs at this tag.'], + ]) + const got = await resolveManifestFromReleaseAssets( + 'token', + [{ name: 'default.tabularium', url: 'https://example.com/default.tabularium' }], + { ref, tag: 'v1.0.0' }, + ) + expect(got?.readmeMarkdown).toContain('Docs at this tag') + // pinned to the tag, not to a branch head + expect(requested.some((u) => u.includes('contents/README.md') && u.includes('ref=v1.0.0'))).toBe(true) + }) + + it('leaves the README null when the caller gives no tag', async () => { + mockFetch([['/default.tabularium', MANIFEST]]) + const got = await resolveManifestFromReleaseAssets('token', [ + { name: 'default.tabularium', url: 'https://example.com/default.tabularium' }, + ]) + expect(got?.parsed.name).toBe('alpha') + expect(got?.readmeMarkdown).toBeNull() + }) + + it('prefers the localized readmes map over the root fallback', async () => { + const withLocales = JSON.stringify({ + name: 'alpha', + version: '1.0.0', + readmes: { en: 'docs/en.md', de: 'docs/de.md' }, + }) + mockFetch([ + ['/default.tabularium', withLocales], + ['docs%2Fen.md', '# English'], + ['docs%2Fde.md', '# Deutsch'], + ]) + const got = await resolveManifestFromReleaseAssets( + 'token', + [{ name: 'default.tabularium', url: 'https://example.com/default.tabularium' }], + { ref, tag: 'v1.0.0' }, + ) + expect(got?.readmeLocales?.en).toContain('English') + expect(got?.readmeLocales?.de).toContain('Deutsch') + expect(readmePayloadOf(got!)).toContain('"de"') + }) +}) + +describe('manifestPatch — README is never blanked', () => { + const base = { raw: MANIFEST, parsed: JSON.parse(MANIFEST) } + + it('omits readme entirely when this pass resolved none', () => { + const patch = manifestPatch( + { ...base, readmeMarkdown: null, readmeLocales: null }, + { + repoBase: 'https://raw.example/', + version: '1.0.0', + }, + ) + // absent, not null — an explicit null would overwrite the stored README + expect('readme' in patch).toBe(false) + }) + + it('sets readme when one was resolved', () => { + const patch = manifestPatch( + { ...base, readmeMarkdown: '# Hi', readmeLocales: null }, + { + repoBase: 'https://raw.example/', + version: '1.0.0', + }, + ) + expect(patch.readme).toBe('# Hi') + }) + + it('stores a locale map as JSON', () => { + const patch = manifestPatch( + { ...base, readmeMarkdown: null, readmeLocales: { en: '# Hi' } }, + { + repoBase: 'https://raw.example/', + version: '1.0.0', + }, + ) + expect(JSON.parse(patch.readme as string)).toEqual({ en: '# Hi' }) + }) +}) diff --git a/apps/frontend/messages/de.json b/apps/frontend/messages/de.json index 7eeac91..560e26b 100644 --- a/apps/frontend/messages/de.json +++ b/apps/frontend/messages/de.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "Langsam — 1× pro Minute", "plugin_detail_refresh_reauth_redirect": "OAuth-Token abgelaufen — leite zur Re-Autorisierung weiter…", "plugin_detail_refresh_running": "Lade neu…", + "plugin_detail_release_readme_missing": "Für dieses Release wurde kein README erfasst.", + "plugin_detail_release_readme_title": "README dieser Version", "plugin_detail_releases": "Releases", "plugin_detail_releases_subtitle": "Die neueste Version ist im Badge oben hervorgehoben. Die SHA256-Spalte zeigt, wenn die Integrität gehasht wurde.", "plugin_detail_report_issue": "Problem melden", diff --git a/apps/frontend/messages/en.json b/apps/frontend/messages/en.json index 181cece..c9c09fc 100644 --- a/apps/frontend/messages/en.json +++ b/apps/frontend/messages/en.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "Slow down — once per minute", "plugin_detail_refresh_reauth_redirect": "OAuth token expired — redirecting to re-authorize…", "plugin_detail_refresh_running": "Refetching…", + "plugin_detail_release_readme_missing": "No README was captured for this release.", + "plugin_detail_release_readme_title": "README at this version", "plugin_detail_releases": "Releases", "plugin_detail_releases_subtitle": "Latest version is highlighted in the badge above. SHA256 column shows when integrity has been hashed.", "plugin_detail_report_issue": "Report issue", diff --git a/apps/frontend/messages/es.json b/apps/frontend/messages/es.json index 8df0fc8..043d6dc 100644 --- a/apps/frontend/messages/es.json +++ b/apps/frontend/messages/es.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "Despacio — una vez por minuto", "plugin_detail_refresh_reauth_redirect": "Token OAuth expirado — redirigiendo para re-autorizar…", "plugin_detail_refresh_running": "Recargando…", + "plugin_detail_release_readme_missing": "No se guardó ningún README para esta versión.", + "plugin_detail_release_readme_title": "README de esta versión", "plugin_detail_releases": "Releases", "plugin_detail_releases_subtitle": "La última versión está resaltada en el badge de arriba. La columna SHA256 muestra cuando la integridad ha sido hasheada.", "plugin_detail_report_issue": "Reportar problema", diff --git a/apps/frontend/messages/fr.json b/apps/frontend/messages/fr.json index 0ec552b..0538c74 100644 --- a/apps/frontend/messages/fr.json +++ b/apps/frontend/messages/fr.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "Doucement — une fois par minute", "plugin_detail_refresh_reauth_redirect": "Jeton OAuth expiré — redirection pour ré-autoriser…", "plugin_detail_refresh_running": "Rechargement…", + "plugin_detail_release_readme_missing": "Aucun README n'a été enregistré pour cette version.", + "plugin_detail_release_readme_title": "README de cette version", "plugin_detail_releases": "Releases", "plugin_detail_releases_subtitle": "La dernière version est mise en avant dans le badge ci-dessus. La colonne SHA256 s'affiche quand l'intégrité a été hachée.", "plugin_detail_report_issue": "Signaler un problème", diff --git a/apps/frontend/messages/it.json b/apps/frontend/messages/it.json index 1bb1f58..9ed1d29 100644 --- a/apps/frontend/messages/it.json +++ b/apps/frontend/messages/it.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "Piano — una volta al minuto", "plugin_detail_refresh_reauth_redirect": "Token OAuth scaduto — reindirizzamento per ri-autorizzare…", "plugin_detail_refresh_running": "Ricaricando…", + "plugin_detail_release_readme_missing": "Nessun README è stato registrato per questa release.", + "plugin_detail_release_readme_title": "README di questa versione", "plugin_detail_releases": "Release", "plugin_detail_releases_subtitle": "L'ultima versione è evidenziata nel badge sopra. La colonna SHA256 si mostra quando l'integrità è stata hashata.", "plugin_detail_report_issue": "Segnala problema", diff --git a/apps/frontend/messages/zh-CN.json b/apps/frontend/messages/zh-CN.json index 3bae6a3..cba6992 100644 --- a/apps/frontend/messages/zh-CN.json +++ b/apps/frontend/messages/zh-CN.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "慢点 — 每分钟一次", "plugin_detail_refresh_reauth_redirect": "OAuth 令牌已过期 — 正在重定向以重新授权…", "plugin_detail_refresh_running": "正在加载…", + "plugin_detail_release_readme_missing": "此发布未记录 README。", + "plugin_detail_release_readme_title": "该版本的 README", "plugin_detail_releases": "Releases", "plugin_detail_releases_subtitle": "最新版本在上方徽章中高亮显示。SHA256 列显示已计算完整性哈希的版本。", "plugin_detail_report_issue": "反馈问题", diff --git a/apps/frontend/src/routes/plugins/[slug]/+page.svelte b/apps/frontend/src/routes/plugins/[slug]/+page.svelte index f001d1e..6974fba 100644 --- a/apps/frontend/src/routes/plugins/[slug]/+page.svelte +++ b/apps/frontend/src/routes/plugins/[slug]/+page.svelte @@ -110,6 +110,27 @@ untrack(() => load(locale)) }) + // README as it stood at each release, loaded the first time a release row is + // expanded. Kept out of the plugin payload so the detail response does not + // have to carry a full README per version. + type VersionReadme = { loading: boolean; html: string | null; captured: boolean } + let versionReadmes = $state>({}) + + async function loadVersionReadme(pluginId: string, version: string) { + if (versionReadmes[version]) return + versionReadmes[version] = { loading: true, html: null, captured: false } + try { + const res = await fetch( + `/api/plugins/${encodeURIComponent(pluginId)}/releases/${encodeURIComponent(version)}/readme?locale=${encodeURIComponent(locale)}`, + ) + if (!res.ok) throw new Error(String(res.status)) + const data = (await res.json()) as { readmeHtml: string | null; captured: boolean } + versionReadmes[version] = { loading: false, html: data.readmeHtml, captured: data.captured } + } catch { + versionReadmes[version] = { loading: false, html: null, captured: false } + } + } + const sortedReleases = $derived( plugin?.releases ? [...plugin.releases].sort((a, b) => b.createdAt - a.createdAt) : [], ) @@ -747,7 +768,14 @@ {#each sortedReleases as release (release.id)} {@const totalSize = Object.values(release.assets).reduce((acc, a) => acc + (a.size ?? 0), 0)} {@const platformCount = Object.keys(release.assets).length} -
+ {@const vr = versionReadmes[release.version]} +
{ + if (plugin && (e.currentTarget as HTMLDetailsElement).open) + void loadVersionReadme(plugin.id, release.version) + }} + > @@ -845,6 +873,22 @@ {/each} +
+

+ {m.plugin_detail_release_readme_title()} +

+ {#if !vr || vr.loading} + + {:else if vr.html} +
+ {@html vr.html} +
+ {:else} +

{m.plugin_detail_release_readme_missing()}

+ {/if} +
{/each} From 414834fef622e7e511b452df6684a7b765cec118 Mon Sep 17 00:00:00 2001 From: NewtTheWolf Date: Tue, 18 Aug 2026 20:45:22 +0200 Subject: [PATCH 2/3] fix(ingest): persist min_runtime_version from the manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The releases table has had a min_runtime_version column since the initial migration, the manifest schema validates the field, and the submit preview echoes it back — but no ingest path ever wrote it. Only the seed script did, which is why it looked wired up. Every release ingested through a webhook, submit or publish stored NULL. The API then served that NULL faithfully: the postgresql plugin declares min_runtime_version 0.20.0 in its .tabularium for v1.0.0-beta.7, and /api/plugins/postgresql/releases/1.0.0-beta.7 answers with min_runtime_version: null. Clients had nothing to gate on, so a plugin could install on a runtime too old for it. Carried through the same opts guard as manifestSha256/manifestRaw, so an asset-only re-ingest (rehash, asset backfill) leaves a stored value intact instead of nulling it. --- apps/api/src/lib/release-ingest.ts | 20 +++++++-- .../api/admin/plugins/[id]/replay-webhook.ts | 1 + apps/api/src/routes/api/publish/[slug].ts | 2 + apps/api/src/routes/api/submit/oauth.ts | 1 + apps/api/src/routes/api/webhooks/release.ts | 1 + apps/api/tests/lib/release-readme.test.ts | 42 ++++++++++++++++++- 6 files changed, 63 insertions(+), 4 deletions(-) diff --git a/apps/api/src/lib/release-ingest.ts b/apps/api/src/lib/release-ingest.ts index 08bcbf9..80fae01 100644 --- a/apps/api/src/lib/release-ingest.ts +++ b/apps/api/src/lib/release-ingest.ts @@ -40,7 +40,12 @@ const log = logger.child({ module: 'release-ingest' }) export async function persistRelease( plugin: { id: string; latestVersion: string | null }, normalized: NormalizedRelease, - opts: { manifestSha256?: string | null; manifestRaw?: string | null; readme?: string | null } = {}, + opts: { + manifestSha256?: string | null + manifestRaw?: string | null + readme?: string | null + minRuntimeVersion?: string | null + } = {}, ): Promise<{ version: string; assetMap: AssetMap }> { // Strict semver gate: rejecting here keeps lax tags ("v1.2", "release-foo") // out of the releases table entirely. Manifest-side validation already @@ -63,12 +68,14 @@ export async function persistRelease( manifestSha256?: string | null manifestRaw?: string | null readme?: string | null + minRuntimeVersion?: string | null } = { assets: serializeAssets(assetMap), } if (opts.manifestSha256 !== undefined) set.manifestSha256 = opts.manifestSha256 if (opts.manifestRaw !== undefined) set.manifestRaw = opts.manifestRaw if (opts.readme !== undefined) set.readme = opts.readme + if (opts.minRuntimeVersion !== undefined) set.minRuntimeVersion = opts.minRuntimeVersion await db .insert(releases) .values({ @@ -79,6 +86,7 @@ export async function persistRelease( manifestSha256: opts.manifestSha256 ?? null, manifestRaw: opts.manifestRaw ?? null, readme: opts.readme ?? null, + minRuntimeVersion: opts.minRuntimeVersion ?? null, }) .onConflictDoUpdate({ target: [releases.pluginId, releases.version], set }) @@ -214,7 +222,7 @@ export async function refreshManifestAtRelease( tag: string, version: string, assets: Array<{ name: string; url: string }> = [], -): Promise<{ sha: string; raw: string; readme: string | null } | null> { +): Promise<{ sha: string; raw: string; readme: string | null; minRuntimeVersion: string | null } | null> { const ref = parseRepoUrl(plugin.repoUrl) if (!ref) return null const ownerIdentity = await db.query.identities.findFirst({ @@ -327,7 +335,12 @@ export async function refreshManifestAtRelease( await cache().del(latestCacheKey(plugin.id)) const sha = manifestSha256(manifest.raw) log.info({ slug: plugin.id, version }, 'manifest refreshed at release') - return { sha, raw: manifest.raw, readme: readmePayloadOf(manifest) } + return { + sha, + raw: manifest.raw, + readme: readmePayloadOf(manifest), + minRuntimeVersion: manifest.parsed.min_runtime_version ?? null, + } } catch (err) { const reason = err instanceof Error ? err.message : String(err) log.warn({ err, slug: plugin.id }, 'manifest apply failed after fetch succeeded') @@ -428,6 +441,7 @@ async function recheckAssetsOnce(plugin: PluginRef, tag: string, version: string manifestSha256: manifestSha256(manifest.raw), manifestRaw: manifest.raw, readme: readmePayloadOf(manifest), + minRuntimeVersion: manifest.parsed.min_runtime_version ?? null, }) .where(and(eq(releases.pluginId, plugin.id), eq(releases.version, expectedVersion))) diff --git a/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts b/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts index 4213d12..a56c192 100644 --- a/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts +++ b/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts @@ -84,6 +84,7 @@ export default new Elysia().use(adminMiddleware).post( manifestSha256: manifest.sha, manifestRaw: manifest.raw, readme: manifest.readme, + minRuntimeVersion: manifest.minRuntimeVersion, }) }) diff --git a/apps/api/src/routes/api/publish/[slug].ts b/apps/api/src/routes/api/publish/[slug].ts index 8e354d9..b86435c 100644 --- a/apps/api/src/routes/api/publish/[slug].ts +++ b/apps/api/src/routes/api/publish/[slug].ts @@ -165,6 +165,7 @@ export default new Elysia().use(publisherTokenMiddleware).post( const { assetMap } = await persistRelease({ id: slug, latestVersion: null }, normalized, { manifestSha256: manifestSha256(body.manifest), manifestRaw: body.manifest, + minRuntimeVersion: parsed.min_runtime_version ?? null, }) // Apply manifest fields (category, tags, icon, extensions, …) to the @@ -267,6 +268,7 @@ export default new Elysia().use(publisherTokenMiddleware).post( const { assetMap } = await persistRelease({ id: slug, latestVersion: existing.latestVersion }, normalized, { manifestSha256: manifestSha256(body.manifest), manifestRaw: body.manifest, + minRuntimeVersion: parsed.min_runtime_version ?? null, }) const patch = manifestPatch( diff --git a/apps/api/src/routes/api/submit/oauth.ts b/apps/api/src/routes/api/submit/oauth.ts index a6fe5e6..be253c2 100644 --- a/apps/api/src/routes/api/submit/oauth.ts +++ b/apps/api/src/routes/api/submit/oauth.ts @@ -158,6 +158,7 @@ export default new Elysia() manifestSha256: manifestSha256(manifest.raw), manifestRaw: manifest.raw, readme: readmePayloadOf(manifest), + minRuntimeVersion: manifest.parsed.min_runtime_version ?? null, } : {} const { version, assetMap } = await persistRelease( diff --git a/apps/api/src/routes/api/webhooks/release.ts b/apps/api/src/routes/api/webhooks/release.ts index a8eea51..dfa62df 100644 --- a/apps/api/src/routes/api/webhooks/release.ts +++ b/apps/api/src/routes/api/webhooks/release.ts @@ -111,6 +111,7 @@ export default new Elysia().use(rateLimit({ bucket: 'webhook-release', limit: 60 manifestSha256: manifest.sha, manifestRaw: manifest.raw, readme: manifest.readme, + minRuntimeVersion: manifest.minRuntimeVersion, }) } }) diff --git a/apps/api/tests/lib/release-readme.test.ts b/apps/api/tests/lib/release-readme.test.ts index 1ae4f66..c8c24a2 100644 --- a/apps/api/tests/lib/release-readme.test.ts +++ b/apps/api/tests/lib/release-readme.test.ts @@ -1,5 +1,8 @@ import { describe, it, expect, spyOn, afterEach, beforeEach } from 'bun:test' -import { clearDb } from '../helpers' +import { clearDb, makeUser, makePlugin } from '../helpers' +import { db } from '../../src/db' +import { persistRelease } from '../../src/lib/release-ingest' +import type { NormalizedRelease } from '../../src/lib/webhook' import { resolveManifestFromReleaseAssets } from '../../src/lib/manifest' import { manifestPatch, readmePayloadOf } from '../../src/lib/manifest-apply' import type { RepoRef } from '../../src/lib/providers' @@ -22,6 +25,13 @@ const ref: RepoRef = { fullName: 'alice/my-plugin', } +const sampleRelease: NormalizedRelease = { + repoUrl: 'https://github.com/alice/my-plugin', + published: true, + tag: 'v1.0.0', + assets: [], +} + let spy: ReturnType | null = null let requested: string[] = [] @@ -133,3 +143,33 @@ describe('manifestPatch — README is never blanked', () => { expect(JSON.parse(patch.readme as string)).toEqual({ en: '# Hi' }) }) }) + +describe('persistRelease — manifest fields reach the release row', () => { + beforeEach(clearDb) + + // Regression: min_runtime_version was declared in the manifest, validated by + // the schema and echoed back by the submit preview, but no ingest path ever + // wrote it to the release row — so /latest served null and clients could + // install a plugin their runtime was too old for. + it('stores min_runtime_version from the manifest', async () => { + const user = await makeUser({ username: 'alice' }) + const plugin = await makePlugin(user.id, { id: 'alpha' }) + await persistRelease({ id: plugin.id, latestVersion: null }, sampleRelease, { + minRuntimeVersion: '0.20.0', + }) + const row = await db.query.releases.findFirst({ where: { pluginId: plugin.id, version: '1.0.0' } }) + expect(row?.minRuntimeVersion).toBe('0.20.0') + }) + + it('leaves a stored min_runtime_version alone on an asset-only re-ingest', async () => { + const user = await makeUser({ username: 'alice' }) + const plugin = await makePlugin(user.id, { id: 'alpha' }) + await persistRelease({ id: plugin.id, latestVersion: null }, sampleRelease, { + minRuntimeVersion: '0.20.0', + }) + // rehash and asset backfills re-persist without a manifest in hand + await persistRelease({ id: plugin.id, latestVersion: '1.0.0' }, sampleRelease) + const row = await db.query.releases.findFirst({ where: { pluginId: plugin.id, version: '1.0.0' } }) + expect(row?.minRuntimeVersion).toBe('0.20.0') + }) +}) From 110051c18c2316214081d3d3c6b9af86b33289aa Mon Sep 17 00:00:00 2001 From: NewtTheWolf Date: Tue, 18 Aug 2026 21:00:10 +0200 Subject: [PATCH 3/3] fix(backfill): recover READMEs without an owner token The README backfill threw when a plugin's owner had no stored or no longer valid OAuth token, which is exactly the case for the older plugins whose history the backfill exists to repair. A README in a public repo needs no credentials, so a missing or expired token now downgrades to an unauthenticated read from the forge's raw-content host instead of failing the run. Private repos answer 404 there and are skipped, same as any other release whose README is gone. --- apps/api/src/lib/manifest.ts | 21 ++++++++++++++++++-- apps/api/src/lib/release-ingest.ts | 13 ++++++++++-- apps/api/tests/lib/release-readme.test.ts | 24 ++++++++++++++++++++++- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/apps/api/src/lib/manifest.ts b/apps/api/src/lib/manifest.ts index 6a36be9..3683117 100644 --- a/apps/api/src/lib/manifest.ts +++ b/apps/api/src/lib/manifest.ts @@ -290,12 +290,29 @@ async function resolveReadme( // README column existed still have their tag on the forge, and a tag is // immutable, so this recovers exactly what that version shipped with. export async function fetchReadmeAtTag( - accessToken: string, + accessToken: string | null, ref: RepoRef, tag: string, parsed: Manifest, ): Promise<{ readmeMarkdown: string | null; readmeLocales: ReadmeMap | null }> { - return resolveReadme(fetcherFor(accessToken, ref, tag), parsed) + return resolveReadme(accessToken ? fetcherFor(accessToken, ref, tag) : makePublicRawFetcher(ref, tag), parsed) +} + +// Unauthenticated read from the forge's raw-content host. A README in a public +// repo needs no credentials, so the backfill can still recover history for a +// plugin whose owner has no usable OAuth token left. A private repo answers +// 404 here, which the caller treats as "nothing to recover" rather than an +// error. +function makePublicRawFetcher(ref: RepoRef, tag: string): FileFetcher { + const base = rawContentBase(ref, tag) + return async (path) => { + const res = await fetch(base + path.split('/').map(encodeURIComponent).join('/')) + if (!res.ok) return null + const len = Number(res.headers.get('content-length') ?? 0) + if (len > MAX_README_BYTES) throw new Error(`${path} exceeds size cap`) + const text = await res.text() + return { content: text, bytes: new TextEncoder().encode(text).length } + } } // Asset-first manifest resolution: scan the release's published assets for diff --git a/apps/api/src/lib/release-ingest.ts b/apps/api/src/lib/release-ingest.ts index 80fae01..fc96abc 100644 --- a/apps/api/src/lib/release-ingest.ts +++ b/apps/api/src/lib/release-ingest.ts @@ -513,11 +513,20 @@ export async function backfillReleaseReadmes(plugin: { }): Promise<{ scanned: number; filled: number; skipped: number }> { const ref = parseRepoUrl(plugin.repoUrl) if (!ref) return { scanned: 0, filled: 0, skipped: 0 } + // A README in a public repo needs no credentials, so a missing or expired + // owner token downgrades to an unauthenticated raw-content read instead of + // failing the whole backfill. Private repos simply yield nothing. const ownerIdentity = await db.query.identities.findFirst({ where: { userId: plugin.ownerId, providerInstanceId: ref.instance.id }, }) - if (!ownerIdentity?.accessToken) throw new Error('owner has no stored access token') - const token = await getValidAccessToken(ownerIdentity, ref.instance) + let token: string | null = null + if (ownerIdentity?.accessToken) { + try { + token = await getValidAccessToken(ownerIdentity, ref.instance) + } catch (err) { + log.warn({ err, slug: plugin.id }, 'owner token unusable — backfilling READMEs unauthenticated') + } + } const rows = await db.query.releases.findMany({ where: { pluginId: plugin.id }, diff --git a/apps/api/tests/lib/release-readme.test.ts b/apps/api/tests/lib/release-readme.test.ts index c8c24a2..0a4e568 100644 --- a/apps/api/tests/lib/release-readme.test.ts +++ b/apps/api/tests/lib/release-readme.test.ts @@ -3,7 +3,7 @@ import { clearDb, makeUser, makePlugin } from '../helpers' import { db } from '../../src/db' import { persistRelease } from '../../src/lib/release-ingest' import type { NormalizedRelease } from '../../src/lib/webhook' -import { resolveManifestFromReleaseAssets } from '../../src/lib/manifest' +import { resolveManifestFromReleaseAssets, fetchReadmeAtTag } from '../../src/lib/manifest' import { manifestPatch, readmePayloadOf } from '../../src/lib/manifest-apply' import type { RepoRef } from '../../src/lib/providers' @@ -173,3 +173,25 @@ describe('persistRelease — manifest fields reach the release row', () => { expect(row?.minRuntimeVersion).toBe('0.20.0') }) }) + +describe('fetchReadmeAtTag — unauthenticated fallback', () => { + beforeEach(clearDb) + + // The backfill has to work for plugins whose owner has no usable OAuth token + // left. A README in a public repo needs no credentials, so it reads from the + // raw-content host instead of the authenticated contents API. + it('reads from the raw host when no token is given', async () => { + mockFetch([['raw.githubusercontent.com', '# Public docs']]) + const got = await fetchReadmeAtTag(null, ref, 'v1.0.0', { name: 'alpha', version: '1.0.0' } as never) + expect(got.readmeMarkdown).toContain('Public docs') + expect(requested.every((u) => !u.includes('api.github.com'))).toBe(true) + expect(requested.some((u) => u.includes('/v1.0.0/README.md'))).toBe(true) + }) + + it('yields nothing when the repo is not public', async () => { + mockFetch([]) + const got = await fetchReadmeAtTag(null, ref, 'v1.0.0', { name: 'alpha', version: '1.0.0' } as never) + expect(got.readmeMarkdown).toBeNull() + expect(got.readmeLocales).toBeNull() + }) +})