diff --git a/apps/api/src/db/migrations.mysql/20260818200000_release_readme/migration.sql b/apps/api/src/db/migrations.mysql/20260818200000_release_readme/migration.sql new file mode 100644 index 0000000..869f83b --- /dev/null +++ b/apps/api/src/db/migrations.mysql/20260818200000_release_readme/migration.sql @@ -0,0 +1,2 @@ +-- Per-release README markdown. See sqlite migration for rationale. +ALTER TABLE `releases` ADD `readme` text; diff --git a/apps/api/src/db/migrations.pg/20260818200000_release_readme/migration.sql b/apps/api/src/db/migrations.pg/20260818200000_release_readme/migration.sql new file mode 100644 index 0000000..748e176 --- /dev/null +++ b/apps/api/src/db/migrations.pg/20260818200000_release_readme/migration.sql @@ -0,0 +1,2 @@ +-- Per-release README markdown. See sqlite migration for rationale. +ALTER TABLE "releases" ADD COLUMN "readme" text; diff --git a/apps/api/src/db/migrations/20260818200000_release_readme/migration.sql b/apps/api/src/db/migrations/20260818200000_release_readme/migration.sql new file mode 100644 index 0000000..338bcb2 --- /dev/null +++ b/apps/api/src/db/migrations/20260818200000_release_readme/migration.sql @@ -0,0 +1,5 @@ +-- README markdown captured at each release's tag, so an older version can be +-- read exactly as it shipped instead of showing whatever the plugin's current +-- README happens to be. Raw markdown, or a JSON locale map when the manifest +-- declares `readmes`. NULL on releases ingested before this column existed. +ALTER TABLE `releases` ADD `readme` text; diff --git a/apps/api/src/db/schema.mysql.ts b/apps/api/src/db/schema.mysql.ts index 712179c..987d0c3 100644 --- a/apps/api/src/db/schema.mysql.ts +++ b/apps/api/src/db/schema.mysql.ts @@ -125,6 +125,11 @@ export const releases = mysqlTable( manifestSha256: varchar('manifest_sha256', { length: 64 }), // Canonical raw .tabularium bytes — see schema.ts for rationale. manifestRaw: text('manifest_raw'), + // README markdown captured at this release's tag. Raw markdown, or a + // JSON locale map when the manifest declares `readmes` — same shape as + // plugins.readme, so pickReadme() handles both. NULL on releases + // ingested before this column existed. + readme: text('readme'), createdAt: ts('created_at').notNull().$defaultFn(now), }, (t) => ({ diff --git a/apps/api/src/db/schema.pg.ts b/apps/api/src/db/schema.pg.ts index d19ff8b..3b1d256 100644 --- a/apps/api/src/db/schema.pg.ts +++ b/apps/api/src/db/schema.pg.ts @@ -125,6 +125,11 @@ export const releases = pgTable( manifestSha256: text('manifest_sha256'), // Canonical raw .tabularium bytes — see schema.ts for rationale. manifestRaw: text('manifest_raw'), + // README markdown captured at this release's tag. Raw markdown, or a + // JSON locale map when the manifest declares `readmes` — same shape as + // plugins.readme, so pickReadme() handles both. NULL on releases + // ingested before this column existed. + readme: text('readme'), createdAt: ts('created_at').notNull().$defaultFn(now), }, (t) => ({ diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 8e214f7..8cb2957 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -134,6 +134,11 @@ export const releases = sqliteTable( // registry can serve the manifest itself — clients verify // sha256(manifest_raw) === manifestSha256 (JWS-signed) without the forge. manifestRaw: text('manifest_raw'), + // README markdown captured at this release's tag. Raw markdown, or a + // JSON locale map when the manifest declares `readmes` — same shape as + // plugins.readme, so pickReadme() handles both. NULL on releases + // ingested before this column existed. + readme: text('readme'), createdAt: integer('created_at').notNull().$defaultFn(now), }, (t) => ({ diff --git a/apps/api/src/lib/manifest-apply.ts b/apps/api/src/lib/manifest-apply.ts index 21562d5..774e27e 100644 --- a/apps/api/src/lib/manifest-apply.ts +++ b/apps/api/src/lib/manifest-apply.ts @@ -46,7 +46,11 @@ export type PluginManifestUpdate = { license: string | null iconUrl: string | null screenshots: string | null - readme: string | null + // Optional on purpose: a pass that resolved no README leaves the stored one + // alone instead of nulling it. Mirrors the manifestSha256/manifestRaw guard + // in persistRelease — a blank overwrite used to wipe the README on every + // asset-resolved release. + readme?: string | null documentationUrl: string | null supportEmail: string | null issuesUrl: string | null @@ -63,12 +67,19 @@ import { jsonArrayOrNull as jsonArray } from './util' * Convert a parsed manifest into a column patch for the `plugins` row. * `repoBase` is used to resolve relative icon/screenshot paths to absolute URLs. */ +// Storage shape for a resolved README: a JSON locale map when the manifest +// declares `readmes`, otherwise plain markdown. Same encoding for the plugin +// column and the per-release column, so pickReadme() reads either one. +export function readmePayloadOf(m: ResolvedManifest): string | null { + return m.readmeLocales ? JSON.stringify(m.readmeLocales) : (m.readmeMarkdown ?? null) +} + export function manifestPatch( m: ResolvedManifest, opts: { repoBase: string; version: string | null }, ): PluginManifestUpdate { - const { parsed, readmeMarkdown, readmeLocales } = m - const readmePayload = readmeLocales ? JSON.stringify(readmeLocales) : (readmeMarkdown ?? null) + const { parsed } = m + const readmePayload = readmePayloadOf(m) const iconUrl = parsed.icon ? resolveAbsolute(opts.repoBase, parsed.icon) : null const screenshots = @@ -91,7 +102,6 @@ export function manifestPatch( license: parsed.license ?? null, iconUrl, screenshots: jsonArray(screenshots), - readme: readmePayload, documentationUrl: parsed.documentation_url ?? null, supportEmail: parsed.support?.email ?? null, issuesUrl: parsed.support?.issues_url ?? null, @@ -100,6 +110,7 @@ export function manifestPatch( manifestVersion: opts.version, updatedAt: Date.now(), } + if (readmePayload !== null) patch.readme = readmePayload if (parsed.name) patch.name = parsed.name if (parsed.description) patch.description = parsed.description if (parsed.homepage) patch.homepage = parsed.homepage diff --git a/apps/api/src/lib/manifest.ts b/apps/api/src/lib/manifest.ts index 100b62a..3683117 100644 --- a/apps/api/src/lib/manifest.ts +++ b/apps/api/src/lib/manifest.ts @@ -238,13 +238,96 @@ async function fetchAssetContent(url: string, accessToken: string): Promise<{ co return { content: text, bytes: new TextEncoder().encode(text).length } } +// Resolves the README a manifest points at: the localized `readmes` map first, +// then a single `readme` path, then the conventional root filenames. Shared by +// both manifest paths so an asset-resolved release captures the same README a +// git-ref-resolved one would. +async function resolveReadme( + fetch: FileFetcher, + parsed: Manifest, +): Promise<{ readmeMarkdown: string | null; readmeLocales: ReadmeMap | null }> { + let readmeMarkdown: string | null = null + let readmeLocales: ReadmeMap | null = null + + if (parsed.readmes && Object.keys(parsed.readmes).length > 0) { + readmeLocales = {} + for (const [locale, path] of Object.entries(parsed.readmes)) { + try { + const r = await fetch(path) + if (r) readmeLocales[locale] = r.content + } catch (err) { + log.warn({ err, locale, path }, 'localized readme fetch failed') + } + } + if (Object.keys(readmeLocales).length === 0) readmeLocales = null + } + if (!readmeLocales && parsed.readme) { + try { + const r = await fetch(parsed.readme) + if (r) readmeMarkdown = r.content + } catch (err) { + log.warn({ err, readme: parsed.readme }, 'readme path fetch failed — manifest still applied') + } + } + if (!readmeLocales && !readmeMarkdown) { + for (const fallback of ['README.md', 'readme.md', 'README.markdown']) { + try { + const r = await fetch(fallback) + if (r) { + readmeMarkdown = r.content + break + } + } catch { + // try next + } + } + } + return { readmeMarkdown, readmeLocales } +} + +// Reads the README for an already-ingested release, given the manifest that +// shipped with it. Used by the backfill: releases stored before the per-release +// README column existed still have their tag on the forge, and a tag is +// immutable, so this recovers exactly what that version shipped with. +export async function fetchReadmeAtTag( + accessToken: string | null, + ref: RepoRef, + tag: string, + parsed: Manifest, +): Promise<{ readmeMarkdown: string | null; readmeLocales: ReadmeMap | null }> { + return resolveReadme(accessToken ? fetcherFor(accessToken, ref, tag) : makePublicRawFetcher(ref, tag), parsed) +} + +// Unauthenticated read from the forge's raw-content host. A README in a public +// repo needs no credentials, so the backfill can still recover history for a +// plugin whose owner has no usable OAuth token left. A private repo answers +// 404 here, which the caller treats as "nothing to recover" rather than an +// error. +function makePublicRawFetcher(ref: RepoRef, tag: string): FileFetcher { + const base = rawContentBase(ref, tag) + return async (path) => { + const res = await fetch(base + path.split('/').map(encodeURIComponent).join('/')) + if (!res.ok) return null + const len = Number(res.headers.get('content-length') ?? 0) + if (len > MAX_README_BYTES) throw new Error(`${path} exceeds size cap`) + const text = await res.text() + return { content: text, bytes: new TextEncoder().encode(text).length } + } +} + // Asset-first manifest resolution: scan the release's published assets for // any filename in the configured candidate list and ingest that. Avoids the // race + auth flakiness of the git-ref fetch path because release assets are // immutable per release and served by the forge's CDN. +// +// The README is not a release asset, so it is read from the repo at the +// release's tag when `readmeAt` is supplied. A tag is immutable, so that is +// the same content the release shipped with — without it the plugin's README +// would be blank for every asset-resolved release. export async function resolveManifestFromReleaseAssets( accessToken: string, assets: ReleaseAsset[], + readmeAt?: { ref: RepoRef; tag: string }, ): Promise { if (assets.length === 0) return null const byName = new Map(assets.map((a) => [a.name, a])) @@ -264,7 +347,12 @@ export async function resolveManifestFromReleaseAssets( continue } const parsed = parseManifestText(got.content) - return { raw: got.content, parsed, readmeMarkdown: null, readmeLocales: null } + if (!readmeAt) return { raw: got.content, parsed, readmeMarkdown: null, readmeLocales: null } + const { readmeMarkdown, readmeLocales } = await resolveReadme( + fetcherFor(accessToken, readmeAt.ref, readmeAt.tag), + parsed, + ) + return { raw: got.content, parsed, readmeMarkdown, readmeLocales } } catch (err) { if (err instanceof UpstreamUnauthorizedError) throw err if (err instanceof ManifestValidationError) { @@ -297,42 +385,7 @@ export async function resolveManifest( continue } const parsed = parseManifestText(got.content) - let readmeMarkdown: string | null = null - let readmeLocales: ReadmeMap | null = null - - if (parsed.readmes && Object.keys(parsed.readmes).length > 0) { - readmeLocales = {} - for (const [locale, path] of Object.entries(parsed.readmes)) { - try { - const r = await fetch(path) - if (r) readmeLocales[locale] = r.content - } catch (err) { - log.warn({ err, locale, path }, 'localized readme fetch failed') - } - } - if (Object.keys(readmeLocales).length === 0) readmeLocales = null - } - if (!readmeLocales && parsed.readme) { - try { - const r = await fetch(parsed.readme) - if (r) readmeMarkdown = r.content - } catch (err) { - log.warn({ err, readme: parsed.readme }, 'readme path fetch failed — manifest still applied') - } - } - if (!readmeLocales && !readmeMarkdown) { - for (const fallback of ['README.md', 'readme.md', 'README.markdown']) { - try { - const r = await fetch(fallback) - if (r) { - readmeMarkdown = r.content - break - } - } catch { - // try next - } - } - } + const { readmeMarkdown, readmeLocales } = await resolveReadme(fetch, parsed) return { raw: got.content, parsed, readmeMarkdown, readmeLocales } } catch (err) { if (err instanceof UpstreamUnauthorizedError) throw err diff --git a/apps/api/src/lib/readme.ts b/apps/api/src/lib/readme.ts new file mode 100644 index 0000000..d57fd72 --- /dev/null +++ b/apps/api/src/lib/readme.ts @@ -0,0 +1,26 @@ +// README storage is either plain markdown or a JSON locale map (written when +// the manifest declares `readmes`). Both the plugin column and the per-release +// column use that shape, so one reader serves both. +export function pickReadme( + raw: string | null, + preferredLocale: string | undefined, +): { markdown: string | null; locale: string | null; available: string[] } { + if (!raw) return { markdown: null, locale: null, available: [] } + if (!raw.startsWith('{')) return { markdown: raw, locale: null, available: [] } + try { + const map = JSON.parse(raw) as Record + const available = Object.keys(map).filter((k) => typeof map[k] === 'string') + if (available.length === 0) return { markdown: null, locale: null, available: [] } + const pick = (locale: string | undefined): string | null => { + if (locale && typeof map[locale] === 'string') return locale + return null + } + const baseLocale = preferredLocale?.split('-')[0] + const chosen = pick(preferredLocale) ?? pick(baseLocale) ?? pick('en') ?? available[0] + return { markdown: map[chosen] as string, locale: chosen, available } + } catch { + return { markdown: raw, locale: null, available: [] } + } +} + +export const README_TTL = 600 diff --git a/apps/api/src/lib/release-ingest.ts b/apps/api/src/lib/release-ingest.ts index f4a7bec..fc96abc 100644 --- a/apps/api/src/lib/release-ingest.ts +++ b/apps/api/src/lib/release-ingest.ts @@ -12,9 +12,18 @@ import { recordAudit } from './audit' import { fetchAttestation } from './attestation' import { logger } from './logger' import { compareSemver, tagToVersion } from './semver' -import { resolveManifest, resolveManifestFromReleaseAssets, fetchReleaseAssetList, rawContentBase } from './manifest' +import { + resolveManifest, + resolveManifestFromReleaseAssets, + fetchReleaseAssetList, + fetchReadmeAtTag, + parseManifestText, + rawContentBase, + type Manifest, +} from './manifest' import { manifestPatch, + readmePayloadOf, applyManifestToPlugin, assertManifestVersionMatches, ManifestVersionMismatchError, @@ -31,7 +40,12 @@ const log = logger.child({ module: 'release-ingest' }) export async function persistRelease( plugin: { id: string; latestVersion: string | null }, normalized: NormalizedRelease, - opts: { manifestSha256?: string | null; manifestRaw?: string | null } = {}, + opts: { + manifestSha256?: string | null + manifestRaw?: string | null + readme?: string | null + minRuntimeVersion?: string | null + } = {}, ): Promise<{ version: string; assetMap: AssetMap }> { // Strict semver gate: rejecting here keeps lax tags ("v1.2", "release-foo") // out of the releases table entirely. Manifest-side validation already @@ -49,11 +63,19 @@ export async function persistRelease( // manifest this pass. Asset-only re-ingests (rehash, asset backfill) leave // a previously-signed manifestSha256/manifestRaw intact instead of nulling // it — a wiped hash would make the integrity endpoint silently 404. - const set: { assets: string; manifestSha256?: string | null; manifestRaw?: string | null } = { + const set: { + assets: string + manifestSha256?: string | null + manifestRaw?: string | null + readme?: string | null + minRuntimeVersion?: string | null + } = { assets: serializeAssets(assetMap), } if (opts.manifestSha256 !== undefined) set.manifestSha256 = opts.manifestSha256 if (opts.manifestRaw !== undefined) set.manifestRaw = opts.manifestRaw + if (opts.readme !== undefined) set.readme = opts.readme + if (opts.minRuntimeVersion !== undefined) set.minRuntimeVersion = opts.minRuntimeVersion await db .insert(releases) .values({ @@ -63,6 +85,8 @@ export async function persistRelease( assets: serializeAssets(assetMap), manifestSha256: opts.manifestSha256 ?? null, manifestRaw: opts.manifestRaw ?? null, + readme: opts.readme ?? null, + minRuntimeVersion: opts.minRuntimeVersion ?? null, }) .onConflictDoUpdate({ target: [releases.pluginId, releases.version], set }) @@ -198,7 +222,7 @@ export async function refreshManifestAtRelease( tag: string, version: string, assets: Array<{ name: string; url: string }> = [], -): Promise<{ sha: string; raw: string } | null> { +): Promise<{ sha: string; raw: string; readme: string | null; minRuntimeVersion: string | null } | null> { const ref = parseRepoUrl(plugin.repoUrl) if (!ref) return null const ownerIdentity = await db.query.identities.findFirst({ @@ -234,7 +258,7 @@ export async function refreshManifestAtRelease( // only when the operator allows it via the require_release_asset setting. let manifest: Awaited> = null try { - manifest = await resolveManifestFromReleaseAssets(token, assets) + manifest = await resolveManifestFromReleaseAssets(token, assets, { ref, tag }) } catch (err) { log.warn({ err, slug: plugin.id, tag }, 'asset-based manifest resolution errored') } @@ -311,7 +335,12 @@ export async function refreshManifestAtRelease( await cache().del(latestCacheKey(plugin.id)) const sha = manifestSha256(manifest.raw) log.info({ slug: plugin.id, version }, 'manifest refreshed at release') - return { sha, raw: manifest.raw } + return { + sha, + raw: manifest.raw, + readme: readmePayloadOf(manifest), + minRuntimeVersion: manifest.parsed.min_runtime_version ?? null, + } } catch (err) { const reason = err instanceof Error ? err.message : String(err) log.warn({ err, slug: plugin.id }, 'manifest apply failed after fetch succeeded') @@ -372,7 +401,7 @@ async function recheckAssetsOnce(plugin: PluginRef, tag: string, version: string } let manifest: Awaited> = null try { - manifest = await resolveManifestFromReleaseAssets(token, assets) + manifest = await resolveManifestFromReleaseAssets(token, assets, { ref, tag }) } catch (err) { log.warn({ err, slug: plugin.id, tag, attempt }, 'asset recheck resolution errored') } @@ -408,7 +437,12 @@ async function recheckAssetsOnce(plugin: PluginRef, tag: string, version: string // the publish webhook, so the original persistRelease saw no manifest). await db .update(releases) - .set({ manifestSha256: manifestSha256(manifest.raw), manifestRaw: manifest.raw }) + .set({ + manifestSha256: manifestSha256(manifest.raw), + manifestRaw: manifest.raw, + readme: readmePayloadOf(manifest), + minRuntimeVersion: manifest.parsed.min_runtime_version ?? null, + }) .where(and(eq(releases.pluginId, plugin.id), eq(releases.version, expectedVersion))) // Back-fill the binary asset URL map. The original webhook race left @@ -465,3 +499,67 @@ async function recheckAssetsOnce(plugin: PluginRef, tag: string, version: string log.warn({ err, slug: plugin.id, tag }, 'asset recheck apply failed') } } + +// One-shot repair for releases ingested before READMEs were captured per +// release. Walks the plugin's releases that have none, reads the README at +// each release's tag, and stores it. Tags are immutable, so this recovers the +// exact docs each version shipped with. Releases whose tag or README is gone +// upstream are skipped, not failed — a partial backfill is still an +// improvement over a blank history. +export async function backfillReleaseReadmes(plugin: { + id: string + ownerId: string + repoUrl: string +}): Promise<{ scanned: number; filled: number; skipped: number }> { + const ref = parseRepoUrl(plugin.repoUrl) + if (!ref) return { scanned: 0, filled: 0, skipped: 0 } + // A README in a public repo needs no credentials, so a missing or expired + // owner token downgrades to an unauthenticated raw-content read instead of + // failing the whole backfill. Private repos simply yield nothing. + const ownerIdentity = await db.query.identities.findFirst({ + where: { userId: plugin.ownerId, providerInstanceId: ref.instance.id }, + }) + let token: string | null = null + if (ownerIdentity?.accessToken) { + try { + token = await getValidAccessToken(ownerIdentity, ref.instance) + } catch (err) { + log.warn({ err, slug: plugin.id }, 'owner token unusable — backfilling READMEs unauthenticated') + } + } + + const rows = await db.query.releases.findMany({ + where: { pluginId: plugin.id }, + columns: { id: true, version: true, readme: true, manifestRaw: true }, + }) + const pending = rows.filter((r) => !r.readme) + + let filled = 0 + let skipped = 0 + for (const row of pending) { + // The stored manifest tells us where that version kept its README; without + // one, resolveReadme still falls back to the conventional root filenames. + let parsed: Manifest + try { + parsed = row.manifestRaw ? parseManifestText(row.manifestRaw) : ({ readme: null } as unknown as Manifest) + } catch { + parsed = { readme: null } as unknown as Manifest + } + // Releases are stored without the "v", tags commonly carry it. + for (const tag of [`v${row.version}`, row.version]) { + try { + const got = await fetchReadmeAtTag(token, ref, tag, parsed) + const payload = readmePayloadOf({ raw: '', parsed, ...got }) + if (!payload) continue + await db.update(releases).set({ readme: payload }).where(eq(releases.id, row.id)) + filled++ + break + } catch (err) { + log.warn({ err, slug: plugin.id, version: row.version, tag }, 'readme backfill failed for tag') + } + } + } + skipped = pending.length - filled + log.info({ slug: plugin.id, scanned: pending.length, filled, skipped }, 'release readme backfill done') + return { scanned: pending.length, filled, skipped } +} diff --git a/apps/api/src/routes/api/admin/plugins/[id]/backfill-readmes.ts b/apps/api/src/routes/api/admin/plugins/[id]/backfill-readmes.ts new file mode 100644 index 0000000..8587d22 --- /dev/null +++ b/apps/api/src/routes/api/admin/plugins/[id]/backfill-readmes.ts @@ -0,0 +1,66 @@ +import { Elysia, t } from 'elysia' +import { adminMiddleware } from '$middleware/admin' +import { db } from '$db' +import { backfillReleaseReadmes } from '$lib/release-ingest' +import { OAuthExpiredError, UpstreamUnauthorizedError, reauthErrorBody } from '$lib/oauth-tokens' +import { recordAudit, actorFromAdmin } from '$lib/audit' +import { logger } from '$lib/logger' + +const log = logger.child({ module: 'readme-backfill' }) + +export default new Elysia().use(adminMiddleware).post( + '/', + async ({ params, set, admin, request }) => { + const plugin = await db.query.plugins.findFirst({ where: { id: params.id } }) + if (!plugin) { + set.status = 404 + return { error: 'Plugin not found' } + } + + let result: Awaited> + try { + result = await backfillReleaseReadmes(plugin) + } catch (e) { + if (e instanceof OAuthExpiredError || e instanceof UpstreamUnauthorizedError) { + set.status = 401 + return reauthErrorBody(e) + } + log.warn({ err: e, slug: plugin.id }, 'readme backfill failed') + set.status = 422 + return { error: e instanceof Error ? e.message : 'Backfill failed' } + } + + await recordAudit({ + ...actorFromAdmin(admin, request), + action: 'plugin.backfill_readmes', + target: `plugin:${plugin.id}`, + meta: result, + }) + + return { ok: true, ...result } + }, + { + detail: { + tags: ['Admin'], + summary: 'Backfill per-release READMEs from each release tag', + description: + 'Releases ingested before READMEs were captured per release have none stored. This reads the README at ' + + "each such release's tag using the owner's stored OAuth token and fills it in. Tags are immutable, so the " + + 'recovered README is what that version actually shipped with. Already-populated releases are left alone.', + operationId: 'backfillReleaseReadmes', + security: [{ bearerAuth: [] }, { cookieAuth: [] }], + }, + params: t.Object({ id: t.String() }), + response: { + 200: t.Object({ + ok: t.Boolean(), + scanned: t.Number(), + filled: t.Number(), + skipped: t.Number(), + }), + 401: t.Object({ error: t.String(), reauthFor: t.String() }), + 404: t.Object({ error: t.String() }), + 422: t.Object({ error: t.String() }), + }, + }, +) diff --git a/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts b/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts index 460e541..a56c192 100644 --- a/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts +++ b/apps/api/src/routes/api/admin/plugins/[id]/replay-webhook.ts @@ -80,7 +80,12 @@ export default new Elysia().use(adminMiddleware).post( queueMicrotask(async () => { const manifest = await refreshManifestAtRelease(plugin, normalized.tag, version) if (manifest) - await persistRelease(plugin, normalized, { manifestSha256: manifest.sha, manifestRaw: manifest.raw }) + await persistRelease(plugin, normalized, { + manifestSha256: manifest.sha, + manifestRaw: manifest.raw, + readme: manifest.readme, + minRuntimeVersion: manifest.minRuntimeVersion, + }) }) queueMicrotask(() => { diff --git a/apps/api/src/routes/api/plugins/[slug]/index.ts b/apps/api/src/routes/api/plugins/[slug]/index.ts index d77dc7d..b18f703 100644 --- a/apps/api/src/routes/api/plugins/[slug]/index.ts +++ b/apps/api/src/routes/api/plugins/[slug]/index.ts @@ -6,6 +6,7 @@ import { authMiddleware } from '$middleware/auth' import { projectPluginDetail } from '$lib/plugin-projection' import { renderMarkdown } from '$lib/markdown' import { cache, isString } from '$lib/cache' +import { pickReadme, README_TTL } from '$lib/readme' import { buildIntegrity } from '$lib/release-integrity' const screenshotSchema = t.Object({ @@ -77,32 +78,8 @@ const pluginDetailSchema = t.Object({ releases: t.Array(releaseSchema), }) -function pickReadme( - raw: string | null, - preferredLocale: string | undefined, -): { markdown: string | null; locale: string | null; available: string[] } { - if (!raw) return { markdown: null, locale: null, available: [] } - if (!raw.startsWith('{')) return { markdown: raw, locale: null, available: [] } - try { - const map = JSON.parse(raw) as Record - const available = Object.keys(map).filter((k) => typeof map[k] === 'string') - if (available.length === 0) return { markdown: null, locale: null, available: [] } - const pick = (locale: string | undefined): string | null => { - if (locale && typeof map[locale] === 'string') return locale - return null - } - const baseLocale = preferredLocale?.split('-')[0] - const chosen = pick(preferredLocale) ?? pick(baseLocale) ?? pick('en') ?? available[0] - return { markdown: map[chosen] as string, locale: chosen, available } - } catch { - return { markdown: raw, locale: null, available: [] } - } -} - const errorSchema = t.Object({ error: t.String() }) -const README_TTL = 600 - export default new Elysia() .get( '/', diff --git a/apps/api/src/routes/api/plugins/[slug]/releases/[version]/readme.ts b/apps/api/src/routes/api/plugins/[slug]/releases/[version]/readme.ts new file mode 100644 index 0000000..8baad0d --- /dev/null +++ b/apps/api/src/routes/api/plugins/[slug]/releases/[version]/readme.ts @@ -0,0 +1,79 @@ +import { Elysia, t } from 'elysia' +import { db } from '$db' +import { renderMarkdown } from '$lib/markdown' +import { cache, isString } from '$lib/cache' +import { pickReadme, README_TTL } from '$lib/readme' + +// The README as it stood at a specific release. Captured at ingest from the +// release's tag, so browsing an older version shows the docs that shipped with +// it instead of the plugin's current README. Releases ingested before the +// column existed have none — the response says so rather than falling back to +// the current one, which would quietly misrepresent the old version. +export default new Elysia().get( + '/', + async ({ params, query, set }) => { + const plugin = await db.query.plugins.findFirst({ + where: { id: params.slug }, + columns: { id: true, status: true }, + }) + if (!plugin || plugin.status !== 'approved') { + set.status = 404 + return { error: 'Plugin not found' } + } + + const release = await db.query.releases.findFirst({ + where: { pluginId: plugin.id, version: params.version }, + columns: { id: true, version: true, readme: true }, + }) + if (!release) { + set.status = 404 + return { error: 'Release not found' } + } + + const picked = pickReadme(release.readme ?? null, query.locale) + if (!picked.markdown) { + return { + version: release.version, + readmeHtml: null, + readmeLocale: null, + readmeAvailableLocales: [], + captured: false, + } + } + + const cacheKey = `release:readme:${plugin.id}:${release.id}:${picked.locale ?? 'default'}` + let readmeHtml = await cache().get(cacheKey, isString) + if (!readmeHtml) { + readmeHtml = await renderMarkdown(picked.markdown) + await cache().set(cacheKey, readmeHtml, README_TTL) + } + + return { + version: release.version, + readmeHtml, + readmeLocale: picked.locale, + readmeAvailableLocales: picked.available, + captured: true, + } + }, + { + detail: { + tags: ['Plugins'], + summary: 'README captured at a specific release', + operationId: 'getReleaseReadme', + }, + params: t.Object({ slug: t.String(), version: t.String() }), + query: t.Object({ locale: t.Optional(t.String({ maxLength: 16 })) }), + response: { + 200: t.Object({ + version: t.String(), + readmeHtml: t.Nullable(t.String()), + readmeLocale: t.Nullable(t.String()), + readmeAvailableLocales: t.Array(t.String()), + // false when this release predates per-release README capture + captured: t.Boolean(), + }), + 404: t.Object({ error: t.String() }), + }, + }, +) diff --git a/apps/api/src/routes/api/publish/[slug].ts b/apps/api/src/routes/api/publish/[slug].ts index 8e354d9..b86435c 100644 --- a/apps/api/src/routes/api/publish/[slug].ts +++ b/apps/api/src/routes/api/publish/[slug].ts @@ -165,6 +165,7 @@ export default new Elysia().use(publisherTokenMiddleware).post( const { assetMap } = await persistRelease({ id: slug, latestVersion: null }, normalized, { manifestSha256: manifestSha256(body.manifest), manifestRaw: body.manifest, + minRuntimeVersion: parsed.min_runtime_version ?? null, }) // Apply manifest fields (category, tags, icon, extensions, …) to the @@ -267,6 +268,7 @@ export default new Elysia().use(publisherTokenMiddleware).post( const { assetMap } = await persistRelease({ id: slug, latestVersion: existing.latestVersion }, normalized, { manifestSha256: manifestSha256(body.manifest), manifestRaw: body.manifest, + minRuntimeVersion: parsed.min_runtime_version ?? null, }) const patch = manifestPatch( diff --git a/apps/api/src/routes/api/submit/oauth.ts b/apps/api/src/routes/api/submit/oauth.ts index 919d0f8..be253c2 100644 --- a/apps/api/src/routes/api/submit/oauth.ts +++ b/apps/api/src/routes/api/submit/oauth.ts @@ -11,7 +11,7 @@ import { getValidAccessToken, OAuthExpiredError } from '$lib/oauth-tokens' import { env } from '$lib/env' import { getSetting } from '$lib/settings' import { resolveManifest, rawContentBase, ManifestValidationError } from '$lib/manifest' -import { manifestPatch, applyManifestToPlugin } from '$lib/manifest-apply' +import { manifestPatch, readmePayloadOf, applyManifestToPlugin } from '$lib/manifest-apply' import { fetchLatestRelease } from '$lib/release-fetch' import { persistRelease, hashReleaseAssetsAsync, manifestSha256 } from '$lib/release-ingest' import { getFeatures } from '$lib/features' @@ -154,7 +154,12 @@ export default new Elysia() // factored out so both call sites stay in sync. if (latestRelease.published) { const manifestOpts = manifest - ? { manifestSha256: manifestSha256(manifest.raw), manifestRaw: manifest.raw } + ? { + manifestSha256: manifestSha256(manifest.raw), + manifestRaw: manifest.raw, + readme: readmePayloadOf(manifest), + minRuntimeVersion: manifest.parsed.min_runtime_version ?? null, + } : {} const { version, assetMap } = await persistRelease( { id: slug, latestVersion: null }, diff --git a/apps/api/src/routes/api/webhooks/release.ts b/apps/api/src/routes/api/webhooks/release.ts index 5c476b4..dfa62df 100644 --- a/apps/api/src/routes/api/webhooks/release.ts +++ b/apps/api/src/routes/api/webhooks/release.ts @@ -107,7 +107,12 @@ export default new Elysia().use(rateLimit({ bucket: 'webhook-release', limit: 60 if (manifest) { // Re-persist the row with the now-known manifest sha + raw bytes // (persistRelease ran without them because the fetch is async). - await persistRelease(plugin, normalized, { manifestSha256: manifest.sha, manifestRaw: manifest.raw }) + await persistRelease(plugin, normalized, { + manifestSha256: manifest.sha, + manifestRaw: manifest.raw, + readme: manifest.readme, + minRuntimeVersion: manifest.minRuntimeVersion, + }) } }) diff --git a/apps/api/tests/lib/release-readme.test.ts b/apps/api/tests/lib/release-readme.test.ts new file mode 100644 index 0000000..0a4e568 --- /dev/null +++ b/apps/api/tests/lib/release-readme.test.ts @@ -0,0 +1,197 @@ +import { describe, it, expect, spyOn, afterEach, beforeEach } from 'bun:test' +import { clearDb, makeUser, makePlugin } from '../helpers' +import { db } from '../../src/db' +import { persistRelease } from '../../src/lib/release-ingest' +import type { NormalizedRelease } from '../../src/lib/webhook' +import { resolveManifestFromReleaseAssets, fetchReadmeAtTag } from '../../src/lib/manifest' +import { manifestPatch, readmePayloadOf } from '../../src/lib/manifest-apply' +import type { RepoRef } from '../../src/lib/providers' + +const MANIFEST = JSON.stringify({ name: 'alpha', version: '1.0.0', description: 'A test plugin.' }) + +const ref: RepoRef = { + instance: { + id: 'github', + kind: 'github', + displayName: 'GitHub', + baseUrl: 'https://github.com', + clientId: '', + clientSecret: '', + logoUrl: null, + enabled: true, + }, + owner: 'alice', + repo: 'my-plugin', + fullName: 'alice/my-plugin', +} + +const sampleRelease: NormalizedRelease = { + repoUrl: 'https://github.com/alice/my-plugin', + published: true, + tag: 'v1.0.0', + assets: [], +} + +let spy: ReturnType | null = null +let requested: string[] = [] + +function mockFetch(byUrlContains: Array<[string, string]>) { + requested = [] + spy = spyOn(global, 'fetch').mockImplementation((async (url: string | URL | Request) => { + const key = String(typeof url === 'string' ? url : url instanceof URL ? url.toString() : url.url) + requested.push(key) + const hit = byUrlContains.find(([needle]) => key.includes(needle)) + if (!hit) return new Response('not found', { status: 404 }) + return new Response(hit[1], { + status: 200, + headers: { 'content-length': String(new TextEncoder().encode(hit[1]).length) }, + }) + }) as unknown as typeof fetch) +} + +afterEach(() => { + spy?.mockRestore() + spy = null +}) + +describe('resolveManifestFromReleaseAssets — README capture', () => { + beforeEach(clearDb) + + // Regression: the asset path used to return readme: null unconditionally, so + // every asset-resolved release stored a blank README — and the patch then + // wrote that null over whatever the plugin already had. + it('reads the README at the release tag when told where to look', async () => { + mockFetch([ + ['/default.tabularium', MANIFEST], + ['contents/README.md', '# Alpha\n\nDocs at this tag.'], + ]) + const got = await resolveManifestFromReleaseAssets( + 'token', + [{ name: 'default.tabularium', url: 'https://example.com/default.tabularium' }], + { ref, tag: 'v1.0.0' }, + ) + expect(got?.readmeMarkdown).toContain('Docs at this tag') + // pinned to the tag, not to a branch head + expect(requested.some((u) => u.includes('contents/README.md') && u.includes('ref=v1.0.0'))).toBe(true) + }) + + it('leaves the README null when the caller gives no tag', async () => { + mockFetch([['/default.tabularium', MANIFEST]]) + const got = await resolveManifestFromReleaseAssets('token', [ + { name: 'default.tabularium', url: 'https://example.com/default.tabularium' }, + ]) + expect(got?.parsed.name).toBe('alpha') + expect(got?.readmeMarkdown).toBeNull() + }) + + it('prefers the localized readmes map over the root fallback', async () => { + const withLocales = JSON.stringify({ + name: 'alpha', + version: '1.0.0', + readmes: { en: 'docs/en.md', de: 'docs/de.md' }, + }) + mockFetch([ + ['/default.tabularium', withLocales], + ['docs%2Fen.md', '# English'], + ['docs%2Fde.md', '# Deutsch'], + ]) + const got = await resolveManifestFromReleaseAssets( + 'token', + [{ name: 'default.tabularium', url: 'https://example.com/default.tabularium' }], + { ref, tag: 'v1.0.0' }, + ) + expect(got?.readmeLocales?.en).toContain('English') + expect(got?.readmeLocales?.de).toContain('Deutsch') + expect(readmePayloadOf(got!)).toContain('"de"') + }) +}) + +describe('manifestPatch — README is never blanked', () => { + const base = { raw: MANIFEST, parsed: JSON.parse(MANIFEST) } + + it('omits readme entirely when this pass resolved none', () => { + const patch = manifestPatch( + { ...base, readmeMarkdown: null, readmeLocales: null }, + { + repoBase: 'https://raw.example/', + version: '1.0.0', + }, + ) + // absent, not null — an explicit null would overwrite the stored README + expect('readme' in patch).toBe(false) + }) + + it('sets readme when one was resolved', () => { + const patch = manifestPatch( + { ...base, readmeMarkdown: '# Hi', readmeLocales: null }, + { + repoBase: 'https://raw.example/', + version: '1.0.0', + }, + ) + expect(patch.readme).toBe('# Hi') + }) + + it('stores a locale map as JSON', () => { + const patch = manifestPatch( + { ...base, readmeMarkdown: null, readmeLocales: { en: '# Hi' } }, + { + repoBase: 'https://raw.example/', + version: '1.0.0', + }, + ) + expect(JSON.parse(patch.readme as string)).toEqual({ en: '# Hi' }) + }) +}) + +describe('persistRelease — manifest fields reach the release row', () => { + beforeEach(clearDb) + + // Regression: min_runtime_version was declared in the manifest, validated by + // the schema and echoed back by the submit preview, but no ingest path ever + // wrote it to the release row — so /latest served null and clients could + // install a plugin their runtime was too old for. + it('stores min_runtime_version from the manifest', async () => { + const user = await makeUser({ username: 'alice' }) + const plugin = await makePlugin(user.id, { id: 'alpha' }) + await persistRelease({ id: plugin.id, latestVersion: null }, sampleRelease, { + minRuntimeVersion: '0.20.0', + }) + const row = await db.query.releases.findFirst({ where: { pluginId: plugin.id, version: '1.0.0' } }) + expect(row?.minRuntimeVersion).toBe('0.20.0') + }) + + it('leaves a stored min_runtime_version alone on an asset-only re-ingest', async () => { + const user = await makeUser({ username: 'alice' }) + const plugin = await makePlugin(user.id, { id: 'alpha' }) + await persistRelease({ id: plugin.id, latestVersion: null }, sampleRelease, { + minRuntimeVersion: '0.20.0', + }) + // rehash and asset backfills re-persist without a manifest in hand + await persistRelease({ id: plugin.id, latestVersion: '1.0.0' }, sampleRelease) + const row = await db.query.releases.findFirst({ where: { pluginId: plugin.id, version: '1.0.0' } }) + expect(row?.minRuntimeVersion).toBe('0.20.0') + }) +}) + +describe('fetchReadmeAtTag — unauthenticated fallback', () => { + beforeEach(clearDb) + + // The backfill has to work for plugins whose owner has no usable OAuth token + // left. A README in a public repo needs no credentials, so it reads from the + // raw-content host instead of the authenticated contents API. + it('reads from the raw host when no token is given', async () => { + mockFetch([['raw.githubusercontent.com', '# Public docs']]) + const got = await fetchReadmeAtTag(null, ref, 'v1.0.0', { name: 'alpha', version: '1.0.0' } as never) + expect(got.readmeMarkdown).toContain('Public docs') + expect(requested.every((u) => !u.includes('api.github.com'))).toBe(true) + expect(requested.some((u) => u.includes('/v1.0.0/README.md'))).toBe(true) + }) + + it('yields nothing when the repo is not public', async () => { + mockFetch([]) + const got = await fetchReadmeAtTag(null, ref, 'v1.0.0', { name: 'alpha', version: '1.0.0' } as never) + expect(got.readmeMarkdown).toBeNull() + expect(got.readmeLocales).toBeNull() + }) +}) diff --git a/apps/frontend/messages/de.json b/apps/frontend/messages/de.json index 7eeac91..560e26b 100644 --- a/apps/frontend/messages/de.json +++ b/apps/frontend/messages/de.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "Langsam — 1× pro Minute", "plugin_detail_refresh_reauth_redirect": "OAuth-Token abgelaufen — leite zur Re-Autorisierung weiter…", "plugin_detail_refresh_running": "Lade neu…", + "plugin_detail_release_readme_missing": "Für dieses Release wurde kein README erfasst.", + "plugin_detail_release_readme_title": "README dieser Version", "plugin_detail_releases": "Releases", "plugin_detail_releases_subtitle": "Die neueste Version ist im Badge oben hervorgehoben. Die SHA256-Spalte zeigt, wenn die Integrität gehasht wurde.", "plugin_detail_report_issue": "Problem melden", diff --git a/apps/frontend/messages/en.json b/apps/frontend/messages/en.json index 181cece..c9c09fc 100644 --- a/apps/frontend/messages/en.json +++ b/apps/frontend/messages/en.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "Slow down — once per minute", "plugin_detail_refresh_reauth_redirect": "OAuth token expired — redirecting to re-authorize…", "plugin_detail_refresh_running": "Refetching…", + "plugin_detail_release_readme_missing": "No README was captured for this release.", + "plugin_detail_release_readme_title": "README at this version", "plugin_detail_releases": "Releases", "plugin_detail_releases_subtitle": "Latest version is highlighted in the badge above. SHA256 column shows when integrity has been hashed.", "plugin_detail_report_issue": "Report issue", diff --git a/apps/frontend/messages/es.json b/apps/frontend/messages/es.json index 8df0fc8..043d6dc 100644 --- a/apps/frontend/messages/es.json +++ b/apps/frontend/messages/es.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "Despacio — una vez por minuto", "plugin_detail_refresh_reauth_redirect": "Token OAuth expirado — redirigiendo para re-autorizar…", "plugin_detail_refresh_running": "Recargando…", + "plugin_detail_release_readme_missing": "No se guardó ningún README para esta versión.", + "plugin_detail_release_readme_title": "README de esta versión", "plugin_detail_releases": "Releases", "plugin_detail_releases_subtitle": "La última versión está resaltada en el badge de arriba. La columna SHA256 muestra cuando la integridad ha sido hasheada.", "plugin_detail_report_issue": "Reportar problema", diff --git a/apps/frontend/messages/fr.json b/apps/frontend/messages/fr.json index 0ec552b..0538c74 100644 --- a/apps/frontend/messages/fr.json +++ b/apps/frontend/messages/fr.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "Doucement — une fois par minute", "plugin_detail_refresh_reauth_redirect": "Jeton OAuth expiré — redirection pour ré-autoriser…", "plugin_detail_refresh_running": "Rechargement…", + "plugin_detail_release_readme_missing": "Aucun README n'a été enregistré pour cette version.", + "plugin_detail_release_readme_title": "README de cette version", "plugin_detail_releases": "Releases", "plugin_detail_releases_subtitle": "La dernière version est mise en avant dans le badge ci-dessus. La colonne SHA256 s'affiche quand l'intégrité a été hachée.", "plugin_detail_report_issue": "Signaler un problème", diff --git a/apps/frontend/messages/it.json b/apps/frontend/messages/it.json index 1bb1f58..9ed1d29 100644 --- a/apps/frontend/messages/it.json +++ b/apps/frontend/messages/it.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "Piano — una volta al minuto", "plugin_detail_refresh_reauth_redirect": "Token OAuth scaduto — reindirizzamento per ri-autorizzare…", "plugin_detail_refresh_running": "Ricaricando…", + "plugin_detail_release_readme_missing": "Nessun README è stato registrato per questa release.", + "plugin_detail_release_readme_title": "README di questa versione", "plugin_detail_releases": "Release", "plugin_detail_releases_subtitle": "L'ultima versione è evidenziata nel badge sopra. La colonna SHA256 si mostra quando l'integrità è stata hashata.", "plugin_detail_report_issue": "Segnala problema", diff --git a/apps/frontend/messages/zh-CN.json b/apps/frontend/messages/zh-CN.json index 3bae6a3..cba6992 100644 --- a/apps/frontend/messages/zh-CN.json +++ b/apps/frontend/messages/zh-CN.json @@ -823,6 +823,8 @@ "plugin_detail_refresh_rate_limited": "慢点 — 每分钟一次", "plugin_detail_refresh_reauth_redirect": "OAuth 令牌已过期 — 正在重定向以重新授权…", "plugin_detail_refresh_running": "正在加载…", + "plugin_detail_release_readme_missing": "此发布未记录 README。", + "plugin_detail_release_readme_title": "该版本的 README", "plugin_detail_releases": "Releases", "plugin_detail_releases_subtitle": "最新版本在上方徽章中高亮显示。SHA256 列显示已计算完整性哈希的版本。", "plugin_detail_report_issue": "反馈问题", diff --git a/apps/frontend/src/routes/plugins/[slug]/+page.svelte b/apps/frontend/src/routes/plugins/[slug]/+page.svelte index f001d1e..6974fba 100644 --- a/apps/frontend/src/routes/plugins/[slug]/+page.svelte +++ b/apps/frontend/src/routes/plugins/[slug]/+page.svelte @@ -110,6 +110,27 @@ untrack(() => load(locale)) }) + // README as it stood at each release, loaded the first time a release row is + // expanded. Kept out of the plugin payload so the detail response does not + // have to carry a full README per version. + type VersionReadme = { loading: boolean; html: string | null; captured: boolean } + let versionReadmes = $state>({}) + + async function loadVersionReadme(pluginId: string, version: string) { + if (versionReadmes[version]) return + versionReadmes[version] = { loading: true, html: null, captured: false } + try { + const res = await fetch( + `/api/plugins/${encodeURIComponent(pluginId)}/releases/${encodeURIComponent(version)}/readme?locale=${encodeURIComponent(locale)}`, + ) + if (!res.ok) throw new Error(String(res.status)) + const data = (await res.json()) as { readmeHtml: string | null; captured: boolean } + versionReadmes[version] = { loading: false, html: data.readmeHtml, captured: data.captured } + } catch { + versionReadmes[version] = { loading: false, html: null, captured: false } + } + } + const sortedReleases = $derived( plugin?.releases ? [...plugin.releases].sort((a, b) => b.createdAt - a.createdAt) : [], ) @@ -747,7 +768,14 @@ {#each sortedReleases as release (release.id)} {@const totalSize = Object.values(release.assets).reduce((acc, a) => acc + (a.size ?? 0), 0)} {@const platformCount = Object.keys(release.assets).length} -
+ {@const vr = versionReadmes[release.version]} +
{ + if (plugin && (e.currentTarget as HTMLDetailsElement).open) + void loadVersionReadme(plugin.id, release.version) + }} + > @@ -845,6 +873,22 @@ {/each} +
+

+ {m.plugin_detail_release_readme_title()} +

+ {#if !vr || vr.loading} + + {:else if vr.html} +
+ {@html vr.html} +
+ {:else} +

{m.plugin_detail_release_readme_missing()}

+ {/if} +
{/each}