From 73e19fbc9c0725e7f603b44ee1960228fdb4d5e9 Mon Sep 17 00:00:00 2001 From: Lukas Bresser Date: Fri, 11 Sep 2026 12:53:44 +0000 Subject: [PATCH 1/4] Recover public AI context and fix prose filtering Replay implementation writes and patches from the exported worker session against the original base, regenerating shared content through its pipeline. Preserve case-sensitive source identifiers and the original test suite. Add failing-then-passing public-prose regressions while retaining private route exclusions. --- content/talks/jbp-allison-duettmann/index.md | 2 +- content/talks/jbp-konrad-kording/index.md | 2 +- content/talks/jbp-tom-oxley/index.md | 2 +- docs/ai-context.md | 45 ++ package.json | 2 +- scripts/ai-access.test.mjs | 40 ++ scripts/ai-content.test.mjs | 65 +++ scripts/ai-pipeline.test.mjs | 57 +++ scripts/ai-routes.test.mjs | 59 +++ scripts/ai-visibility.test.mjs | 121 ++++++ scripts/build-content.mjs | 20 +- scripts/content-visibility.mjs | 16 + scripts/readable-markdown.mjs | 42 ++ src/app/ai/index.json/route.ts | 4 + src/app/ai/markdown/[kind]/[slug]/route.ts | 9 + src/app/ai/page.tsx | 69 +++ src/app/ai/records/[kind]/[slug]/route.ts | 7 + src/app/ai/topics/[slug]/route.ts | 9 + src/app/api/ai/search/route.ts | 4 + src/app/layout.tsx | 1 + src/app/llms-full.txt/route.ts | 5 + src/app/llms.txt/route.ts | 5 + src/components/AiStarterPrompt.tsx | 22 + src/components/SiteFooter.tsx | 1 + src/data/generated/areas.json | 20 + src/data/generated/authors.json | 405 ++++++++++++++++++ src/data/generated/blog.json | 78 ++++ src/data/generated/publications.json | 395 ++++++++++++++++++ src/data/generated/talks.json | 416 ++++++++++++++++++- src/data/generated/tutorials.json | 18 + src/lib/ai-access.ts | 84 ++++ src/lib/ai-content.ts | 149 +++++++ src/lib/ai-markdown.ts | 53 +++ src/lib/content.ts | 15 + 34 files changed, 2233 insertions(+), 9 deletions(-) create mode 100644 docs/ai-context.md create mode 100644 scripts/ai-access.test.mjs create mode 100644 scripts/ai-content.test.mjs create mode 100644 scripts/ai-pipeline.test.mjs create mode 100644 scripts/ai-routes.test.mjs create mode 100644 scripts/ai-visibility.test.mjs create mode 100644 scripts/content-visibility.mjs create mode 100644 scripts/readable-markdown.mjs create mode 100644 src/app/ai/index.json/route.ts create mode 100644 src/app/ai/markdown/[kind]/[slug]/route.ts create mode 100644 src/app/ai/page.tsx create mode 100644 src/app/ai/records/[kind]/[slug]/route.ts create mode 100644 src/app/ai/topics/[slug]/route.ts create mode 100644 src/app/api/ai/search/route.ts create mode 100644 src/app/llms-full.txt/route.ts create mode 100644 src/app/llms.txt/route.ts create mode 100644 src/components/AiStarterPrompt.tsx create mode 100644 src/lib/ai-access.ts create mode 100644 src/lib/ai-content.ts create mode 100644 src/lib/ai-markdown.ts diff --git a/content/talks/jbp-allison-duettmann/index.md b/content/talks/jbp-allison-duettmann/index.md index afbf6f1f..099ec3f6 100644 --- a/content/talks/jbp-allison-duettmann/index.md +++ b/content/talks/jbp-allison-duettmann/index.md @@ -6,7 +6,7 @@ venue_url: "https://www.youtube.com/@JuanBenetPodcast" venue_location: "Podcast" authors: - juan-benet - - allison-duettmann + - "Allison Duettmann" areas: - neurotech abstract: "Juan Benet speaks with Allison Duettmann, CEO of the Foresight Institute, about why cryonics and mind emulation could mean we don't have to die, how whole-brain emulation and connectomics could keep humans compatible with superintelligent AI, how nanotechnology could rebuild the physical world, and the case for Existential Hope — imagining flourishing futures as a precondition for building them." diff --git a/content/talks/jbp-konrad-kording/index.md b/content/talks/jbp-konrad-kording/index.md index a7ab6f00..6b19d4a1 100644 --- a/content/talks/jbp-konrad-kording/index.md +++ b/content/talks/jbp-konrad-kording/index.md @@ -6,7 +6,7 @@ venue_url: "https://www.youtube.com/@JuanBenetPodcast" venue_location: "Podcast" authors: - juan-benet - - konrad-kording + - "Konrad Kording" areas: - neurotech abstract: "Juan Benet speaks with Konrad Kording of the University of Pennsylvania about the surprising computational power of a single neuron, reading the brain's wiring down to molecular detail, and building compilers and simulations as a path toward understanding — and ultimately simulating — the brain." diff --git a/content/talks/jbp-tom-oxley/index.md b/content/talks/jbp-tom-oxley/index.md index 431399b9..b6217b44 100644 --- a/content/talks/jbp-tom-oxley/index.md +++ b/content/talks/jbp-tom-oxley/index.md @@ -6,7 +6,7 @@ venue_url: "https://www.youtube.com/@JuanBenetPodcast" venue_location: "Podcast" authors: - juan-benet - - tom-oxley + - "Tom Oxley" areas: - neurotech abstract: "Juan Benet speaks with Tom Oxley, co-founder and CEO of Synchron, about the Stentrode — a brain-computer interface that reaches the motor cortex through a blood vessel without opening the skull — restoring independence to people with motor impairment and the path toward higher-channel-count neural interfaces." diff --git a/docs/ai-context.md b/docs/ai-context.md new file mode 100644 index 00000000..1c6db540 --- /dev/null +++ b/docs/ai-context.md @@ -0,0 +1,45 @@ +# Public AI context + +This is additive read-only access, not a chatbot, crawler, embeddings service or permission grant. The human entry point is `/ai/`; the footer and HTML `rel="describedby"` link lead to it or `/llms.txt`. + +## Content contract + +`content/` → existing `scripts/build-content.mjs` mappers → checked-in generated JSON → `src/lib/content.ts` → `src/lib/ai-content.ts` → text and JSON route handlers. There is no second content catalog, runtime Markdown parser, arbitrary-file route, CMS dump or URL-fetching API. Build-time `readable-markdown.mjs` uses the already-installed jsdom package to turn the same native HTML body into readable Markdown. Source folder/file IDs are preserved exactly, including historical mixed-case IDs. + +Covered: + +- Native blog articles: source text and attribution. CSS/scripts and interactive widgets are omitted; images and diagrams use supplied descriptions. Existing disclosures remain part of the text. +- External blog entries: the local summary and byline only. `canonicalKind: external` and `canonicalUrl` identify the original article; `sourceUrl` identifies the local metadata page. External full text is not fetched or republished. +- Publications: abstracts, named authors, dates, venue, publication types, DOI/PDF/source links. No full-paper claim. +- Talks: source summaries, venue metadata and available recording links. No transcripts, generated or otherwise. +- Tutorials: top-level text when present, otherwise metadata. Nested tutorial lessons are not included. +- Authors: exact display names and profile metadata. Slugs resolve exactly; exact unique display names can also resolve. Unmatched strings remain unchanged and unlinked. No fuzzy identity matching or inferred affiliations. +- Four public focus areas: the shared `FOCUS_AREA_DESCRIPTIONS` strings and navigation titles, plus current page links. No full overview-body export. + +Not covered: landing/About body copy, live ATProto posts or edits, interactive dashboards/data, nested lessons, unpublished material, non-discoverable routes, repository documentation/source, or operational interfaces. This is an explicit allowlist, not a recursive site dump. A short topic list reflects source tagging, not the amount of research in that field. + +### Snapshot versus live pages + +The generic area template currently renders repository content, while the special Economies & Governance page, landing page and About page also read live indexer sections with hardcoded fallbacks. They do not share one full-page loader. The export deliberately does not serialize seed files or stale Markdown overview bodies as live copy: it uses the same short shared descriptors and links readers to the rendered pages. This keeps static exports reproducible and avoids importing unrestricted live CMS records. The scope appears in the guide, index, complete text, focused resources and record coverage fields. + +Dates are source publication dates, not export or last-modified timestamps. Undated records and descriptor-only area records have null dates. Coverage reports the actual included counts and source date range. No new license is asserted; original terms and exceptions remain in force. + +## HTTP interface (schema 1.0) + +- `GET /llms.txt`: compact plain-text entry index. +- `GET /llms-full.txt`: downloadable plain-text covered context. +- `GET /ai/index.json`: metadata for all covered records, counts, dates, limits and resource links. Bodies are excluded from this index. +- `GET /api/ai/search/?q=connectome&kind=publication&limit=5`: bounded JSON search. +- `GET /ai/records/{kind}/{slug}/`: a covered record as JSON. +- `GET /ai/markdown/{kind}/{slug}/`: the same record as `text/markdown`. +- `GET /ai/topics/{area}/`: descriptor and focused resource links as `text/markdown`. + +Every route works with plain HTTP GET. Internal page/dynamic-resource URLs require trailing slashes; file-like `/llms.txt`, `/llms-full.txt` and `/ai/index.json` do not. GET is the only defined method (Next supplies HEAD/OPTIONS and rejects mutation methods). No credentialed CORS or runtime upstream requests are added. Success responses are public-cacheable for 300 seconds with nosniff; errors use no-store. Static files are generated with the deployment, while search/details reapply the visibility gate on the same repository snapshot. + +Search accepts only `q`, `kind`, `area`, `limit`, `offset`, once each. q is at most 200 characters with no control characters. kind is area/author/blog/publication/talk/tutorial. area is a covered focus-area slug. limit defaults to 10 and must be an integer 1–50; offset defaults to 0 and must be an integer 0–10000. Matching is case-insensitive AND substrings across title, summary, covered body and author names; ordering is stable ID ascending, never a relevance or quality score. `total` is the number of matches, `nextOffset` is null at the end. Valid searches without matches return 200 with an empty list. Unknown, malformed or duplicate parameters return 400. Unknown or denied detail/topic resources return an indistinguishable 404. + +## Visibility and verification + +Every shared mapper retains normalized visibility (version, denied flag, source not-before date). Hidden, draft, unlisted, preview, unaffiliated, private, noindex/robots metadata, nonpublic statuses, future dates and malformed dates are denied. Unsupported visibility versions fail closed. Only six explicit collections and four approved area descriptors can enter the export. Reserved operational paths, nonpublic paths, credential-bearing URLs and repository-document links are not export sources. Discovery and direct details use the same filtered view; guessing an ID does not bypass it. Existing HTML publishing behavior is not changed by the AI access layer. + +`npm test` includes the AI behavioral suite. It exercises native-source equality, collection coverage, exact bylines/canonicals, source-ID preservation, deterministic real mapper builds with synthetic hidden frontmatter, hidden data at discovery/detail boundaries, query bounds/pagination/errors, text MIME/headers, route wiring and server-rendered guide discovery. No browser checks are implied by these tests. Run `node_modules/.bin/tsc --noEmit` and the normal production build as well. Install dependencies with pnpm 10 and the frozen lockfile; no dependency or lockfile change is needed for this feature. diff --git a/package.json b/package.json index 6d23f956..928bb4a9 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "build": "node scripts/build-content.mjs && next build", "start": "next start", "generate-jwk": "node scripts/generate-jwk.js", - "test": "node --test scripts/velocity/*.test.mjs scripts/neuro-content.test.mjs scripts/neuro-publication.test.mjs scripts/blog-metadata.test.mjs" + "test": "node --test scripts/velocity/*.test.mjs scripts/neuro-content.test.mjs scripts/neuro-publication.test.mjs scripts/blog-metadata.test.mjs scripts/ai-*.test.mjs" }, "dependencies": { "@atproto/api": "^0.18.17", diff --git a/scripts/ai-access.test.mjs b/scripts/ai-access.test.mjs new file mode 100644 index 00000000..321cef9d --- /dev/null +++ b/scripts/ai-access.test.mjs @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { existsSync } from 'node:fs' +import { source } from './velocity/test-source-loader.mjs' + +const request = (query = '') => new Request(`https://www.plrd.org/api/ai/search/${query}`) +test('read-only access returns deterministic bounded search and canonical detail', async () => { + assert.ok(existsSync('src/lib/ai-access.ts'), 'public HTTP access is required') + const { createAiAccess } = source('lib/ai-access.ts') + const api = createAiAccess() + const response = api.search(request('?q=connectome&kind=publication&limit=2')) + assert.equal(response.status, 200) + assert.match(response.headers.get('content-type'), /^application\/json/) + assert.equal(response.headers.get('x-content-type-options'), 'nosniff') + const result = await response.json() + assert.equal(result.schemaVersion, '1.0') + assert.ok(result.total > 0) + assert.ok(result.results.length <= 2) + assert.deepEqual(result, await api.search(request('?q=connectome&kind=publication&limit=2')).json()) + const r = result.results[0] + const detail = await api.detail(r.kind, r.slug).json() + assert.equal(detail.record.canonicalUrl, r.canonicalUrl) + assert.equal(detail.record.visibility, undefined) + assert.ok(r.markdownUrl.endsWith('/')) + assert.ok(!('body' in r)) + assert.equal((await api.search(request('?q=zzzznomatchzzzz')).json()).total, 0) + for (const query of ['?limit=0', '?limit=51', '?limit=1.5', '?offset=-1', '?offset=10001', '?kind=other', '?q=a&q=b', '?url=http://127.0.0.1', '?q='+ 'a'.repeat(201), '?area=bogus', '?offset=01']) { + assert.equal(api.search(request(query)).status, 400, query) + } + assert.equal(api.detail('blog', 'does-not-exist').status, 404) + assert.equal(api.detail('admin', 'account').status, 404) + assert.equal(api.detail('blog', '../about').status, 404) + const page1 = await api.search(request('?limit=2')).json() + const page2 = await api.search(request('?limit=2&offset=2')).json() + assert.equal(page1.nextOffset, 2) + assert.ok(page1.results.every(a => !page2.results.some(b => b.id === a.id))) + const end = await api.search(request('?offset=10000')).json() + assert.deepEqual(end.results, []) + assert.equal(end.nextOffset, null) +}) diff --git a/scripts/ai-content.test.mjs b/scripts/ai-content.test.mjs new file mode 100644 index 00000000..a5465335 --- /dev/null +++ b/scripts/ai-content.test.mjs @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { existsSync } from 'node:fs' +import { source } from './velocity/test-source-loader.mjs' + +const content = source('lib/content.ts') + +test('public context maps native bodies, exact authors, external canonicals and shared area descriptors', () => { + assert.ok(existsSync('src/lib/ai-content.ts'), 'typed public context layer is required') + const { aiRecords } = source('lib/ai-content.ts') + const post = aiRecords.find(r => r.id === 'blog/neurotech-frontier-human-flourishing') + assert.ok(post) + assert.equal(post.body, content.blogPosts.find(p => p.slug === post.slug).markdown) + assert.deepEqual(post.authors.map(a => a.name), ['Sean Escola', 'David A. Markowitz']) + assert.ok(post.authors.every(a => a.url.endsWith('/'))) + const external = aiRecords.find(r => r.kind === 'blog' && r.canonicalKind === 'external') + assert.ok(external) + assert.equal(external.body, '') + assert.equal(external.canonicalUrl, content.blogPosts.find(p => p.slug === external.slug).external_url) + const pub = aiRecords.find(r => r.id === 'publication/how-to-obtain-complete-human-connectome') + assert.equal(pub.summary, content.publications.find(p => p.slug === pub.slug).abstract) + assert.ok(pub.sources.some(s => s.label === 'DOI')) + const talk = aiRecords.find(r => r.slug === 'jbp-adam-marblestone') + assert.equal(talk.body, '') + assert.ok(talk.sources.some(s => s.url === 'https://www.youtube.com/watch?v=X0B_GWTuEFo')) + for (const area of aiRecords.filter(r => r.kind === 'area')) { + assert.equal(area.body, '') + assert.equal(area.date, null) + assert.equal(area.summary, source('lib/focus-area-descriptions.ts').FOCUS_AREA_DESCRIPTIONS[area.slug]) + } + for (const t of content.tutorials) assert.equal(typeof t.markdown, 'string') + for (const items of [content.blogPosts.filter(p => p.unlisted)]) { + for (const hidden of items) assert.ok(!aiRecords.some(r => r.slug === hidden.slug)) + } +}) + +test('coverage includes every public source record without lowercasing historical slugs', () => { + const { aiRecords } = source('lib/ai-content.ts') + for (const [key, kind] of [['publications','publication'], ['authors','author'], ['talks','talk'], ['tutorials','tutorial'], ['areas','area'], ['blogPosts','blog']]) { + const expected = content[key].filter(r => !r.visibility.denied && (!r.visibility.notBefore || Date.parse(r.visibility.notBefore) <= Date.now())).map(r => r.slug).sort() + assert.deepEqual(aiRecords.filter(r => r.kind === kind).map(r => r.slug).sort(), expected) + } + for (const slug of ['Azouvi2022', 'Azouvi2022a']) assert.ok(aiRecords.some(r => r.canonicalUrl.endsWith(`/publications/${slug}/`))) +}) + +test('visibility survives every shared mapper; unlisted content remains denied', () => { + for (const items of [content.blogPosts, content.publications, content.talks, content.tutorials, content.authors, content.areas]) { + for (const item of items) { + assert.equal(typeof item.visibility, 'object', `${item.slug}: missing visibility metadata`) + assert.equal(item.visibility.version, 1) + } + } + const hidden = content.blogPosts.filter(p => p.unlisted) + assert.ok(hidden.length) + for (const p of hidden) assert.equal(p.visibility.denied, true) +}) + +test('shared mapper retains the native article as readable Markdown, including source links and disclosure', () => { + const post = content.blogPosts.find(p => p.slug === 'neurotech-frontier-human-flourishing') + assert.equal(typeof post.markdown, 'string', 'shared blog mapper must retain a readable body') + assert.match(post.markdown, /Neural Augmentation/) + assert.match(post.markdown, /https:\/\/paradromics.com\/news\//) + assert.match(post.markdown, /financial interests in some of the companies mentioned/) + assert.doesNotMatch(post.markdown, /