From 76525b8216c8b98bca779eea34b696f3affdbb19 Mon Sep 17 00:00:00 2001 From: BP <11394934+benjipeng@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:18:56 -0400 Subject: [PATCH] Catalog projects at the address their About URL states The catalog required every About URL to equal appautomaton.com//, so delineate, which lives on its own subdomain, failed the sync and blocked every deployment since October 20. - Accept any https About URL; still refuse other schemes and non-URLs. - Compare canonical links after URL normalization, so a bare origin matches its root path. - Look for a project sitemap only at its conventional central address, and list only same-host pages in the central sitemap, robots.txt and registry. A project on its own host keeps its discovery files there. --- sites/home/scripts/build.mjs | 3 ++- sites/home/scripts/catalog-policy.mjs | 12 +++++++++--- sites/home/scripts/check-publication.mjs | 3 ++- sites/home/scripts/discovery.test.mjs | 5 +++-- sites/home/scripts/page-audit.mjs | 9 ++++++++- sites/home/scripts/sync-catalog.mjs | 7 ++++--- tooling/sync-registry.mjs | 3 ++- 7 files changed, 30 insertions(+), 12 deletions(-) diff --git a/sites/home/scripts/build.mjs b/sites/home/scripts/build.mjs index df97ed8..4bb22b5 100644 --- a/sites/home/scripts/build.mjs +++ b/sites/home/scripts/build.mjs @@ -3,6 +3,7 @@ import {mkdir,writeFile,readFile,cp,rm} from 'node:fs/promises' import {dirname,join} from 'node:path' import {fileURLToPath} from 'node:url' import {catalog,ORIGIN,unitCount} from '../src/data/catalog.ts' +import {servedFrom} from './catalog-policy.mjs' import {renderLlms} from './discovery-files.mjs' import {artworks,artworkManifest,assignArtwork} from './artwork.mjs' import {escapeHTML as e,editorial,icon,logo} from './interface.mjs' @@ -107,7 +108,7 @@ const html=`${head('App Au
A change of scenery

Same workshop.
A different perspective.

RenoCrypt’s App Automaton field guide offers another way into the projects, with a character all its own.

Visit the field guide${icon('arrow')}
` await writeFile(join(OUT,'index.html'),html) -const sitemapUrls=[rootUrl,...hasSite.map(p=>p.site),rootUrl+'colophon.html'] +const sitemapUrls=[rootUrl,...hasSite.map(p=>p.site).filter(url=>servedFrom(ORIGIN,url)),rootUrl+'colophon.html'] await writeFile(join(OUT,'sitemap.xml'),`${sitemapUrls.map(url=>{const p=projects.find(p=>p.site===url);return `${e(url)}${p?.lastmod?`${p.lastmod}`:''}`}).join('')}\n`) const sitemaps=[...new Set([rootUrl+'sitemap.xml',...projects.map(p=>p.sitemap).filter(Boolean)])].sort() await writeFile(join(OUT,'robots.txt'),`User-agent: *\nAllow: /\n\n${sitemaps.map(url=>'Sitemap: '+url).join('\n')}\n`) diff --git a/sites/home/scripts/catalog-policy.mjs b/sites/home/scripts/catalog-policy.mjs index a0a8364..de0a0d1 100644 --- a/sites/home/scripts/catalog-policy.mjs +++ b/sites/home/scripts/catalog-policy.mjs @@ -2,12 +2,18 @@ // fallback without requiring a source edit for every newly published website. export const projectAddress=(origin,name)=>`${origin}/${encodeURIComponent(name)}/` -export function homepageProblem(repo,origin){ +// A project lives wherever its About URL says. The catalog refuses only an +// address it cannot safely link to. +export function homepageProblem(repo){ if(!repo.homepage)return null - const expected=projectAddress(origin,repo.name) - return repo.homepage===expected?null:`${repo.name} About URL is ${JSON.stringify(repo.homepage)}, expected ${expected}` + let url + try{url=new URL(repo.homepage)}catch{return `${repo.name} About URL ${JSON.stringify(repo.homepage)} is not a URL`} + return url.protocol==='https:'?null:`${repo.name} About URL ${JSON.stringify(repo.homepage)} is not an https address` } +// The central sitemap and robots.txt may list only addresses on their own host. +export const servedFrom=(origin,url)=>typeof url==='string'&&url.startsWith(`${origin}/`) + export function automaticGroup(repo){ const name=repo.name.toLowerCase() const topics=repo.topics.join(' ').toLowerCase() diff --git a/sites/home/scripts/check-publication.mjs b/sites/home/scripts/check-publication.mjs index a73c351..8bdd8b2 100644 --- a/sites/home/scripts/check-publication.mjs +++ b/sites/home/scripts/check-publication.mjs @@ -2,6 +2,7 @@ import assert from 'node:assert/strict' import {readFile,access} from 'node:fs/promises' import {gzipSync} from 'node:zlib' import {catalog,ORIGIN,unitCount} from '../src/data/catalog.ts' +import {servedFrom} from './catalog-policy.mjs' import {escapeHTML as e} from './interface.mjs' import {assignArtwork} from './artwork.mjs' import {validateState} from './catalog-state.mjs' @@ -38,7 +39,7 @@ for(const match of html.matchAll(/srcset="([^"]+)"/g))for(const candidate of mat for(const name of ['UncutSans-Regular.woff2','UncutSans-Medium.woff2','UncutSans-LICENSE.txt'])await access('dist/assets/fonts/'+name) const sitemap=await readFile('dist/sitemap.xml','utf8') const robots=await readFile('dist/robots.txt','utf8') -for(const url of [ORIGIN+'/',...projects.map(p=>p.site).filter(Boolean),ORIGIN+'/colophon.html'])assert(sitemap.includes(`${e(url)}`),`Sitemap omitted ${url}`) +for(const url of [ORIGIN+'/',...projects.map(p=>p.site).filter(url=>servedFrom(ORIGIN,url)),ORIGIN+'/colophon.html'])assert(sitemap.includes(`${e(url)}`),`Sitemap omitted ${url}`) for(const url of [ORIGIN+'/sitemap.xml',...projects.map(p=>p.sitemap).filter(Boolean)])assert(robots.includes(`Sitemap: ${url}`),`robots.txt omitted ${url}`) assert(!/^Disallow:\s*\/$/m.test(robots),'Production blocks crawling') assert(html.includes('href="https://appautomaton.renocrypt.com/"'),'Missing the related field guide') diff --git a/sites/home/scripts/discovery.test.mjs b/sites/home/scripts/discovery.test.mjs index 6de70fe..4da02f4 100644 --- a/sites/home/scripts/discovery.test.mjs +++ b/sites/home/scripts/discovery.test.mjs @@ -70,8 +70,8 @@ test('new public project websites enter the catalog automatically while editoria assert.equal(selected.length,5) const result=resolveShelves(shelves,selected) assert.deepEqual(result.map(s=>s.items.map(p=>p.repo)),[['new-skill'],['source-only','new-system'],['mlx-new'],['studio']]) - assert.equal(homepageProblem(repo('mlx-new'),'https://appautomaton.com'),null) - for(const homepage of ['https://appautomaton.renocrypt.com/mlx-new/','https://appautomaton.com/another/','http://appautomaton.com/mlx-new/','javascript:alert(1)'])assert(homepageProblem(repo('mlx-new',{homepage}),'https://appautomaton.com')) + for(const homepage of ['https://appautomaton.com/mlx-new/','https://mlx-new.appautomaton.com','https://appautomaton.renocrypt.com/mlx-new/'])assert.equal(homepageProblem(repo('mlx-new',{homepage})),null) + for(const homepage of ['http://appautomaton.com/mlx-new/','javascript:alert(1)','not a url'])assert(homepageProblem(repo('mlx-new',{homepage}))) assert.throws(()=>selectExhibits([],shelves,{}),/unavailable repositories/) }) @@ -83,6 +83,7 @@ test('project indexing guards inspect canonical metadata and both HTML and HTTP assert.deepEqual(audit(),[]) assert.match(audit(page.replace(url,'https://appautomaton.renocrypt.com/example/')).join(' '),/canonical/) assert.match(audit('Missing canonical').join(' '),/missing canonical/) + assert.doesNotMatch(auditProjectPage('https://example.appautomaton.com',page.replace(url,'https://example.appautomaton.com/'),new Headers()).problems.join(' '),/canonical/) for(const meta of ['','',''])assert.match(audit(page+meta).join(' '),/forbids indexing/) assert.match(audit(page,new Headers({'X-Robots-Tag':'googlebot: noindex'})).join(' '),/forbids indexing/) assert.deepEqual(audit(page+''),[]) diff --git a/sites/home/scripts/page-audit.mjs b/sites/home/scripts/page-audit.mjs index eb59cd5..f7dcf95 100644 --- a/sites/home/scripts/page-audit.mjs +++ b/sites/home/scripts/page-audit.mjs @@ -1,4 +1,11 @@ // Read static project metadata without executing page scripts. + +// An address and its normalized form name the same page, e.g. a bare origin +// and the origin with its root path. +const sameAddress = (href, url) => { + try { return new URL(href, url).href === new URL(url).href } catch { return false } +} + export function auditProjectPage(url, html, headers) { const problems = [] const warnings = [] @@ -6,7 +13,7 @@ export function auditProjectPage(url, html, headers) { const canonical = html.match(/]+rel=["']canonical["'][^>]*>/i)?.[0] const href = canonical?.match(/href=["']([^"']+)["']/i)?.[1] if (!href) problems.push('missing canonical') - else if (href !== url) problems.push(`canonical points at ${href}, not ${url}`) + else if (!sameAddress(href, url)) problems.push(`canonical points at ${href}, not ${url}`) const directives=[headers.get('x-robots-tag')||''] for(const tag of html.match(/]*>/gi)||[]) { diff --git a/sites/home/scripts/sync-catalog.mjs b/sites/home/scripts/sync-catalog.mjs index e5a5fc6..131f6cd 100644 --- a/sites/home/scripts/sync-catalog.mjs +++ b/sites/home/scripts/sync-catalog.mjs @@ -17,7 +17,7 @@ import { join, dirname } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' import { serves } from './probe-response.mjs' import { readRepositories } from './github-repositories.mjs' -import { homepageProblem, projectAddress, selectExhibits } from './catalog-policy.mjs' +import { homepageProblem, projectAddress, selectExhibits, servedFrom } from './catalog-policy.mjs' import { auditProjectPage } from './page-audit.mjs' import {emptyState,loadPublishedState,reconcileState} from './catalog-state.mjs' const { shelves, notShown } = await import('../src/data/shelves.ts') @@ -60,7 +60,8 @@ async function probe(repo) { two conventional names answers is how that list stops being a thing somebody has to remember to update. */ let sitemap = null - for (const name_ of ['sitemap-index.xml', 'sitemap.xml']) { + // A project on its own host keeps its sitemap there, outside this catalog. + if (url === projectAddress(ORIGIN, name)) for (const name_ of ['sitemap-index.xml', 'sitemap.xml']) { const url = `${ORIGIN}/${name}/${name_}` try { const response=await serves(url,'GET') @@ -118,7 +119,7 @@ try { process.exit(0) } -const invalidHomepages=repos.filter(r=>!Object.hasOwn(notShown,r.name)).map(r=>homepageProblem(r,ORIGIN)).filter(Boolean) +const invalidHomepages=repos.filter(r=>!Object.hasOwn(notShown,r.name)).map(r=>homepageProblem(r)).filter(Boolean) if(invalidHomepages.length)throw new Error(invalidHomepages.join('\n')) const probes = await Promise.all(repos.map(r => Object.hasOwn(notShown,r.name)||(!r.homepage&&!placed.has(r.name))?{hasSite:false}:probe(r))) diff --git a/tooling/sync-registry.mjs b/tooling/sync-registry.mjs index 28615e3..62c12d9 100644 --- a/tooling/sync-registry.mjs +++ b/tooling/sync-registry.mjs @@ -10,7 +10,8 @@ const registry = await readJSON(path) const byID = new Map(registry.sites.map(site => [site.id, site])) for (const project of catalog.flatMap(group => group.items).filter(project => project.site)) { const url = new URL(project.site) - assert.equal(url.origin, registry.origin, `Project uses a different origin: ${project.repo}`) + // The registry records mounts on this origin; a project on its own host has none. + if (url.origin !== registry.origin) continue const existing = byID.get(project.repo) if (existing) { assert.equal(existing.publicPath, url.pathname, `A public mount changed and needs review: ${project.repo}`)