Catalog projects at the address their About URL states

sites/home/scripts/build.mjs

@@ -3,6 +3,7 @@ import {mkdir,writeFile,readFile,cp,rm} from 'node:fs/promises'
 import {dirname,join} from 'node:path'
 import {fileURLToPath} from 'node:url'
 import {catalog,ORIGIN,unitCount} from '../src/data/catalog.ts'
+import {servedFrom} from './catalog-policy.mjs'
 import {renderLlms} from './discovery-files.mjs'
 import {artworks,artworkManifest,assignArtwork} from './artwork.mjs'
 import {escapeHTML as e,editorial,icon,logo} from './interface.mjs'
@@ -107,7 +108,7 @@ const html=`<!doctype html><html lang="en" data-theme="day"><head>${head('App Au
 <section class="elsewhere" aria-labelledby="elsewhere-title"><div><span class="eyebrow">A change of scenery</span><h2 id="elsewhere-title">Same workshop.<br>A different perspective.</h2><p>RenoCrypt’s App Automaton field guide offers another way into the projects, with a character all its own.</p><a class="button" href="${fieldGuide}"><span>Visit the field guide</span><span class="button-symbol">${icon('arrow')}</span></a></div><div class="elsewhere-drawing" aria-hidden="true"><svg viewBox="0 0 320 320" fill="none" stroke="currentColor"><circle cx="160" cy="160" r="138" stroke-dasharray="1 9"/>${Array.from({length:8},(_,i)=>`<ellipse cx="160" cy="111" rx="32" ry="71" transform="rotate(${i*45} 160 160)"/>`).join('')}<circle cx="160" cy="160" r="24"/></svg></div></section></main>
 <footer data-reveal><div class="footer-top"><a class="brand" href="#top">${logo()}<span>App<br>Automaton</span></a><p>Independent thinking.<br>Shared possibilities.</p><div><a href="https://github.com/appautomaton">GitHub organization ${icon('arrow')}</a><a href="https://huggingface.co/appautomaton">Hugging Face ${icon('arrow')}</a><a href="llms.txt">Text edition ${icon('arrow')}</a><a href="colophon.html">Art & colophon ${icon('arrow')}</a><button type="button" class="share-workshop" hidden><span>Share this page</span>${icon('share')}</button><span class="share-status sr-only" role="status"></span><input class="share-url" aria-label="Link to share" readonly hidden value="https://appautomaton.com/"></div></div><div class="footer-word" aria-hidden="true"><span>Keep making.</span></div><div class="footer-bottom"><span>An <a href="https://www.appcubic.com/">AppCubic</a> workshop.</span><a href="https://www.renocrypt.com/">Research & writing at RenoCrypt ${icon('arrow')}</a><a href="#top">Back to the top ${icon('arrow')}</a></div></footer></body></html>`
 await writeFile(join(OUT,'index.html'),html)
-const sitemapUrls=[rootUrl,...hasSite.map(p=>p.site),rootUrl+'colophon.html']
+const sitemapUrls=[rootUrl,...hasSite.map(p=>p.site).filter(url=>servedFrom(ORIGIN,url)),rootUrl+'colophon.html']
 await writeFile(join(OUT,'sitemap.xml'),`<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">${sitemapUrls.map(url=>{const p=projects.find(p=>p.site===url);return `<url><loc>${e(url)}</loc>${p?.lastmod?`<lastmod>${p.lastmod}</lastmod>`:''}</url>`}).join('')}</urlset>\n`)
 const sitemaps=[...new Set([rootUrl+'sitemap.xml',...projects.map(p=>p.sitemap).filter(Boolean)])].sort()
 await writeFile(join(OUT,'robots.txt'),`User-agent: *\nAllow: /\n\n${sitemaps.map(url=>'Sitemap: '+url).join('\n')}\n`)

sites/home/scripts/catalog-policy.mjs

@@ -2,12 +2,18 @@
 // fallback without requiring a source edit for every newly published website.
 export const projectAddress=(origin,name)=>`${origin}/${encodeURIComponent(name)}/`
 
-export function homepageProblem(repo,origin){
+// A project lives wherever its About URL says. The catalog refuses only an
+// address it cannot safely link to.
+export function homepageProblem(repo){
  if(!repo.homepage)return null
- const expected=projectAddress(origin,repo.name)
- return repo.homepage===expected?null:`${repo.name} About URL is ${JSON.stringify(repo.homepage)}, expected ${expected}`
+ let url
+ try{url=new URL(repo.homepage)}catch{return `${repo.name} About URL ${JSON.stringify(repo.homepage)} is not a URL`}
+ return url.protocol==='https:'?null:`${repo.name} About URL ${JSON.stringify(repo.homepage)} is not an https address`
 }
 
+// The central sitemap and robots.txt may list only addresses on their own host.
+export const servedFrom=(origin,url)=>typeof url==='string'&&url.startsWith(`${origin}/`)
+
 export function automaticGroup(repo){
  const name=repo.name.toLowerCase()
  const topics=repo.topics.join(' ').toLowerCase()

sites/home/scripts/check-publication.mjs

@@ -2,6 +2,7 @@ import assert from 'node:assert/strict'
 import {readFile,access} from 'node:fs/promises'
 import {gzipSync} from 'node:zlib'
 import {catalog,ORIGIN,unitCount} from '../src/data/catalog.ts'
+import {servedFrom} from './catalog-policy.mjs'
 import {escapeHTML as e} from './interface.mjs'
 import {assignArtwork} from './artwork.mjs'
 import {validateState} from './catalog-state.mjs'
@@ -38,7 +39,7 @@ for(const match of html.matchAll(/srcset="([^"]+)"/g))for(const candidate of mat
 for(const name of ['UncutSans-Regular.woff2','UncutSans-Medium.woff2','UncutSans-LICENSE.txt'])await access('dist/assets/fonts/'+name)
 const sitemap=await readFile('dist/sitemap.xml','utf8')
 const robots=await readFile('dist/robots.txt','utf8')
-for(const url of [ORIGIN+'/',...projects.map(p=>p.site).filter(Boolean),ORIGIN+'/colophon.html'])assert(sitemap.includes(`<loc>${e(url)}</loc>`),`Sitemap omitted ${url}`)
+for(const url of [ORIGIN+'/',...projects.map(p=>p.site).filter(url=>servedFrom(ORIGIN,url)),ORIGIN+'/colophon.html'])assert(sitemap.includes(`<loc>${e(url)}</loc>`),`Sitemap omitted ${url}`)
 for(const url of [ORIGIN+'/sitemap.xml',...projects.map(p=>p.sitemap).filter(Boolean)])assert(robots.includes(`Sitemap: ${url}`),`robots.txt omitted ${url}`)
 assert(!/^Disallow:\s*\/$/m.test(robots),'Production blocks crawling')
 assert(html.includes('href="https://appautomaton.renocrypt.com/"'),'Missing the related field guide')

sites/home/scripts/discovery.test.mjs

@@ -70,8 +70,8 @@ test('new public project websites enter the catalog automatically while editoria
  assert.equal(selected.length,5)
  const result=resolveShelves(shelves,selected)
  assert.deepEqual(result.map(s=>s.items.map(p=>p.repo)),[['new-skill'],['source-only','new-system'],['mlx-new'],['studio']])
- assert.equal(homepageProblem(repo('mlx-new'),'https://appautomaton.com'),null)
- for(const homepage of ['https://appautomaton.renocrypt.com/mlx-new/','https://appautomaton.com/another/','http://appautomaton.com/mlx-new/','javascript:alert(1)'])assert(homepageProblem(repo('mlx-new',{homepage}),'https://appautomaton.com'))
+ for(const homepage of ['https://appautomaton.com/mlx-new/','https://mlx-new.appautomaton.com','https://appautomaton.renocrypt.com/mlx-new/'])assert.equal(homepageProblem(repo('mlx-new',{homepage})),null)
+ for(const homepage of ['http://appautomaton.com/mlx-new/','javascript:alert(1)','not a url'])assert(homepageProblem(repo('mlx-new',{homepage})))
  assert.throws(()=>selectExhibits([],shelves,{}),/unavailable repositories/)
 })
 
@@ -83,6 +83,7 @@ test('project indexing guards inspect canonical metadata and both HTML and HTTP
  assert.deepEqual(audit(),[])
  assert.match(audit(page.replace(url,'https://appautomaton.renocrypt.com/example/')).join(' '),/canonical/)
  assert.match(audit('<title>Missing canonical</title>').join(' '),/missing canonical/)
+ assert.doesNotMatch(auditProjectPage('https://example.appautomaton.com',page.replace(url,'https://example.appautomaton.com/'),new Headers()).problems.join(' '),/canonical/)
  for(const meta of ['<meta name="robots" content="noindex,follow">','<meta content="NONE" name="googlebot">','<meta name="bingbot" content="noindex">'])assert.match(audit(page+meta).join(' '),/forbids indexing/)
  assert.match(audit(page,new Headers({'X-Robots-Tag':'googlebot: noindex'})).join(' '),/forbids indexing/)
  assert.deepEqual(audit(page+'<meta name="robots" content="index, follow, max-image-preview:large">'),[])

sites/home/scripts/page-audit.mjs

@@ -1,12 +1,19 @@
 // Read static project metadata without executing page scripts.
+
+// An address and its normalized form name the same page, e.g. a bare origin
+// and the origin with its root path.
+const sameAddress = (href, url) => {
+  try { return new URL(href, url).href === new URL(url).href } catch { return false }
+}
+
 export function auditProjectPage(url, html, headers) {
   const problems = []
   const warnings = []
 
   const canonical = html.match(/<link[^>]+rel=["']canonical["'][^>]*>/i)?.[0]
   const href = canonical?.match(/href=["']([^"']+)["']/i)?.[1]
   if (!href) problems.push('missing canonical')
-  else if (href !== url) problems.push(`canonical points at ${href}, not ${url}`)
+  else if (!sameAddress(href, url)) problems.push(`canonical points at ${href}, not ${url}`)
 
   const directives=[headers.get('x-robots-tag')||'']
   for(const tag of html.match(/<meta\b[^>]*>/gi)||[]) {

sites/home/scripts/sync-catalog.mjs

@@ -17,7 +17,7 @@ import { join, dirname } from 'node:path'
 import { fileURLToPath, pathToFileURL } from 'node:url'
 import { serves } from './probe-response.mjs'
 import { readRepositories } from './github-repositories.mjs'
-import { homepageProblem, projectAddress, selectExhibits } from './catalog-policy.mjs'
+import { homepageProblem, projectAddress, selectExhibits, servedFrom } from './catalog-policy.mjs'
 import { auditProjectPage } from './page-audit.mjs'
 import {emptyState,loadPublishedState,reconcileState} from './catalog-state.mjs'
 const { shelves, notShown } = await import('../src/data/shelves.ts')
@@ -60,7 +60,8 @@ async function probe(repo) {
        two conventional names answers is how that list stops being a thing
        somebody has to remember to update. */
     let sitemap = null
-    for (const name_ of ['sitemap-index.xml', 'sitemap.xml']) {
+    // A project on its own host keeps its sitemap there, outside this catalog.
+    if (url === projectAddress(ORIGIN, name)) for (const name_ of ['sitemap-index.xml', 'sitemap.xml']) {
       const url = `${ORIGIN}/${name}/${name_}`
       try {
         const response=await serves(url,'GET')
@@ -118,7 +119,7 @@ try {
   process.exit(0)
 }
 
-const invalidHomepages=repos.filter(r=>!Object.hasOwn(notShown,r.name)).map(r=>homepageProblem(r,ORIGIN)).filter(Boolean)
+const invalidHomepages=repos.filter(r=>!Object.hasOwn(notShown,r.name)).map(r=>homepageProblem(r)).filter(Boolean)
 if(invalidHomepages.length)throw new Error(invalidHomepages.join('\n'))
 const probes = await Promise.all(repos.map(r => Object.hasOwn(notShown,r.name)||(!r.homepage&&!placed.has(r.name))?{hasSite:false}:probe(r)))
 

tooling/sync-registry.mjs

@@ -10,7 +10,8 @@ const registry = await readJSON(path)
 const byID = new Map(registry.sites.map(site => [site.id, site]))
 for (const project of catalog.flatMap(group => group.items).filter(project => project.site)) {
   const url = new URL(project.site)
-  assert.equal(url.origin, registry.origin, `Project uses a different origin: ${project.repo}`)
+  // The registry records mounts on this origin; a project on its own host has none.
+  if (url.origin !== registry.origin) continue
   const existing = byID.get(project.repo)
   if (existing) {
     assert.equal(existing.publicPath, url.pathname, `A public mount changed and needs review: ${project.repo}`)