Filter the sitemap on noindex rather than on directory name

The exclusion list was growing one framework route at a time. #123
added 404 after it appeared in the sitemap; this branch added _not-found
for the same reason; and the versioned-docs work in progress adds a
third hand-written skip for the archived version directories. Each is
the same rule discovered again: do not advertise a page that tells
crawlers not to index it.

That property is readable from the page itself, so collectPages now
reads each index.html and skips the ones carrying a noindex robots meta.
The name list keeps only the directories that hold no pages at all -
_next, deb, rpm, images - where it is a traversal concern rather than an
indexing decision.

The tag is matched in either attribute order and tolerates content lists
such as "noindex,nofollow", since Next.js and hand-written metadata do
not agree on either. The final log line now reports how many pages were
skipped, so a filter that starts matching too much is visible in the
build output instead of silently shrinking the sitemap.

Verified against a synthetic export carrying an indexable root, /docs,
/docs/versions, /packages and /samples, plus three noindex pages written
in three different tag forms and an excluded _next directory: five URLs
emitted, three skipped, and out/404.html left alone as the file it is.
Note that the archived-version case is now covered by this rule, so the
docs/versions skip on the versioned-docs branch can go when it lands.

Author: GuanzhouSong

scripts/generate-sitemap.mjs

@@ -11,25 +11,41 @@ import path from 'node:path';
 const siteUrl = 'https://documentdb.io';
 const outDir = path.join(process.cwd(), 'out');
 
-// Top-level build outputs that are not HTML pages: Next.js assets, the APT/RPM
-// package repositories, images, and the two forms the not-found page takes
-// (with trailingSlash the export emits out/404/index.html alongside
-// out/404.html, plus out/_not-found/index.html for the App Router's not-found
-// route - both serve noindex, so listing either one submits a URL that tells
-// crawlers not to index it). The packages workflow adds deb/ and rpm/ after
-// this script runs in the deploy job, but they are excluded here too so local
-// full builds behave identically. Note that out/packages/ is NOT excluded: it
-// is the exported /packages download page; the workflow only adds
-// release-info.json (not a page) next to it.
+// Top-level build outputs that are not pages at all: Next.js assets, the
+// APT/RPM package repositories, and images. The packages workflow adds deb/
+// and rpm/ after this script runs in the deploy job, but they are excluded
+// here too so local full builds behave identically. Note that out/packages/ is
+// NOT excluded: it is the exported /packages download page; the workflow only
+// adds release-info.json (not a page) next to it.
+//
+// Pages that exist but must not be advertised are handled by isNoindex()
+// rather than by name. That covers both forms the not-found route takes - with
+// trailingSlash the export writes out/404/index.html alongside out/404.html,
+// and the App Router adds out/_not-found/index.html - and anything else the
+// framework starts emitting later. Listing a noindex URL is what earns the
+// "submitted URL marked noindex" warning in Search Console, and a page already
+// states that about itself, so there is no second list to keep in sync.
 const excludedTopLevelDirectories = new Set([
   '_next',
   'deb',
   'rpm',
   'images',
-  '404',
-  '_not-found',
 ]);
 
+let noindexPagesSkipped = 0;
+
+/**
+ * True when the page asks crawlers not to index it. Reads the meta tag in
+ * either attribute order and tolerates content lists such as "noindex,nofollow".
+ */
+function isNoindex(html) {
+  return (html.match(/<meta\b[^>]*>/gi) ?? []).some(
+    (tag) =>
+      /\bname=["']?robots["']?/i.test(tag) &&
+      /\bcontent=["'][^"']*\bnoindex\b/i.test(tag),
+  );
+}
+
 function xmlEscape(value) {
   return value
     .replace(/&/g, '&amp;')
@@ -49,10 +65,14 @@ function collectPages(directory, relativePath = '') {
   const indexFile = path.join(directory, 'index.html');
 
   if (fs.existsSync(indexFile)) {
-    pages.push({
-      url: relativePath === '' ? '/' : `/${relativePath}/`,
-      lastModified: fs.statSync(indexFile).mtime,
-    });
+    if (isNoindex(fs.readFileSync(indexFile, 'utf8'))) {
+      noindexPagesSkipped += 1;
+    } else {
+      pages.push({
+        url: relativePath === '' ? '/' : `/${relativePath}/`,
+        lastModified: fs.statSync(indexFile).mtime,
+      });
+    }
   }
 
   for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
@@ -117,4 +137,8 @@ if (fs.existsSync(robotsPath)) {
   fs.writeFileSync(robotsPath, `User-agent: *\nAllow: /\n\n${sitemapLine}\n`);
 }
 
-console.log(`Wrote out/sitemap.xml with ${pages.length} URLs and advertised it in out/robots.txt.`);
+console.log(
+  `Wrote out/sitemap.xml with ${pages.length} URLs (skipped ${noindexPagesSkipped} noindex ${
+    noindexPagesSkipped === 1 ? 'page' : 'pages'
+  }) and advertised it in out/robots.txt.`,
+);