Sitemap fixes

Author: webstackdev

astro.config.ts

@@ -35,6 +35,7 @@ import { testimonialsLengthWarning } from './src/integrations/TestimonialsLength
 import { fixContentAssetPropagation } from './src/lib/plugins/fixContentAssetPropagation'
 import { pwaDevAssetServer } from './src/lib/plugins/pwaDevAssetServer'
 import { createSerializeFunction, pagesJsonWriter } from './src/integrations/sitemapSerialize'
+import { getDeepDiveArticleAliasPaths } from './src/integrations/sitemapSerialize/deepDiveAliases'
 
 // Ensure Vite's HMR websocket connects through the same exposed dev server port used by Astro.
 const devServerPort = Number(process.env['DEV_SERVER_PORT'] ?? 4321)
@@ -44,6 +45,7 @@ const shouldEnableSentryIntegration = Boolean(sentryAuthToken)
 const sentryReleaseName = getPackageRelease()
 const astroEnvConfig = environmentalVariablesConfig as NonNullable<AstroUserConfig['env']>
 const astroMarkdownConfig = markdownConfig as unknown as NonNullable<AstroUserConfig['markdown']>
+const deepDiveArticleAliasPaths = getDeepDiveArticleAliasPaths()
 
 const shouldSuppressViteWarning = (message: string): boolean => {
   return (
@@ -104,7 +106,18 @@ const standardIntegrations = [
   })] : []),
   sitemap({
     serialize: createSerializeFunction({
-      exclude: ['deep-dive', 'downloads', 'print', '/articles/demo', 'testing', 'hero', 'links'],
+      exclude: [
+        'downloads',
+        'offline',
+        'print',
+        '/articles/demo',
+        'search',
+        'tags',
+        'testing',
+        'hero',
+        'links',
+        ...deepDiveArticleAliasPaths,
+      ],
     }),
   }),
   /**

src/integrations/sitemapSerialize/__tests__/deepDiveAliases.spec.ts

@@ -0,0 +1,37 @@
+import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { tmpdir } from 'node:os'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import {
+  getDeepDiveArticleAliasPaths,
+  normalizeDeepDiveEntryId,
+} from '../deepDiveAliases'
+
+describe('deepDiveAliases', () => {
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('normalizes deep-dive pdf entries to route ids', () => {
+    expect(normalizeDeepDiveEntryId('platform-architecture/pdf.mdx')).toBe('platform-architecture')
+    expect(normalizeDeepDiveEntryId('nested/guide/pdf.mdx')).toBe('nested/guide')
+    expect(normalizeDeepDiveEntryId('nested\\guide\\pdf.mdx')).toBe('nested/guide')
+  })
+
+  it('returns article alias paths for deep-dive pdf files', () => {
+    const contentRoot = mkdtempSync(join(tmpdir(), 'deep-dive-aliases-'))
+
+    mkdirSync(join(contentRoot, 'first-article'), { recursive: true })
+    mkdirSync(join(contentRoot, 'nested', 'second-article'), { recursive: true })
+    mkdirSync(join(contentRoot, 'ignored'), { recursive: true })
+
+    writeFileSync(join(contentRoot, 'first-article', 'pdf.mdx'), '---\ntitle: First\n---')
+    writeFileSync(join(contentRoot, 'nested', 'second-article', 'pdf.mdx'), '---\ntitle: Second\n---')
+    writeFileSync(join(contentRoot, 'ignored', 'index.mdx'), '---\ntitle: Ignored\n---')
+
+    expect(getDeepDiveArticleAliasPaths({ contentRoot })).toEqual([
+      '/articles/first-article',
+      '/articles/nested/second-article',
+    ])
+  })
+})

src/integrations/sitemapSerialize/deepDiveAliases.ts

@@ -0,0 +1,49 @@
+import { readdirSync } from 'node:fs'
+import { relative, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const deepDiveContentRoot = fileURLToPath(new URL('../../content/articles', import.meta.url))
+
+export interface DeepDiveAliasOptions {
+  contentRoot?: string
+}
+
+export function getDeepDiveArticleAliasPaths(
+  options: DeepDiveAliasOptions = {}
+): string[] {
+  const contentRoot = options.contentRoot ?? deepDiveContentRoot
+  const pdfEntries = collectDeepDiveEntries(contentRoot)
+
+  return pdfEntries
+    .map(entryPath => `/articles/${normalizeDeepDiveEntryId(entryPath)}`)
+    .sort((left, right) => left.localeCompare(right))
+}
+
+export function normalizeDeepDiveEntryId(entryPath: string): string {
+  const normalizedEntry = entryPath.replace(/\\/g, '/')
+  const withoutExtension = normalizedEntry.replace(/\.[^/.]+$/, '')
+
+  return withoutExtension.endsWith('/pdf')
+    ? withoutExtension.slice(0, -'/pdf'.length)
+    : withoutExtension
+}
+
+function collectDeepDiveEntries(contentRoot: string, currentDir = contentRoot): string[] {
+  const entries = readdirSync(currentDir, { withFileTypes: true })
+  const deepDiveEntries: string[] = []
+
+  for (const entry of entries) {
+    const entryPath = join(currentDir, entry.name)
+
+    if (entry.isDirectory()) {
+      deepDiveEntries.push(...collectDeepDiveEntries(contentRoot, entryPath))
+      continue
+    }
+
+    if (entry.isFile() && entry.name === 'pdf.mdx') {
+      deepDiveEntries.push(relative(contentRoot, entryPath))
+    }
+  }
+
+  return deepDiveEntries
+}

src/pages/robots.txt.ts

@@ -4,9 +4,6 @@ const getRobotsTxt = (sitemapURL: URL) => `\
 User-agent: *
 Allow: /
 Disallow: /downloads/
-Disallow: /articles$
-Disallow: /case-studies$
-Disallow: /deep-dive$
 Disallow: /offline
 Disallow: /print
 Disallow: /search