From 4dee91c020bacf1df3043e444ec58901901fcd09 Mon Sep 17 00:00:00 2001 From: Logan Lindquist Land Date: Wed, 9 Sep 2026 23:30:39 -0500 Subject: [PATCH 1/3] feat(nav): honor frontmatter order and description, add prev/next links Navigation was derived entirely from the database with no ordering controls: articles sorted alphabetically by title, folders appeared in whatever order their first article happened to sort into, and every page's meta description was its own title repeated because libsql-search parses frontmatter description into the embedding text and then discards it. Two nullable columns now carry sort_order and description, added by a PRAGMA-guarded ALTER so existing deployments upgrade without a manual migration, populated by a post-index pass and read through one query. Folder sequence lives in a single config file. Article pages emit the real description and prev/next links within their folder. --- README.md | 3 +- content/features/semantic-search.md | 2 + content/getting-started/welcome.md | 2 + content/theme/overview.md | 2 + docs/DEPLOYMENT.md | 7 + docs/GETTING_STARTED.md | 11 ++ package.json | 1 + pnpm-lock.yaml | 3 + scripts/index-content-runner.test.ts | 54 ++++++++ scripts/index-content-runner.ts | 9 ++ scripts/index-content.ts | 32 ++++- scripts/init-db.ts | 12 ++ scripts/nav-frontmatter.test.ts | 178 ++++++++++++++++++++++++ scripts/nav-frontmatter.ts | 118 ++++++++++++++++ src/components/DocsSidebar.astro | 34 ++--- src/config/nav.ts | 15 +++ src/lib/articles.test.ts | 129 ++++++++++++++++++ src/lib/articles.ts | 80 +++++++++++ src/lib/nav.test.ts | 195 +++++++++++++++++++++++++++ src/lib/nav.ts | 116 ++++++++++++++++ src/lib/navSchema.test.ts | 110 +++++++++++++++ src/lib/navSchema.ts | 73 ++++++++++ src/pages/content/[...slug].astro | 45 ++++++- src/types/article.ts | 20 +++ 24 files changed, 1217 insertions(+), 34 deletions(-) create mode 100644 scripts/nav-frontmatter.test.ts create mode 100644 scripts/nav-frontmatter.ts create mode 100644 src/config/nav.ts create mode 100644 src/lib/articles.test.ts create mode 100644 src/lib/articles.ts create mode 100644 src/lib/nav.test.ts create mode 100644 src/lib/nav.ts create mode 100644 src/lib/navSchema.test.ts create mode 100644 src/lib/navSchema.ts diff --git a/README.md b/README.md index a1d13c0..6abf961 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,8 @@ content/ └── api.md ``` -Folders become sidebar groups, and frontmatter can define titles and tags. +Folders become sidebar groups, and frontmatter can define `title`, `tags`, +`description`, and `order`. Folder order is configured in `src/config/nav.ts`. ## Development Checks diff --git a/content/features/semantic-search.md b/content/features/semantic-search.md index b484f68..f321598 100644 --- a/content/features/semantic-search.md +++ b/content/features/semantic-search.md @@ -1,5 +1,7 @@ --- title: Semantic Search - AI-Powered Search +description: How vector embeddings let search match on meaning rather than keywords, and how this theme wires them up. +order: 1 tags: [semantic-search, embeddings, ai, vector-search] --- diff --git a/content/getting-started/welcome.md b/content/getting-started/welcome.md index e519644..4f3aa08 100644 --- a/content/getting-started/welcome.md +++ b/content/getting-started/welcome.md @@ -1,5 +1,7 @@ --- title: Welcome to Your Docs +description: Set up semantic-docs, index your first markdown files, and learn how the theme fits together. +order: 1 tags: [getting-started, introduction] --- diff --git a/content/theme/overview.md b/content/theme/overview.md index eedd155..63527ed 100644 --- a/content/theme/overview.md +++ b/content/theme/overview.md @@ -1,5 +1,7 @@ --- title: Semantic Docs Theme Overview +description: A tour of the Astro theme - hybrid rendering, the libSQL-backed sidebar, and the pieces you customize. +order: 1 tags: [astro, theme, documentation, semantic-search] --- diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index be131d5..87dd6ce 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -100,6 +100,13 @@ After upgrading, confirm the index is populated rather than assuming it: SELECT count(*) FROM articles_cf_bgem3_1024_fts; ``` +Sidebar ordering adds two columns, `sort_order` and `description`. Both +`pnpm db:init` and `pnpm index` add them to an existing table, so upgrading +needs no separate migration; existing rows are preserved and the values are +filled in on the next `pnpm index`. Rendering reads these columns, so a +deployment serving pages from a database that has never run either command +fails with `no such column: sort_order` rather than quietly dropping the order. + ## Containers Two Dockerfiles are provided. Both index content during the build, and indexing diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index 4b4c2e4..32a7e66 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -62,6 +62,8 @@ Example: ```markdown --- title: Getting Started +description: What this page covers, used for the meta description and link previews. +order: 1 tags: [tutorial, beginner] --- @@ -70,6 +72,15 @@ tags: [tutorial, beginner] Your content here. ``` +`order` sets the position in the sidebar within its folder, ascending. Articles +without one sort after those that have one, by title. `description` fills the +page's `` and `og:description`, falling back to the +title when absent. Both are read at index time, so rerun `pnpm index` after +changing them. + +Folder order is configured in `src/config/nav.ts`. Folders left out of that +list render after the listed ones, alphabetically. + After adding or changing content, rebuild the index before building or deploying: diff --git a/package.json b/package.json index e37c625..bf39555 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "astro": "^7.2.10", "clsx": "^2.1.1", "cmdk": "^1.1.1", + "gray-matter": "^4.0.3", "logan-logger": "^2.5.3", "lucide-react": "^1.41.0", "marked": "^18.0.11", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 590be10..007783b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -51,6 +51,9 @@ importers: cmdk: specifier: ^1.1.1 version: 1.1.1(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) + gray-matter: + specifier: ^4.0.3 + version: 4.0.3 logan-logger: specifier: ^2.5.3 version: 2.5.3 diff --git a/scripts/index-content-runner.test.ts b/scripts/index-content-runner.test.ts index af3b394..5d867bb 100644 --- a/scripts/index-content-runner.test.ts +++ b/scripts/index-content-runner.test.ts @@ -126,4 +126,58 @@ describe('runContentIndexing', () => { expect(await runContentIndexing(operations, logger)).toBe(1); expect(rebuildKeywordIndex).not.toHaveBeenCalled(); }); + + it('applies navigation frontmatter after the rows are written', async () => { + const order: string[] = []; + const logger = createLogger(); + const operations: IndexingOperations = { + createTable: vi.fn().mockResolvedValue(undefined), + indexContent: vi.fn(async () => { + order.push('indexContent'); + return { success: 1, total: 1, failed: 0 }; + }), + applyNavFrontmatter: vi.fn(async () => { + order.push('applyNavFrontmatter'); + return 3; + }), + }; + + expect(await runContentIndexing(operations, logger)).toBe(0); + // indexContent clears and reinserts the table, so an earlier pass would be + // overwritten. + expect(order).toEqual(['indexContent', 'applyNavFrontmatter']); + expect(logger.info).toHaveBeenCalledWith( + 'Applied navigation frontmatter to 3 documents', + ); + }); + + it('skips the frontmatter pass when no operation is supplied', async () => { + const logger = createLogger(); + const operations: IndexingOperations = { + createTable: vi.fn().mockResolvedValue(undefined), + indexContent: vi.fn().mockResolvedValue({ + success: 1, + total: 1, + failed: 0, + }), + }; + + expect(await runContentIndexing(operations, logger)).toBe(0); + expect(logger.info).not.toHaveBeenCalledWith( + expect.stringContaining('navigation frontmatter'), + ); + }); + + it('does not apply frontmatter when indexing throws', async () => { + const logger = createLogger(); + const applyNavFrontmatter = vi.fn(); + const operations: IndexingOperations = { + createTable: vi.fn().mockResolvedValue(undefined), + indexContent: vi.fn().mockRejectedValue(new Error('embedding failed')), + applyNavFrontmatter, + }; + + expect(await runContentIndexing(operations, logger)).toBe(1); + expect(applyNavFrontmatter).not.toHaveBeenCalled(); + }); }); diff --git a/scripts/index-content-runner.ts b/scripts/index-content-runner.ts index 858c0e0..8599aad 100644 --- a/scripts/index-content-runner.ts +++ b/scripts/index-content-runner.ts @@ -15,6 +15,8 @@ export interface IndexingOperations { indexContent: (onProgress: IndexProgress) => Promise; /** Repopulate the keyword index from the rows just written. */ rebuildKeywordIndex?: () => Promise; + /** Persist frontmatter fields the library parses but discards. Returns rows updated. */ + applyNavFrontmatter?: () => Promise; } export interface IndexingLogger { @@ -39,6 +41,13 @@ export async function runContentIndexing( logger.info(`[${current}/${total}] Indexing: ${file}`); }); + // After the vector rows land, because indexContent clears and reinserts the + // table and would otherwise discard these values. + if (operations.applyNavFrontmatter) { + const updated = await operations.applyNavFrontmatter(); + logger.info(`Applied navigation frontmatter to ${updated} documents`); + } + // After the vector rows land, so a failed embedding run never leaves a // keyword index describing content that is not in the articles table. if (operations.rebuildKeywordIndex) { diff --git a/scripts/index-content.ts b/scripts/index-content.ts index 8d40d47..ca27d54 100644 --- a/scripts/index-content.ts +++ b/scripts/index-content.ts @@ -8,6 +8,7 @@ import { createClient } from '@libsql/client'; import { createTable, indexContent } from '@logan/libsql-search'; import { logger } from 'logan-logger'; import { env } from '../src/lib/env'; +import { ensureNavColumns } from '../src/lib/navSchema'; import { EMBEDDING_DIMENSIONS, getEmbeddingOptions, @@ -15,6 +16,9 @@ import { SEARCH_TABLE_NAME, } from '../src/lib/searchConfig'; import { runContentIndexing } from './index-content-runner'; +import { applyNavFrontmatter, collectNavFrontmatter } from './nav-frontmatter'; + +const CONTENT_PATH = './content'; // Initialize client (Turso or local libSQL) const url = process.env.TURSO_DB_URL; @@ -66,16 +70,38 @@ async function rebuildKeywordIndex(): Promise { process.exitCode = await runContentIndexing( { - createTable: () => - createTable(client, SEARCH_TABLE_NAME, EMBEDDING_DIMENSIONS), + // Indexing runs against databases created before the navigation columns + // existed, so it migrates rather than assuming db:init was rerun. + createTable: async () => { + await createTable(client, SEARCH_TABLE_NAME, EMBEDDING_DIMENSIONS); + await ensureNavColumns(client, SEARCH_TABLE_NAME); + }, indexContent: (onProgress) => indexContent({ client, - contentPath: './content', + contentPath: CONTENT_PATH, tableName: SEARCH_TABLE_NAME, embeddingOptions, onProgress, }), + applyNavFrontmatter: async () => { + const records = await collectNavFrontmatter(CONTENT_PATH); + const updated = await applyNavFrontmatter( + client, + SEARCH_TABLE_NAME, + records, + ); + + // A shortfall means a file on disk has no indexed row under the slug this + // pass derived, so its ordering and description are silently missing. + if (updated < records.length) { + logger.warn( + `Navigation frontmatter matched ${updated} of ${records.length} content files; the rest are not in the index under the expected slug`, + ); + } + + return updated; + }, rebuildKeywordIndex, }, logger, diff --git a/scripts/init-db.ts b/scripts/init-db.ts index 5010cca..11fca32 100644 --- a/scripts/init-db.ts +++ b/scripts/init-db.ts @@ -7,6 +7,7 @@ import { createClient } from '@libsql/client'; import { createTable } from '@logan/libsql-search'; import { logger } from 'logan-logger'; +import { ensureNavColumns } from '../src/lib/navSchema'; import { EMBEDDING_DIMENSIONS, SEARCH_FTS_TABLE_NAME, @@ -35,6 +36,17 @@ try { `Created ${SEARCH_TABLE_NAME} with ${EMBEDDING_DIMENSIONS}-dimension embeddings`, ); + // createTable is CREATE TABLE IF NOT EXISTS, so it leaves an existing table + // alone. This is what upgrades a deployment indexed before the navigation + // columns existed, without a separate migration step. + const addedColumns = await ensureNavColumns(client, SEARCH_TABLE_NAME); + + if (addedColumns.length > 0) { + logger.info( + `Added navigation columns to ${SEARCH_TABLE_NAME}: ${addedColumns.join(', ')} (reindex to populate)`, + ); + } + // Keyword half of hybrid search. Kept as its own table rather than an // external-content one: the indexer clears and repopulates the articles // table wholesale, and a standalone index is rebuilt from it in one step diff --git a/scripts/nav-frontmatter.test.ts b/scripts/nav-frontmatter.test.ts new file mode 100644 index 0000000..4e3c9a1 --- /dev/null +++ b/scripts/nav-frontmatter.test.ts @@ -0,0 +1,178 @@ +/** + * @vitest-environment node + */ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { type Client, createClient } from '@libsql/client'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { applyNavFrontmatter, collectNavFrontmatter } from './nav-frontmatter'; + +const TABLE = 'articles_test'; + +let contentDir: string; +let client: Client; + +async function writeContent( + relativePath: string, + frontmatter: string, +): Promise { + const fullPath = join(contentDir, relativePath); + await mkdir(join(fullPath, '..'), { recursive: true }); + await writeFile(fullPath, `---\n${frontmatter}\n---\n\nBody text.\n`, 'utf8'); +} + +function bySlug(records: Awaited>) { + return Object.fromEntries(records.map((record) => [record.slug, record])); +} + +beforeEach(async () => { + contentDir = await mkdtemp(join(tmpdir(), 'nav-frontmatter-')); + client = createClient({ url: ':memory:' }); + await client.execute(` + CREATE TABLE "${TABLE}" ( + slug TEXT UNIQUE NOT NULL, + sort_order INTEGER, + description TEXT + ) + `); +}); + +afterEach(async () => { + client.close(); + await rm(contentDir, { recursive: true, force: true }); +}); + +describe('collectNavFrontmatter', () => { + it('derives the slug from the path the way libsql-search does', async () => { + await writeContent('guides/intro.md', 'title: Intro'); + await writeContent('top.markdown', 'title: Top'); + + const slugs = (await collectNavFrontmatter(contentDir)) + .map((record) => record.slug) + .sort(); + + expect(slugs).toEqual(['guides/intro', 'top']); + }); + + it('reads order and description', async () => { + await writeContent( + 'guides/intro.md', + 'title: Intro\norder: 3\ndescription: The opening chapter', + ); + + expect( + bySlug(await collectNavFrontmatter(contentDir))['guides/intro'], + ).toEqual({ + slug: 'guides/intro', + order: 3, + description: 'The opening chapter', + }); + }); + + it('accepts a quoted numeric order', async () => { + await writeContent('a.md', "order: '4'"); + + expect((await collectNavFrontmatter(contentDir))[0].order).toBe(4); + }); + + it('ignores a non-numeric order rather than failing the run', async () => { + await writeContent('a.md', 'order: first'); + + expect((await collectNavFrontmatter(contentDir))[0].order).toBeNull(); + }); + + it('ignores a non-string description', async () => { + await writeContent('a.md', 'description:\n - a list'); + + expect((await collectNavFrontmatter(contentDir))[0].description).toBeNull(); + }); + + it('handles a description containing a colon', async () => { + await writeContent('a.md', 'description: "Search: how it works"'); + + expect((await collectNavFrontmatter(contentDir))[0].description).toBe( + 'Search: how it works', + ); + }); + + it('yields nulls for a file with no frontmatter', async () => { + await writeFile(join(contentDir, 'plain.md'), '# Plain\n', 'utf8'); + + expect(await collectNavFrontmatter(contentDir)).toEqual([ + { slug: 'plain', order: null, description: null }, + ]); + }); + + it('skips non-markdown files and dot directories', async () => { + await writeContent('a.md', 'order: 1'); + await writeFile(join(contentDir, 'notes.txt'), 'ignored', 'utf8'); + await mkdir(join(contentDir, '.hidden'), { recursive: true }); + await writeFile(join(contentDir, '.hidden', 'b.md'), '# Hidden\n', 'utf8'); + + expect( + (await collectNavFrontmatter(contentDir)).map((r) => r.slug), + ).toEqual(['a']); + }); +}); + +describe('applyNavFrontmatter', () => { + beforeEach(async () => { + await client.batch( + [ + `INSERT INTO "${TABLE}" (slug) VALUES ('guides/intro')`, + `INSERT INTO "${TABLE}" (slug) VALUES ('other')`, + ], + 'write', + ); + }); + + it('writes the fields onto the matching row only', async () => { + const updated = await applyNavFrontmatter(client, TABLE, [ + { slug: 'guides/intro', order: 2, description: 'Opening' }, + ]); + + expect(updated).toBe(1); + + const rows = await client.execute( + `SELECT slug, sort_order, description FROM "${TABLE}" ORDER BY slug`, + ); + expect( + rows.rows.map((row) => [row.slug, row.sort_order, row.description]), + ).toEqual([ + ['guides/intro', 2, 'Opening'], + ['other', null, null], + ]); + }); + + it('counts rows matched, not records supplied', async () => { + await expect( + applyNavFrontmatter(client, TABLE, [ + { slug: 'never-indexed', order: 1, description: 'x' }, + ]), + ).resolves.toBe(0); + }); + + it('is a no-op for an empty content directory', async () => { + await expect(applyNavFrontmatter(client, TABLE, [])).resolves.toBe(0); + }); + + it('stores a slug containing SQL metacharacters as a value, not syntax', async () => { + await client.execute( + `INSERT INTO "${TABLE}" (slug) VALUES ('a''); DROP TABLE "${TABLE}"; --')`, + ); + + await applyNavFrontmatter(client, TABLE, [ + { + slug: `a'); DROP TABLE "${TABLE}"; --`, + order: 7, + description: null, + }, + ]); + + const rows = await client.execute( + `SELECT sort_order FROM "${TABLE}" WHERE sort_order = 7`, + ); + expect(rows.rows).toHaveLength(1); + }); +}); diff --git a/scripts/nav-frontmatter.ts b/scripts/nav-frontmatter.ts new file mode 100644 index 0000000..b7c08f7 --- /dev/null +++ b/scripts/nav-frontmatter.ts @@ -0,0 +1,118 @@ +/** + * Post-index pass that persists the frontmatter fields libsql-search parses but + * does not store. + * + * The library folds `description` into the embedding text and discards it, and + * has no concept of `order`. Both are written back here by slug, after indexing, + * so pages and the sidebar read them from the database like every other field + * rather than reaching for ./content at render time. + */ + +import { readdir, readFile } from 'node:fs/promises'; +import { extname, join, relative } from 'node:path'; +import type { Client } from '@libsql/client'; +import matter from 'gray-matter'; +import { quoteIdentifier } from '../src/lib/navSchema'; + +/** Matches the extensions libsql-search indexes, so slugs cannot diverge. */ +const CONTENT_EXTENSIONS = ['.md', '.markdown']; + +/** The indexer's own default excludes; collecting more would count files it never indexed. */ +const EXCLUDED_DIRECTORIES = ['node_modules', '.git', 'dist', 'build']; + +export interface NavFrontmatter { + slug: string; + order: number | null; + description: string | null; +} + +function parseOrder(value: unknown): number | null { + if (typeof value === 'number') return Number.isFinite(value) ? value : null; + if (typeof value === 'string' && value.trim() !== '') { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; + } + return null; +} + +function parseDescription(value: unknown): string | null { + if (typeof value !== 'string') return null; + const trimmed = value.trim(); + return trimmed === '' ? null : trimmed; +} + +async function findContentFiles( + dir: string, + baseDir: string, +): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + const files: string[] = []; + + for (const entry of entries) { + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name.startsWith('.')) continue; + if (EXCLUDED_DIRECTORIES.includes(entry.name)) continue; + files.push(...(await findContentFiles(fullPath, baseDir))); + } else if (CONTENT_EXTENSIONS.includes(extname(entry.name))) { + files.push(relative(baseDir, fullPath)); + } + } + + return files; +} + +/** + * Read `order` and `description` from every content file, keyed by the same + * slug libsql-search derives: the relative path minus its extension. + */ +export async function collectNavFrontmatter( + contentPath: string, +): Promise { + // Sorted for the same reason the indexer sorts: when two files claim one + // slug, the winner must not depend on directory read order. + const relativePaths = (await findContentFiles(contentPath, contentPath)).sort( + (a, b) => a.localeCompare(b, 'en'), + ); + const records: NavFrontmatter[] = []; + + for (const relativePath of relativePaths) { + const raw = await readFile(join(contentPath, relativePath), 'utf8'); + const { data } = matter(raw); + + records.push({ + slug: relativePath.replace(/\.(md|markdown)$/, '').replaceAll('\\', '/'), + order: parseOrder(data.order), + description: parseDescription(data.description), + }); + } + + return records; +} + +/** + * Write the collected fields onto the indexed rows. Only sets values: the + * indexer clears and reinserts the whole table on every run, so a field removed + * from frontmatter is already back to NULL by the time this runs. + */ +export async function applyNavFrontmatter( + client: Client, + tableName: string, + records: readonly NavFrontmatter[], +): Promise { + if (records.length === 0) return 0; + + const quotedTableName = quoteIdentifier(tableName); + + const results = await client.batch( + records.map((record) => ({ + sql: `UPDATE ${quotedTableName} SET sort_order = ?, description = ? WHERE slug = ?`, + args: [record.order, record.description, record.slug], + })), + 'write', + ); + + // Rows matched, not files read. A slug derived differently here than by the + // indexer updates nothing, and this count is the only signal that happened. + return results.reduce((total, result) => total + result.rowsAffected, 0); +} diff --git a/src/components/DocsSidebar.astro b/src/components/DocsSidebar.astro index 96ca233..4475ee8 100644 --- a/src/components/DocsSidebar.astro +++ b/src/components/DocsSidebar.astro @@ -4,42 +4,26 @@ * Collapsible navigation populated from Turso database */ -import { getAllArticles } from '@logan/libsql-search'; +import { getNavArticles } from '@/lib/articles'; +import { groupArticlesByFolder } from '@/lib/nav'; import { SEARCH_TABLE_NAME } from '@/lib/searchConfig'; import { getTursoClient } from '@/lib/turso'; import { formatFolderName } from '@/lib/utils'; -import type { ArticleSummary } from '@/types/article'; +import type { ArticleNavSummary } from '@/types/article'; interface Props { - articles?: ArticleSummary[]; + articles?: ArticleNavSummary[]; } // Get current path for active state const currentPath = Astro.url.pathname; // Use provided articles or fetch from database -let allArticles: ArticleSummary[]; -if (Astro.props.articles) { - allArticles = Astro.props.articles; -} else { - const client = getTursoClient(); - allArticles = (await getAllArticles( - client, - SEARCH_TABLE_NAME, - )) as ArticleSummary[]; -} +const allArticles: ArticleNavSummary[] = + Astro.props.articles ?? + (await getNavArticles(getTursoClient(), SEARCH_TABLE_NAME)); -const articlesByFolder = allArticles.reduce( - (acc, article) => { - const folder = article.folder || 'root'; - if (!acc[folder]) { - acc[folder] = []; - } - acc[folder].push(article); - return acc; - }, - {} as Record, -); +const folderGroups = groupArticlesByFolder(allArticles); ---