feat: implement cache tag invalidation

README.md

@@ -103,7 +103,7 @@ export default async function Dog({ params }) {
 
 ## `withHarper()`
 
-`withHarper(config: NextConfig, harperConfig?: HarperConfig): NextConfig`
+`withHarper(config?: NextConfig): NextConfig`
 
 A configuration helper that wraps your Next.js config. It automatically adds `harper` and `harper-pro` to `serverExternalPackages` so Harper's native dependencies are treated correctly by the bundler.
 
@@ -118,19 +118,6 @@ module.exports = withHarper({
 });
 ```
 
-### `experimentalHarperCache: boolean`
-
-Enables the built-in Harper [cache handler](#caching-work-in-progress). Defaults to `false`.
-
-```js
-export default withHarper(
-	{
-		/* Next.js config */
-	},
-	{ experimentalHarperCache: true }
-);
-```
-
 ## Options
 
 All plugin options are configured in `config.yaml` under the `@harperfast/nextjs` key. All options are optional.
@@ -185,21 +172,100 @@ Glob pattern specifying which files Harper should watch for changes. Example: `'
 
 ## Caching (Work In Progress)
 
-> This custom caching handler is currently a WIP and is actively being developed.
+`@harperfast/nextjs` includes a Harper-backed cache handler for Next.js [Incremental Static Regeneration (ISR)](https://nextjs.org/docs/app/guides/incremental-static-regeneration), the [Data Cache (`fetch()`)](https://nextjs.org/docs/app/deep-dive/caching#data-cache), and [`unstable_cache`](https://nextjs.org/docs/app/api-reference/functions/unstable_cache). Cached entries live in Harper instead of the worker's local filesystem, so a cache write on one node is visible to every node in the cluster.
+
+### Enabling
+
+Set the `cacheHandler` path using the `cacheHandlerPath()` helper. This helper resolves the cache handler relative to your config file, which is required by Turbopack:
 
-`@harperfast/nextjs` includes a built-in cache handler for Next.js [Incremental Static Regeneration (ISR)](https://nextjs.org/docs/app/guides/incremental-static-regeneration). Instead of storing cached pages on the file system, cached data is stored in Harper's database, making it available across all nodes in your Harper cluster.
+```js
+// next.config.js (CommonJS)
+const { withHarper, cacheHandlerPath } = require('@harperfast/nextjs');
 
-Enable it via the `experimentalHarperCache` option in [`withHarper()`](#withharper):
+module.exports = withHarper({
+	cacheHandler: cacheHandlerPath(__dirname),
+});
+```
 
 ```js
-export default withHarper(
-	{
-		/* Next.js config */
+// next.config.mjs (ESM)
+import { withHarper, cacheHandlerPath } from '@harperfast/nextjs';
+
+export default withHarper({
+	cacheHandler: cacheHandlerPath(import.meta.dirname),
+});
+```
+
+```ts
+// next.config.ts (TypeScript)
+import { withHarper, cacheHandlerPath } from '@harperfast/nextjs';
+
+export default withHarper({
+	cacheHandler: cacheHandlerPath(import.meta.dirname),
+});
+```
+
+### Tag invalidation
+
+[`revalidateTag()`](https://nextjs.org/docs/app/api-reference/functions/revalidateTag) is supported and propagates across the cluster automatically. A typical flow:
+
+```js
+// app/products/[id]/page.js
+import { unstable_cache } from 'next/cache';
+
+const getProduct = unstable_cache(
+	async (id) => {
+		const res = await fetch(`https://api.example.com/products/${id}`);
+		return res.json();
 	},
-	{ experimentalHarperCache: true }
+	['product'],
+	{ tags: ['products'], revalidate: 3600 }
 );
+
+export default async function ProductPage({ params }) {
+	const product = await getProduct(params.id);
+	return <h1>{product.name}</h1>;
+}
 ```
 
+```js
+// app/api/revalidate/route.js
+import { revalidateTag } from 'next/cache';
+import { NextResponse } from 'next/server';
+
+export async function POST(request) {
+	const tag = new URL(request.url).searchParams.get('tag');
+	revalidateTag(tag);
+	return NextResponse.json({ revalidated: true });
+}
+```
+
+`fetch()` calls with `next: { tags: [...] }` and the `'use cache'` directive (with `cacheTag()`) are also supported — anywhere Next.js attaches tags to a cached value, the handler will pick them up.
+
+### How invalidation works
+
+The cache handler uses a **soft-invalidation** model:
+
+1. `revalidateTag(tag)` writes a `{ tag, timestamp }` row to the `nextjs_cache_invalidation` table and updates an in-memory map in the calling worker.
+2. Every other Harper worker subscribes to that table and updates its own map when the row is replicated — typically within milliseconds.
+3. On the next `cache.get()`, if any of the cached entry's tags has an invalidation timestamp newer than the entry's `lastModified`, the handler returns `null` and Next.js regenerates the entry. The new write replaces the row with a fresh `lastModified`, naturally restoring "fresh" status.
+
+There is no background sweep that hard-deletes invalidated rows; stale rows are overwritten by Next.js the next time the entry is regenerated. The `nextjs_cache_invalidation` rows themselves expire after 7 days so abandoned tags don't accumulate.
+
+### Schema
+
+Enabling the cache handler adds two tables to the `harperfast_nextjs` database:
+
+| Table | Purpose |
+| --- | --- |
+| `nextjs_isr_cache` | One row per cached entry. Stores `data` (the Next.js `IncrementalCacheValue`), `tags` (the tags attached to the entry), and `lastModified`. |
+| `nextjs_cache_invalidation` | One row per invalidated tag. `id` is the tag itself; `timestamp` is when `revalidateTag` was called. Auto-expires after 7 days. |
+
+### Limitations
+
+- `revalidatePath()` is not yet implemented.
+- Group-based invalidation (revalidate everything in a logical bucket) is not exposed; tag the entries with a shared tag and call `revalidateTag()` instead.
+
 ## Contributing
 
 See [CONTRIBUTING.md](CONTRIBUTING.md).

fixtures/next-16-caching/app/api/revalidate/route.js

@@ -0,0 +1,11 @@
+import { revalidateTag } from 'next/cache';
+import { NextResponse } from 'next/server';
+
+export async function POST(request) {
+	const tag = new URL(request.url).searchParams.get('tag');
+	if (!tag) {
+		return NextResponse.json({ error: 'tag required' }, { status: 400 });
+	}
+	revalidateTag(tag);
+	return NextResponse.json({ revalidated: true, tag });
+}

fixtures/next-16-caching/app/tagged/page.js

@@ -0,0 +1,17 @@
+import { unstable_cache } from 'next/cache';
+
+const getNonce = unstable_cache(
+	async () => Math.random().toString(36).slice(2),
+	['tagged-nonce'],
+	{ tags: ['test-tag'], revalidate: 3600 }
+);
+
+export default async function TaggedPage() {
+	const nonce = await getNonce();
+	return (
+		<div>
+			<h1>Tagged Page</h1>
+			<p data-testid="nonce">{nonce}</p>
+		</div>
+	);
+}

fixtures/next-16-caching/next.config.mjs

@@ -1,9 +1,5 @@
-import { join } from 'path';
+import { withHarper, cacheHandlerPath } from '@harperfast/nextjs';
 
-export default {
-	// turbopack: {
-	// 	root: '../../../../',
-	// },
-	serverExternalPackages: ['harper', '@harperfast/nextjs'],
-	cacheHandler: join(import.meta.dirname, 'node_modules', '@harperfast', 'nextjs', 'dist', 'CacheHandler.js'),
-};
+export default withHarper({
+	cacheHandler: cacheHandlerPath(import.meta.dirname),
+});

integrationTests/fixture.ts

@@ -34,8 +34,9 @@ export function makeHarperFixture(fixtureName: string) {
 					},
 					applications: {
 						lockdown: 'none',
-						moduleLoader: 'native',
+						moduleLoader: 'none',
 						dependencyLoader: 'native',
+						allowedDirectory: 'any'
 					},
 				},
 			});

integrationTests/next-16-caching.pw.ts

@@ -0,0 +1,170 @@
+import { fixture } from './fixture.ts';
+
+const { test, expect } = fixture('next-16-caching');
+
+test('ISR page serves cached response and revalidates after expiry', async ({ page, harper }) => {
+	const url = `${harper.httpURL}/isr`;
+
+	// Warm the cache. The very first render after boot may be a MISS or STALE
+	// depending on whether Next.js prerendered the page at build time. We do an
+	// initial throw-away request to ensure the page is in the cache before we
+	// start asserting behaviour.
+	await page.goto(url);
+
+	// ── First cached request ──────────────────────────────────────────────────
+	const response1 = await page.goto(url);
+	const timestamp1 = await page.getByTestId('timestamp').innerText();
+
+	// Should be a cache HIT (served from the Harper-backed ISR cache).
+	expect(response1!.headers()['x-nextjs-cache']).toBe('HIT');
+
+	// ── Second request within revalidation window ─────────────────────────────
+	const response2 = await page.goto(url);
+	const timestamp2 = await page.getByTestId('timestamp').innerText();
+
+	expect(response2!.headers()['x-nextjs-cache']).toBe('HIT');
+	// Content must be identical — the cached page has not been regenerated.
+	expect(timestamp1).toBe(timestamp2);
+
+	// ── Wait for the revalidation window to expire (revalidate = 2s) ──────────
+	await page.waitForTimeout(2500);
+
+	// ── Stale request ─────────────────────────────────────────────────────────
+	// Next.js serves the stale cached page while triggering a background regen.
+	const response3 = await page.goto(url);
+	const timestamp3 = await page.getByTestId('timestamp').innerText();
+
+	expect(response3!.headers()['x-nextjs-cache']).toBe('STALE');
+	// Still the old content while revalidation is in flight.
+	expect(timestamp2).toBe(timestamp3);
+
+	// ── Revalidated request ───────────────────────────────────────────────────
+	// Background revalidation should have completed; next hit is the fresh page.
+	const response4 = await page.goto(url);
+	const timestamp4 = await page.getByTestId('timestamp').innerText();
+
+	expect(response4!.headers()['x-nextjs-cache']).toBe('HIT');
+	// Content must have changed — the page was regenerated with a new timestamp.
+	expect(timestamp3).not.toBe(timestamp4);
+});
+
+test('ISR cache record is persisted in Harper', async ({ request, harper }) => {
+	// Hit the ISR page so Next.js writes to the cache handler.
+	await request.get(`${harper.httpURL}/isr`);
+	// Second request ensures the cache is populated (first may be a build-time miss).
+	await request.get(`${harper.httpURL}/isr`);
+
+	// Query the Harper Operations API to inspect the nextjs_isr_cache table.
+	// The key Next.js uses for app-router pages is the route path (e.g. "/isr").
+	const response = await request.post(harper.operationsAPIURL, {
+		headers: {
+			'Content-Type': 'application/json',
+			'Authorization': `Basic ${Buffer.from(`${harper.admin.username}:${harper.admin.password}`).toString('base64')}`,
+		},
+		data: {
+			operation: 'search_by_value',
+			database: 'harperfast_nextjs',
+			table: 'nextjs_isr_cache',
+			search_attribute: 'id',
+			search_value: '/isr',
+			get_attributes: ['id', 'lastModified'],
+		},
+	});
+
+	expect(response.status()).toBe(200);
+
+	const records = await response.json();
+	expect(records).toHaveLength(1);
+
+	const record = records[0];
+	expect(record.id).toBe('/isr');
+	// lastModified should be a recent Unix timestamp in milliseconds.
+	expect(typeof record.lastModified).toBe('number');
+	expect(record.lastModified).toBeGreaterThan(Date.now() - 60_000);
+});
+
+test('ISR cache record is updated after revalidation', async ({ request, harper }) => {
+	const isrURL = `${harper.httpURL}/isr`;
+
+	// Warm the cache.
+	await request.get(isrURL);
+	await request.get(isrURL);
+
+	// Capture the initial lastModified timestamp from the DB.
+	const authHeader = `Basic ${Buffer.from(`${harper.admin.username}:${harper.admin.password}`).toString('base64')}`;
+	const queryPayload = {
+		operation: 'search_by_value',
+		database: 'harperfast_nextjs',
+		table: 'nextjs_isr_cache',
+		search_attribute: 'id',
+		search_value: '/isr',
+		get_attributes: ['id', 'lastModified'],
+	};
+
+	const before = await request.post(harper.operationsAPIURL, {
+		headers: { 'Content-Type': 'application/json', 'Authorization': authHeader },
+		data: queryPayload,
+	});
+	const [beforeRecord] = await before.json();
+	const lastModifiedBefore: number = beforeRecord.lastModified;
+
+	// Wait past the revalidation window and trigger a stale response (which
+	// kicks off background regeneration).
+	await request.get(isrURL); // ensure we have a fresh HIT first
+	await new Promise((resolve) => setTimeout(resolve, 2500));
+	await request.get(isrURL); // STALE — triggers background regen
+	// Give Next.js a moment to complete background regeneration and write to the cache.
+	await new Promise((resolve) => setTimeout(resolve, 500));
+
+	// Query again.
+	const after = await request.post(harper.operationsAPIURL, {
+		headers: { 'Content-Type': 'application/json', 'Authorization': authHeader },
+		data: queryPayload,
+	});
+	const [afterRecord] = await after.json();
+	const lastModifiedAfter: number = afterRecord.lastModified;
+
+	// The record's lastModified timestamp must have advanced.
+	expect(lastModifiedAfter).toBeGreaterThan(lastModifiedBefore);
+});
+
+test('revalidateTag writes invalidation row and forces regeneration', async ({ request, harper, page }) => {
+	const taggedURL = `${harper.httpURL}/tagged`;
+	const revalidateURL = `${harper.httpURL}/api/revalidate?tag=test-tag`;
+	const authHeader = `Basic ${Buffer.from(`${harper.admin.username}:${harper.admin.password}`).toString('base64')}`;
+
+	// Warm the cache.
+	await page.goto(taggedURL);
+	const nonceBefore = await page.getByTestId('nonce').innerText();
+
+	// Sanity: a second hit returns the cached value (same nonce).
+	await page.goto(taggedURL);
+	const nonceCached = await page.getByTestId('nonce').innerText();
+	expect(nonceCached).toBe(nonceBefore);
+
+	// Trigger revalidateTag('test-tag') via the route handler.
+	const revalidateResponse = await request.post(revalidateURL);
+	expect(revalidateResponse.status()).toBe(200);
+
+	// The invalidation row should now exist in Harper.
+	const invalidationRow = await request.post(harper.operationsAPIURL, {
+		headers: { 'Content-Type': 'application/json', 'Authorization': authHeader },
+		data: {
+			operation: 'search_by_value',
+			database: 'harperfast_nextjs',
+			table: 'nextjs_cache_invalidation',
+			search_attribute: 'id',
+			search_value: 'test-tag',
+			get_attributes: ['id', 'timestamp'],
+		},
+	});
+	const rows = await invalidationRow.json();
+	expect(rows).toHaveLength(1);
+	expect(rows[0].id).toBe('test-tag');
+	expect(typeof rows[0].timestamp).toBe('number');
+
+	// Next page request must regenerate (new nonce).
+	await page.goto(taggedURL);
+	const nonceAfter = await page.getByTestId('nonce').innerText();
+	expect(nonceAfter).not.toBe(nonceBefore);
+});