diff --git a/README.md b/README.md index 5c15c43..74aa811 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ export default async function Dog({ params }) { ## `withHarper()` -`withHarper(config: NextConfig, harperConfig?: HarperConfig): NextConfig` +`withHarper(config?: NextConfig): NextConfig` A configuration helper that wraps your Next.js config. It automatically adds `harper` and `harper-pro` to `serverExternalPackages` so Harper's native dependencies are treated correctly by the bundler. @@ -118,19 +118,6 @@ module.exports = withHarper({ }); ``` -### `experimentalHarperCache: boolean` - -Enables the built-in Harper [cache handler](#caching-work-in-progress). Defaults to `false`. - -```js -export default withHarper( - { - /* Next.js config */ - }, - { experimentalHarperCache: true } -); -``` - ## Options All plugin options are configured in `config.yaml` under the `@harperfast/nextjs` key. All options are optional. @@ -185,21 +172,100 @@ Glob pattern specifying which files Harper should watch for changes. Example: `' ## Caching (Work In Progress) -> This custom caching handler is currently a WIP and is actively being developed. +`@harperfast/nextjs` includes a Harper-backed cache handler for Next.js [Incremental Static Regeneration (ISR)](https://nextjs.org/docs/app/guides/incremental-static-regeneration), the [Data Cache (`fetch()`)](https://nextjs.org/docs/app/deep-dive/caching#data-cache), and [`unstable_cache`](https://nextjs.org/docs/app/api-reference/functions/unstable_cache). Cached entries live in Harper instead of the worker's local filesystem, so a cache write on one node is visible to every node in the cluster. + +### Enabling + +Set the `cacheHandler` path using the `cacheHandlerPath()` helper. This helper resolves the cache handler relative to your config file, which is required by Turbopack: -`@harperfast/nextjs` includes a built-in cache handler for Next.js [Incremental Static Regeneration (ISR)](https://nextjs.org/docs/app/guides/incremental-static-regeneration). Instead of storing cached pages on the file system, cached data is stored in Harper's database, making it available across all nodes in your Harper cluster. +```js +// next.config.js (CommonJS) +const { withHarper, cacheHandlerPath } = require('@harperfast/nextjs'); -Enable it via the `experimentalHarperCache` option in [`withHarper()`](#withharper): +module.exports = withHarper({ + cacheHandler: cacheHandlerPath(__dirname), +}); +``` ```js -export default withHarper( - { - /* Next.js config */ +// next.config.mjs (ESM) +import { withHarper, cacheHandlerPath } from '@harperfast/nextjs'; + +export default withHarper({ + cacheHandler: cacheHandlerPath(import.meta.dirname), +}); +``` + +```ts +// next.config.ts (TypeScript) +import { withHarper, cacheHandlerPath } from '@harperfast/nextjs'; + +export default withHarper({ + cacheHandler: cacheHandlerPath(import.meta.dirname), +}); +``` + +### Tag invalidation + +[`revalidateTag()`](https://nextjs.org/docs/app/api-reference/functions/revalidateTag) is supported and propagates across the cluster automatically. A typical flow: + +```js +// app/products/[id]/page.js +import { unstable_cache } from 'next/cache'; + +const getProduct = unstable_cache( + async (id) => { + const res = await fetch(`https://api.example.com/products/${id}`); + return res.json(); }, - { experimentalHarperCache: true } + ['product'], + { tags: ['products'], revalidate: 3600 } ); + +export default async function ProductPage({ params }) { + const product = await getProduct(params.id); + return

{product.name}

; +} ``` +```js +// app/api/revalidate/route.js +import { revalidateTag } from 'next/cache'; +import { NextResponse } from 'next/server'; + +export async function POST(request) { + const tag = new URL(request.url).searchParams.get('tag'); + revalidateTag(tag); + return NextResponse.json({ revalidated: true }); +} +``` + +`fetch()` calls with `next: { tags: [...] }` and the `'use cache'` directive (with `cacheTag()`) are also supported — anywhere Next.js attaches tags to a cached value, the handler will pick them up. + +### How invalidation works + +The cache handler uses a **soft-invalidation** model: + +1. `revalidateTag(tag)` writes a `{ tag, timestamp }` row to the `nextjs_cache_invalidation` table and updates an in-memory map in the calling worker. +2. Every other Harper worker subscribes to that table and updates its own map when the row is replicated — typically within milliseconds. +3. On the next `cache.get()`, if any of the cached entry's tags has an invalidation timestamp newer than the entry's `lastModified`, the handler returns `null` and Next.js regenerates the entry. The new write replaces the row with a fresh `lastModified`, naturally restoring "fresh" status. + +There is no background sweep that hard-deletes invalidated rows; stale rows are overwritten by Next.js the next time the entry is regenerated. The `nextjs_cache_invalidation` rows themselves expire after 7 days so abandoned tags don't accumulate. + +### Schema + +Enabling the cache handler adds two tables to the `harperfast_nextjs` database: + +| Table | Purpose | +| --- | --- | +| `nextjs_isr_cache` | One row per cached entry. Stores `data` (the Next.js `IncrementalCacheValue`), `tags` (the tags attached to the entry), and `lastModified`. | +| `nextjs_cache_invalidation` | One row per invalidated tag. `id` is the tag itself; `timestamp` is when `revalidateTag` was called. Auto-expires after 7 days. | + +### Limitations + +- `revalidatePath()` is not yet implemented. +- Group-based invalidation (revalidate everything in a logical bucket) is not exposed; tag the entries with a shared tag and call `revalidateTag()` instead. + ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/fixtures/next-16-caching/app/api/revalidate/route.js b/fixtures/next-16-caching/app/api/revalidate/route.js new file mode 100644 index 0000000..71fd14a --- /dev/null +++ b/fixtures/next-16-caching/app/api/revalidate/route.js @@ -0,0 +1,11 @@ +import { revalidateTag } from 'next/cache'; +import { NextResponse } from 'next/server'; + +export async function POST(request) { + const tag = new URL(request.url).searchParams.get('tag'); + if (!tag) { + return NextResponse.json({ error: 'tag required' }, { status: 400 }); + } + revalidateTag(tag); + return NextResponse.json({ revalidated: true, tag }); +} diff --git a/fixtures/next-16-caching/app/tagged/page.js b/fixtures/next-16-caching/app/tagged/page.js new file mode 100644 index 0000000..a52b1b4 --- /dev/null +++ b/fixtures/next-16-caching/app/tagged/page.js @@ -0,0 +1,17 @@ +import { unstable_cache } from 'next/cache'; + +const getNonce = unstable_cache( + async () => Math.random().toString(36).slice(2), + ['tagged-nonce'], + { tags: ['test-tag'], revalidate: 3600 } +); + +export default async function TaggedPage() { + const nonce = await getNonce(); + return ( +
+

Tagged Page

+

{nonce}

+
+ ); +} diff --git a/fixtures/next-16-caching/next.config.mjs b/fixtures/next-16-caching/next.config.mjs index 206a478..0f39fc2 100644 --- a/fixtures/next-16-caching/next.config.mjs +++ b/fixtures/next-16-caching/next.config.mjs @@ -1,9 +1,5 @@ -import { join } from 'path'; +import { withHarper, cacheHandlerPath } from '@harperfast/nextjs'; -export default { - // turbopack: { - // root: '../../../../', - // }, - serverExternalPackages: ['harper', '@harperfast/nextjs'], - cacheHandler: join(import.meta.dirname, 'node_modules', '@harperfast', 'nextjs', 'dist', 'CacheHandler.js'), -}; +export default withHarper({ + cacheHandler: cacheHandlerPath(import.meta.dirname), +}); diff --git a/integrationTests/fixture.ts b/integrationTests/fixture.ts index e0b48f6..e1aeef6 100644 --- a/integrationTests/fixture.ts +++ b/integrationTests/fixture.ts @@ -34,8 +34,9 @@ export function makeHarperFixture(fixtureName: string) { }, applications: { lockdown: 'none', - moduleLoader: 'native', + moduleLoader: 'none', dependencyLoader: 'native', + allowedDirectory: 'any' }, }, }); diff --git a/integrationTests/next-16-caching.pw.ts b/integrationTests/next-16-caching.pw.ts new file mode 100644 index 0000000..5b4aa7c --- /dev/null +++ b/integrationTests/next-16-caching.pw.ts @@ -0,0 +1,170 @@ +import { fixture } from './fixture.ts'; + +const { test, expect } = fixture('next-16-caching'); + +test('ISR page serves cached response and revalidates after expiry', async ({ page, harper }) => { + const url = `${harper.httpURL}/isr`; + + // Warm the cache. The very first render after boot may be a MISS or STALE + // depending on whether Next.js prerendered the page at build time. We do an + // initial throw-away request to ensure the page is in the cache before we + // start asserting behaviour. + await page.goto(url); + + // ── First cached request ────────────────────────────────────────────────── + const response1 = await page.goto(url); + const timestamp1 = await page.getByTestId('timestamp').innerText(); + + // Should be a cache HIT (served from the Harper-backed ISR cache). + expect(response1!.headers()['x-nextjs-cache']).toBe('HIT'); + + // ── Second request within revalidation window ───────────────────────────── + const response2 = await page.goto(url); + const timestamp2 = await page.getByTestId('timestamp').innerText(); + + expect(response2!.headers()['x-nextjs-cache']).toBe('HIT'); + // Content must be identical — the cached page has not been regenerated. + expect(timestamp1).toBe(timestamp2); + + // ── Wait for the revalidation window to expire (revalidate = 2s) ────────── + await page.waitForTimeout(2500); + + // ── Stale request ───────────────────────────────────────────────────────── + // Next.js serves the stale cached page while triggering a background regen. + const response3 = await page.goto(url); + const timestamp3 = await page.getByTestId('timestamp').innerText(); + + expect(response3!.headers()['x-nextjs-cache']).toBe('STALE'); + // Still the old content while revalidation is in flight. + expect(timestamp2).toBe(timestamp3); + + // ── Revalidated request ─────────────────────────────────────────────────── + // Background revalidation should have completed; next hit is the fresh page. + const response4 = await page.goto(url); + const timestamp4 = await page.getByTestId('timestamp').innerText(); + + expect(response4!.headers()['x-nextjs-cache']).toBe('HIT'); + // Content must have changed — the page was regenerated with a new timestamp. + expect(timestamp3).not.toBe(timestamp4); +}); + +test('ISR cache record is persisted in Harper', async ({ request, harper }) => { + // Hit the ISR page so Next.js writes to the cache handler. + await request.get(`${harper.httpURL}/isr`); + // Second request ensures the cache is populated (first may be a build-time miss). + await request.get(`${harper.httpURL}/isr`); + + // Query the Harper Operations API to inspect the nextjs_isr_cache table. + // The key Next.js uses for app-router pages is the route path (e.g. "/isr"). + const response = await request.post(harper.operationsAPIURL, { + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Basic ${Buffer.from(`${harper.admin.username}:${harper.admin.password}`).toString('base64')}`, + }, + data: { + operation: 'search_by_value', + database: 'harperfast_nextjs', + table: 'nextjs_isr_cache', + search_attribute: 'id', + search_value: '/isr', + get_attributes: ['id', 'lastModified'], + }, + }); + + expect(response.status()).toBe(200); + + const records = await response.json(); + expect(records).toHaveLength(1); + + const record = records[0]; + expect(record.id).toBe('/isr'); + // lastModified should be a recent Unix timestamp in milliseconds. + expect(typeof record.lastModified).toBe('number'); + expect(record.lastModified).toBeGreaterThan(Date.now() - 60_000); +}); + +test('ISR cache record is updated after revalidation', async ({ request, harper }) => { + const isrURL = `${harper.httpURL}/isr`; + + // Warm the cache. + await request.get(isrURL); + await request.get(isrURL); + + // Capture the initial lastModified timestamp from the DB. + const authHeader = `Basic ${Buffer.from(`${harper.admin.username}:${harper.admin.password}`).toString('base64')}`; + const queryPayload = { + operation: 'search_by_value', + database: 'harperfast_nextjs', + table: 'nextjs_isr_cache', + search_attribute: 'id', + search_value: '/isr', + get_attributes: ['id', 'lastModified'], + }; + + const before = await request.post(harper.operationsAPIURL, { + headers: { 'Content-Type': 'application/json', 'Authorization': authHeader }, + data: queryPayload, + }); + const [beforeRecord] = await before.json(); + const lastModifiedBefore: number = beforeRecord.lastModified; + + // Wait past the revalidation window and trigger a stale response (which + // kicks off background regeneration). + await request.get(isrURL); // ensure we have a fresh HIT first + await new Promise((resolve) => setTimeout(resolve, 2500)); + await request.get(isrURL); // STALE — triggers background regen + // Give Next.js a moment to complete background regeneration and write to the cache. + await new Promise((resolve) => setTimeout(resolve, 500)); + + // Query again. + const after = await request.post(harper.operationsAPIURL, { + headers: { 'Content-Type': 'application/json', 'Authorization': authHeader }, + data: queryPayload, + }); + const [afterRecord] = await after.json(); + const lastModifiedAfter: number = afterRecord.lastModified; + + // The record's lastModified timestamp must have advanced. + expect(lastModifiedAfter).toBeGreaterThan(lastModifiedBefore); +}); + +test('revalidateTag writes invalidation row and forces regeneration', async ({ request, harper, page }) => { + const taggedURL = `${harper.httpURL}/tagged`; + const revalidateURL = `${harper.httpURL}/api/revalidate?tag=test-tag`; + const authHeader = `Basic ${Buffer.from(`${harper.admin.username}:${harper.admin.password}`).toString('base64')}`; + + // Warm the cache. + await page.goto(taggedURL); + const nonceBefore = await page.getByTestId('nonce').innerText(); + + // Sanity: a second hit returns the cached value (same nonce). + await page.goto(taggedURL); + const nonceCached = await page.getByTestId('nonce').innerText(); + expect(nonceCached).toBe(nonceBefore); + + // Trigger revalidateTag('test-tag') via the route handler. + const revalidateResponse = await request.post(revalidateURL); + expect(revalidateResponse.status()).toBe(200); + + // The invalidation row should now exist in Harper. + const invalidationRow = await request.post(harper.operationsAPIURL, { + headers: { 'Content-Type': 'application/json', 'Authorization': authHeader }, + data: { + operation: 'search_by_value', + database: 'harperfast_nextjs', + table: 'nextjs_cache_invalidation', + search_attribute: 'id', + search_value: 'test-tag', + get_attributes: ['id', 'timestamp'], + }, + }); + const rows = await invalidationRow.json(); + expect(rows).toHaveLength(1); + expect(rows[0].id).toBe('test-tag'); + expect(typeof rows[0].timestamp).toBe('number'); + + // Next page request must regenerate (new nonce). + await page.goto(taggedURL); + const nonceAfter = await page.getByTestId('nonce').innerText(); + expect(nonceAfter).not.toBe(nonceBefore); +}); diff --git a/integrationTests/next-16.pw.ts b/integrationTests/next-16.pw.ts index 80ff3a9..d209ec1 100644 --- a/integrationTests/next-16.pw.ts +++ b/integrationTests/next-16.pw.ts @@ -16,132 +16,3 @@ test('status endpoint returns 200', async ({ request, harper }) => { const response = await request.get(`${harper.operationsAPIURL}/health`); expect(response.status()).toBe(200); }); - -// These are meant for `next-16-caching` when we get that all working -test.describe.skip('ISR caching', () => { - test('ISR page serves cached response and revalidates after expiry', async ({ page, harper }) => { - const url = `${harper.httpURL}/isr`; - - // Warm the cache. The very first render after boot may be a MISS or STALE - // depending on whether Next.js prerendered the page at build time. We do an - // initial throw-away request to ensure the page is in the cache before we - // start asserting behaviour. - await page.goto(url); - - // ── First cached request ────────────────────────────────────────────────── - const response1 = await page.goto(url); - const nonce1 = await page.getByTestId('nonce').innerText(); - - // Should be a cache HIT (served from the Harper-backed ISR cache). - expect(response1!.headers()['x-nextjs-cache']).toBe('HIT'); - - // ── Second request within revalidation window ───────────────────────────── - const response2 = await page.goto(url); - const nonce2 = await page.getByTestId('nonce').innerText(); - - expect(response2!.headers()['x-nextjs-cache']).toBe('HIT'); - // Content must be identical — the cached page has not been regenerated. - expect(nonce1).toBe(nonce2); - - // ── Wait for the revalidation window to expire (revalidate = 2s) ────────── - await page.waitForTimeout(2500); - - // ── Stale request ───────────────────────────────────────────────────────── - // Next.js serves the stale cached page while triggering a background regen. - const response3 = await page.goto(url); - const nonce3 = await page.getByTestId('nonce').innerText(); - - expect(response3!.headers()['x-nextjs-cache']).toBe('STALE'); - // Still the old content while revalidation is in flight. - expect(nonce2).toBe(nonce3); - - // ── Revalidated request ─────────────────────────────────────────────────── - // Background revalidation should have completed; next hit is the fresh page. - const response4 = await page.goto(url); - const nonce4 = await page.getByTestId('nonce').innerText(); - - expect(response4!.headers()['x-nextjs-cache']).toBe('HIT'); - // Content must have changed — the page was regenerated with a new nonce. - expect(nonce3).not.toBe(nonce4); - }); - - test('ISR cache record is persisted in Harper', async ({ request, harper }) => { - // Hit the ISR page so Next.js writes to the cache handler. - await request.get(`${harper.httpURL}/isr`); - // Second request ensures the cache is populated (first may be a build-time miss). - await request.get(`${harper.httpURL}/isr`); - - // Query the Harper Operations API to inspect the nextjs_isr_cache table. - // The key Next.js uses for app-router pages is the route path (e.g. "/isr"). - const response = await request.post(harper.operationsAPIURL, { - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Basic ${Buffer.from(`${harper.admin.username}:${harper.admin.password}`).toString('base64')}`, - }, - data: { - operation: 'search_by_value', - database: 'harperfast_nextjs', - table: 'nextjs_isr_cache', - search_attribute: 'id', - search_value: '/isr', - get_attributes: ['id', 'lastModified'], - }, - }); - - expect(response.status()).toBe(200); - - const records = await response.json(); - expect(records).toHaveLength(1); - - const record = records[0]; - expect(record.id).toBe('/isr'); - // lastModified should be a recent Unix nonce in milliseconds. - expect(typeof record.lastModified).toBe('number'); - expect(record.lastModified).toBeGreaterThan(Date.now() - 60_000); - }); - - test('ISR cache record is updated after revalidation', async ({ request, harper }) => { - const isrURL = `${harper.httpURL}/isr`; - - // Warm the cache. - await request.get(isrURL); - await request.get(isrURL); - - // Capture the initial lastModified nonce from the DB. - const authHeader = `Basic ${Buffer.from(`${harper.admin.username}:${harper.admin.password}`).toString('base64')}`; - const queryPayload = { - operation: 'search_by_value', - database: 'harperfast_nextjs', - table: 'nextjs_isr_cache', - search_attribute: 'id', - search_value: '/isr', - get_attributes: ['id', 'lastModified'], - }; - - const before = await request.post(harper.operationsAPIURL, { - headers: { 'Content-Type': 'application/json', 'Authorization': authHeader }, - data: queryPayload, - }); - const [beforeRecord] = await before.json(); - const lastModifiedBefore: number = beforeRecord.lastModified; - - // Wait past the revalidation window and trigger a stale response (which - // kicks off background regeneration). - await request.get(isrURL); // ensure we have a fresh HIT first - await new Promise((resolve) => setTimeout(resolve, 2500)); - await request.get(isrURL); // STALE — triggers background regen - // Give Next.js a moment to complete background regeneration and write to the cache. - await new Promise((resolve) => setTimeout(resolve, 500)); - - // Query again. - const after = await request.post(harper.operationsAPIURL, { - headers: { 'Content-Type': 'application/json', 'Authorization': authHeader }, - data: queryPayload, - }); - const [afterRecord] = await after.json(); - const lastModifiedAfter: number = afterRecord.lastModified; - - // The record's lastModified nonce must have advanced. - expect(lastModifiedAfter).toBeGreaterThan(lastModifiedBefore); - }); -}); diff --git a/package-lock.json b/package-lock.json index ad106db..97bef66 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,15 @@ { "name": "@harperfast/nextjs", - "version": "2.1.2", + "version": "2.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@harperfast/nextjs", - "version": "2.1.2", + "version": "2.1.1", "license": "Apache-2.0", "devDependencies": { - "@harperfast/integration-testing": "0.3.0", + "@harperfast/integration-testing": "0.3.1", "@playwright/test": "1.59.1", "@types/node": "^20", "harper": "5.0.9", @@ -1244,6 +1244,7 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1565,9 +1566,9 @@ "license": "Apache-2.0" }, "node_modules/@harperfast/integration-testing": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@harperfast/integration-testing/-/integration-testing-0.3.0.tgz", - "integrity": "sha512-q8R6k+aYtYQ7iyVuiWFJ9uB2f1OPEh4hXd07VTv12LxsmUY3XFXGuiLh2buDi36SAB4Y5++IZcF7lZQ/CIDbvA==", + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@harperfast/integration-testing/-/integration-testing-0.3.1.tgz", + "integrity": "sha512-hW7XsSTRWv38pK0nY4GZhGmmWAeQg/2eSSHAdwOO+niL7QORLExGjKCYxySylpKbWRdORQ7JjG5RMkFH+LQc9g==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -1777,6 +1778,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -1799,6 +1801,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -1821,6 +1824,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1837,6 +1841,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1853,6 +1858,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1869,6 +1875,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1885,6 +1892,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1901,6 +1909,7 @@ "cpu": [ "riscv64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1917,6 +1926,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1933,6 +1943,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1949,6 +1960,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1965,6 +1977,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1981,6 +1994,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2003,6 +2017,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2025,6 +2040,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2047,6 +2063,7 @@ "cpu": [ "riscv64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2069,6 +2086,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2091,6 +2109,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2113,6 +2132,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2135,6 +2155,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2157,6 +2178,7 @@ "cpu": [ "wasm32" ], + "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { @@ -2176,6 +2198,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -2195,6 +2218,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -2214,6 +2238,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -5469,6 +5494,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, diff --git a/package.json b/package.json index ba25648..1831ce3 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ } }, "devDependencies": { - "@harperfast/integration-testing": "0.3.0", + "@harperfast/integration-testing": "0.3.1", "@playwright/test": "1.59.1", "@types/node": "^20", "harper": "5.0.9", diff --git a/schema.graphql b/schema.graphql index b50aafb..19d1a29 100644 --- a/schema.graphql +++ b/schema.graphql @@ -6,6 +6,12 @@ type NextBuildInfo @table(database: "harperfast_nextjs", table: "nextjs_build_in type NextISRCache @table(database: "harperfast_nextjs", table: "nextjs_isr_cache") { id: String @primaryKey - data: String + data: Any + tags: [String] lastModified: Long @updatedTime } + +type NextCacheInvalidation @table(database: "harperfast_nextjs", table: "nextjs_cache_invalidation", expiration: 604800) { + id: String @primaryKey + timestamp: Long +} diff --git a/src/CacheHandler.cts b/src/CacheHandler.cts index ab9565d..e010f92 100644 --- a/src/CacheHandler.cts +++ b/src/CacheHandler.cts @@ -1,4 +1,8 @@ -import type { CacheHandler, CacheHandlerValue } from 'next/dist/server/lib/incremental-cache/index.d.ts'; +import type { + CacheHandler, + CacheHandlerContext, + CacheHandlerValue, +} from 'next/dist/server/lib/incremental-cache/index.d.ts'; import type { IncrementalCacheValue, @@ -8,27 +12,119 @@ import type { SetIncrementalResponseCacheContext, } from 'next/dist/server/response-cache/index.d.ts'; -// Type-only import: erased at compile time so `harper` is never resolved when this -// module is loaded from a non-Harper context (e.g. Next.js's build worker, which -// requires the cacheHandler path directly via Node's `require`, bypassing webpack -// externals and serverExternalPackages). import type { databases as DatabasesType } from 'harper'; -// `databases` is a Harper-provided global registered when the plugin is loaded -// in-process by Harper. Resolve it lazily so loading this file in a build worker -// does not pull in the harper runtime — which would otherwise try to register -// native worker hooks a second time and crash with "Worker creator already -// registered", restarting the HTTP worker until Harper gives up. +const NEXT_CACHE_TAGS_HEADER = 'x-next-cache-tags'; + +// Map of tag → invalidation timestamp (ms). Hydrated from the +// nextjs_cache_invalidation table on first construction and kept fresh via a +// Harper subscription so any worker observes invalidations from any other. +const cacheInvalidations = new Map(); + +let subscriptionInitialized = false; + +// `databases` is a Harper-provided global. Access it lazily so that loading +// this module from a non-Harper context (e.g. a turbopack build worker that +// resolves the cacheHandler path) does not pull in the harper runtime — which +// would register native worker hooks a second time and crash with +// "Worker creator already registered". function getDatabases(): typeof DatabasesType | undefined { return (globalThis as { databases?: typeof DatabasesType }).databases; } +async function initializeSubscription(): Promise { + if (subscriptionInitialized) return; + const databases = getDatabases(); + if (!databases) return; + subscriptionInitialized = true; + + // Harper's TypeScript types require RequestTarget/SubscriptionRequest objects, + // but the runtime accepts plain object literals (and search() accepts no args). + const table = databases.harperfast_nextjs.nextjs_cache_invalidation as unknown as { + search: () => AsyncIterable<{ id: string; timestamp: number }>; + subscribe: (req: { omitCurrent?: boolean }) => Promise<{ + on: (event: string, listener: (e: { type: string; id: string; value?: { timestamp: number } }) => void) => void; + }>; + }; + + try { + for await (const row of table.search()) { + cacheInvalidations.set(row.id, row.timestamp); + } + + const subscription = await table.subscribe({ omitCurrent: true }); + + subscription.on('data', (event) => { + if (!event.id) return; + if (event.type === 'delete') { + cacheInvalidations.delete(event.id); + } else if (event.type === 'put' && event.value) { + cacheInvalidations.set(event.id, event.value.timestamp); + } + }); + + subscription.on('error', (error) => { + console.error('[CacheHandler] invalidation subscription error', error); + }); + } catch (error) { + // Reset so a future construction can retry — failure here means we lose + // cross-worker visibility, but the cache still works (just falls back to + // per-request revalidatedTags). + subscriptionInitialized = false; + console.error('[CacheHandler] failed to initialize invalidation subscription', error); + } +} + +function extractTags(data: IncrementalCacheValue | null, ctx: SetIncrementalFetchCacheContext | SetIncrementalResponseCacheContext): string[] { + if (!data) return []; + + // FETCH entries carry tags via ctx.tags (set context) and data.tags. + if ('fetchCache' in ctx && ctx.fetchCache && 'tags' in ctx && ctx.tags) { + return ctx.tags; + } + + // APP_PAGE / APP_ROUTE / PAGES carry tags via the NEXT_CACHE_TAGS_HEADER + // header that Next.js writes into the cached value. + const headers = (data as { headers?: Record }).headers; + const tagsHeader = headers?.[NEXT_CACHE_TAGS_HEADER]; + if (typeof tagsHeader === 'string' && tagsHeader.length > 0) { + return tagsHeader.split(',').map((t) => t.trim()).filter(Boolean); + } + + const dataTags = (data as { tags?: unknown }).tags; + if (Array.isArray(dataTags)) { + return dataTags.filter((t): t is string => typeof t === 'string'); + } + + return []; +} + +function isInvalidated( + recordTags: string[], + lastModified: number, + revalidatedTags: string[], + ctxTags: string[] +): boolean { + const allTags = recordTags.length > 0 ? recordTags : ctxTags; + for (const tag of allTags) { + if (revalidatedTags.includes(tag)) return true; + const invalidatedAt = cacheInvalidations.get(tag); + if (invalidatedAt !== undefined && invalidatedAt > lastModified) return true; + } + return false; +} + export default class HarperCacheHandler implements CacheHandler { - constructor() {} + private revalidatedTags: string[]; + + constructor(ctx?: CacheHandlerContext) { + this.revalidatedTags = ctx?.revalidatedTags ?? []; + void initializeSubscription(); + } async get( key: string, - _ctx: GetIncrementalFetchCacheContext | GetIncrementalResponseCacheContext + ctx: GetIncrementalFetchCacheContext | GetIncrementalResponseCacheContext ): Promise { const databases = getDatabases(); if (!databases) return null; @@ -37,32 +133,53 @@ export default class HarperCacheHandler implements CacheHandler { const record = await table.get(key); if (!record) return null; - try { - return { - value: record.data, - lastModified: record.lastModified, - }; - } catch { + const recordTags = Array.isArray(record.tags) ? (record.tags as string[]) : []; + + const ctxTags = + 'tags' in ctx && Array.isArray(ctx.tags) + ? [...ctx.tags, ...(('softTags' in ctx && Array.isArray(ctx.softTags)) ? ctx.softTags : [])] + : []; + + if (isInvalidated(recordTags, record.lastModified ?? 0, this.revalidatedTags, ctxTags)) { return null; } + + return { + value: record.data as IncrementalCacheValue | null, + lastModified: record.lastModified, + }; } async set( key: string, data: IncrementalCacheValue | null, - _ctx: SetIncrementalFetchCacheContext | SetIncrementalResponseCacheContext + ctx: SetIncrementalFetchCacheContext | SetIncrementalResponseCacheContext ): Promise { const databases = getDatabases(); if (!databases) return; const table = databases.harperfast_nextjs.nextjs_isr_cache; - await table.put(key, { - data, - }); + const tags = extractTags(data, ctx); + await table.put(key, { data, tags }); } - async revalidateTag(_tag: string | string[]): Promise { - // TODO: implement tag-based invalidation + async revalidateTag(tags: string | string[]): Promise { + const tagList = typeof tags === 'string' ? [tags] : tags; + if (tagList.length === 0) return; + + const databases = getDatabases(); + if (!databases) return; + + const table = databases.harperfast_nextjs.nextjs_cache_invalidation; + const timestamp = Date.now(); + + // Update the local map immediately so reads on this worker see the + // invalidation without waiting for the subscription roundtrip. + for (const tag of tagList) { + cacheInvalidations.set(tag, timestamp); + } + + await Promise.all(tagList.map((tag) => table.put(tag, { timestamp }))); } resetRequestCache(): void {} diff --git a/src/withHarper.cts b/src/withHarper.cts index 9aa90fa..7ede026 100644 --- a/src/withHarper.cts +++ b/src/withHarper.cts @@ -1,19 +1,24 @@ import { join } from 'node:path'; import type { NextConfig } from 'next'; -export interface HarperConfig { - experimentalHarperCache?: boolean; +/** + * Returns the path to the Harper cache handler module, resolved relative to the + * caller's directory. Pass `import.meta.dirname` (ESM) or `__dirname` (CJS). + * + * This avoids `require.resolve`, which dereferences symlinks and produces paths + * outside Turbopack's filesystem root when the package is linked. + */ +export function cacheHandlerPath(configDir: string): string { + return join(configDir, 'node_modules', '@harperfast', 'nextjs', 'dist', 'CacheHandler.cjs'); } -export function withHarper(config: NextConfig, harperConfig: HarperConfig = {}): NextConfig { - const { experimentalHarperCache = false } = harperConfig; - +export function withHarper(config: NextConfig = {}): NextConfig { // TODO: Do things like `serverExternalPackage` work with Next.js v14? If not, how can we // detect version reliably and apply? What if we added properties specific to v14? Would // they be okay with v15 and v16 or do this all need to be guarded? // Potential solution: To avoid version detection (if thats complicated), add a `version` // option or provide separate exports for each unique Next.js major. Something like: - // `withHarperNext14()` or `withHarper({}, {}, 14)` + // `withHarperNext14()` or `withHarper({}, 14)` // TODO: We should inspect the Next.js config for properties such as `turbo` and then apply // specific options when present. I think things like `serverExternalPackages` used to be @@ -36,6 +41,5 @@ export function withHarper(config: NextConfig, harperConfig: HarperConfig = {}): ...config.turbopack, }, serverExternalPackages: [...(config.serverExternalPackages ?? []), 'harperdb', 'harper', 'harper-pro'], - ...(experimentalHarperCache && { cacheHandler: join(__dirname, 'CacheHandler.cjs') }), }; }