perf: let metadata and section-pages start on their dependency's first chunk

.changeset/stream-metadata.md

@@ -0,0 +1,6 @@
+---
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+---
+
+`metadata` and `section-pages` stream their input: with more than one thread, they start on their dependency's first chunk, so parsing, metadata and section pages overlap. The generated files are unchanged. `toBatches`, from `@doc-kit/core/utils/generators.mjs`, gives an input that is whole or streamed as batches.

docs/creating-generators.md

@@ -357,8 +357,12 @@ export default {
 ```
 
 - A dependency that does not stream arrives as a single chunk.
-- With one thread, `input` is still the collected array, so accept both.
+- With one thread, `input` is still the collected array, so accept both:
+  `toBatches(input)`, from `@doc-kit/core/utils/generators.mjs`, gives either
+  as batches.
 - `input` can be iterated only once.
+- `metadata`'s chunks hold whole files, and a file is one module, so a chunk
+  holds all the entries of each module in it.
 
 ### Non-Streaming Generators
 

packages/core/src/generators/metadata/index.mjs

@@ -16,6 +16,8 @@ export default {
 
   hasParallelProcessor: true,
 
+  streamsInput: true,
+
   markdown: {
     remarkPlugins: [
       'remark-parse',

packages/core/src/generators/metadata/types.d.ts

@@ -3,7 +3,10 @@ import type { Position } from 'unist';
 
 export type Generator = GeneratorMetadata<
   {},
-  Generate<Array<Root>, AsyncGenerator<MetadataEntry>>,
+  Generate<
+    Array<Root> | AsyncIterable<Array<Root>>,
+    AsyncGenerator<MetadataEntry>
+  >,
   ProcessChunk<Root, MetadataEntry, Record<string, string>>
 >;
 

packages/core/src/generators/types.d.ts

@@ -73,7 +73,8 @@ declare global {
     /**
      * Receive the dependency's output as an async iterable of its chunks, as
      * they are produced, instead of one collected array. With one thread, the
-     * input is still a collected array, so accept both.
+     * input is still a collected array, so accept both: `toBatches` from
+     * `@doc-kit/core/utils/generators.mjs` gives either as batches.
      */
     streamsInput?: boolean;
 

packages/core/src/utils/__tests__/generators.test.mjs

@@ -7,6 +7,7 @@ import {
   getVersionFromSemVer,
   coerceSemVer,
   getCompatibleVersions,
+  toBatches,
 } from '../generators.mjs';
 
 describe('getEntryDescription', () => {
@@ -138,3 +139,12 @@ describe('getCompatibleVersions', () => {
     assert.equal(result.length, 2);
   });
 });
+
+describe('toBatches', () => {
+  it('gives an array as a single batch, and batches as they are', () => {
+    const streamed = (async function* () {})();
+
+    assert.deepEqual(toBatches([1, 2]), [[1, 2]]);
+    assert.equal(toBatches(streamed), streamed);
+  });
+});

packages/core/src/utils/generators.mjs

@@ -2,6 +2,7 @@
 
 import { coerce, major } from 'semver';
 
+import { isAsyncIterable } from './misc.mjs';
 import { transformNodeToString } from './unist.mjs';
 
 /**
@@ -42,6 +43,22 @@ export const groupNodesByModule = nodes => {
   return Map.groupBy(nodes, node => node.api);
 };
 
+/**
+ * Gives a generator's input as batches: as they arrive if it streams, or as a
+ * single batch if it arrives whole, as it does with one thread.
+ *
+ * @template T
+ * @param {Array<T> | AsyncIterable<Array<T>>} input
+ * @returns {Array<Array<T>> | AsyncIterable<Array<T>>}
+ */
+export const toBatches = input => {
+  if (isAsyncIterable(input)) {
+    return input;
+  }
+
+  return [input];
+};
+
 /**
  * Parses the SemVer string into a Node.js-alike version
  *

packages/react/src/html/__tests__/generate.test.mjs

@@ -5,14 +5,15 @@ import { join } from 'node:path';
 import { describe, it } from 'node:test';
 import { pathToFileURL } from 'node:url';
 
+import { collectAsyncGenerator } from '@doc-kit/core/caching.mjs';
 import { loadGenerator } from '@doc-kit/core/generators/loader.mjs';
 import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs';
 import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs';
 import { jsx, toJs } from 'estree-util-to-js';
 
 import buildContent from '../../jsx-ast/utils/buildContent.mjs';
 import { buildNotFoundPage } from '../../jsx-ast/utils/synthetic/404.mjs';
-import { generate as chunk } from '../../section-pages/generate.mjs';
+import { generate as sectionPages } from '../../section-pages/generate.mjs';
 import { compile, createViteBundler } from '../bundlers/vite.mjs';
 import { FONTS } from '../constants.mjs';
 import { generate } from '../generate.mjs';
@@ -196,7 +197,10 @@ describe('web generate', () => {
       ],
     });
 
-    const pages = Map.groupBy(await chunk(entries), entry => entry.api);
+    const pages = Map.groupBy(
+      await collectAsyncGenerator(sectionPages(entries)),
+      entry => entry.api
+    );
     const contents = await Promise.all(
       [...pages.values()].map(group => buildContent(group, group[0]))
     );

packages/react/src/section-pages/__tests__/generate.test.mjs

@@ -1,12 +1,31 @@
 import assert from 'node:assert/strict';
 import { before, describe, it } from 'node:test';
 
+import { collectAsyncGenerator } from '@doc-kit/core/caching.mjs';
 import getConfig, {
   setConfig,
 } from '@doc-kit/core/utils/configuration/index.mjs';
 
 import { generate } from '../generate.mjs';
 
+/**
+ * Runs the generator over the given batches of entries, streamed in as
+ * `metadata` yields them, and collects what it yields.
+ *
+ * @param {...Array<object>} batches
+ * @returns {Promise<Array<object>>}
+ */
+const run = (...batches) => collectAsyncGenerator(generate(streamOf(batches)));
+
+/**
+ * Yields each batch in turn, as a streamed input.
+ *
+ * @param {Array<Array<object>>} batches
+ */
+async function* streamOf(batches) {
+  yield* batches;
+}
+
 /**
  * Builds a metadata-shaped entry whose heading node is shared between
  * `heading` and `content`, exactly like the metadata generator produces.
@@ -72,7 +91,7 @@ describe('section-pages generate', () => {
 
   it('keeps every original entry and appends the chunk pages', async () => {
     const input = createModule();
-    const output = await generate(input);
+    const output = await run(input);
 
     assert.deepEqual(output.slice(0, input.length), input);
 
@@ -97,7 +116,7 @@ describe('section-pages generate', () => {
   });
 
   it('describes where each chunk came from', async () => {
-    const output = await generate(createModule());
+    const output = await run(createModule());
     const readFile = output.find(e => e.path === '/fs/readFile');
 
     assert.deepEqual(readFile.chunk, {
@@ -111,7 +130,7 @@ describe('section-pages generate', () => {
   });
 
   it('promotes headings so the chunk reads as its own page', async () => {
-    const output = await generate(createModule());
+    const output = await run(createModule());
 
     const dir = output.filter(e => e.api === 'fs-Dir');
     assert.deepEqual(
@@ -134,13 +153,13 @@ describe('section-pages generate', () => {
     const input = createModule();
     const snapshot = structuredClone(input);
 
-    await generate(input);
+    await run(input);
 
     assert.deepEqual(input, snapshot);
   });
 
   it('leaves modules with a single section alone', async () => {
-    const output = await generate([
+    const output = await run([
       createEntry('tiny', 1, 'Tiny'),
       createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }),
     ]);
@@ -149,7 +168,7 @@ describe('section-pages generate', () => {
   });
 
   it('leaves modules that document no API entry alone', async () => {
-    const output = await generate([
+    const output = await run([
       createEntry('guide', 1, 'About this documentation'),
       createEntry('guide', 2, 'Contributing'),
       createEntry('guide', 2, 'Stability index'),
@@ -172,7 +191,7 @@ describe('section-pages generate', () => {
       link('https://example.com')
     );
 
-    const output = await generate(input);
+    const output = await run(input);
     const chunk = output.find(e => e.path === '/fs/readFile');
 
     assert.deepEqual(
@@ -206,7 +225,7 @@ describe('section-pages generate', () => {
       },
     });
 
-    const output = await generate(input);
+    const output = await run(input);
     const chunk = output.find(e => e.path === '/fs/readFile');
 
     assert.deepEqual(
@@ -225,7 +244,9 @@ describe('section-pages generate', () => {
     getConfig('section-pages').exclude = ['fs'];
 
     try {
-      assert.equal((await generate(createModule())).length, 6);
+      const output = await run(createModule());
+
+      assert.equal(output.length, 6);
     } finally {
       getConfig('section-pages').exclude = [];
     }
@@ -235,7 +256,7 @@ describe('section-pages generate', () => {
     getConfig('section-pages').maxDepth = 2;
 
     try {
-      const output = await generate(createModule());
+      const output = await run(createModule());
 
       assert.deepEqual(
         output.filter(e => e.chunk && e.heading.depth === 1).map(e => e.path),
@@ -245,4 +266,55 @@ describe('section-pages generate', () => {
       getConfig('section-pages').maxDepth = 3;
     }
   });
+
+  it('builds the same output from several batches as from its whole input', async () => {
+    const tiny = [
+      createEntry('tiny', 1, 'Tiny'),
+      createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }),
+    ];
+
+    const batched = await run(createModule(), tiny);
+
+    // The whole input, as it comes with one thread
+    const whole = await collectAsyncGenerator(
+      generate([...createModule(), ...tiny])
+    );
+
+    assert.deepEqual(batched, whole);
+  });
+
+  it(
+    'passes each batch on before the next one arrives',
+    // The next batch waits for the test, so a batch held back until it
+    // arrives would never be passed on
+    { timeout: 10_000 },
+    async () => {
+      const gate = Promise.withResolvers();
+      const first = [...createModule(), createEntry('tiny', 1, 'Tiny')];
+
+      async function* batches() {
+        yield first;
+        await gate.promise;
+        yield [createEntry('zlib', 1, 'Zlib')];
+      }
+
+      const output = generate(batches());
+      const { value } = await output.next();
+
+      assert.deepEqual(value, first);
+
+      gate.resolve();
+
+      const rest = await collectAsyncGenerator(output);
+
+      assert.deepEqual(
+        rest.filter(e => !e.chunk).map(e => e.api),
+        ['zlib']
+      );
+      assert.equal(
+        rest.filter(e => e.chunk && e.heading.depth === 1).length,
+        3
+      );
+    }
+  );
 });

packages/react/src/section-pages/generate.mjs

@@ -1,7 +1,10 @@
 'use strict';
 
 import getConfig from '@doc-kit/core/utils/configuration/index.mjs';
-import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs';
+import {
+  groupNodesByModule,
+  toBatches,
+} from '@doc-kit/core/utils/generators.mjs';
 import { visit } from 'unist-util-visit';
 
 import { headingLabel } from '../jsx-ast/utils/buildBarProps.mjs';
@@ -100,14 +103,14 @@ const buildChunkEntries = (module, chunk, index, anchors) => {
 };
 
 /**
- * Adds per-section chunk pages for every module, next to the full pages.
+ * Builds the chunk pages of every module among the given entries.
  *
- * @type {import('./types').Generator['generate']}
+ * @param {Array<import('@doc-kit/core/generators/metadata/types').MetadataEntry>} input - Whole modules' entries
+ * @param {import('./types').Configuration} config
+ * @returns {Array<import('@doc-kit/core/generators/metadata/types').MetadataEntry>}
  */
-export async function generate(input) {
-  const { maxDepth, exclude } = getConfig('section-pages');
-
-  const output = [...input];
+const buildChunkPages = (input, { maxDepth, exclude }) => {
+  const pages = [];
 
   for (const entries of groupNodesByModule(input).values()) {
     const module = entries.find(entry => entry.heading.depth === 1);
@@ -135,9 +138,37 @@ export async function generate(input) {
     const anchors = buildAnchorMap(chunks);
 
     chunks.forEach((chunk, index) => {
-      output.push(...buildChunkEntries(module, chunk, index, anchors));
+      pages.push(...buildChunkEntries(module, chunk, index, anchors));
     });
   }
 
-  return output;
+  return pages;
+};
+
+/**
+ * Adds per-section chunk pages for every module, next to the full pages.
+ *
+ * The entries are passed on as their modules arrive, so the pages built from
+ * them can start while `metadata` still runs, and the chunk pages follow them
+ * all, in module order, as if built from the whole input at once.
+ *
+ * @type {import('./types').Generator['generate']}
+ */
+export async function* generate(input) {
+  const config = getConfig('section-pages');
+
+  const chunkPages = [];
+
+  // A module's chunk pages need all of its entries, which arrive in the same
+  // batch: each batch `metadata` yields holds whole files, and a file is one
+  // module
+  for await (const entries of toBatches(input)) {
+    yield entries;
+
+    chunkPages.push(...buildChunkPages(entries, config));
+  }
+
+  if (chunkPages.length > 0) {
+    yield chunkPages;
+  }
 }

packages/react/src/section-pages/index.mjs

@@ -29,6 +29,8 @@ export default {
 
   dependsOn: '@doc-kit/core/metadata',
 
+  streamsInput: true,
+
   dependent: [
     '@doc-kit/generator-react/html',
     '@doc-kit/generator-react/sitemap',