fix(server): pf-4402 allow custom resource load limits

src/__tests__/server.getResources.test.ts

@@ -12,20 +12,10 @@ import {
   resolveLocalPathFunction
 } from '../server.getResources';
 import { runWithOptions } from '../options.context';
-import { type GlobalOptions } from '../options';
 import { DEFAULT_OPTIONS } from '../options.defaults';
 
 // Mock dependencies
 jest.mock('node:fs/promises');
-jest.mock('../server.caching', () => ({
-  memo: jest.fn(fn => {
-    const memoized = fn;
-
-    memoized.clear = jest.fn();
-
-    return memoized;
-  })
-}));
 
 const mockReadFile = readFile as jest.MockedFunction<typeof readFile>;
 
@@ -364,10 +354,6 @@ describe('promiseQueue', () => {
 describe('processDocsFunction', () => {
   beforeEach(() => {
     jest.clearAllMocks();
-
-    // Mock the memo functions
-    readLocalFileFunction.memo = jest.fn().mockResolvedValue('local file content') as any;
-    fetchUrlFunction.memo = jest.fn().mockResolvedValue('fetched content') as any;
   });
 
   it.each([
@@ -377,7 +363,6 @@ describe('processDocsFunction', () => {
         'local-file.md',
         'https://example.com/remote.md'
       ],
-      options: {},
       fileMemoHits: 1,
       fetchMemoHits: 1
     },
@@ -390,7 +375,6 @@ describe('processDocsFunction', () => {
         'https://example.com/remote.md',
         'https://example.com/remote.md'
       ],
-      options: {},
       fileMemoHits: 1,
       fetchMemoHits: 1
     },
@@ -403,7 +387,6 @@ describe('processDocsFunction', () => {
         '   ',
         'file2.md'
       ],
-      options: {},
       fileMemoHits: 2
     },
     {
@@ -412,61 +395,95 @@ describe('processDocsFunction', () => {
         { doc: 'file.md', lorem: 'ipsum' },
         { doc: 'file.md', dolor: 'sit' }
       ],
-      options: {},
       fileMemoHits: 1
     },
     {
       description: 'metadata passthrough',
       inputs: [
         { doc: 'file.md', lorem: 'ispum', dolor: 'sit' }
       ],
-      options: {},
       fileMemoHits: 1
     }
-  ])('should process local and remote inputs, $description', async ({ inputs, options, fileMemoHits = 0, fetchMemoHits = 0 }) => {
-    const result = await processDocsFunction(inputs, {
-      isHttp: false,
-      minMax: {
-        docsToLoad: {
-          min: 1,
-          max: 10
-        }
-      },
-      separator: '\n\n---\n\n',
-      urlRegex: /^(https?:)\/\//i,
-      ...options
-    } as GlobalOptions);
+  ])('should process local and remote inputs, $description', async ({ inputs, fileMemoHits = 0, fetchMemoHits = 0 }) => {
+    readLocalFileFunction.memo = jest.fn().mockResolvedValue('local file content') as any;
+    fetchUrlFunction.memo = jest.fn().mockResolvedValue('fetched content') as any;
+
+    const result = await processDocsFunction(inputs, { loadLimit: 10 });
 
     expect(result).toMatchSnapshot();
     expect(readLocalFileFunction.memo).toHaveBeenCalledTimes(fileMemoHits);
     expect(fetchUrlFunction.memo).toHaveBeenCalledTimes(fetchMemoHits);
   });
 
   it('should handle errors gracefully', async () => {
-    const mockOptions = {
-      isHttp: false,
-      minMax: {
-        docsToLoad: {
-          min: 1,
-          max: 10
-        }
-      },
-      separator: '\n\n---\n\n',
-      urlRegex: /^(https?:)\/\//i
-    };
-
     // Mock one success and one failure
     readLocalFileFunction.memo = jest.fn()
       .mockResolvedValueOnce('success content')
       .mockRejectedValueOnce(new Error('File not found')) as any;
+    fetchUrlFunction.memo = jest.fn().mockResolvedValue('fetched content') as any;
 
     const inputs = [
       'good-file.md',
       'bad-file.md'
     ];
 
-    const result = await processDocsFunction(inputs, mockOptions as GlobalOptions);
+    const result = await processDocsFunction(inputs, { loadLimit: 10 });
 
     expect(result).toMatchSnapshot('errors');
   });
+
+  it.each([
+    {
+      description: 'truncate input list to custom loadLimit',
+      inputs: ['file1.md', 'file2.md', 'file3.md', 'file4.md'],
+      settings: { loadLimit: 2 },
+      expectedCount: 2,
+      expectedPaths: ['file1.md', 'file2.md']
+    },
+    {
+      description: 'handle loadLimit of 0',
+      inputs: ['file1.md', 'file2.md'],
+      settings: { loadLimit: 0 },
+      expectedCount: 0,
+      expectedPaths: []
+    },
+    {
+      description: 'fall back to default loadLimit (100) when settings are omitted or empty',
+      inputs: ['file1.md', 'file2.md'],
+      settings: {},
+      expectedCount: 2,
+      expectedPaths: ['file1.md', 'file2.md']
+    },
+    {
+      description: 'process items with custom concurrency and throttle options',
+      inputs: ['file1.md', 'file2.md'],
+      settings: { parallelLoadLimit: 1, parallelLoadThrottleMs: 10 },
+      expectedCount: 2,
+      expectedPaths: ['file1.md', 'file2.md']
+    }
+  ])('should handle custom settings: $description', async ({ inputs, settings, expectedCount, expectedPaths }) => {
+    readLocalFileFunction.memo = jest.fn().mockImplementation(async (path: string) => `content of ${path}`) as any;
+    fetchUrlFunction.memo = jest.fn().mockImplementation(async (url: string) => `content of ${url}`) as any;
+
+    const result = await processDocsFunction(inputs, settings);
+
+    expect(result).toHaveLength(expectedCount);
+    expect(result.map(doc => doc.path)).toEqual(expectedPaths);
+  });
+
+  it('should return cached results on subsequent calls with identical settings', async () => {
+    processDocsFunction.memo.clear();
+    readLocalFileFunction.memo = jest.fn().mockImplementation(async (path: string) => `content of ${path}`) as any;
+
+    const inputs = ['file1.md', 'file2.md'];
+    const settings = { loadLimit: 5 };
+
+    const firstResult = await processDocsFunction.memo(inputs, settings);
+    const secondResult = await processDocsFunction.memo(inputs, { ...settings });
+
+    expect(firstResult).toEqual(secondResult);
+    expect(readLocalFileFunction.memo).toHaveBeenCalledTimes(2); // Only executed during the first invocation
+
+    processDocsFunction.memo.clear();
+  });
 });

src/logger.ts

@@ -126,6 +126,10 @@ const formatLogEvent = (event: LogEvent) => {
 
   const rest = event?.args?.map(arg => {
     try {
+      if (arg instanceof Error) {
+        return arg.stack || arg.message;
+      }
+
       return typeof arg === 'string' ? arg : JSON.stringify(arg);
     } catch {
       return String(arg);

src/resource.patternFlyDocsTemplate.ts

@@ -137,7 +137,7 @@ const resourceCallback = async (passedUri: URL, variables: Record<string, string
 
     if (docPaths.length > 0) {
       // `processDocsFunction` has de-dup docs baked in
-      const processedDocs = await processDocsFunction.memo(docPaths);
+      const processedDocs = await processDocsFunction.memo(docPaths, { loadLimit: options.minMax.docsToLoad.max });
 
       // Failures are `log.debugged` in `processDocsFunction`.
       for (const response of processedDocs) {

src/server.getResources.ts

@@ -326,7 +326,10 @@ const promiseQueue = async (queue: string[], { limit = 5, throttleMs = 250 } = {
  *
  * @template T - Metadata fields on `{ doc, ...metadata }` inputs, merged into each result.
  * @param inputs - List of paths or URLs to load
- * @param options - Optional options
+ * @param [settings] - Configuration options.
+ * @param [settings.loadLimit] - Maximum number of docs to load.
+ * @param [settings.parallelLoadLimit] - Maximum number of docs to load in parallel.
+ * @param [settings.parallelLoadThrottleMs] - Throttle time in ms between parallel loads.
  * @returns An array of {@link ProcessedDoc} entries:
  *   - `content` is the loaded content string.
  *   - `path` is the original input path or URL.
@@ -335,7 +338,9 @@ const promiseQueue = async (queue: string[], { limit = 5, throttleMs = 250 } = {
  */
 const processDocsFunction = async <T extends Record<string, unknown> = Record<string, unknown>>(
   inputs: (string | ({ doc: string } & T))[],
-  options = getOptions()
+  {
+    loadLimit = 100, parallelLoadLimit = 10, parallelLoadThrottleMs = 100
+  }: { loadLimit?: number; parallelLoadLimit?: number; parallelLoadThrottleMs?: number; } = {}
 ): Promise<ProcessedDoc<Omit<T, 'doc'>>[]> => {
   const normalizeInputs = inputs.map(input =>
     (typeof input === 'string' ? { doc: input } : input) as { doc: string } & T);
@@ -354,10 +359,10 @@ const processDocsFunction = async <T extends Record<string, unknown> = Record<st
     }
   }
 
-  const uniqueInputsList = Array.from(uniqueInputsMap.values()).slice(0, options.minMax.docsToLoad.max);
+  const uniqueInputsList = Array.from(uniqueInputsMap.values()).slice(0, loadLimit);
   const list = uniqueInputsList.map(input => input.doc);
 
-  const settled = await promiseQueue(list);
+  const settled = await promiseQueue(list, { limit: parallelLoadLimit, throttleMs: parallelLoadThrottleMs });
   const docs: ProcessedDoc<Omit<T, 'doc'>>[] = [];
 
   settled.forEach((res, index) => {

src/tool.patternFlyDocs.ts

@@ -162,7 +162,7 @@ const usePatternFlyDocsTool = (options = getOptions()): McpTool => {
     }
 
     try {
-      const processedDocs = await processDocsFunction.memo([...finalUrlList]);
+      const processedDocs = await processDocsFunction.memo([...finalUrlList], { loadLimit: options.minMax.docsToLoad.max });
       const primaryDocs: ProcessedDoc[] = [];
       const secondaryDocs: ProcessedDoc[] = [];
       const tertiaryDocs: ProcessedDoc[] = [];