feat(collections): pf-4402 pf api embedded json

eslint.config.js

@@ -124,6 +124,17 @@ export default [
       'no-console': 0
     }
   },
+  {
+    files: [
+      'scripts/**/*.ts',
+      'scripts/**/*.js'
+    ],
+    rules: {
+      'no-console': 0,
+      'n/no-process-exit': 0,
+      'import/no-unresolved': 0
+    }
+  },
   {
     files: [
       'docs/**/*.ts',

jest.config.ts

@@ -12,7 +12,20 @@ const baseConfig = {
     '^.+\\.(ts|tsx)$': [
       'ts-jest',
       {
-        ...tsConfig
+        ...tsConfig,
+        diagnostics: {
+          // See codes https://github.com/Microsoft/TypeScript/blob/main/src/compiler/diagnosticMessages.json
+          // 1324 - Dynamic imports only support a second argument when the '--module' option is...
+          // 1343 - The 'import.meta' meta-property is only allowed when the '--module' option is...
+          ignoreCodes: [1324, 1343]
+        },
+        astTransformers: {
+          before: [
+            {
+              path: 'ts-jest-mock-import-meta'
+            }
+          ]
+        }
       }
     ]
   }
@@ -40,28 +53,7 @@ export default {
       roots: ['<rootDir>/src'],
       testMatch: ['<rootDir>/src/**/*.test.ts'],
       setupFilesAfterEnv: ['<rootDir>/jest.setupTests.ts'],
-      ...baseConfig,
-      transform: {
-        '^.+\\.(ts|tsx)$': [
-          'ts-jest',
-          {
-            ...tsConfig,
-            diagnostics: {
-              // See codes https://github.com/Microsoft/TypeScript/blob/main/src/compiler/diagnosticMessages.json
-              // 1324 - Dynamic imports only support a second argument when the '--module' option is...
-              // 1343 - The 'import.meta' meta-property is only allowed when the '--module' option is...
-              ignoreCodes: [1324, 1343]
-            },
-            astTransformers: {
-              before: [
-                {
-                  path: 'ts-jest-mock-import-meta'
-                }
-              ]
-            }
-          }
-        ]
-      }
+      ...baseConfig
     },
     {
       displayName: 'package',
@@ -79,6 +71,12 @@ export default {
       ],
       ...baseConfig
     },
+    {
+      displayName: 'collections',
+      roots: ['<rootDir>/tests/scripts'],
+      testMatch: ['<rootDir>/tests/scripts/**/*collection*.test.ts'],
+      ...baseConfig
+    },
     {
       displayName: 'audit',
       roots: ['<rootDir>/tests/audit'],

package.json

@@ -5,6 +5,7 @@
   "main": "dist/index.js",
   "type": "module",
   "imports": {
+    "~apiCatalog": "./src/collection.patternFlyApi.json",
     "~docsCatalog": "./src/docs.json",
     "#toolsHost": "./dist/server.toolsHost.js",
     "#workerEntry": "./dist/server.workerEntry.js",
@@ -34,13 +35,14 @@
     "build": "npm run build:clean; npm run test:types; pkgroll",
     "build:clean": "rm -rf dist",
     "build:watch": "npm run build -- --watch",
+    "build:collections": "tsx ./scripts/update.collection.patternFlyApi.ts && NODE_OPTIONS='--experimental-vm-modules' jest --selectProjects collections",
     "container:build": "bash ./scripts/container.build.sh",
     "container:start": "bash ./scripts/container.run.sh",
     "release": "changelog --non-cc --link-url https://github.com/patternfly/patternfly-mcp.git",
     "start": "node dist/cli.js --log-stderr",
     "start:dev": "tsx watch src/cli.ts --verbose --log-stderr",
     "test": "npm run test:spell && npm run test:spell-docs && npm run test:lint && npm run test:types && jest --selectProjects unit",
-    "test:audit": "jest --selectProjects audit",
+    "test:audit": "NODE_OPTIONS='--experimental-vm-modules' jest --selectProjects audit",
     "test:audit-container": "npm run container:build && jest --selectProjects audit:container",
     "test:ci": "npm test -- --coverage",
     "test:dev": "npm test -- --watchAll",

scripts/update.collection.patternFlyApi.ts

@@ -0,0 +1,177 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  apiSpider,
+  contentMetadata,
+  type ApiCrawler,
+  type ApiEmbedded,
+  type ApiEmbeddedCollection
+} from '../src/collection.patternFlyApi';
+import { getOptions, runWithOptions } from '../src/options.context';
+
+/**
+ * Create a light diff report between old and new collections.
+ *
+ * @param oldRecords - Previous collection
+ * @param newRecords - Updated collection
+ */
+const diffCollections = (oldRecords: ApiEmbedded[], newRecords: ApiEmbedded[]) => {
+  const oldMap = new Map(oldRecords.map(record => [record.p, record]));
+  const newMap = new Map(newRecords.map(record => [record.p, record]));
+
+  const added = newRecords.filter(record => !oldMap.has(record.p));
+  const removed = oldRecords.filter(record => !newMap.has(record.p));
+  const modified = newRecords.filter(record => {
+    const prev = oldMap.get(record.p);
+
+    return prev && (prev.q !== record.q || prev.n !== record.n || prev.d !== record.d || prev.c !== record.c);
+  });
+
+  return { added, removed, modified };
+};
+
+/**
+ * Create a light diff report between old and new collections.
+ *
+ * @param diff - Diff report
+ */
+const diffReport = (diff: ReturnType<typeof diffCollections>) => {
+  const { added, removed, modified } = diff;
+  const hasChanges = added.length > 0 || removed.length > 0 || modified.length > 0;
+
+  console.log('\n📊 Collection Diff Report:');
+
+  if (!hasChanges) {
+    console.log('   ✨ No record additions, removals, or property modifications detected.');
+
+    return;
+  }
+
+  if (added.length > 0) {
+    console.log(`   ➕ Added (${added.length}):`);
+    added.slice(0, 10).forEach(record => console.log(`      + ${record.p} (Q: ${record.q})`));
+
+    if (added.length > 10) {
+      console.log(`      ... and ${added.length - 10} more`);
+    }
+  }
+
+  if (removed.length > 0) {
+    console.log(`   ➖ Removed (${removed.length}):`);
+    removed.slice(0, 10).forEach(record => console.log(`      - ${record.p}`));
+
+    if (removed.length > 10) {
+      console.log(`      ... and ${removed.length - 10} more`);
+    }
+  }
+
+  if (modified.length > 0) {
+    console.log(`   🔄 Modified (${modified.length}):`);
+    modified.slice(0, 10).forEach(record => console.log(`      ~ ${record.p} (Q: ${record.q})`));
+
+    if (modified.length > 10) {
+      console.log(`      ... and ${modified.length - 10} more`);
+    }
+  }
+};
+
+/**
+ * Run apiSpider directly and transform crawler entries into compressed embedded JSON.
+ *
+ * @param [options] - Optional configuration options.
+ * @param [options.isPrettyPrint=true] - Whether to pretty-print the JSON output.
+ * @param [options.filterLowQualityRecords=false] - Whether to filter low-quality records based on the collection's criteria.
+ */
+const run = async (
+  {
+    isPrettyPrint = true,
+    filterLowQualityRecords = false
+  }: { isPrettyPrint?: boolean; filterLowQualityRecords?: boolean; } = {}
+) => {
+  console.log('🚀 Generating PatternFly API embedded collection...');
+  const keepAlive = setTimeout(() => {}, 86_400_000);
+
+  const startTime = Date.now();
+  const options = getOptions();
+  const { base } = options.patternflyOptions.api;
+
+  try {
+    const entries: ApiCrawler[] = await runWithOptions(options, async () => apiSpider(options));
+
+    if (!entries.length) {
+      console.error('❌ Crawl failed or returned 0 entries. Aborting update.');
+      process.exit(1);
+    }
+
+    const recordsMap = new Map<string, ApiEmbedded>();
+
+    for (const entry of entries) {
+      // Generate full metadata using the shared contentMetadata function
+      const metadata = contentMetadata(entry, options);
+
+      if (filterLowQualityRecords && (metadata.isDeferred || metadata.isLowQuality)) {
+        continue;
+      }
+
+      const relativePath = metadata.path.replace(base, '').replace(/^\//, '');
+
+      if (recordsMap.has(relativePath)) {
+        continue;
+      }
+
+      recordsMap.set(relativePath, {
+        p: relativePath,
+        n: metadata.displayName,
+        d: metadata.description,
+        c: metadata.contentType,
+        q: entry.qualityScore
+      });
+    }
+
+    const records = [...recordsMap.values()].sort((a, b) => a.p.localeCompare(b.p));
+
+    const payload: ApiEmbeddedCollection = {
+      version: '1',
+      generated: new Date().toISOString(),
+      base,
+      records
+    };
+
+    const outputPath = resolve(fileURLToPath(new URL('../src/collection.patternFlyApi.json', import.meta.url)));
+    const jsonContent = isPrettyPrint ? JSON.stringify(payload, null, 2) : JSON.stringify(payload);
+    let oldRecords: ApiEmbedded[] = [];
+
+    try {
+      const existingContent = await readFile(outputPath, 'utf-8');
+      const parsedExisting: ApiEmbeddedCollection = JSON.parse(existingContent);
+
+      oldRecords = parsedExisting.records || [];
+    } catch {
+      // File might not exist yet on initial run
+    }
+
+    await writeFile(outputPath, jsonContent + '\n', 'utf-8');
+
+    const durationSec = ((Date.now() - startTime) / 1000).toFixed(1);
+    const sizeKb = (Buffer.byteLength(jsonContent, 'utf-8') / 1024).toFixed(1);
+
+    console.log(`✅ Updated src/collection.patternFlyApi.json:`);
+    console.log(`   - Total Crawled: ${entries.length} endpoints`);
+    console.log(`   - Admitted Records: ${records.length}`);
+    console.log(`   - File Size: ${sizeKb} KB`);
+    console.log(`   - Time Elapsed: ${durationSec}s`);
+
+    diffReport(diffCollections(oldRecords, records));
+  } finally {
+    clearTimeout(keepAlive);
+  }
+};
+
+/**
+ * Configurable options for maintainers.
+ */
+run({ isPrettyPrint: true, filterLowQualityRecords: true }).catch(error => {
+  console.error('❌ Failed to update API collection:', error);
+  process.exit(1);
+});

src/__tests__/__snapshots__/collection.patternFlyApi.test.ts.snap

@@ -8,7 +8,6 @@ exports[`collectionCallback should match snapshot for collection result 1`] = `
         "card": [
           {
             "category": "css",
-            "content": "Card css content",
             "contentType": "",
             "description": "PatternFly variables and tokens for Card CSS.",
             "displayName": "Card CSS",
@@ -28,3 +27,45 @@ exports[`collectionCallback should match snapshot for collection result 1`] = `
   ],
 }
 `;
+
+exports[`expandApiEmbeddedCollection should expand compressed records: expanded 1`] = `
+[
+  {
+    "content": "",
+    "contentType": "text/markdown",
+    "description": "A standard button component.",
+    "displayName": "Button",
+    "path": "https://main.patternfly-org.pages.dev/api/v1/components/Button/react",
+    "qualityScore": 1,
+    "resolvedPath": "https://main.patternfly-org.pages.dev/api/v1/components/Button/react",
+  },
+]
+`;
+
+exports[`getPatternFlyApiRecords should attempt to convert expanded embedded records into McpCollectionResult records, high quality score 1`] = `
+[
+  {
+    "data": {
+      "dolor": [
+        {
+          "category": "react",
+          "contentType": "text/markdown",
+          "description": "Dolor sit component description",
+          "displayName": "Dolor",
+          "id": "api::v1::components::dolor::react",
+          "path": "https://main.patternfly-org.pages.dev/api/v1/components/Dolor/react",
+          "pathSlug": "components-dolor-react",
+          "section": "components",
+          "source": "api",
+          "version": "v1",
+        },
+      ],
+    },
+    "id": "api::v1::components::dolor::react",
+    "sourceId": "https://main.patternfly-org.pages.dev/api/v1/components/Dolor/react",
+    "sourceType": "api",
+  },
+]
+`;
+
+exports[`getPatternFlyApiRecords should attempt to convert expanded embedded records into McpCollectionResult records, low quality score 1`] = `[]`;

src/__tests__/__snapshots__/options.defaults.test.ts.snap

@@ -60,10 +60,11 @@ exports[`options defaults should return specific properties: defaults 1`] = `
       "enabled": false,
       "schedule": {
         "continueOnError": true,
+        "delayStartMs": 21600000,
         "intervalMs": 604800000,
         "repeat": Infinity,
       },
-      "timeoutMs": 120000,
+      "timeoutMs": 300000,
       "traversalPaths": [
         "examples",
       ],
@@ -149,8 +150,8 @@ exports[`options defaults should return specific properties: defaults 1`] = `
     },
     "usePatternFlyDocs": {
       "cacheErrors": false,
-      "cacheLimit": 10,
-      "expire": 60000,
+      "cacheLimit": 25,
+      "expire": 600000,
     },
   },
   "toolModules": [],