Retrieves all matching CVE records after applying the JSON request body. This endpoint is deprecated and can be disabled by the service operator. Use POST /search instead.
", + "summary": "Raw query search", + "description": "Retrieves matching CVE records using the raw query expression supplied in the JSON request body. Service operators can disable this endpoint through runtime configuration.
", "operationId": "webSearch", "responses": { "200": { "description": "A list of matching CVE records", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebSearchResponse" + } + } + }, "headers": { "X-Request-ID": { "description": "Request correlation ID", @@ -224,10 +290,34 @@ "description": "Bad Request" }, "410": { - "description": "The deprecated endpoint is disabled; use POST /search" + "description": "The endpoint is disabled by service configuration", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "502": { + "description": "The search backend returned an error or invalid response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebSearchResponse" + } + } + } }, "504": { - "description": "Search exceeded its OpenSearch request deadline" + "description": "Search exceeded its OpenSearch request deadline", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebSearchResponse" + } + } + } } }, "requestBody": { @@ -278,13 +368,180 @@ } } } - }, - "deprecated": true + } } } }, "components": { "schemas": { + "ApiError": { + "type": "object", + "additionalProperties": false, + "required": [ + "error", + "message" + ], + "properties": { + "error": { + "type": "string" + }, + "message": { + "type": "string" + } + } + }, + "ApiErrorResponse": { + "oneOf": [ + { + "$ref": "#/components/schemas/ApiError" + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/ApiError" + } + } + ] + }, + "SearchHit": { + "type": "object", + "additionalProperties": true, + "required": [ + "_source" + ], + "properties": { + "_id": { + "type": "string" + }, + "_source": { + "type": "object", + "additionalProperties": true + } + } + }, + "SearchHits": { + "type": "object", + "additionalProperties": true, + "required": [ + "total", + "hits" + ], + "properties": { + "total": { + "type": "object", + "additionalProperties": true, + "required": [ + "value" + ], + "properties": { + "value": { + "type": "integer", + "minimum": 0 + } + } + }, + "hits": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SearchHit" + } + } + } + }, + "SearchResponse": { + "type": "object", + "additionalProperties": false, + "required": [ + "page", + "pageSize", + "totalPages", + "status", + "data" + ], + "properties": { + "page": { + "type": "integer", + "minimum": 0, + "description": "One-based page number, or 0 when the search has no matches." + }, + "pageSize": { + "type": "integer", + "minimum": 0, + "description": "Number of CVE records returned in this response." + }, + "totalPages": { + "type": "integer", + "minimum": 0 + }, + "status": { + "type": "string", + "enum": [ + "ok" + ] + }, + "data": { + "type": "object", + "additionalProperties": true, + "required": [ + "hits" + ], + "properties": { + "took": { + "type": "number", + "minimum": 0 + }, + "hits": { + "$ref": "#/components/schemas/SearchHits" + } + } + } + } + }, + "WebSearchResponse": { + "type": "object", + "additionalProperties": false, + "required": [ + "searchMetadata", + "resultsTotal", + "data" + ], + "properties": { + "searchMetadata": { + "type": "object", + "additionalProperties": true, + "required": [ + "searchStatus", + "version", + "errors" + ], + "properties": { + "searchStatus": { + "type": "string" + }, + "version": { + "type": "string" + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "resultsTotal": { + "type": "integer", + "minimum": 0 + }, + "data": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + } + } + } + }, "SearchRequest": { "type": "object", "description": "Optional filters and pagination controls for a CVE record search. Enum-valued fields are accepted case-insensitively and normalized to the documented canonical values. When both boundaries of a date range are supplied, the start must be before or equal to the end.", @@ -307,9 +564,20 @@ "description": "Returns CVE records where a word or phrase is included in a description selected by scope. Multiple keywords operate like an AND statement, returning records whose descriptions include every keyword. Limited to 512 characters and 64 whitespace-delimited terms." }, "keywordExactMatch": { - "type": "boolean", + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "enum": [ + "true", + "false" + ] + } + ], "default": false, - "description": "When true, a multi-word keywordSearch value must appear exactly and in the same order. keywordSearch is required when keywordExactMatch is true." + "description": "When true, a multi-word keywordSearch value must appear exactly and in the same order. keywordSearch is required when keywordExactMatch is true. JSON booleans and the lowercase strings \"true\" and \"false\" are accepted." }, "cveIds": { "type": "string", @@ -318,9 +586,20 @@ "description": "Returns one or more CVE records identified by a comma-separated list of CVE IDs. Limited to 100 CVE IDs, 64 characters per ID, and 4096 characters overall." }, "noRejected": { - "type": "boolean", + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "enum": [ + "true", + "false" + ] + } + ], "default": false, - "description": "When true, excludes CVE records with REJECT, REJECTED, or Rejected status." + "description": "When true, excludes CVE records with REJECT, REJECTED, or Rejected status. JSON booleans and the lowercase strings \"true\" and \"false\" are accepted." }, "cweId": { "type": "string", @@ -414,9 +693,20 @@ "description": "Returns CVE records whose CPE Match Criteria apply to the provided CPE 2.3 asset name. CPE string literal comparisons are insensitive to lexical case. The part, vendor, product, and version components must contain values other than *. Every concrete non-version component must be compatible with the same CPE Match Criteria entry that satisfies the version and vulnerability checks; optional * components remain broad matches. The concrete version can match stored criteria with the same version or criteria with a wildcard version and version range fields. Numeric version segments in ranges ignore leading zeros; exact CPE Match Criteria preserve leading zeros. Version qualifiers use Maven-style ordering, with prerelease qualifiers before an unqualified release and sp or unknown qualifiers after it. The version - represents the CPE logical value NA; numeric range boundaries on wildcard-version criteria do not exclude it. Invalid cpeName values return HTTP 404. Use virtualMatchString for partial CPE Match String searches." }, "isVulnerable": { - "type": "boolean", + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "enum": [ + "true", + "false" + ] + } + ], "default": false, - "description": "When true, returns CVE records where the same CPE Match Criteria entry both matches cpeName and is marked vulnerable. False or omitted does not apply a vulnerability restriction. cpeName is required when isVulnerable is true. virtualMatchString cannot be provided with isVulnerable." + "description": "When true, returns CVE records where the same CPE Match Criteria entry both matches cpeName and is marked vulnerable. False or omitted does not apply a vulnerability restriction. cpeName is required when isVulnerable is true. virtualMatchString cannot be provided with isVulnerable. JSON booleans and the lowercase strings \"true\" and \"false\" are accepted." }, "lastModStartDate": { "type": "string", diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index 8adb8d3..b3c5d0a 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -74,8 +74,18 @@ describe('OpenAPI contract', () => { Object.entries(constants.SEARCH_STRING_LENGTH_LIMITS).forEach(([field, maxLength]) => { expect(schema.properties[field].maxLength).toBe(maxLength); }); - expect(schema.properties.noRejected.type).toBe('boolean'); + expect(schema.properties.noRejected.oneOf).toStrictEqual([ + { + type: 'boolean' + }, + { + type: 'string', + enum: ['true', 'false'] + } + ]); expect(schema.properties.noRejected.description).toContain('When true'); + expect(schema.properties.keywordExactMatch.oneOf).toStrictEqual(schema.properties.noRejected.oneOf); + expect(schema.properties.isVulnerable.oneOf).toStrictEqual(schema.properties.noRejected.oneOf); expect(schema.properties.scope.enum).toStrictEqual(['FULL', 'CNA_ONLY', 'ADP_ONLY']); expect(schema.properties.scope.default).toBe('FULL'); expect(schema.properties.scope.description).toContain('does not remove data'); @@ -128,8 +138,8 @@ describe('OpenAPI contract', () => { const operation = openapi.paths['/webSearch'].post; const schema = operation.requestBody.content['application/json'].schema; - expect(operation.deprecated).toBe(true); - expect(operation.description).toContain('Use POST /search instead'); + expect(operation.deprecated).toBeUndefined(); + expect(operation.description).toContain('raw query expression'); expect(operation.parameters || []).toStrictEqual([]); expect(operation.requestBody.required).toBe(true); expect(schema.required).toStrictEqual(['query']); @@ -138,7 +148,7 @@ describe('OpenAPI contract', () => { expect(schema.properties.sort.additionalProperties).toBe(false); expect(schema.properties.sort.properties.property.enum).toStrictEqual(['cveId', 'relevancy']); expect(schema.properties.size.maximum).toBe(constants.MAX_RESULTS_LIMIT); - expect(operation.responses['410'].description).toContain('use POST /search'); + expect(operation.responses['410'].description).toContain('disabled by service configuration'); }); test('documents cpeName as a CPE Match Criteria filter with required components', () => { @@ -163,6 +173,7 @@ describe('OpenAPI contract', () => { expect(responses['400'].description).toContain('Malformed JSON'); expect(responses['415'].description).toContain('application/json'); + expect(responses['502'].description).toContain('invalid or unexpected response'); expect(responses['503'].description).toContain('OpenSearch'); expect(responses['503'].headers['Retry-After']).toStrictEqual({ description: 'Seconds before retrying an unavailable OpenSearch dependency', @@ -172,6 +183,24 @@ describe('OpenAPI contract', () => { }); }); + test('documents reusable success and error response schemas', () => { + const schemas = openapi.components.schemas; + const searchResponse = schemas.SearchResponse; + + expect(searchResponse.required).toStrictEqual(['page', 'pageSize', 'totalPages', 'status', 'data']); + expect(searchResponse.properties.pageSize.description).toContain('returned in this response'); + expect(searchResponse.properties.data.properties.hits.$ref).toBe('#/components/schemas/SearchHits'); + expect(schemas.SearchHits.properties.hits.items.$ref).toBe('#/components/schemas/SearchHit'); + expect(schemas.ApiError.required).toStrictEqual(['error', 'message']); + expect(schemas.ApiErrorResponse.oneOf).toHaveLength(2); + expect(openapi.paths['/search'].post.responses['200'].content['application/json'].schema.$ref).toBe( + '#/components/schemas/SearchResponse' + ); + expect(openapi.paths['/webSearch'].post.responses['200'].content['application/json'].schema.$ref).toBe( + '#/components/schemas/WebSearchResponse' + ); + }); + test('documents request correlation headers', () => { expect(openapi.paths['/search'].post.responses['200'].headers['X-Request-ID']).toStrictEqual({ description: 'Request correlation ID', diff --git a/config/default.jsonc b/config/default.jsonc index 8759707..45d6a0a 100644 --- a/config/default.jsonc +++ b/config/default.jsonc @@ -8,8 +8,6 @@ "appConfig": { // constants for search capability "search": { - // minimum versions for servers that are compatible with current code - "minServer": ["elasticsearch:7.10.2", "opensearch:3.5.0"], // setting this to FALSE (recommended) requires an SSL cert to access the search server // The only time this should be allowed to be true is when developing or testing // using containers that do not have SSL certs @@ -28,9 +26,9 @@ // Set to TRUE to retain successful health-check access logs. "healthChecks": "FALSE" }, - // Feature flags used for controlled endpoint migrations. + // Feature flags used for operator-controlled endpoint availability. "features": { - // Keep enabled until existing consumers have migrated to POST /search. + // Set to FALSE only when an environment must disable POST /webSearch. "webSearchEnabled": "TRUE" } } diff --git a/config/devel.jsonc b/config/devel.jsonc index 0a6569f..0eb0fc4 100644 --- a/config/devel.jsonc +++ b/config/devel.jsonc @@ -6,8 +6,6 @@ "appConfig": { // constants for search capability "search": { - // minimum versions for servers that are compatible with current code - "minServer": ["opensearch:3.5.0"], // URL to reach search server "providerEndpoint": "https://localhost:9200", // index on search server related to searching CVEs diff --git a/config/prod.jsonc b/config/prod.jsonc index 5aeb014..eb4d49c 100644 --- a/config/prod.jsonc +++ b/config/prod.jsonc @@ -5,7 +5,6 @@ // NOTE for consistency, all values need to be strings for proper type when using .env overrides "appConfig": { "search": { - "minServer": ["elasticsearch:7.10.2", "opensearch:3.5.0"], "providerEndpoint": "", // preference is to specify this in a (secret) environment variable on production platforms "index": "", // preference is to specify this in a (secret) environment variable on production platforms "allowUnknownSslCerts": "FALSE" diff --git a/config/runtime-config.test.js b/config/runtime-config.test.js index c65066b..c94173e 100644 --- a/config/runtime-config.test.js +++ b/config/runtime-config.test.js @@ -97,12 +97,11 @@ describe('runtime container and compose configuration', () => { }); test.each([['config/default.jsonc'], ['config/devel.jsonc'], ['config/prod.jsonc']])( - '%s references OpenSearch 3.5 as the minimum OpenSearch version', + '%s does not retain the unused minServer capability configuration', (filePath) => { const config = readProjectFile(filePath); - expect(config).toContain('"opensearch:3.5.0"'); - expect(config).not.toContain('"opensearch:2.10.0"'); + expect(config).not.toContain('"minServer"'); } ); diff --git a/controllers/searchController.js b/controllers/searchController.js index 572a925..b8d97a8 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -72,6 +72,26 @@ function buildWebSearchSort(sort) { return undefined; } +function getWebSearchSuccessData(searchResponse) { + if (typeof searchResponse?.isOk !== 'function' || searchResponse.isOk() !== true) return null; + + const hits = searchResponse?.data?.hits; + const totalValue = hits?.total?.value; + const hasValidMetadata = + typeof searchResponse.status === 'string' && + Array.isArray(searchResponse.notes) && + Array.isArray(searchResponse.errors); + + if (!hasValidMetadata || !Array.isArray(hits?.hits) || !Number.isSafeInteger(totalValue) || totalValue < 0) { + return null; + } + + return { + hits: hits.hits, + totalValue + }; +} + function getHitCveId(hit) { return hit?._source?.cveMetadata?.cveId ?? hit?._id; } @@ -198,7 +218,7 @@ async function searchHandler(req, res, next) { //Calculate current page and total pages to append to JSON output const totalValue = resp.data.hits.total.value; const requestedPageSize = pagingOptions.resultsPerPage; - const pageSize = Math.min(totalValue, requestedPageSize); + const pageSize = resp.data.hits.hits.length; const page = totalValue === 0 ? 0 : Math.floor(pagingOptions.from / requestedPageSize) + 1; const totalPages = totalValue === 0 ? 0 : Math.ceil(totalValue / requestedPageSize); @@ -261,21 +281,14 @@ async function webSearchHandler(req, res, next) { const queryTime = Date.now() - _startTime; body.searchMetadata.queryTime_ms = queryTime; - /* build response */ - const respData = searchResponse?.data; - let resultsTotal = 0; - let data = []; - if (respData && respData['hits'] && respData['hits']['hits']) { - resultsTotal = searchResponse?.data['hits']['total']['value'] || 0; - data = searchResponse?.data['hits']['hits'] || []; - } - if (searchResponse.isOk()) { + const successData = getWebSearchSuccessData(searchResponse); + if (successData) { status = 200; body.searchMetadata.searchStatus = searchResponse.status; body.searchMetadata.notes = searchResponse.notes; body.searchMetadata.errors = searchResponse.errors; - body.resultsTotal = resultsTotal; - body.data = data; + body.resultsTotal = successData.totalValue; + body.data = successData.hits; } else { status = 502; body.searchMetadata.errors = [error.upstreamSearchError()]; diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index ade95e3..c7d1702 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -74,6 +74,17 @@ function basicResponse({ ok = true, status = 'ok', total = 0, hits = [], notes = }; } +function cvssTokenRegexp(field, token) { + return { + regexp: { + [`${field}.keyword`]: { + value: `(.*/)?${token}(/.*)?`, + case_insensitive: true + } + } + }; +} + function captureTimers() { const timers = []; jest.spyOn(global, 'setTimeout').mockImplementation((callback, delay) => { @@ -192,7 +203,14 @@ describe('Search controller SEARCH handler', () => { }, track_total_hits: true, size: 10, - from: 20 + from: 20, + sort: [ + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ] }, expect.objectContaining({ signal: expect.any(AbortSignal) @@ -202,7 +220,7 @@ describe('Search controller SEARCH handler', () => { expect(res.status).toHaveBeenCalledWith(200); expect(res.json).toHaveBeenCalledWith({ page: 3, - pageSize: 10, + pageSize: 1, totalPages: 3, ...resp }); @@ -488,7 +506,14 @@ describe('Search controller SEARCH handler', () => { }, track_total_hits: true, size: 1, - from: 0 + from: 0, + sort: [ + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ] }, expect.objectContaining({ signal: expect.any(AbortSignal) @@ -847,27 +872,9 @@ describe('Search controller SEARCH handler', () => { { bool: { filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*AC:L*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*Au:N*' - } - } - } + cvssTokenRegexp('containers.cna.metrics.cvssV2_0.vectorString', 'AV:N'), + cvssTokenRegexp('containers.cna.metrics.cvssV2_0.vectorString', 'AC:L'), + cvssTokenRegexp('containers.cna.metrics.cvssV2_0.vectorString', 'AU:N') ] } } @@ -894,40 +901,16 @@ describe('Search controller SEARCH handler', () => { { bool: { filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV3_0.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV3_0.vectorString.keyword': { - value: '*AC:L*' - } - } - } + cvssTokenRegexp('containers.cna.metrics.cvssV3_0.vectorString', 'AV:N'), + cvssTokenRegexp('containers.cna.metrics.cvssV3_0.vectorString', 'AC:L') ] } }, { bool: { filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV3_1.vectorString.keyword': { - value: '*PR:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV3_1.vectorString.keyword': { - value: '*UI:R*' - } - } - } + cvssTokenRegexp('containers.cna.metrics.cvssV3_1.vectorString', 'PR:N'), + cvssTokenRegexp('containers.cna.metrics.cvssV3_1.vectorString', 'UI:R') ] } } @@ -1090,7 +1073,7 @@ describe('Search controller SEARCH handler', () => { expect(res.status).toHaveBeenCalledWith(200); expect(res.json).toHaveBeenCalledWith({ page: 2, - pageSize: 5, + pageSize: 0, totalPages: 1, ...resp }); @@ -1467,6 +1450,46 @@ describe('Search controller WEB_SEARCH handler', () => { }); }); + test('WEB_SEARCH fails closed when cve-core reports success without a complete hits response', async () => { + mockBasicSearch.mockResolvedValue({ + status: 'ok', + notes: [], + errors: [], + data: {}, + isOk: jest.fn(() => true) + }); + + await controller.WEB_SEARCH( + { + body: { + query: 'apache' + } + }, + res, + next + ); + + expect(res.status).toHaveBeenCalledWith(502); + expect(res.json.mock.calls[0][0]).toStrictEqual({ + searchMetadata: { + searchStatus: 'errors', + version: '2025-08-26', + cveCore: 'test-core-version', + cveMitreCore: 'test-core-version', + notes: [], + errors: [ + { + error: 'UPSTREAM_SEARCH_ERROR', + message: 'Search backend returned an error. Please try again later.' + } + ], + queryTime_ms: expect.any(Number) + }, + resultsTotal: 0, + data: [] + }); + }); + test('WEB_SEARCH returns a sanitized error when BasicSearchManager throws', async () => { mockBasicSearch.mockRejectedValue(new Error('boom')); diff --git a/jest.config.js b/jest.config.js index dd834cb..36247ba 100644 --- a/jest.config.js +++ b/jest.config.js @@ -2,13 +2,7 @@ const config = { clearMocks: true, collectCoverage: false, - collectCoverageFrom: [ - 'controllers/**/*.js', - 'routes/index.js', - 'server.js', - 'utils/**/*.js', - '!controllers/wrapper.js' - ], + collectCoverageFrom: ['controllers/**/*.js', 'routes/index.js', 'server.js', 'utils/**/*.js'], coverageDirectory: 'coverage', coverageProvider: 'v8', coverageReporters: ['json', 'text', 'lcov'], diff --git a/package-lock.json b/package-lock.json index 48d2e11..cfad6cc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,11 +10,9 @@ "dependencies": { "cors": "^2.8.5", "cve-core": "github:CVEProject/cve-core.git#dev-package-test", - "debug": "~3.2.7", "dotenv": "^6.1.0", "express": "^4.22.2", "express-validator": "^7.2.1", - "http-errors": "~1.8.1", "morgan": "~1.11.0", "swagger-ui-express": "^5.0.1" }, @@ -2319,15 +2317,6 @@ "date-fns": "2.x" } }, - "node_modules/debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.1" - } - }, "node_modules/dedent": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz", @@ -3413,40 +3402,6 @@ "dev": true, "license": "MIT" }, - "node_modules/http-errors": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", - "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", - "license": "MIT", - "dependencies": { - "depd": "~1.1.2", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/http-errors/node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/http-errors/node_modules/statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/https-proxy-agent": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", diff --git a/package.json b/package.json index 1cd2c2b..1f07d5b 100644 --- a/package.json +++ b/package.json @@ -16,11 +16,9 @@ "dependencies": { "cors": "^2.8.5", "cve-core": "github:CVEProject/cve-core.git#dev-package-test", - "debug": "~3.2.7", "dotenv": "^6.1.0", "express": "^4.22.2", "express-validator": "^7.2.1", - "http-errors": "~1.8.1", "morgan": "~1.11.0", "swagger-ui-express": "^5.0.1" }, diff --git a/routes/index.js b/routes/index.js index 9a62b1f..1858700 100644 --- a/routes/index.js +++ b/routes/index.js @@ -280,6 +280,13 @@ router.post( } #swagger.responses[200] = { description: 'A list of matching CVE records', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/SearchResponse" + } + } +}, headers: { "X-Request-ID": { description: "Request correlation ID", @@ -291,15 +298,53 @@ headers: { } #swagger.responses[400] = { description: 'Malformed JSON, invalid request body, parameter combination, or version range', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ApiErrorResponse" + } + } +}, } #swagger.responses[404] = { description: 'Invalid cpeName value', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ApiErrorResponse" + } + } +}, } #swagger.responses[415] = { description: 'A non-empty request body did not use the application/json media type with UTF-8 encoding', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ApiError" + } + } +}, +} +#swagger.responses[502] = { +description: 'OpenSearch returned an invalid or unexpected response', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ApiError" + } + } +}, } #swagger.responses[503] = { description: 'OpenSearch is unavailable', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ApiError" + } + } +}, headers: { "Retry-After": { description: "Seconds before retrying an unavailable OpenSearch dependency", @@ -311,6 +356,13 @@ headers: { } #swagger.responses[504] = { description: 'Search exceeded its OpenSearch or overall processing deadline', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ApiError" + } + } +}, } */ requireSearchJsonContentType, @@ -375,7 +427,7 @@ router.get( /* #swagger.operationId = 'readinessCheck' #swagger.summary = "OpenSearch readiness check" -#swagger.description = "Sends a minimal zero-result search to the configured CVE index. Returns 200 when the index is available with search permission, or a sanitized 503 response when configuration, connectivity, timeout, permission, or index availability checks fail." +#swagger.description = "Validates the field capabilities required by the search contract, then sends a minimal zero-result search to the configured CVE index. Returns 200 when mappings, index availability, and search permission are valid, or a sanitized 503 response when a readiness check fails." #swagger.responses[200] = { description: 'API is ready to serve search requests', headers: { @@ -389,6 +441,13 @@ headers: { } #swagger.responses[503] = { description: 'Search service is not ready', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ApiError" + } + } +}, headers: { "Retry-After": { description: "Seconds before retrying the readiness check", @@ -413,11 +472,10 @@ router.post( /* #swagger.tags = ['CVE Record'] #swagger.operationId = 'webSearch' -#swagger.deprecated = true -#swagger.summary = "Deprecated raw query search" +#swagger.summary = "Raw query search" #swagger.description = "Retrieves all matching CVE records after applying the JSON request body. This endpoint is deprecated and can be disabled by the service operator. Use POST /search instead.
+Retrieves matching CVE records using the raw query expression supplied in the JSON request body. Service operators can disable this endpoint through runtime configuration.
#swagger.requestBody = { required: true, content: { @@ -461,6 +519,13 @@ router.post( } #swagger.responses[200] = { description: 'A list of matching CVE records', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/WebSearchResponse" + } + } +}, headers: { "X-Request-ID": { description: "Request correlation ID", @@ -472,9 +537,33 @@ headers: { } #swagger.responses[504] = { description: 'Search exceeded its OpenSearch request deadline', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/WebSearchResponse" + } + } +}, } #swagger.responses[410] = { -description: 'The deprecated endpoint is disabled; use POST /search', +description: 'The endpoint is disabled by service configuration', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ApiError" + } + } +}, +} +#swagger.responses[502] = { +description: 'The search backend returned an error or invalid response', +content: { + "application/json": { + schema: { + $ref: "#/components/schemas/WebSearchResponse" + } + } +}, } */ requireWebSearchEnabled, diff --git a/routes/index.test.js b/routes/index.test.js index a041d7a..9be9c4d 100644 --- a/routes/index.test.js +++ b/routes/index.test.js @@ -401,7 +401,7 @@ describe('routes/index router', () => { expect(response.status).toBe(410); expect(response.body).toStrictEqual({ error: 'WEB_SEARCH_DISABLED', - message: 'The /webSearch endpoint is disabled. Use POST /search instead.' + message: 'The /webSearch endpoint is disabled by service configuration.' }); expect(controller.WEB_SEARCH).not.toHaveBeenCalled(); }); diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index 22b0c81..df381d0 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -9,8 +9,7 @@ jest.mock('../utils/searchProvider', () => ({ search: mockBasicSearch })), getSearchRequestDeadlineMs: jest.fn(() => 30000), - searchOpenSearch: mockSearchOpenSearch, - shouldLogSearchPerformance: jest.fn(() => false) + searchOpenSearch: mockSearchOpenSearch })); const { createApp } = require('../server'); @@ -83,7 +82,14 @@ describe('search route integration', () => { }, track_total_hits: true, size: 1, - from: 0 + from: 0, + sort: [ + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ] }, { signal: expect.any(AbortSignal) diff --git a/routes/swagger.js b/routes/swagger.js index 025ddd3..0e0c273 100644 --- a/routes/swagger.js +++ b/routes/swagger.js @@ -16,6 +16,21 @@ function searchRequestProperty(description, schema) { }; } +function coercibleBooleanSchema(defaultValue = false) { + return { + oneOf: [ + { + type: 'boolean' + }, + { + type: 'string', + enum: ['true', 'false'] + } + ], + default: defaultValue + }; +} + const searchRequestProperties = { scope: searchRequestProperty( 'Controls which containers participate in filter matching. FULL searches base record fields, CNA data, and all ADP providers. CNA_ONLY searches base record fields and CNA data. ADP_ONLY searches base record fields and all ADP providers. Scope does not remove data from returned records. Values are case-insensitive. CPE applicability filters are not available with ADP_ONLY.', @@ -34,11 +49,8 @@ const searchRequestProperties = { } ), keywordExactMatch: searchRequestProperty( - 'When true, a multi-word keywordSearch value must appear exactly and in the same order. keywordSearch is required when keywordExactMatch is true.', - { - type: 'boolean', - default: false - } + 'When true, a multi-word keywordSearch value must appear exactly and in the same order. keywordSearch is required when keywordExactMatch is true. JSON booleans and the lowercase strings "true" and "false" are accepted.', + coercibleBooleanSchema() ), cveIds: searchRequestProperty( `Returns one or more CVE records identified by a comma-separated list of CVE IDs. Limited to ${constants.MAX_CVE_IDS} CVE IDs, ${constants.MAX_CVE_ID_LENGTH} characters per ID, and ${constants.SEARCH_STRING_LENGTH_LIMITS.cveIds} characters overall.`, @@ -48,10 +60,10 @@ const searchRequestProperties = { example: 'CVE-2024-1234,CVE-2024-5678' } ), - noRejected: searchRequestProperty('When true, excludes CVE records with REJECT, REJECTED, or Rejected status.', { - type: 'boolean', - default: false - }), + noRejected: searchRequestProperty( + 'When true, excludes CVE records with REJECT, REJECTED, or Rejected status. JSON booleans and the lowercase strings "true" and "false" are accepted.', + coercibleBooleanSchema() + ), cweId: searchRequestProperty( 'Returns CVE records whose scope-selected containers include the provided Common Weakness Enumeration ID.', { @@ -144,11 +156,8 @@ const searchRequestProperties = { } ), isVulnerable: searchRequestProperty( - 'When true, returns CVE records where the same CPE Match Criteria entry both matches cpeName and is marked vulnerable. False or omitted does not apply a vulnerability restriction. cpeName is required when isVulnerable is true. virtualMatchString cannot be provided with isVulnerable.', - { - type: 'boolean', - default: false - } + 'When true, returns CVE records where the same CPE Match Criteria entry both matches cpeName and is marked vulnerable. False or omitted does not apply a vulnerability restriction. cpeName is required when isVulnerable is true. virtualMatchString cannot be provided with isVulnerable. JSON booleans and the lowercase strings "true" and "false" are accepted.', + coercibleBooleanSchema() ), lastModStartDate: searchRequestProperty( 'Returns CVE records with dateUpdated timestamps on or after this value. Use the format yyyy-MM-ddTHH:mm:ss.SSSZ.', @@ -341,6 +350,146 @@ const doc = { ], components: { '@schemas': { + ApiError: { + type: 'object', + additionalProperties: false, + required: ['error', 'message'], + properties: { + error: { + type: 'string' + }, + message: { + type: 'string' + } + } + }, + ApiErrorResponse: { + oneOf: [ + { + $ref: '#/components/schemas/ApiError' + }, + { + type: 'array', + items: { + $ref: '#/components/schemas/ApiError' + } + } + ] + }, + SearchHit: { + type: 'object', + additionalProperties: true, + required: ['_source'], + properties: { + _id: { + type: 'string' + }, + _source: { + type: 'object', + additionalProperties: true + } + } + }, + SearchHits: { + type: 'object', + additionalProperties: true, + required: ['total', 'hits'], + properties: { + total: { + type: 'object', + additionalProperties: true, + required: ['value'], + properties: { + value: { + type: 'integer', + minimum: 0 + } + } + }, + hits: { + type: 'array', + items: { + $ref: '#/components/schemas/SearchHit' + } + } + } + }, + SearchResponse: { + type: 'object', + additionalProperties: false, + required: ['page', 'pageSize', 'totalPages', 'status', 'data'], + properties: { + page: { + type: 'integer', + minimum: 0, + description: 'One-based page number, or 0 when the search has no matches.' + }, + pageSize: { + type: 'integer', + minimum: 0, + description: 'Number of CVE records returned in this response.' + }, + totalPages: { + type: 'integer', + minimum: 0 + }, + status: { + type: 'string', + enum: ['ok'] + }, + data: { + type: 'object', + additionalProperties: true, + required: ['hits'], + properties: { + took: { + type: 'number', + minimum: 0 + }, + hits: { + $ref: '#/components/schemas/SearchHits' + } + } + } + } + }, + WebSearchResponse: { + type: 'object', + additionalProperties: false, + required: ['searchMetadata', 'resultsTotal', 'data'], + properties: { + searchMetadata: { + type: 'object', + additionalProperties: true, + required: ['searchStatus', 'version', 'errors'], + properties: { + searchStatus: { + type: 'string' + }, + version: { + type: 'string' + }, + errors: { + type: 'array', + items: { + $ref: '#/components/schemas/ApiError' + } + } + } + }, + resultsTotal: { + type: 'integer', + minimum: 0 + }, + data: { + type: 'array', + items: { + type: 'object', + additionalProperties: true + } + } + } + }, SearchRequest: { type: 'object', description: diff --git a/server.js b/server.js index 4c94a56..4680dcc 100644 --- a/server.js +++ b/server.js @@ -5,7 +5,6 @@ const swaggerUi = require('swagger-ui-express'); const openApiSpecification = require('./api-docs/openapi.json'); const http = require('http'); const logger = require('morgan'); -const path = require('path'); const router = require('./routes/index'); const errors = require('./utils/errors'); const requestLogging = require('./utils/requestLogging'); @@ -56,7 +55,6 @@ function createApp() { app.use(requestLogging.assignRequestId); app.use(createRequestLogger()); app.use(cors()); - app.use(express.static(path.join(__dirname, 'public'))); app.use(express.json()); app.get('/api-docs/openapi.json', (req, res) => res.json(openApiSpecification)); diff --git a/test/postman/CVE-Search-API.postman_collection.json b/test/postman/CVE-Search-API.postman_collection.json index d4a90fa..14b8c4e 100644 --- a/test/postman/CVE-Search-API.postman_collection.json +++ b/test/postman/CVE-Search-API.postman_collection.json @@ -2,7 +2,7 @@ "info": { "_postman_id": "28911974-2b15-4ddf-a88f-3fa6fe001a50", "name": "CVE Search API - Manual Contract Tests", - "description": "Positive filter coverage, intentional combinations, validation failures, and opt-in fault-injection checks for POST /search. The deprecated /webSearch endpoint is intentionally excluded.", + "description": "Positive filter coverage, intentional combinations, validation failures, and opt-in fault-injection checks for POST /search. The separate /webSearch contract is intentionally excluded.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, "variable": [ diff --git a/test/postman/README.md b/test/postman/README.md index 4aa8afc..d9b4cf5 100644 --- a/test/postman/README.md +++ b/test/postman/README.md @@ -1,6 +1,6 @@ # Postman Manual Contract Tests -This directory contains an importable Postman collection for manually validating the CVE Search API. It covers every documented `POST /search` request field, representative intentional filter combinations, and the API's deterministic validation errors. The deprecated `/webSearch` endpoint is intentionally excluded. +This directory contains an importable Postman collection for manually validating the CVE Search API. It covers every documented `POST /search` request field, representative intentional filter combinations, and the API's deterministic validation errors. The separate `/webSearch` contract is intentionally excluded. ## Files diff --git a/test/postman/buildCollection.js b/test/postman/buildCollection.js index f58b62f..94b2095 100644 --- a/test/postman/buildCollection.js +++ b/test/postman/buildCollection.js @@ -1145,7 +1145,7 @@ const collection = { _postman_id: '28911974-2b15-4ddf-a88f-3fa6fe001a50', name: 'CVE Search API - Manual Contract Tests', description: - 'Positive filter coverage, intentional combinations, validation failures, and opt-in fault-injection checks for POST /search. The deprecated /webSearch endpoint is intentionally excluded.', + 'Positive filter coverage, intentional combinations, validation failures, and opt-in fault-injection checks for POST /search. The separate /webSearch contract is intentionally excluded.', schema: 'https://schema.getpostman.com/json/collection/v2.1.0/collection.json' }, variable: [ diff --git a/utils/cvss.js b/utils/cvss.js index d3f2ca5..0641e7d 100644 --- a/utils/cvss.js +++ b/utils/cvss.js @@ -13,12 +13,9 @@ const PARAMS = Object.freeze(Object.keys(VECTOR_FIELDS)); function parseMetricTokens(inputVectorString) { return String(inputVectorString ?? '') .split('/') - .map((token) => token.trim()) + .map((token) => token.trim().toUpperCase()) .filter(Boolean) - .filter((token) => { - const normalized = token.toUpperCase(); - return !normalized.startsWith('CVSS:') && !normalized.startsWith('CVSS#'); - }); + .filter((token) => !token.startsWith('CVSS:') && !token.startsWith('CVSS#')); } function buildPostFilter(params) { @@ -34,7 +31,10 @@ function buildPostFilter(params) { } function vectorContainsEveryToken(vector, tokens) { - return typeof vector === 'string' && tokens.every((token) => vector.includes(token)); + if (typeof vector !== 'string') return false; + + const vectorTokens = new Set(parseMetricTokens(vector)); + return tokens.every((token) => vectorTokens.has(String(token).trim().toUpperCase())); } function metricMatches(metric, fieldNames, tokens) { diff --git a/utils/cvss.test.js b/utils/cvss.test.js index 7d961e8..98b6a6f 100644 --- a/utils/cvss.test.js +++ b/utils/cvss.test.js @@ -36,6 +36,16 @@ describe('CVSS semantic vector matching', () => { expect(cvss.hitMatches(hit([{ cvssV3_1: { vectorString: 'CVSS:3.1/AV:N/AC:L' } }]), filter)).toBe(true); }); + test('matches complete metric tokens without confusing base and modified metrics', () => { + const filter = cvss.buildPostFilter({ + scope: 'ADP_ONLY', + cvssV3_1Metrics: 'AV:N' + }); + + expect(cvss.hitMatches(hit([{ cvssV3_1: { vectorString: 'CVSS:3.1/AV:L/MAV:N' } }]), filter)).toBe(false); + expect(cvss.hitMatches(hit([{ cvssV3_1: { vectorString: 'CVSS:3.1/AV:N/MAV:L' } }]), filter)).toBe(true); + }); + test('returns only source fields required by supplied vector filters and scope', () => { const filter = cvss.buildPostFilter({ scope: 'CNA_ONLY', diff --git a/utils/errors.js b/utils/errors.js index d942253..292c161 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -301,7 +301,7 @@ class errors { } webSearchDisabledError() { - return makeError('WEB_SEARCH_DISABLED', 'The /webSearch endpoint is disabled. Use POST /search instead.'); + return makeError('WEB_SEARCH_DISABLED', 'The /webSearch endpoint is disabled by service configuration.'); } openSearchError() { diff --git a/utils/errors.test.js b/utils/errors.test.js index 128e6ad..aae9e04 100644 --- a/utils/errors.test.js +++ b/utils/errors.test.js @@ -368,7 +368,7 @@ describe('error response helpers', () => { [], { error: 'WEB_SEARCH_DISABLED', - message: 'The /webSearch endpoint is disabled. Use POST /search instead.' + message: 'The /webSearch endpoint is disabled by service configuration.' } ], [ diff --git a/utils/searchProvider.js b/utils/searchProvider.js index 98e373d..8616d73 100644 --- a/utils/searchProvider.js +++ b/utils/searchProvider.js @@ -1,6 +1,7 @@ const { AppConfig, BasicSearchManager, SearchProviderSpec, SearchReader } = require('cve-core'); const constants = require('./constants'); const searchDeadline = require('./searchDeadline'); +const searchReadiness = require('./searchReadiness'); const searchResponseValidator = require('./searchResponseValidator'); class SearchProviderContractError extends Error { @@ -95,8 +96,9 @@ function getBasicSearchManager() { function validateSearchReader(reader) { const hasIndex = typeof reader?._cveIndex === 'string' && reader._cveIndex.trim() !== ''; const hasSearchClient = typeof reader?._client?.search === 'function'; + const hasFieldCapsClient = typeof reader?._client?.fieldCaps === 'function'; - if (!hasIndex || !hasSearchClient) { + if (!hasIndex || !hasSearchClient || !hasFieldCapsClient) { throw new SearchProviderContractError(); } @@ -181,6 +183,25 @@ async function searchOpenSearch(body, { signal } = {}) { async function checkOpenSearchReadiness({ signal } = {}) { searchDeadline.throwIfAborted(signal); const reader = getSearchReader(); + const requestOptions = { + maxRetries: 0, + requestTimeout: getOpenSearchReadinessTimeoutMs() + }; + const fieldCapsResponse = await executeOpenSearchRequest( + () => + reader._client.fieldCaps( + { + index: reader._cveIndex, + fields: searchReadiness.getRequiredFieldNames().join(','), + include_unmapped: true + }, + requestOptions + ), + { signal } + ); + searchReadiness.validateFieldCapabilities(fieldCapsResponse.body); + searchDeadline.throwIfAborted(signal); + const response = await executeOpenSearchRequest( () => reader._client.search( @@ -196,10 +217,7 @@ async function checkOpenSearchReadiness({ signal } = {}) { track_total_hits: false } }, - { - maxRetries: 0, - requestTimeout: getOpenSearchReadinessTimeoutMs() - } + requestOptions ), { signal } ); @@ -210,7 +228,12 @@ async function checkOpenSearchReadiness({ signal } = {}) { } function isSearchProviderContractError(err) { - return err instanceof SearchProviderContractError || err?.code === 'SEARCH_PROVIDER_CONTRACT'; + return ( + err instanceof SearchProviderContractError || + err instanceof searchReadiness.SearchMappingContractError || + err?.code === 'SEARCH_PROVIDER_CONTRACT' || + err?.code === 'SEARCH_MAPPING_CONTRACT' + ); } function getOpenSearchReadinessTimeoutMs() { diff --git a/utils/searchProvider.test.js b/utils/searchProvider.test.js index d18f6e5..dcaada4 100644 --- a/utils/searchProvider.test.js +++ b/utils/searchProvider.test.js @@ -10,11 +10,13 @@ const mockBasicSearchManager = jest.fn(function BasicSearchManager(spec) { }; }); const mockClientSearch = jest.fn(); +const mockClientFieldCaps = jest.fn(); const mockSearchReader = jest.fn(function SearchReader(endpoint, index, options) { this._openSearchDomainEndpoint = endpoint; this._cveIndex = index; this._options = options; this._client = { + fieldCaps: mockClientFieldCaps, search: mockClientSearch, transport: { maxRetries: 3, @@ -41,6 +43,7 @@ jest.mock('cve-core', () => ({ const searchProvider = require('./searchProvider'); const searchDeadline = require('./searchDeadline'); +const searchReadiness = require('./searchReadiness'); function deferred() { const state = {}; @@ -69,12 +72,32 @@ function openSearchResponse(hits = []) { }; } +function fieldCapsResponse() { + return { + body: { + fields: Object.fromEntries( + searchReadiness.FIELD_CAPABILITY_REQUIREMENTS.map((requirement) => [ + requirement.field, + { + [requirement.types[0]]: { + searchable: true, + aggregatable: requirement.aggregatable === true + } + } + ]) + ) + } + }; +} + describe('searchProvider', () => { beforeEach(() => { searchProvider.resetSearchProviderForTests(); mockBasicSearchManager.mockClear(); mockSearchReader.mockClear(); mockClientSearch.mockReset(); + mockClientFieldCaps.mockReset(); + mockClientFieldCaps.mockResolvedValue(fieldCapsResponse()); mockGetDefaultSearchProviderSpec.mockReset(); mockGetDefaultSearchProviderSpec.mockReturnValue({ providerEndpoint: 'https://search.example', @@ -160,6 +183,17 @@ describe('searchProvider', () => { await expect(searchProvider.checkOpenSearchReadiness()).resolves.toBe(true); + expect(mockClientFieldCaps).toHaveBeenCalledWith( + { + index: 'test-index', + fields: searchReadiness.getRequiredFieldNames().join(','), + include_unmapped: true + }, + { + maxRetries: 0, + requestTimeout: 5000 + } + ); expect(mockClientSearch).toHaveBeenCalledWith( { index: 'test-index', @@ -187,11 +221,24 @@ describe('searchProvider', () => { await expect(searchProvider.checkOpenSearchReadiness()).rejects.toBe(unavailable); }); - test('aborts the readiness transport request when its lifecycle is canceled', async () => { + test('fails readiness before searching when required field mappings are unavailable', async () => { + mockClientFieldCaps.mockResolvedValue({ + body: { + fields: {} + } + }); + + await expect(searchProvider.checkOpenSearchReadiness()).rejects.toMatchObject({ + code: 'SEARCH_MAPPING_CONTRACT' + }); + expect(mockClientSearch).not.toHaveBeenCalled(); + }); + + test('aborts the readiness field-capabilities request when its lifecycle is canceled', async () => { const pending = deferred(); const request = pending.promise; request.abort = jest.fn(); - mockClientSearch.mockReturnValue(request); + mockClientFieldCaps.mockReturnValue(request); const controller = new AbortController(); const readiness = searchProvider.checkOpenSearchReadiness({ signal: controller.signal diff --git a/utils/searchQueryBuilder.js b/utils/searchQueryBuilder.js index 8e24b8d..1a95324 100644 --- a/utils/searchQueryBuilder.js +++ b/utils/searchQueryBuilder.js @@ -59,7 +59,14 @@ function buildOpenSearchBody(searchOptions, queryStringObjects = [], { sourceInc }, track_total_hits: trackTotalHits, size: searchOptions.resultsPerPage, - from: searchOptions.from + from: searchOptions.from, + sort: [ + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ] }; if (sourceIncludes?.length) { @@ -242,36 +249,20 @@ function escapeQueryStringTerm(term, { escapeWildcards = false } = {}) { } function buildQueryStrings(params, scope = searchScope.SEARCH_SCOPES.CNA_ONLY) { - const queryStringObjects = []; - - Object.keys(params).forEach((param) => { - if (param === 'virtualMatchString') return; + if (params?.keywordSearch === undefined) return []; - const fieldSpec = searchScope.resolveFields(param, scope); - const fieldList = Array.isArray(fieldSpec) ? fieldSpec : [fieldSpec]; - if (!fieldList.length) return; - - if (param === 'keywordSearch') { - queryStringObjects.push({ - simple_query_string: { - query: params[param], - default_operator: 'AND', - fields: fieldList - } - }); - return; - } + const fieldList = searchScope.resolveFields('keywordSearch', scope); + if (!fieldList.length) return []; - queryStringObjects.push({ - query_string: { - query: params[param], + return [ + { + simple_query_string: { + query: params.keywordSearch, default_operator: 'AND', fields: fieldList } - }); - }); - - return queryStringObjects; + } + ]; } function getFilterFields(params, scope = searchScope.SEARCH_SCOPES.CNA_ONLY) { @@ -369,6 +360,10 @@ function escapeForWildcard(s) { return String(s).replace(/([\\*?])/g, '\\$1'); } +function escapeForRegexp(s) { + return String(s).replace(/([\\.?+*|{}[\]()"#@&<>~])/g, '\\$1'); +} + function parseCveIds(value) { return String(value ?? '') .split(',') @@ -639,8 +634,11 @@ function buildCvssWildcardFilter(inputVectorString, fieldSpec) { return { bool: { filter: tokens.map((t) => ({ - wildcard: { - [keywordField]: { value: `*${escapeForWildcard(t)}*` } + regexp: { + [keywordField]: { + value: `(.*/)?${escapeForRegexp(t)}(/.*)?`, + case_insensitive: true + } } })) } diff --git a/utils/searchQueryBuilder.test.js b/utils/searchQueryBuilder.test.js index c039a05..335aae7 100644 --- a/utils/searchQueryBuilder.test.js +++ b/utils/searchQueryBuilder.test.js @@ -20,6 +20,17 @@ function prefixShould(prefixes) { }; } +function cvssTokenRegexp(field, token) { + return { + regexp: { + [`${field}.keyword`]: { + value: `(.*/)?${token}(/.*)?`, + case_insensitive: true + } + } + }; +} + describe('searchQueryBuilder buildSearchRequest()', () => { test('buildSearchRequest builds filters, query strings, date ranges, and pagination options', () => { const output = queryBuilder.buildSearchRequest({ @@ -225,6 +236,13 @@ describe('searchQueryBuilder buildSearchRequest()', () => { track_total_hits: false, size: 10000, from: 0, + sort: [ + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ], _source: [ 'cveMetadata.cveId', 'containers.cna.cpeApplicability.nodes.cpeMatch.criteria', @@ -269,7 +287,14 @@ describe('searchQueryBuilder buildSearchRequest()', () => { }, track_total_hits: true, size: 10, - from: 20 + from: 20, + sort: [ + { + 'cveMetadata.cveId.keyword': { + order: 'asc' + } + } + ] }); }); @@ -433,7 +458,7 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds order-independent wildcard filters for cvssV2 metrics.', () => { + test('getFilterFields() builds token-boundary filters for cvssV2 metrics.', () => { const output = queryBuilder.getFilterFields({ cvssV2Metrics: 'AV:N/AC:L/Au:N' }); @@ -442,27 +467,9 @@ describe('searchQueryBuilder getFilterFields() function', () => { { bool: { filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*AC:L*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV2_0.vectorString.keyword': { - value: '*Au:N*' - } - } - } + cvssTokenRegexp('containers.cna.metrics.cvssV2_0.vectorString', 'AV:N'), + cvssTokenRegexp('containers.cna.metrics.cvssV2_0.vectorString', 'AC:L'), + cvssTokenRegexp('containers.cna.metrics.cvssV2_0.vectorString', 'AU:N') ] } } @@ -638,7 +645,7 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() builds order-independent wildcard filters for combined cvssV3 metrics.', () => { + test('getFilterFields() builds token-boundary filters for combined cvssV3 metrics.', () => { const output = queryBuilder.getFilterFields({ cvssV3Metrics: 'CVSS:3.1/AV:N/AC:L' }); @@ -650,40 +657,16 @@ describe('searchQueryBuilder getFilterFields() function', () => { { bool: { filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV3_0.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV3_0.vectorString.keyword': { - value: '*AC:L*' - } - } - } + cvssTokenRegexp('containers.cna.metrics.cvssV3_0.vectorString', 'AV:N'), + cvssTokenRegexp('containers.cna.metrics.cvssV3_0.vectorString', 'AC:L') ] } }, { bool: { filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV3_1.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV3_1.vectorString.keyword': { - value: '*AC:L*' - } - } - } + cvssTokenRegexp('containers.cna.metrics.cvssV3_1.vectorString', 'AV:N'), + cvssTokenRegexp('containers.cna.metrics.cvssV3_1.vectorString', 'AC:L') ] } } @@ -694,7 +677,7 @@ describe('searchQueryBuilder getFilterFields() function', () => { ]); }); - test('getFilterFields() escapes wildcard metacharacters in CVSS metric filters.', () => { + test('getFilterFields() escapes regexp metacharacters in CVSS metric filters.', () => { const output = queryBuilder.getFilterFields({ cvssV4Metrics: 'CVSS#4.0/AV:N/AC:*?/UI:P' }); @@ -703,27 +686,9 @@ describe('searchQueryBuilder getFilterFields() function', () => { { bool: { filter: [ - { - wildcard: { - 'containers.cna.metrics.cvssV4_0.vectorString.keyword': { - value: '*AV:N*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV4_0.vectorString.keyword': { - value: '*AC:\\*\\?*' - } - } - }, - { - wildcard: { - 'containers.cna.metrics.cvssV4_0.vectorString.keyword': { - value: '*UI:P*' - } - } - } + cvssTokenRegexp('containers.cna.metrics.cvssV4_0.vectorString', 'AV:N'), + cvssTokenRegexp('containers.cna.metrics.cvssV4_0.vectorString', 'AC:\\*\\?'), + cvssTokenRegexp('containers.cna.metrics.cvssV4_0.vectorString', 'UI:P') ] } } diff --git a/utils/searchReadiness.js b/utils/searchReadiness.js new file mode 100644 index 0000000..095ccfb --- /dev/null +++ b/utils/searchReadiness.js @@ -0,0 +1,133 @@ +const searchScope = require('./searchScope'); + +const NUMERIC_FIELD_TYPES = Object.freeze([ + 'byte', + 'short', + 'integer', + 'long', + 'half_float', + 'float', + 'double', + 'scaled_float' +]); + +function unique(values) { + return [...new Set(values)]; +} + +function keywordRequirements(params) { + return unique(params.flatMap((param) => searchScope.resolveFields(param))).map((field) => ({ + field: `${field}.keyword`, + types: ['keyword'] + })); +} + +function containerKeywordRequirements(relativePaths) { + return unique(relativePaths.flatMap((path) => searchScope.resolveContainerPaths(path))).map((field) => ({ + field: `${field}.keyword`, + types: ['keyword'] + })); +} + +function containerDateRequirements(relativePaths) { + return unique(relativePaths.flatMap((path) => searchScope.resolveContainerPaths(path))).map((field) => ({ + field, + types: ['date'] + })); +} + +const FIELD_CAPABILITY_REQUIREMENTS = Object.freeze([ + { + field: 'cveMetadata.cveId.keyword', + types: ['keyword'], + aggregatable: true + }, + { field: 'cveMetadata.datePublished', types: ['date'] }, + { field: 'cveMetadata.dateUpdated', types: ['date'] }, + ...keywordRequirements([ + 'noRejected', + 'cvssV3Severity', + 'cvssV3_0Severity', + 'cvssV3_1Severity', + 'cvssV4Severity', + 'cvssV2Metrics', + 'cvssV3Metrics', + 'cvssV3_0Metrics', + 'cvssV3_1Metrics', + 'cvssV4Metrics', + 'cpeName', + 'virtualMatchString', + 'referenceURL', + 'disputed' + ]), + ...unique(searchScope.resolveFields('keywordSearch')).map((field) => ({ + field, + types: ['text', 'keyword'] + })), + ...unique(searchScope.resolveFields('cweId')).map((field) => ({ + field, + types: ['text', 'keyword'] + })), + ...unique(searchScope.resolveFields('cvssV2Severity')).map((field) => ({ + field, + types: NUMERIC_FIELD_TYPES + })), + ...unique(searchScope.resolveFields('isVulnerable')).map((field) => ({ + field, + types: ['boolean'] + })), + ...containerKeywordRequirements([ + 'metrics.other.type', + 'metrics.other.content.options.Exploitation', + 'metrics.other.content.options.Automatable', + 'metrics.other.content.options.Technical Impact', + 'metrics.other.content.role' + ]), + ...containerDateRequirements(['metrics.other.content.timestamp', 'metrics.other.content.dateAdded']) +]); + +class SearchMappingContractError extends Error { + constructor(fields) { + super('Configured CVE index does not expose the field mappings required by the Search API.'); + this.name = 'SearchMappingContractError'; + this.code = 'SEARCH_MAPPING_CONTRACT'; + this.fields = fields; + } +} + +function supportsRequirement(capabilities, requirement) { + if (!capabilities || typeof capabilities !== 'object') return false; + + return requirement.types.some((type) => { + const capability = capabilities[type]; + return ( + capability?.searchable === true && (requirement.aggregatable !== true || capability.aggregatable === true) + ); + }); +} + +function validateFieldCapabilities(body) { + const capabilities = body?.fields; + if (!capabilities || typeof capabilities !== 'object' || Array.isArray(capabilities)) { + throw new SearchMappingContractError(FIELD_CAPABILITY_REQUIREMENTS.map(({ field }) => field)); + } + + const incompatibleFields = FIELD_CAPABILITY_REQUIREMENTS.filter( + (requirement) => !supportsRequirement(capabilities[requirement.field], requirement) + ).map(({ field }) => field); + + if (incompatibleFields.length) throw new SearchMappingContractError(incompatibleFields); + + return true; +} + +function getRequiredFieldNames() { + return FIELD_CAPABILITY_REQUIREMENTS.map(({ field }) => field); +} + +module.exports = { + FIELD_CAPABILITY_REQUIREMENTS, + SearchMappingContractError, + getRequiredFieldNames, + validateFieldCapabilities +}; diff --git a/utils/searchReadiness.test.js b/utils/searchReadiness.test.js new file mode 100644 index 0000000..ebbc643 --- /dev/null +++ b/utils/searchReadiness.test.js @@ -0,0 +1,48 @@ +const searchReadiness = require('./searchReadiness'); + +function validCapabilities() { + return { + fields: Object.fromEntries( + searchReadiness.FIELD_CAPABILITY_REQUIREMENTS.map((requirement) => [ + requirement.field, + { + [requirement.types[0]]: { + searchable: true, + aggregatable: requirement.aggregatable === true + } + } + ]) + ) + }; +} + +describe('search readiness mapping contract', () => { + test('accepts searchable fields with the required types', () => { + expect(searchReadiness.validateFieldCapabilities(validCapabilities())).toBe(true); + }); + + test('rejects missing fields', () => { + const body = validCapabilities(); + const missingField = searchReadiness.getRequiredFieldNames()[0]; + delete body.fields[missingField]; + + expect(() => searchReadiness.validateFieldCapabilities(body)).toThrow( + searchReadiness.SearchMappingContractError + ); + }); + + test('rejects a non-searchable or incompatible field type', () => { + const body = validCapabilities(); + const requirement = searchReadiness.FIELD_CAPABILITY_REQUIREMENTS.find(({ aggregatable }) => aggregatable); + body.fields[requirement.field] = { + text: { + searchable: true, + aggregatable: false + } + }; + + expect(() => searchReadiness.validateFieldCapabilities(body)).toThrow( + searchReadiness.SearchMappingContractError + ); + }); +});