Retrieves matching CVE records using the raw query expression supplied in the JSON request body. Service operators can disable this endpoint through runtime configuration. Standalone dates and date ranges search the supported record/container date fields; other expressions use raw query syntax with an AND default operator.
", + "description": "Retrieves matching CVE records using the validated query expression supplied in the JSON request body. Service operators can disable this endpoint through runtime configuration. Standalone dates and date ranges search the supported record/container date fields; other expressions use query syntax with an AND default operator. Every token is validated before execution. Regular expressions are not supported.
", "operationId": "webSearch", "responses": { "200": { @@ -297,11 +297,18 @@ } }, "400": { - "description": "Malformed JSON, unreadable request body, or invalid raw-query request body, sort, or pagination", + "description": "Malformed JSON, unreadable request body, invalid request fields, sort or pagination, or unsupported query syntax. WEB_SEARCH_QUERY_SYNTAX uses the WebSearchResponse envelope with searchMetadata.errors; request validation retains its existing error format.", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ApiErrorResponse" + "oneOf": [ + { + "$ref": "#/components/schemas/ApiErrorResponse" + }, + { + "$ref": "#/components/schemas/WebSearchResponse" + } + ] } } } @@ -380,7 +387,7 @@ "properties": { "query": { "type": "string", - "description": "Raw search expression. Examples: remote code execution, apache*, 2024-01-01, or 2024-01-01..2024-01-31. Date ranges are inclusive and allow year/month endpoints; standalone years/months and quoted dates remain text queries. Date-query years must be between 1900 and 2100.", + "description": "Search expression. Supports ordinary multi-term text, quoted phrases, Boolean expressions with parentheses, and * or ? wildcards, including leading wildcards. Recognized URLs containing slashes and simple relative paths are treated as literal text. Quote or escape literal special characters, including absolute paths. Regex expressions (including field-qualified regex), unsupported tokens in any position, unescaped braces, unclosed quotes, unbalanced parentheses, dangling escapes, and reversed date ranges return HTTP 400 WEB_SEARCH_QUERY_SYNTAX in searchMetadata.errors before backend execution. These checks do not implement the complete OpenSearch query grammar or bound the cost of allowed wildcard searches. Examples: remote code execution, apache*, 2024-01-01, or 2024-01-01..2024-01-31. Date ranges are inclusive and allow year/month endpoints; standalone years/months and quoted dates remain text queries. Date-query years must be between 1900 and 2100.", "minLength": 1 }, "sort": { diff --git a/api-docs/openapi.test.js b/api-docs/openapi.test.js index a88fbf6..521e06e 100644 --- a/api-docs/openapi.test.js +++ b/api-docs/openapi.test.js @@ -159,12 +159,20 @@ describe('OpenAPI contract', () => { const schema = operation.requestBody.content['application/json'].schema; expect(operation.deprecated).toBeUndefined(); - expect(operation.description).toContain('raw query expression'); + expect(operation.description).toContain('validated query expression'); expect(operation.parameters || []).toStrictEqual([]); expect(operation.requestBody.required).toBe(true); expect(schema.required).toStrictEqual(['query']); expect(schema.additionalProperties).toBe(false); expect(schema.properties.query.type).toBe('string'); + expect(schema.properties.query.description).toContain('WEB_SEARCH_QUERY_SYNTAX'); + expect(schema.properties.query.description).toContain('unsupported tokens in any position'); + expect(schema.properties.query.description).toContain('including leading wildcards'); + expect(schema.properties.query.description).toContain('treated as literal text'); + expect(operation.responses['400'].content['application/json'].schema.oneOf).toStrictEqual([ + { $ref: '#/components/schemas/ApiErrorResponse' }, + { $ref: '#/components/schemas/WebSearchResponse' } + ]); expect(schema.properties.sort.additionalProperties).toBe(false); expect(schema.properties.sort.properties.property.enum).toStrictEqual(['cveId', 'relevancy']); expect(schema.properties.size.maximum).toBe(constants.MAX_RESULTS_LIMIT); diff --git a/controllers/searchController.js b/controllers/searchController.js index 244e15c..74f3f51 100644 --- a/controllers/searchController.js +++ b/controllers/searchController.js @@ -230,6 +230,11 @@ async function webSearchHandler(req, res, next) { body.data = searchData.hits.hits; return res.status(200).json(body); } catch (ex) { + if (ex instanceof webSearchQueryBuilder.WebSearchQueryError) { + body.searchMetadata.errors = [error.webSearchQuerySyntaxError()]; + return res.status(400).json(body); + } + const timedOut = searchDeadline.isSearchTimeoutError(ex); const err = timedOut ? error.searchTimeoutError() : error.upstreamSearchError(); body.searchMetadata.errors = [err]; diff --git a/controllers/searchController.test.js b/controllers/searchController.test.js index 8f08d04..c110e2c 100644 --- a/controllers/searchController.test.js +++ b/controllers/searchController.test.js @@ -1403,6 +1403,8 @@ describe('Search controller WEB_SEARCH handler', () => { test.each([ [new Error('sensitive backend details'), 502, 'UPSTREAM_SEARCH_ERROR'], + [{ name: 'WebSearchQueryError', message: 'sensitive backend details' }, 502, 'UPSTREAM_SEARCH_ERROR'], + [{ statusCode: 400, message: 'sensitive backend parse error' }, 502, 'UPSTREAM_SEARCH_ERROR'], [{ name: 'TimeoutError', message: 'sensitive timeout details' }, 504, 'SEARCH_TIMEOUT'] ])('WEB_SEARCH sanitizes transport errors: %p', async (err, status, code) => { mockApiSearch.mockRejectedValue(err); @@ -1417,10 +1419,29 @@ describe('Search controller WEB_SEARCH handler', () => { expect(console.log).not.toHaveBeenCalled(); }); - test('WEB_SEARCH keeps unsupported expressions sanitized without contacting OpenSearch', async () => { - await controller.WEB_SEARCH({ body: { query: '{invalid}' } }, res, next); - expect(res.status).toHaveBeenCalledWith(502); + test.each([ + '{invalid}', + '{invalid} apache', + 'apache {invalid}', + '/fre[e]type/', + '/fre[e]type/ apache', + 'apache /fre[e]type/', + 'containers.cna.descriptions.value:/fre[e]type/ apache', + '2024-02-01..2024-01-01', + '"unterminated', + '(apache OR nginx' + ])('WEB_SEARCH rejects unsupported expressions before contacting OpenSearch: %s', async (query) => { + await controller.WEB_SEARCH({ body: { query } }, res, next); + expect(res.status).toHaveBeenCalledWith(400); expect(mockSearchOpenSearch).not.toHaveBeenCalled(); - expect(res.json.mock.calls[0][0].searchMetadata.errors[0].error).toBe('UPSTREAM_SEARCH_ERROR'); + expect(res.json.mock.calls[0][0]).toMatchObject({ + searchMetadata: { + searchStatus: 'errors', + notes: [], + errors: [{ error: 'WEB_SEARCH_QUERY_SYNTAX' }] + }, + resultsTotal: 0, + data: [] + }); }); }); diff --git a/routes/index.js b/routes/index.js index 0d69c3c..905fd9e 100644 --- a/routes/index.js +++ b/routes/index.js @@ -485,7 +485,7 @@ router.post( #swagger.summary = "Raw query search" #swagger.description = "Retrieves matching CVE records using the raw query expression supplied in the JSON request body. Service operators can disable this endpoint through runtime configuration. Standalone dates and date ranges search the supported record/container date fields; other expressions use raw query syntax with an AND default operator.
+Retrieves matching CVE records using the validated query expression supplied in the JSON request body. Service operators can disable this endpoint through runtime configuration. Standalone dates and date ranges search the supported record/container date fields; other expressions use query syntax with an AND default operator. Every token is validated before execution. Regular expressions are not supported.
#swagger.requestBody = { required: true, content: { @@ -497,7 +497,7 @@ router.post( properties: { query: { type: "string", - description: "Raw search expression. Examples: remote code execution, apache*, 2024-01-01, or 2024-01-01..2024-01-31. Date ranges are inclusive and allow year/month endpoints; standalone years/months and quoted dates remain text queries. Date-query years must be between 1900 and 2100.", + description: "Search expression. Supports ordinary multi-term text, quoted phrases, Boolean expressions with parentheses, and * or ? wildcards, including leading wildcards. Recognized URLs containing slashes and simple relative paths are treated as literal text. Quote or escape literal special characters, including absolute paths. Regex expressions (including field-qualified regex), unsupported tokens in any position, unescaped braces, unclosed quotes, unbalanced parentheses, dangling escapes, and reversed date ranges return HTTP 400 WEB_SEARCH_QUERY_SYNTAX in searchMetadata.errors before backend execution. These checks do not implement the complete OpenSearch query grammar or bound the cost of allowed wildcard searches. Examples: remote code execution, apache*, 2024-01-01, or 2024-01-01..2024-01-31. Date ranges are inclusive and allow year/month endpoints; standalone years/months and quoted dates remain text queries. Date-query years must be between 1900 and 2100.", minLength: 1 }, sort: { @@ -548,11 +548,14 @@ headers: { } } #swagger.responses[400] = { -description: 'Malformed JSON, unreadable request body, or invalid raw-query request body, sort, or pagination', +description: 'Malformed JSON, unreadable request body, invalid request fields, sort or pagination, or unsupported query syntax. WEB_SEARCH_QUERY_SYNTAX uses the WebSearchResponse envelope with searchMetadata.errors; request validation retains its existing error format.', content: { "application/json": { schema: { - $ref: "#/components/schemas/ApiErrorResponse" + oneOf: [ + { $ref: "#/components/schemas/ApiErrorResponse" }, + { $ref: "#/components/schemas/WebSearchResponse" } + ] } } }, diff --git a/routes/search.integration.test.js b/routes/search.integration.test.js index b16faee..e115030 100644 --- a/routes/search.integration.test.js +++ b/routes/search.integration.test.js @@ -555,6 +555,51 @@ describe('search route integration', () => { } ); + test.each([ + '/fre[e]type/', + '/fre[e]type/ apache', + 'apache /fre[e]type/', + '(apache OR /fre[e]type/)', + 'containers.cna.descriptions.value:/fre[e]type/ apache', + '{invalid} apache', + 'apache {invalid}', + 'field:value apache', + '"unterminated', + '(apache OR nginx', + '2024-02-01..2024-01-01' + ])('POST /webSearch rejects unsupported syntax before searching: %s', async (query) => { + const response = await request(createApp(), { method: 'POST', path: '/webSearch', body: { query } }); + + expect(response.status).toBe(400); + expect(response.body).toMatchObject({ + searchMetadata: { searchStatus: 'errors', errors: [{ error: 'WEB_SEARCH_QUERY_SYNTAX' }] }, + resultsTotal: 0, + data: [] + }); + expect(mockSearchOpenSearch).not.toHaveBeenCalled(); + }); + + test.each([ + ['remote code execution', 'remote code execution'], + ['apache*', 'apache*'], + ['*etype', '*etype'], + ['ap?che', 'ap?che'], + ['apache AND (nginx OR "remote code")', 'apache AND (nginx OR "remote code")'], + ['"/fre[e]type/"', '"/fre[e]type/"'], + ['\\/fre[e]type\\/', '\\/fre[e]type\\/'], + ['https://example.com/a?b=1', '"https://example.com/a?b=1"'] + ])('POST /webSearch preserves supported query behavior: %s', async (query, executedQuery) => { + mockSearchOpenSearch.mockResolvedValue(advancedResponse(0).data); + const response = await request(createApp(), { method: 'POST', path: '/webSearch', body: { query } }); + + expect(response.status).toBe(200); + expect(mockSearchOpenSearch).toHaveBeenCalledWith( + expect.objectContaining({ + query: { query_string: { query: executedQuery, default_operator: 'AND' } } + }) + ); + }); + test('POST /webSearch normalizes fields, builds DSL, and formats OpenSearch responses', async () => { mockSearchOpenSearch.mockResolvedValue(advancedResponse(1, [{ _id: 'CVE-2024-0002' }]).data); diff --git a/utils/errors.js b/utils/errors.js index 6710d26..17449e9 100644 --- a/utils/errors.js +++ b/utils/errors.js @@ -337,6 +337,13 @@ class errors { return makeError('WEB_SEARCH_QUERY_REQUIRED', 'The query field is required and must be a non-empty string.'); } + webSearchQuerySyntaxError() { + return makeError( + 'WEB_SEARCH_QUERY_SYNTAX', + 'The query contains unsupported or malformed syntax. Regular expressions and unescaped braces are not supported. Quote or escape literal special characters.' + ); + } + webSearchSortPropertyError() { return makeError('WEB_SEARCH_SORT_PROPERTY', 'The sort.property field must be cveId or relevancy.'); } diff --git a/utils/errors.test.js b/utils/errors.test.js index 9e1b0c2..7c0767e 100644 --- a/utils/errors.test.js +++ b/utils/errors.test.js @@ -323,6 +323,15 @@ describe('error response helpers', () => { message: 'The query field is required and must be a non-empty string.' } ], + [ + 'webSearchQuerySyntaxError', + [], + { + error: 'WEB_SEARCH_QUERY_SYNTAX', + message: + 'The query contains unsupported or malformed syntax. Regular expressions and unescaped braces are not supported. Quote or escape literal special characters.' + } + ], [ 'webSearchSortPropertyError', [], diff --git a/utils/webSearchQueryBuilder.js b/utils/webSearchQueryBuilder.js index ecf8a3b..c475c90 100644 --- a/utils/webSearchQueryBuilder.js +++ b/utils/webSearchQueryBuilder.js @@ -16,6 +16,13 @@ const DATE_FIELDS = Object.freeze([ ]); const URL_OPTIONS = { protocols: ['http', 'https', 'ftp', 'app', 'wss', 'ws', 'file'] }; +class WebSearchQueryError extends Error { + constructor() { + super('Unsupported or malformed web search expression.'); + this.name = 'WebSearchQueryError'; + } +} + function parseDate(value, allowPartial = false) { const text = value.trim(); const match = @@ -59,7 +66,7 @@ function parseDateRange(text) { if (parts.length !== 2) return null; const [start, end] = parts.map((part) => parseDate(part, true)); if (!start || !end) return null; - if (start.timestamp > end.timestamp) throw new Error('Invalid web search date range.'); + if (start.timestamp > end.timestamp) throw new WebSearchQueryError(); return [start, end]; } @@ -73,7 +80,6 @@ function isUrl(text) { } function describeToken(token) { - if (token.includes('{')) return null; if (/"[^"]*"/.test(token)) return 'search text is a phrase (surrounded by double quotes)'; if (parseDate(token)) return 'search text is a date (ISO 8601)'; if (parseDateRange(token)) return 'search text is a date range (ISO 8601)'; @@ -96,25 +102,95 @@ function describeToken(token) { return 'general search text string'; } -function describeQuery(query) { - const cleaned = query.trim().replace(/([^\p{L}0-9.-])\1{2,}/gu, ''); - const tokens = (cleaned.match(/"[^"]*"|[\S]+/g) ?? []).filter((token) => token !== ','); - let note; - for (let token of tokens) { - if (token.startsWith('"') && token.endsWith('"')) { - token = token.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, '\\'); +function validateToken(token) { + // URLs and relative paths are literal text, not slash-delimited query operators. + const url = token.includes('/') && isUrl(token) && !/["\\]/.test(token); + const relativePath = /^[\p{L}\p{N}_.-]+(?:\/[\p{L}\p{N}_.-]+)+$/u.test(token); + if (url || relativePath) { + return { + query: `"${token}"`, + note: url ? 'search text is a URL' : 'search text is a literal path' + }; + } + + let quoted = false; + let escaped = false; + let syntax = ''; + for (const character of token) { + if (escaped) { + syntax += 'x'; + escaped = false; + } else if (character === '\\') { + escaped = true; + } else if (character === '"') { + quoted = !quoted; + syntax += '"'; + } else if (quoted) { + syntax += 'x'; + } else { + if ('/{}'.includes(character)) throw new WebSearchQueryError(); + syntax += character; + } + } + if (quoted || escaped) throw new WebSearchQueryError(); + + const note = describeToken(syntax); + if (!note) throw new WebSearchQueryError(); + return { query: token, note }; +} + +function validateQuery(query) { + const parts = []; + const notes = new Set(); + let token = ''; + let quoted = false; + let escaped = false; + let depth = 0; + function flushToken() { + if (!token) return; + const validated = validateToken(token); + parts.push(validated.query); + if (!['AND', 'OR', 'NOT', '&&', '||'].includes(token)) notes.add(validated.note); + token = ''; + } + + // Preserve quoted and escaped characters while checking every unquoted token. + for (const character of query) { + if (escaped) { + token += character; + escaped = false; + } else if (character === '\\') { + token += character; + escaped = true; + } else if (character === '"') { + token += character; + quoted = !quoted; + } else if (!quoted && /\s/.test(character)) { + flushToken(); + parts.push(character); + } else if (!quoted && '()'.includes(character)) { + flushToken(); + depth += character === '(' ? 1 : -1; + if (depth < 0) throw new WebSearchQueryError(); + parts.push(character); + } else { + token += character; } - // Preserve the legacy final-token note/acceptance behavior; the DSL uses the original query. - note = describeToken(token); } - if (!note) throw new Error('Unsupported web search expression.'); - return [note]; + if (quoted || escaped || depth !== 0) throw new WebSearchQueryError(); + flushToken(); + if (!notes.size || parts.every((part) => /^[\s,()]*$/.test(part))) throw new WebSearchQueryError(); + return { query: parts.join(''), notes: [...notes] }; } function buildWebSearchRequest(query, { sort, from = 0, size = WEB_SEARCH_RESULTS_PER_PAGE } = {}) { - const notes = describeQuery(query); + if (typeof query !== 'string' || !query.trim()) throw new WebSearchQueryError(); + query = query.trim(); const date = parseDate(query); const range = date ? [date, date] : parseDateRange(query); + const validated = range + ? { notes: [date ? 'search text is a date (ISO 8601)' : 'search text is a date range (ISO 8601)'] } + : validateQuery(query); const searchQuery = range ? { bool: { @@ -124,7 +200,7 @@ function buildWebSearchRequest(query, { sort, from = 0, size = WEB_SEARCH_RESULT minimum_should_match: 1 } } - : { query_string: { query, default_operator: 'AND' } }; + : { query_string: { query: validated.query, default_operator: 'AND' } }; const offset = Math.max(0, from); return { @@ -135,8 +211,8 @@ function buildWebSearchRequest(query, { sort, from = 0, size = WEB_SEARCH_RESULT from: offset, size }, - notes + notes: validated.notes }; } -module.exports = { buildWebSearchRequest }; +module.exports = { buildWebSearchRequest, WebSearchQueryError }; diff --git a/utils/webSearchQueryBuilder.test.js b/utils/webSearchQueryBuilder.test.js index 5c85777..7dd6086 100644 --- a/utils/webSearchQueryBuilder.test.js +++ b/utils/webSearchQueryBuilder.test.js @@ -1,4 +1,4 @@ -const { buildWebSearchRequest } = require('./webSearchQueryBuilder'); +const { buildWebSearchRequest, WebSearchQueryError } = require('./webSearchQueryBuilder'); const dateFields = [ 'cveMetadata.datePublished', @@ -20,18 +20,28 @@ describe('raw web search compatibility', () => { 'remote code execution', '"remote code execution"', 'apache*', + '*etype', 'ap?che', 'apache AND nginx', '(apache OR nginx)', 'cveMetadata.cveId.keyword:CVE-2024-0001', 'CVE-2024-0001', - 'https://example.com/a?b=1', '192.0.2.1', '2001:db8::1', '2024', '2024-01', '2024-02-30', - 'prefix\\:value apache' + 'prefix\\:value apache', + '"/fre[e]type/"', + 'containers.cna.descriptions.value:"/fre[e]type/"', + '\\/fre[e]type\\/', + '"/usr/local/bin"', + 'src\\/components\\/search.js', + '"{reserved}" apache', + '\\{reserved\\} apache', + '"a \\"quoted\\" phrase"', + '"(apache OR nginx)"', + 'apache AND (nginx OR "remote code")' ])('preserves the original query expression: %s', (query) => { expect(buildWebSearchRequest(query).body).toStrictEqual({ query: { query_string: { query, default_operator: 'AND' } }, @@ -44,7 +54,7 @@ describe('raw web search compatibility', () => { test.each([ ['apache', 'general search text string'], - ['"remote code"', 'general search text string'], + ['"remote code"', 'search text is a phrase (surrounded by double quotes)'], ['CVE-2024-0001', 'search text is a CVE ID'], ['CVE-2024', 'search text is a CVE ID year series'], ['CWE-79', 'search text is a CWE ID'], @@ -59,12 +69,37 @@ describe('raw web search compatibility', () => { ['some-file', 'search text is a version string'], ['file.txt', 'search text is a URL'], ['a&b', 'search text is a phrase (surrounded by double quotes)'], - ['"a\\\\b"', 'general search text string'], - ['{reserved} apache', 'general search text string'] - ])('preserves the legacy description for %s', (query, note) => { + ['"a\\\\b"', 'search text is a phrase (surrounded by double quotes)'] + ])('describes supported input: %s', (query, note) => { expect(buildWebSearchRequest(query).notes).toStrictEqual([note]); }); + test('describes every query term rather than just the final token', () => { + expect(buildWebSearchRequest('apache* AND "remote code" OR CVE-2024-0001 apache').notes).toStrictEqual([ + "search as wildcard ('*')", + 'search text is a phrase (surrounded by double quotes)', + 'search text is a CVE ID', + 'general search text string' + ]); + }); + + test.each([ + 'https://example.com/a?b=1', + 'https://example.com/fre[e]type/', + 'https://example.com/*', + 'src/components/search.js' + ])('serializes URLs and relative paths as literal text: %s', (query) => { + expect(buildWebSearchRequest(query).body.query).toStrictEqual({ + query_string: { query: `"${query}"`, default_operator: 'AND' } + }); + }); + + test('executes the validated URL representation within a multi-term query', () => { + expect(buildWebSearchRequest('apache https://example.com/a?b=1').body.query).toStrictEqual({ + query_string: { query: 'apache "https://example.com/a?b=1"', default_operator: 'AND' } + }); + }); + test.each([ ['2024-02-29', '2024-02-29||/d', '2024-02-29||/d'], ['2024-01-01T12:30:00Z', '2024-01-01T12:30:00Z', '2024-01-01T12:30:00Z'], @@ -83,12 +118,52 @@ describe('raw web search compatibility', () => { }); }); - test.each(['', ' ', ',', '{reserved}', 'field:value', '2024-02-01..2024-01-01'])( - 'rejects unsupported input without sending a query: %s', - (query) => { - expect(() => buildWebSearchRequest(query)).toThrow(); - } - ); + test.each([ + '', + ' ', + ',', + '()', + 'AND', + '{reserved}', + '{reserved} apache', + 'apache {reserved}', + '***{reserved} apache', + '"safe"{reserved} apache', + 'apache }', + 'field:value', + 'field:value apache', + 'apache field:value', + '2024-02-01..2024-01-01', + '/fre[e]type/', + '/fre[e]type/ apache', + 'apache /fre[e]type/', + 'apache OR /fre[e]type/ nginx', + 'apache\n/fre[e]type/\tnginx', + '(/fre[e]type/ OR apache)', + 'containers.cna.descriptions.value:/fre[e]type/', + 'containers.cna.descriptions.value:/fre[e]type/ apache', + '*/fre[e]type/ apache', + '"safe"/fre[e]type/ apache', + '+/fre[e]type/ apache', + '/apache/', + '/apache', + '\\\\/apache/ nginx', + '\\/apache/ nginx', + '"unterminated apache', + 'apache\\', + '(apache OR nginx', + 'apache) nginx' + ])('rejects unsupported input without sending a query: %s', (query) => { + expect(() => buildWebSearchRequest(query)).toThrow(WebSearchQueryError); + }); + + test.each([null, undefined, 1, [], {}])('rejects non-string input: %j', (query) => { + expect(() => buildWebSearchRequest(query)).toThrow(WebSearchQueryError); + }); + + test('uses the same trimmed query for validation and execution', () => { + expect(buildWebSearchRequest(' apache* ').body.query.query_string.query).toBe('apache*'); + }); test('quoted dates remain literal query expressions', () => { expect(buildWebSearchRequest('"2024-01-01"').body.query).toStrictEqual({