From 9d263b4e39199ac69bd288c28614be0d542e75cf Mon Sep 17 00:00:00 2001 From: mango <333602023+mangomac@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:58:12 -0500 Subject: [PATCH 01/11] feat: optionally include leading comments in statement text Use tokenizer offsets and keep defaults unchanged. --- src/index.ts | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/index.ts b/src/index.ts index e4288d8..9b32e2c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,7 +24,7 @@ export function identify(query: string, options: IdentifyOptions = {}): Identify // Default parameter types for each dialect const paramTypes = options.paramTypes || defaultParamTypesFor(dialect); - const result = parse( + const parsed = parse( query, isStrict, dialect, @@ -34,11 +34,40 @@ export function identify(query: string, options: IdentifyOptions = {}): Identify ); const sort = dialect === 'psql' && !options.paramTypes; - return result.body.map((statement) => { + // Use token boundaries so comment markers inside strings cannot become headers. + let tokenIndex = 0; + let previousEnd = -1; + return parsed.body.map((statement) => { + let start = statement.start; + let leadingStart: number | undefined; + let hasComment = false; + while ( + options.includeLeadingComments === true && + tokenIndex < parsed.tokens.length && + parsed.tokens[tokenIndex].start < statement.start + ) { + const token = parsed.tokens[tokenIndex++]; + if (token.end <= previousEnd) continue; + if ( + token.type === 'whitespace' || + token.type === 'comment-inline' || + token.type === 'comment-block' + ) { + if (leadingStart === undefined) leadingStart = token.start; + if (token.type !== 'whitespace') hasComment = true; + } else { + leadingStart = undefined; + hasComment = false; + } + } + if (options.includeLeadingComments === true && hasComment && leadingStart !== undefined) { + start = leadingStart; + } + previousEnd = statement.end; const result: IdentifyResult = { - start: statement.start, + start, end: statement.end, - text: query.substring(statement.start, statement.end + 1), + text: query.substring(start, statement.end + 1), type: statement.type, executionType: statement.executionType, // we want to sort the postgres params: $1 $2 $3, regardless of the order they appear From 6d0cb77da5cd644ab53649207f9a0d04a399c5fe Mon Sep 17 00:00:00 2001 From: mango <333602023+mangomac@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:59:07 -0500 Subject: [PATCH 02/11] feat: expose includeLeadingComments option --- src/defines.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/defines.ts b/src/defines.ts index c2d4ba0..be6dc1d 100644 --- a/src/defines.ts +++ b/src/defines.ts @@ -136,6 +136,7 @@ export interface TableReference { export interface IdentifyOptions { strict?: boolean; + includeLeadingComments?: boolean; dialect?: Dialect; identifyTables?: boolean; identifyColumns?: boolean; From 49116defedf175cd2858a657688c08fc2ec4de77 Mon Sep 17 00:00:00 2001 From: mango <333602023+mangomac@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:00:21 -0500 Subject: [PATCH 03/11] test: cover leading comments and unchanged defaults --- test/identifier/single-statement.spec.ts | 46 +++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/test/identifier/single-statement.spec.ts b/test/identifier/single-statement.spec.ts index c5b7f7e..8057ab3 100644 --- a/test/identifier/single-statement.spec.ts +++ b/test/identifier/single-statement.spec.ts @@ -1,6 +1,6 @@ import { expect } from 'chai'; -import { Dialect, identify } from '../../src'; +import { Dialect, identify, Options } from '../../src'; describe('identifier', () => { describe('given queries with a single statement', () => { @@ -1898,3 +1898,47 @@ describe('identifier', () => { }); }); }); + +const options = { dialect: 'mssql', includeLeadingComments: true } as Options; + +describe('includeLeadingComments', () => { + it('keeps the default keyword offsets and text', () => { + const sql = '-- header\nCREATE PROCEDURE p AS SELECT 1;'; + const actual = identify(sql, { dialect: 'mssql' })[0]; + expect(actual.start).to.equal(sql.indexOf('CREATE')); + expect(actual.text).to.equal(sql.slice(actual.start, actual.end + 1)); + }); + + it('includes leading comments and whitespace when enabled', () => { + const sql = ' -- header\n/* detail */\nCREATE PROCEDURE p AS SELECT 1;'; + const actual = identify(sql, options)[0]; + expect(actual.start).to.equal(0); + expect(actual.text).to.equal(sql); + expect(actual.type).to.equal('CREATE_PROCEDURE'); + }); + + it('does not include empty statement delimiters', () => { + const sql = '; ;\n-- header\nSELECT 1;'; + const actual = identify(sql, options)[0]; + expect(actual.start).to.equal(3); + expect(actual.text).to.equal(sql.slice(3)); + }); + + it('keeps each header with its own statement', () => { + const sql = '-- first\nCREATE VIEW a AS SELECT 1;\n/* second */\nCREATE VIEW b AS SELECT 2;'; + const actual = identify(sql, options); + expect(actual).to.have.length(2); + expect(actual[0].text).to.equal(sql.slice(0, sql.indexOf(';') + 1)); + expect(actual[1].start).to.equal(sql.indexOf(';') + 1); + expect(actual[1].text).to.equal(sql.slice(actual[1].start)); + }); + + it('does not change whitespace-only prefixes', () => { + expect(identify(' SELECT 1;', options)[0].start).to.equal(2); + }); + + it('does not treat comment markers in strings as headers', () => { + const actual = identify("SELECT '-- not a header'; SELECT 2;", options); + expect(actual[1].text).to.equal('SELECT 2;'); + }); +}); From 535bf500c22bbe388e55b4a8b1122024a51976c6 Mon Sep 17 00:00:00 2001 From: mango <333602023+mangomac@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:01:11 -0500 Subject: [PATCH 04/11] docs: explain leading comment offsets and defaults Add documentation for including leading comments in statements. --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index d8e7bd5..71e2885 100644 --- a/README.md +++ b/README.md @@ -182,3 +182,20 @@ npm test Copyright (c) 2016-2021 The SQLECTRON Team. This software is licensed under the [MIT License](https://github.com/sqlectron/sql-query-identifier/blob/master/LICENSE). + +## Leading comments + +Pass `includeLeadingComments: true` to include a statement's leading comments in +its `text`. Its `start` moves to the beginning of the comment prefix, including +adjacent whitespace. `end` does not change. Semicolons before the prefix stay out +of the statement. Whitespace alone does not change `start`. + +```js +identify('-- header\nCREATE VIEW v AS SELECT 1;', { + dialect: 'mssql', + includeLeadingComments: true, +}); +``` + +The option defaults to `false`. Existing callers still receive text and offsets +that start at the first keyword. From 3907300e04abefad26fb7882226886dec7208839 Mon Sep 17 00:00:00 2001 From: mango <333602023+mangomac@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:02:30 -0500 Subject: [PATCH 05/11] Document includeLeadingComments under API options Added option to include leading comments in SQL statements and updated related documentation. --- README.md | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 71e2885..15fbc47 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,24 @@ console.log(statements); 1. `options (object)`: allow to set different configurations 1. `strict (bool)`: allow disable strict mode which will ignore unknown types *(default=true)* 2. `dialect (string)`: Specify your database dialect, values: `generic`, `mysql`, `oracle`, `psql`, `sqlite`, `mssql`, `bigquery` and `dynamodb`. *(default=generic)* + 3. `includeLeadingComments (bool)`: Include leading comments and adjacent whitespace in statement `text` and `start`. Whitespace alone does not change `start`; `end` stays unchanged. *(default=false)* + +### Leading comments + +Pass `includeLeadingComments: true` to include a statement's leading comments in +its `text`. Its `start` moves to the beginning of the comment prefix, including +adjacent whitespace. `end` does not change. Semicolons before the prefix stay out +of the statement. Whitespace alone does not change `start`. + +```js +identify('-- header\nCREATE VIEW v AS SELECT 1;', { + dialect: 'mssql', + includeLeadingComments: true, +}); +``` + +The option defaults to `false`. Existing callers still receive text and offsets +that start at the first keyword. ### DynamoDB (PartiQL) @@ -182,20 +200,3 @@ npm test Copyright (c) 2016-2021 The SQLECTRON Team. This software is licensed under the [MIT License](https://github.com/sqlectron/sql-query-identifier/blob/master/LICENSE). - -## Leading comments - -Pass `includeLeadingComments: true` to include a statement's leading comments in -its `text`. Its `start` moves to the beginning of the comment prefix, including -adjacent whitespace. `end` does not change. Semicolons before the prefix stay out -of the statement. Whitespace alone does not change `start`. - -```js -identify('-- header\nCREATE VIEW v AS SELECT 1;', { - dialect: 'mssql', - includeLeadingComments: true, -}); -``` - -The option defaults to `false`. Existing callers still receive text and offsets -that start at the first keyword. From 9dcb972ddf3f3ec9e4dfd4b72bd7e88a39ca7adf Mon Sep 17 00:00:00 2001 From: mango <333602023+mangomac@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:34:25 -0500 Subject: [PATCH 06/11] Separate leading comment and whitespace boundaries --- src/index.ts | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/index.ts b/src/index.ts index 9b32e2c..ac87197 100644 --- a/src/index.ts +++ b/src/index.ts @@ -40,28 +40,31 @@ export function identify(query: string, options: IdentifyOptions = {}): Identify return parsed.body.map((statement) => { let start = statement.start; let leadingStart: number | undefined; - let hasComment = false; + let firstComment: number | undefined; while ( - options.includeLeadingComments === true && + (options.includeLeadingComments === true || options.includeLeadingWhitespace === true) && tokenIndex < parsed.tokens.length && parsed.tokens[tokenIndex].start < statement.start ) { const token = parsed.tokens[tokenIndex++]; if (token.end <= previousEnd) continue; - if ( - token.type === 'whitespace' || - token.type === 'comment-inline' || - token.type === 'comment-block' + if (token.type === 'whitespace') { + if (leadingStart === undefined) leadingStart = token.start; + } else if ( + options.includeLeadingComments === true && + (token.type === 'comment-inline' || token.type === 'comment-block') ) { if (leadingStart === undefined) leadingStart = token.start; - if (token.type !== 'whitespace') hasComment = true; + if (firstComment === undefined) firstComment = token.start; } else { leadingStart = undefined; - hasComment = false; + firstComment = undefined; } } - if (options.includeLeadingComments === true && hasComment && leadingStart !== undefined) { + if (options.includeLeadingWhitespace === true && leadingStart !== undefined) { start = leadingStart; + } else if (options.includeLeadingComments === true && firstComment !== undefined) { + start = firstComment; } previousEnd = statement.end; const result: IdentifyResult = { From 38de456edbbdca1430244ccc6103daa644480f2e Mon Sep 17 00:00:00 2001 From: mango <333602023+mangomac@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:34:31 -0500 Subject: [PATCH 07/11] Add includeLeadingWhitespace option to IdentifyOptions --- src/defines.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/defines.ts b/src/defines.ts index be6dc1d..a17647f 100644 --- a/src/defines.ts +++ b/src/defines.ts @@ -137,6 +137,7 @@ export interface TableReference { export interface IdentifyOptions { strict?: boolean; includeLeadingComments?: boolean; + includeLeadingWhitespace?: boolean; dialect?: Dialect; identifyTables?: boolean; identifyColumns?: boolean; From 40376bccc7d5d01d2fb8e9b8e58962c4801e3883 Mon Sep 17 00:00:00 2001 From: mango <333602023+mangomac@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:34:49 -0500 Subject: [PATCH 08/11] Refactor tests for leading comments and whitespace --- test/identifier/single-statement.spec.ts | 92 ++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 6 deletions(-) diff --git a/test/identifier/single-statement.spec.ts b/test/identifier/single-statement.spec.ts index 8057ab3..d45793b 100644 --- a/test/identifier/single-statement.spec.ts +++ b/test/identifier/single-statement.spec.ts @@ -1909,19 +1909,19 @@ describe('includeLeadingComments', () => { expect(actual.text).to.equal(sql.slice(actual.start, actual.end + 1)); }); - it('includes leading comments and whitespace when enabled', () => { + it('starts at the first comment without leading whitespace', () => { const sql = ' -- header\n/* detail */\nCREATE PROCEDURE p AS SELECT 1;'; const actual = identify(sql, options)[0]; - expect(actual.start).to.equal(0); - expect(actual.text).to.equal(sql); + expect(actual.start).to.equal(2); + expect(actual.text).to.equal(sql.slice(2)); expect(actual.type).to.equal('CREATE_PROCEDURE'); }); it('does not include empty statement delimiters', () => { const sql = '; ;\n-- header\nSELECT 1;'; const actual = identify(sql, options)[0]; - expect(actual.start).to.equal(3); - expect(actual.text).to.equal(sql.slice(3)); + expect(actual.start).to.equal(4); + expect(actual.text).to.equal(sql.slice(4)); }); it('keeps each header with its own statement', () => { @@ -1929,7 +1929,7 @@ describe('includeLeadingComments', () => { const actual = identify(sql, options); expect(actual).to.have.length(2); expect(actual[0].text).to.equal(sql.slice(0, sql.indexOf(';') + 1)); - expect(actual[1].start).to.equal(sql.indexOf(';') + 1); + expect(actual[1].start).to.equal(sql.indexOf('/* second */')); expect(actual[1].text).to.equal(sql.slice(actual[1].start)); }); @@ -1942,3 +1942,83 @@ describe('includeLeadingComments', () => { expect(actual[1].text).to.equal('SELECT 2;'); }); }); + +describe('includeLeadingWhitespace', () => { + for (const includeLeadingComments of [false, true]) { + for (const includeLeadingWhitespace of [false, true]) { + const flags = { + dialect: 'mssql', + includeLeadingComments, + includeLeadingWhitespace, + } as Options; + it(`keeps independent boundaries with comments=${String( + includeLeadingComments, + )}, whitespace=${String(includeLeadingWhitespace)}`, () => { + const sql = ' -- header\n /* detail */\n CREATE VIEW v AS SELECT 1;'; + const actual = identify(sql, flags)[0]; + const start = includeLeadingComments + ? includeLeadingWhitespace + ? 0 + : sql.indexOf('--') + : includeLeadingWhitespace + ? sql.indexOf('\n CREATE') + : sql.indexOf('CREATE'); + expect(actual.start).to.equal(start); + expect(actual.text).to.equal(sql.slice(start)); + expect(actual.end).to.equal(sql.length - 1); + expect(actual.type).to.equal('CREATE_VIEW'); + expect(actual.parameters).to.deep.equal([]); + }); + } + } + + it('includes whitespace-only prefixes when enabled', () => { + const sql = ' \n\tSELECT 1;'; + const actual = identify(sql, { includeLeadingWhitespace: true })[0]; + expect(actual.start).to.equal(0); + expect(actual.text).to.equal(sql); + }); + + it('keeps delimiters out and preserves each contiguous prefix with both options', () => { + const sql = '; ;\n -- first\nSELECT 1;\n /* second */\nSELECT 2;'; + const actual = identify(sql, { includeLeadingComments: true, includeLeadingWhitespace: true }); + expect(actual).to.have.length(2); + expect(actual[0].start).to.equal(3); + expect(actual[0].text).to.equal(sql.slice(3, sql.indexOf('SELECT 1;') + 9)); + expect(actual[1].start).to.equal(sql.indexOf('SELECT 1;') + 9); + expect(actual[1].text).to.equal(sql.slice(actual[1].start)); + }); +}); + +describe('leading boundary compatibility', () => { + it('does not cross a comment when only whitespace is enabled', () => { + const sql = ' /* header */SELECT 1;'; + const actual = identify(sql, { includeLeadingWhitespace: true })[0]; + expect(actual.start).to.equal(sql.indexOf('SELECT')); + expect(actual.text).to.equal('SELECT 1;'); + }); + + it('does not cross a delimiter when only whitespace is enabled', () => { + const sql = '; ; \nSELECT 1;'; + const actual = identify(sql, { includeLeadingWhitespace: true })[0]; + expect(actual.start).to.equal(3); + expect(actual.text).to.equal(sql.slice(3)); + }); + + it('keeps parameter extraction unchanged with both options enabled', () => { + const sql = " -- header\nSELECT '-- literal', ?;"; + const flags = { + dialect: 'mysql', + includeLeadingComments: true, + includeLeadingWhitespace: true, + } as Options; + const expected = identify(sql, { dialect: 'mysql' })[0]; + const actual = identify(sql, flags)[0]; + expect(actual.start).to.equal(0); + expect(actual.text).to.equal(sql); + expect(actual.end).to.equal(expected.end); + expect(actual.type).to.equal(expected.type); + expect(actual.executionType).to.equal(expected.executionType); + expect(actual.parameters).to.deep.equal(expected.parameters); + }); +}); From ddb9e6f6af3ec957fb34f8d4730be1a9e350e8ff Mon Sep 17 00:00:00 2001 From: mango <333602023+mangomac@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:34:54 -0500 Subject: [PATCH 09/11] Clarify leading comments and whitespace options Updated descriptions for options related to leading comments and whitespace in SQL processing. --- README.md | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 15fbc47..b2ecaac 100644 --- a/README.md +++ b/README.md @@ -152,25 +152,37 @@ console.log(statements); 1. `options (object)`: allow to set different configurations 1. `strict (bool)`: allow disable strict mode which will ignore unknown types *(default=true)* 2. `dialect (string)`: Specify your database dialect, values: `generic`, `mysql`, `oracle`, `psql`, `sqlite`, `mssql`, `bigquery` and `dynamodb`. *(default=generic)* - 3. `includeLeadingComments (bool)`: Include leading comments and adjacent whitespace in statement `text` and `start`. Whitespace alone does not change `start`; `end` stays unchanged. *(default=false)* + 3. `includeLeadingComments (bool)`: Include leading comments in statement `text` and `start`, starting at the first comment unless `includeLeadingWhitespace` is also enabled. *(default=false)* + 4. `includeLeadingWhitespace (bool)`: Include whitespace immediately before the statement, or before its leading comments when `includeLeadingComments` is enabled. *(default=false)* -### Leading comments +### Leading comments and whitespace -Pass `includeLeadingComments: true` to include a statement's leading comments in -its `text`. Its `start` moves to the beginning of the comment prefix, including -adjacent whitespace. `end` does not change. Semicolons before the prefix stay out -of the statement. Whitespace alone does not change `start`. +Both options default to `false`, so existing callers still receive text and +offsets that start at the first keyword. Neither option changes `end`. + +For ` -- header\n SELECT 1;`: + +| Options | Returned `text` starts at | +| --- | --- | +| Neither | `SELECT` | +| `includeLeadingComments` | `-- header` | +| `includeLeadingWhitespace` | The newline and two spaces after the comment | +| Both | The two spaces before `-- header` | + +Comment-only mode keeps whitespace between comments and the keyword because +`text` remains a contiguous source substring. Whitespace-only mode does not cross +or include a comment. Semicolon delimiters are excluded in every mode. + +To preserve a module's header comments and their leading whitespace: ```js -identify('-- header\nCREATE VIEW v AS SELECT 1;', { +identify(' -- header\nCREATE VIEW v AS SELECT 1;', { dialect: 'mssql', includeLeadingComments: true, + includeLeadingWhitespace: true, }); ``` -The option defaults to `false`. Existing callers still receive text and offsets -that start at the first keyword. - ### DynamoDB (PartiQL) DynamoDB exposes a [PartiQL](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.html)-compatible From 6a80871cc5d60ce84842a5101dbe938d5a5f0a4f Mon Sep 17 00:00:00 2001 From: mango <333602023+mangomac@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:35:40 -0500 Subject: [PATCH 10/11] Add test for line comment whitespace boundary --- test/identifier/single-statement.spec.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/identifier/single-statement.spec.ts b/test/identifier/single-statement.spec.ts index d45793b..bad719c 100644 --- a/test/identifier/single-statement.spec.ts +++ b/test/identifier/single-statement.spec.ts @@ -2022,3 +2022,12 @@ describe('leading boundary compatibility', () => { expect(actual.parameters).to.deep.equal(expected.parameters); }); }); + +describe('line comment whitespace boundary', () => { + it('does not include the newline owned by an excluded line comment', () => { + const sql = ' -- header\n SELECT 1;'; + const actual = identify(sql, { includeLeadingWhitespace: true })[0]; + expect(actual.start).to.equal(sql.indexOf(' SELECT')); + expect(actual.text).to.equal(' SELECT 1;'); + }); +}); From 903de896ef7438da1526eb45dbdc172f12907ed6 Mon Sep 17 00:00:00 2001 From: mango <333602023+mangomac@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:35:46 -0500 Subject: [PATCH 11/11] Clarify line comment newline ownership --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b2ecaac..50605ee 100644 --- a/README.md +++ b/README.md @@ -166,12 +166,14 @@ For ` -- header\n SELECT 1;`: | --- | --- | | Neither | `SELECT` | | `includeLeadingComments` | `-- header` | -| `includeLeadingWhitespace` | The newline and two spaces after the comment | +| `includeLeadingWhitespace` | The two spaces after the comment line | | Both | The two spaces before `-- header` | Comment-only mode keeps whitespace between comments and the keyword because `text` remains a contiguous source substring. Whitespace-only mode does not cross -or include a comment. Semicolon delimiters are excluded in every mode. +or include a comment. The tokenizer includes a line comment's terminating newline +in that comment token, so whitespace-only mode does not retain that newline. +Semicolon delimiters are excluded in every mode. To preserve a module's header comments and their leading whitespace: