diff --git a/.changeset/fix-word-tokenizer-punctuation.md b/.changeset/fix-word-tokenizer-punctuation.md new file mode 100644 index 000000000..8253e7be2 --- /dev/null +++ b/.changeset/fix-word-tokenizer-punctuation.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Strip all declared punctuation from words in the basic word tokenizer, including backslash, which the previous data-built regex escaped away. diff --git a/agents/src/tokenize/basic/word.ts b/agents/src/tokenize/basic/word.ts index 3e48948d6..dcb0a6959 100644 --- a/agents/src/tokenize/basic/word.ts +++ b/agents/src/tokenize/basic/word.ts @@ -3,6 +3,12 @@ // SPDX-License-Identifier: Apache-2.0 import { PUNCTUATIONS } from '../tokenizer.js'; +// Strip punctuation by set membership rather than a regex built from the joined +// list: concatenating the characters into a `[...]` class mis-parses the members +// that are regex-significant (e.g. `\` followed by `]` becomes an escaped `]`, so +// backslash was never stripped, and `,-.` silently forms a range). +const PUNCTUATION_SET = new Set(PUNCTUATIONS); + /** * Split the text into words. */ @@ -17,7 +23,9 @@ export const splitWords = (text: string, ignorePunctuation = true): [string, num const end = start + word.length; if (ignorePunctuation) { - word = word.replace(new RegExp(`[${PUNCTUATIONS.join('')}]`, 'g'), ''); + word = Array.from(word) + .filter((c) => !PUNCTUATION_SET.has(c)) + .join(''); } words.push([word, start, end]); diff --git a/agents/src/tokenize/tokenizer.test.ts b/agents/src/tokenize/tokenizer.test.ts index 05a7778cd..a60a18e39 100644 --- a/agents/src/tokenize/tokenizer.test.ts +++ b/agents/src/tokenize/tokenizer.test.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from 'vitest'; -import { SentenceTokenizer, WordTokenizer, hyphenateWord } from './basic/index.js'; +import { SentenceTokenizer, WordTokenizer, hyphenateWord, splitWords } from './basic/index.js'; import { splitParagraphs } from './basic/paragraph.js'; const TEXT = @@ -256,4 +256,17 @@ describe('tokenizer', () => { }); }); }); + describe('splitWords', () => { + it('strips backslash, a declared punctuation the joined-list regex escaped away', () => { + // Regression: PUNCTUATIONS includes both '\\' and ']', but building a regex + // from the joined list produced the fragment `[\]` inside the character + // class, where `\]` is an escaped literal ']' — consuming the backslash so + // it was never a class member and never stripped. + expect(splitWords('c\\d', true)).toStrictEqual([['cd', 0, 3]]); + }); + + it('keeps punctuation when ignorePunctuation is false', () => { + expect(splitWords('c\\d', false)).toStrictEqual([['c\\d', 0, 3]]); + }); + }); });