From b579a6f6341884fb6e313107a5c6f04fab03975e Mon Sep 17 00:00:00 2001
From: Rohit <71192000+rohitsux@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:18:43 +0530
Subject: [PATCH] fix(tokenize): strip backslash and all declared punctuation
 from words
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

splitWords built its filter by joining PUNCTUATIONS into a regex character
class. Because the list contains both `\` and `]`, the joined class held the
fragment `[\]`, where `\]` is an escaped literal `]` — so the backslash was
consumed as an escape and never a class member, and was never stripped when
ignorePunctuation is on. Strip by Set membership instead, removing exactly the
declared characters and eliminating the escaping/range fragility.
---
 .changeset/fix-word-tokenizer-punctuation.md |  5 +++++
 agents/src/tokenize/basic/word.ts            | 10 +++++++++-
 agents/src/tokenize/tokenizer.test.ts        | 15 ++++++++++++++-
 3 files changed, 28 insertions(+), 2 deletions(-)
 create mode 100644 .changeset/fix-word-tokenizer-punctuation.md

diff --git a/.changeset/fix-word-tokenizer-punctuation.md b/.changeset/fix-word-tokenizer-punctuation.md
new file mode 100644
index 0000000000..8253e7be2b
--- /dev/null
+++ b/.changeset/fix-word-tokenizer-punctuation.md
@@ -0,0 +1,5 @@
+---
+'@livekit/agents': patch
+---
+
+Strip all declared punctuation from words in the basic word tokenizer, including backslash, which the previous data-built regex escaped away.
diff --git a/agents/src/tokenize/basic/word.ts b/agents/src/tokenize/basic/word.ts
index 3e48948d6a..dcb0a69598 100644
--- a/agents/src/tokenize/basic/word.ts
+++ b/agents/src/tokenize/basic/word.ts
@@ -3,6 +3,12 @@
 // SPDX-License-Identifier: Apache-2.0
 import { PUNCTUATIONS } from '../tokenizer.js';
 
+// Strip punctuation by set membership rather than a regex built from the joined
+// list: concatenating the characters into a `[...]` class mis-parses the members
+// that are regex-significant (e.g. `\` followed by `]` becomes an escaped `]`, so
+// backslash was never stripped, and `,-.` silently forms a range).
+const PUNCTUATION_SET = new Set(PUNCTUATIONS);
+
 /**
  * Split the text into words.
  */
@@ -17,7 +23,9 @@ export const splitWords = (text: string, ignorePunctuation = true): [string, num
     const end = start + word.length;
 
     if (ignorePunctuation) {
-      word = word.replace(new RegExp(`[${PUNCTUATIONS.join('')}]`, 'g'), '');
+      word = Array.from(word)
+        .filter((c) => !PUNCTUATION_SET.has(c))
+        .join('');
     }
 
     words.push([word, start, end]);
diff --git a/agents/src/tokenize/tokenizer.test.ts b/agents/src/tokenize/tokenizer.test.ts
index 05a7778cd7..a60a18e39a 100644
--- a/agents/src/tokenize/tokenizer.test.ts
+++ b/agents/src/tokenize/tokenizer.test.ts
@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { describe, expect, it } from 'vitest';
-import { SentenceTokenizer, WordTokenizer, hyphenateWord } from './basic/index.js';
+import { SentenceTokenizer, WordTokenizer, hyphenateWord, splitWords } from './basic/index.js';
 import { splitParagraphs } from './basic/paragraph.js';
 
 const TEXT =
@@ -256,4 +256,17 @@ describe('tokenizer', () => {
       });
     });
   });
+  describe('splitWords', () => {
+    it('strips backslash, a declared punctuation the joined-list regex escaped away', () => {
+      // Regression: PUNCTUATIONS includes both '\\' and ']', but building a regex
+      // from the joined list produced the fragment `[\]` inside the character
+      // class, where `\]` is an escaped literal ']' — consuming the backslash so
+      // it was never a class member and never stripped.
+      expect(splitWords('c\\d', true)).toStrictEqual([['cd', 0, 3]]);
+    });
+
+    it('keeps punctuation when ignorePunctuation is false', () => {
+      expect(splitWords('c\\d', false)).toStrictEqual([['c\\d', 0, 3]]);
+    });
+  });
 });