From 0827bee4dd0e7b734313915efcaa753a1724540a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 2 Aug 2026 14:39:20 +0000 Subject: [PATCH 1/6] Make SourceTextData thread-safe with ConcurrentDictionary Co-authored-by: xperiandri <2365592+xperiandri@users.noreply.github.com> --- .../LanguageService/Tokenizer.fs | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs index 6901ceb97b1..02448fcaddc 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs @@ -459,30 +459,28 @@ module internal Tokenizer = && let lineContents = textLine.Text.ToString(textLine.Span) in data.HashCode = lineContents.GetHashCode() + // Shared by concurrent editor operations (classification and symbol lookup), so must be thread-safe. type private SourceTextData(approxLines: int) = - let data = ResizeArray(approxLines) - - let extendTo i = - if i >= data.Count then - data.Capacity <- i + 1 - - for j in data.Count .. i do - data.Add(None) + let data = + ConcurrentDictionary(Environment.ProcessorCount, approxLines) member x.Item with get (i: int) = - extendTo i - data.[i] + match data.TryGetValue(i) with + | true, v -> Some v + | _ -> None and set (i: int) v = - extendTo i - data.[i] <- v + match v with + | Some v -> data.[i] <- v + | None -> data.TryRemove(i) |> ignore member x.ClearFrom(n) = let mutable i = n + let mutable cont = true - while i < data.Count && data.[i].IsSome do - data.[i] <- None - i <- i + 1 + while cont do + let removed, _ = data.TryRemove(i) + if removed then i <- i + 1 else cont <- false /// This saves the tokenization data for a file for as long as the DocumentId object is alive. /// This seems risky - if one single thing leaks a DocumentId (e.g. stores it in some global table of documents From a46b6a8cd3f600235586e0e5c2efd5f21477d628 Mon Sep 17 00:00:00 2001 From: Andrii Chebukin Date: Mon, 17 Aug 2026 03:47:24 +0200 Subject: [PATCH 2/6] Avoid redundant allocations in Tokenizer.fs hot paths - Reuse already-materialized line contents in SourceTextData cache lookups instead of re-stringifying the line via IsValid - processToken now takes the token as a parameter instead of reading a mutable voption repeatedly; scanAndColorNextToken returns whether a token was scanned, removing the tokenInfoOption mutable - tokensUnderCursor uses List.filter instead of a list comprehension - Hoist forbiddenSymbolNameChars to a module-level private array so it is not recreated on every isValidNameForSymbol call --- .../LanguageService/Tokenizer.fs | 127 +++++++++--------- 1 file changed, 61 insertions(+), 66 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs index 02448fcaddc..c98004c3b30 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs @@ -454,12 +454,18 @@ module internal Tokenizer = member val ClassifiedSpans = classifiedSpans member val SavedTokens = savedTokens + member data.IsValid(textLine: TextLine, lineContents: string) = + data.LineStart = textLine.Start && data.HashCode = lineContents.GetHashCode() + member data.IsValid(textLine: TextLine) = data.LineStart = textLine.Start && let lineContents = textLine.Text.ToString(textLine.Span) in data.HashCode = lineContents.GetHashCode() - // Shared by concurrent editor operations (classification and symbol lookup), so must be thread-safe. + // Shared by concurrent editor operations (classification and symbol lookup), so each entry access + // must be thread-safe. This only guarantees per-index atomicity, not a coherent snapshot across a + // range of lines: concurrent scans over overlapping line ranges can still interleave, but the cache + // self-heals on the next read via the IsValid/LexStateAtStartOfLine checks. type private SourceTextData(approxLines: int) = let data = ConcurrentDictionary(Environment.ProcessorCount, approxLines) @@ -467,12 +473,12 @@ module internal Tokenizer = member x.Item with get (i: int) = match data.TryGetValue(i) with - | true, v -> Some v - | _ -> None + | true, v -> ValueSome v + | _ -> ValueNone and set (i: int) v = match v with - | Some v -> data.[i] <- v - | None -> data.TryRemove(i) |> ignore + | ValueSome v -> data.[i] <- v + | ValueNone -> data.TryRemove(i) |> ignore member x.ClearFrom(n) = let mutable i = n @@ -511,66 +517,54 @@ module internal Tokenizer = let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.Text let lineTokenizer = sourceTokenizer.CreateLineTokenizer(lineContents) let tokens = ResizeArray() - let mutable tokenInfoOption = None let mutable previousLexState = lexState - let processToken () = - let classificationType = - compilerTokenToRoslynToken (tokenInfoOption.Value.ColorClass) + let processToken token = + let classificationType = compilerTokenToRoslynToken token.ColorClass - for i = tokenInfoOption.Value.LeftColumn to tokenInfoOption.Value.RightColumn do + for i = token.LeftColumn to token.RightColumn do Array.set colorMap i classificationType - let token = tokenInfoOption.Value - let savedToken = SavedTokenInfo.Create token - - tokens.Add savedToken + tokens.Add(SavedTokenInfo.Create token) let scanAndColorNextToken () = let info, nextLexState = lineTokenizer.ScanToken(previousLexState) - tokenInfoOption <- info previousLexState <- nextLexState // Apply some hacks to clean up the token stream (we apply more later) match info with | Some info when info.Tag = FSharpTokenTag.INT32_DOT_DOT -> - tokenInfoOption <- - Some - { - LeftColumn = info.LeftColumn - RightColumn = info.RightColumn - 2 - ColorClass = FSharpTokenColorKind.Number - CharClass = FSharpTokenCharKind.Literal - FSharpTokenTriggerClass = info.FSharpTokenTriggerClass - Tag = info.Tag - TokenName = "INT32" - FullMatchedLength = info.FullMatchedLength - 2 - } - - processToken () - - tokenInfoOption <- - Some - { - LeftColumn = info.RightColumn - 1 - RightColumn = info.RightColumn - ColorClass = FSharpTokenColorKind.Operator - CharClass = FSharpTokenCharKind.Operator - FSharpTokenTriggerClass = info.FSharpTokenTriggerClass - Tag = FSharpTokenTag.DOT_DOT - TokenName = "DOT_DOT" - FullMatchedLength = 2 - } - - processToken () - - | Some _ -> processToken () + processToken + { + LeftColumn = info.LeftColumn + RightColumn = info.RightColumn - 2 + ColorClass = FSharpTokenColorKind.Number + CharClass = FSharpTokenCharKind.Literal + FSharpTokenTriggerClass = info.FSharpTokenTriggerClass + Tag = info.Tag + TokenName = "INT32" + FullMatchedLength = info.FullMatchedLength - 2 + } + + processToken + { + LeftColumn = info.RightColumn - 1 + RightColumn = info.RightColumn + ColorClass = FSharpTokenColorKind.Operator + CharClass = FSharpTokenCharKind.Operator + FSharpTokenTriggerClass = info.FSharpTokenTriggerClass + Tag = FSharpTokenTag.DOT_DOT + TokenName = "DOT_DOT" + FullMatchedLength = 2 + } + + | Some info -> processToken info | _ -> () - scanAndColorNextToken () + info.IsSome - while tokenInfoOption.IsSome do - scanAndColorNextToken () + while scanAndColorNextToken () do + () let mutable startPosition = 0 let mutable endPosition = startPosition @@ -633,8 +627,8 @@ module internal Tokenizer = while i > 0 && (match sourceTextDataCache.[i] with - | Some data -> not (data.IsValid(lines.[i])) - | None -> true) do + | ValueSome data -> not (data.IsValid(lines.[i])) + | ValueNone -> true) do i <- i - 1 i @@ -656,11 +650,15 @@ module internal Tokenizer = // 2. the hash codes match // 3. the start-of-line lex states are the same match sourceTextDataCache.[i] with - | Some data when data.IsValid(textLine) && data.LexStateAtStartOfLine.Equals(lexState) -> data + | ValueSome data when + data.IsValid(textLine, lineContents) + && data.LexStateAtStartOfLine.Equals(lexState) + -> + data | _ -> // Otherwise, we recompute let newData = scanSourceLine (sourceTokenizer, textLine, lineContents, lexState) - sourceTextDataCache.[i] <- Some newData + sourceTextDataCache.[i] <- ValueSome newData newData lexState <- lineData.LexStateAtEndOfLine @@ -671,10 +669,10 @@ module internal Tokenizer = // If necessary, invalidate all subsequent lines after endLine if endLine < lines.Count - 1 then match sourceTextDataCache.[endLine + 1] with - | Some data -> + | ValueSome data -> if not (data.LexStateAtStartOfLine.Equals(lexState)) then sourceTextDataCache.ClearFrom(endLine + 1) - | None -> () + | ValueNone -> () ] /// Generates a list of Classified Spans for tokens which undergo syntactic classification (i.e., are not typechecked). @@ -842,14 +840,10 @@ module internal Tokenizer = | SymbolLookupKind.Precise -> 0 | SymbolLookupKind.Greedy -> 1 - [ - for x in draftTokens do - if - x.LeftColumn <= linePos.Character - && (x.RightColumn + rightColumnCorrection) >= linePos.Character - then - yield x - ] + draftTokens + |> List.filter (fun x -> + x.LeftColumn <= linePos.Character + && (x.RightColumn + rightColumnCorrection) >= linePos.Character) // Select IDENT token. If failed, select OPERATOR token. let symbol = @@ -1022,6 +1016,9 @@ module internal Tokenizer = else false + let private forbiddenSymbolNameChars = + [| '.'; '+'; '$'; '&'; '['; ']'; '/'; '\\'; '*'; '"' |] + let isValidNameForSymbol (lexerSymbolKind: LexerSymbolKind, symbol: FSharpSymbol, name: string) : bool = let inline isIdentifier (ident: string) = @@ -1040,11 +1037,9 @@ module internal Tokenizer = not (String.IsNullOrEmpty s) && FSharpKeywords.NormalizeIdentifierBackticks s |> isIdentifier - let forbiddenChars = [| '.'; '+'; '$'; '&'; '['; ']'; '/'; '\\'; '*'; '\"' |] - let inline isTypeNameIdent (s: string) = not (String.IsNullOrEmpty s) - && s.IndexOfAny forbiddenChars = -1 + && s.IndexOfAny forbiddenSymbolNameChars = -1 && isFixableIdentifier s let inline isUnionCaseIdent (s: string) = From 3bc9ea7efa061e74e616704ea36be4dfb5262cf2 Mon Sep 17 00:00:00 2001 From: Andrii Chebukin Date: Mon, 17 Aug 2026 04:06:23 +0200 Subject: [PATCH 3/6] Convert FSharpLineTokenizer.ScanToken to allocation-free voption struct return Change ScanToken's signature from 'FSharpTokenInfo option * FSharpTokenizerLexState' to 'struct (FSharpTokenInfo voption * FSharpTokenizerLexState)', eliminating a per-token option allocation on the hot tokenization path. Update all in-tree callers: FSharpChecker.TokenizeLine, the editor Tokenizer, and the deprecated FSharp.LanguageService colorizer/scanner (Colorize.fs, Intellisense.fs). Also clean up incidental trailing whitespace in Intellisense.fs picked up by formatting. --- docs/fcs/tokenizer.fsx | 6 ++--- .../.FSharp.Compiler.Service/11.0.100.md | 2 ++ src/Compiler/Service/ServiceLexing.fs | 26 +++++++++---------- src/Compiler/Service/ServiceLexing.fsi | 2 +- src/Compiler/Service/service.fs | 16 ++++++++---- ...iler.Service.SurfaceArea.netstandard20.bsl | 2 +- .../TokenizerTests.fs | 8 +++--- .../LanguageService/Tokenizer.fs | 6 ++--- .../src/FSharp.LanguageService/Colorize.fs | 16 ++++++------ .../FSharp.LanguageService/Intellisense.fs | 8 +++--- 10 files changed, 50 insertions(+), 42 deletions(-) diff --git a/docs/fcs/tokenizer.fsx b/docs/fcs/tokenizer.fsx index 690374e64ce..afec54d9d19 100644 --- a/docs/fcs/tokenizer.fsx +++ b/docs/fcs/tokenizer.fsx @@ -57,18 +57,18 @@ on the `FSharpSourceTokenizer` object that we created earlier: let tokenizer = sourceTok.CreateLineTokenizer("let answer=42") (** Now, we can write a simple recursive function that calls `ScanToken` on the `tokenizer` -until it returns `None` (indicating the end of line). When the function succeeds, it +until it returns `ValueNone` (indicating the end of line). When the function succeeds, it returns an `FSharpTokenInfo` object with all the interesting details: *) /// Tokenize a single line of F# code let rec tokenizeLine (tokenizer:FSharpLineTokenizer) state = match tokenizer.ScanToken(state) with - | Some tok, state -> + | ValueSome tok, state -> // Print token name printf "%s " tok.TokenName // Tokenize the rest, in the new state tokenizeLine tokenizer state - | None, state -> state + | ValueNone, state -> state (** The function returns the new state, which is needed if you need to tokenize multiple lines and an earlier line ends with a multi-line comment. As an initial state, we can use `0L`: diff --git a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md index 27f2c0070d7..62262186353 100644 --- a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md +++ b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md @@ -183,6 +183,7 @@ ### Improved +* `FSharpLineTokenizer.ScanToken` no longer allocates an `option` box per token; it returns a `struct (FSharpTokenInfo voption * FSharpTokenizerLexState)` instead, removing per-token heap allocations on the hot tokenization path used by classification, brace matching, and the deprecated `FSharp.LanguageService` colorizer. ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) * IL: cache C# extension methods per CCU ([PR #20256](https://github.com/dotnet/fsharp/pull/20256)) * Nullness warning FS3261 on dotted method or property access (e.g. `x.Member`) now underlines the receiver expression and includes the member name and (when known) the binding name in the message. ([Issue #19658](https://github.com/dotnet/fsharp/issues/19658), [PR #19814](https://github.com/dotnet/fsharp/pull/19814)) * Import: share assembly CCUs between projects ([PR #20296](https://github.com/dotnet/fsharp/pull/20296)) @@ -219,5 +220,6 @@ * `FSharp.Compiler.Syntax.SynInterpolatedStringPart.FillExpr` now carries a `SynInterpolationFormatting` value (separating .NET alignment/format from printf specifiers) instead of an `Ident option`. ([PR #19971](https://github.com/dotnet/fsharp/pull/19971)) * `FSharp.Compiler.Syntax.SynComponentInfo` now holds the type name as `synType: SynType option` instead of the previous `longId: LongIdent` field, so tuple-type extensions such as `type ('T1 * 'T2) with` can be represented. A `member LongIdent` compatibility property returns the long identifier for named types and an empty list for tuple or erroneous type names. AST consumers that pattern-matched on the `longId` field must switch to the `synType` field or the `LongIdent` member. ([PR #19602](https://github.com/dotnet/fsharp/pull/19602)) +* `FSharpLineTokenizer.ScanToken: lexState -> FSharpTokenInfo option * FSharpTokenizerLexState` now returns `struct (FSharpTokenInfo voption * FSharpTokenizerLexState)`. All in-tree callers (`FSharpChecker.TokenizeLine`, the editor `Tokenizer`, and the deprecated `FSharp.LanguageService` colorizer) have been updated accordingly. ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) * Optimizer: don't inline named functions in debug builds ([PR #19548](https://github.com/dotnet/fsharp/pull/19548) * LexFilter: drop non-strict mode ([PR #20106](https://github.com/dotnet/fsharp/pull/20106)) diff --git a/src/Compiler/Service/ServiceLexing.fs b/src/Compiler/Service/ServiceLexing.fs index ce501ac7755..2ea04beb6f2 100644 --- a/src/Compiler/Service/ServiceLexing.fs +++ b/src/Compiler/Service/ServiceLexing.fs @@ -1011,7 +1011,7 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi false, (EOF LexerStateEncoding.revertToDefaultLexCont, 0, 0) // Scan a token starting with the given lexer state - member x.ScanToken(lexState: FSharpTokenizerLexState) : FSharpTokenInfo option * FSharpTokenizerLexState = + member x.ScanToken(lexState: FSharpTokenizerLexState) : struct (FSharpTokenInfo voption * FSharpTokenizerLexState) = use _ = UseBuildPhase BuildPhase.Parse use _ = UseDiagnosticsLogger DiscardErrorsLogger @@ -1022,12 +1022,12 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi let isCached, (token, leftc, rightc) = getTokenWithPosition lexcont // Check for end-of-string and failure - let tokenDataOption, lexcontFinal, tokenTag = + let struct (tokenDataOption, lexcontFinal, tokenTag) = match token with | EOF lexcont -> // End of text! No more tokens. - None, lexcont, 0 - | LEX_FAILURE _ -> None, LexerStateEncoding.revertToDefaultLexCont, 0 + struct (ValueNone, lexcont, 0) + | LEX_FAILURE _ -> struct (ValueNone, LexerStateEncoding.revertToDefaultLexCont, 0) | _ -> // Get the information about the token let colorClass, charClass, triggerClass = TokenClassifications.tokenInfo token @@ -1058,14 +1058,14 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi FullMatchedLength = fullMatchedLength } - Some tokenData, lexcontFinal, tokenTag + struct (ValueSome tokenData, lexcontFinal, tokenTag) // Check for patterns like #-IDENT and see if they look like meta commands for .fsx files. If they do then merge them into a single token. - let tokenDataOption, lexintFinal = + let struct (tokenDataOption, lexintFinal) = let lexintFinal = LexerStateEncoding.encodeLexInt lexcontFinal match tokenDataOption, singleLineTokenState, tokenTagToTokenId tokenTag with - | Some tokenData, SingleLineTokenState.BeforeHash, TOKEN_HASH -> + | ValueSome tokenData, SingleLineTokenState.BeforeHash, TOKEN_HASH -> // Don't allow further matches. singleLineTokenState <- SingleLineTokenState.NoFurtherMatchPossible // Peek at the next token @@ -1110,17 +1110,17 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi let lexintFinal = LexerStateEncoding.encodeLexInt lexcontFinal - Some tokenData, lexintFinal - | _ -> tokenDataOption, lexintFinal - | _ -> tokenDataOption, lexintFinal + struct (ValueSome tokenData, lexintFinal) + | _ -> struct (tokenDataOption, lexintFinal) + | _ -> struct (tokenDataOption, lexintFinal) | _, SingleLineTokenState.BeforeHash, TOKEN_WHITESPACE -> // Allow leading whitespace. - tokenDataOption, lexintFinal + struct (tokenDataOption, lexintFinal) | _ -> singleLineTokenState <- SingleLineTokenState.NoFurtherMatchPossible - tokenDataOption, lexintFinal + struct (tokenDataOption, lexintFinal) - tokenDataOption, lexintFinal + struct (tokenDataOption, lexintFinal) static member ColorStateOfLexState(lexState: FSharpTokenizerLexState) = LexerStateEncoding.colorStateOfLexState lexState diff --git a/src/Compiler/Service/ServiceLexing.fsi b/src/Compiler/Service/ServiceLexing.fsi index ea7d05b60fe..bcfc56c545a 100755 --- a/src/Compiler/Service/ServiceLexing.fsi +++ b/src/Compiler/Service/ServiceLexing.fsi @@ -310,7 +310,7 @@ type FSharpTokenInfo = [] type FSharpLineTokenizer = /// Scan one token from the line - member ScanToken: lexState: FSharpTokenizerLexState -> FSharpTokenInfo option * FSharpTokenizerLexState + member ScanToken: lexState: FSharpTokenizerLexState -> struct (FSharpTokenInfo voption * FSharpTokenizerLexState) /// Get the color state from the lexer state static member ColorStateOfLexState: FSharpTokenizerLexState -> FSharpTokenizerColorState diff --git a/src/Compiler/Service/service.fs b/src/Compiler/Service/service.fs index a07f5f90e4e..4bd65fb29ba 100644 --- a/src/Compiler/Service/service.fs +++ b/src/Compiler/Service/service.fs @@ -638,16 +638,22 @@ type FSharpChecker member _.TokenizeLine(line: string, state: FSharpTokenizerLexState) = let tokenizer = FSharpSourceTokenizer([], None, None) let lineTokenizer = tokenizer.CreateLineTokenizer line - let mutable state = (None, state) + let mutable lexState = state + let mutable token = ValueNone + + let scanNext () = + let struct (t, s) = lineTokenizer.ScanToken(lexState) + token <- t + lexState <- s + token.IsSome let tokens = [| - while (state <- lineTokenizer.ScanToken(snd state) - (fst state).IsSome) do - yield (fst state).Value + while scanNext () do + yield token.Value |] - tokens, snd state + tokens, lexState /// Tokenize an entire file, line by line member x.TokenizeFile(source: string) : FSharpTokenInfo[][] = diff --git a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl index 5c9c346b613..0e896db381f 100644 --- a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl +++ b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl @@ -11505,7 +11505,7 @@ FSharp.Compiler.Tokenization.FSharpLexerFlags: FSharp.Compiler.Tokenization.FSha FSharp.Compiler.Tokenization.FSharpLexerFlags: Int32 value__ FSharp.Compiler.Tokenization.FSharpLineTokenizer: FSharp.Compiler.Tokenization.FSharpTokenizerColorState ColorStateOfLexState(FSharp.Compiler.Tokenization.FSharpTokenizerLexState) FSharp.Compiler.Tokenization.FSharpLineTokenizer: FSharp.Compiler.Tokenization.FSharpTokenizerLexState LexStateOfColorState(FSharp.Compiler.Tokenization.FSharpTokenizerColorState) -FSharp.Compiler.Tokenization.FSharpLineTokenizer: System.Tuple`2[Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Tokenization.FSharpTokenInfo],FSharp.Compiler.Tokenization.FSharpTokenizerLexState] ScanToken(FSharp.Compiler.Tokenization.FSharpTokenizerLexState) +FSharp.Compiler.Tokenization.FSharpLineTokenizer: System.ValueTuple`2[Microsoft.FSharp.Core.FSharpValueOption`1[FSharp.Compiler.Tokenization.FSharpTokenInfo],FSharp.Compiler.Tokenization.FSharpTokenizerLexState] ScanToken(FSharp.Compiler.Tokenization.FSharpTokenizerLexState) FSharp.Compiler.Tokenization.FSharpSourceTokenizer: FSharp.Compiler.Tokenization.FSharpLineTokenizer CreateBufferTokenizer(Microsoft.FSharp.Core.FSharpFunc`2[System.Tuple`3[System.Char[],System.Int32,System.Int32],System.Int32]) FSharp.Compiler.Tokenization.FSharpSourceTokenizer: FSharp.Compiler.Tokenization.FSharpLineTokenizer CreateLineTokenizer(System.String) FSharp.Compiler.Tokenization.FSharpSourceTokenizer: Void .ctor(Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.String]) diff --git a/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs b/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs index 805a159c8f4..8bdbde460b3 100644 --- a/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs @@ -7,12 +7,12 @@ open Xunit let rec parseLine(line: string, state: FSharpTokenizerLexState ref, tokenizer: FSharpLineTokenizer) = seq { match tokenizer.ScanToken(state.Value) with - | Some(tok), nstate -> + | ValueSome(tok), nstate -> let str = line.Substring(tok.LeftColumn, tok.RightColumn - tok.LeftColumn + 1) yield str, tok state.Value <- nstate yield! parseLine(line, state, tokenizer) - | None, nstate -> + | ValueNone, nstate -> state.Value <- nstate } let tokenizeLines (lines:string[]) = @@ -30,8 +30,8 @@ let scanTokens (defines: string list) (source: string) = let tokenizer = sourceTok.CreateLineTokenizer(source) let rec loop (state: FSharpTokenizerLexState) acc = match tokenizer.ScanToken(state) with - | Some tok, nstate -> loop nstate (tok :: acc) - | None, _ -> List.rev acc + | ValueSome tok, nstate -> loop nstate (tok :: acc) + | ValueNone, _ -> List.rev acc loop FSharpTokenizerLexState.Initial [] [] diff --git a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs index c98004c3b30..9c80bf12a63 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs @@ -528,12 +528,12 @@ module internal Tokenizer = tokens.Add(SavedTokenInfo.Create token) let scanAndColorNextToken () = - let info, nextLexState = lineTokenizer.ScanToken(previousLexState) + let struct (info, nextLexState) = lineTokenizer.ScanToken(previousLexState) previousLexState <- nextLexState // Apply some hacks to clean up the token stream (we apply more later) match info with - | Some info when info.Tag = FSharpTokenTag.INT32_DOT_DOT -> + | ValueSome info when info.Tag = FSharpTokenTag.INT32_DOT_DOT -> processToken { LeftColumn = info.LeftColumn @@ -558,7 +558,7 @@ module internal Tokenizer = FullMatchedLength = 2 } - | Some info -> processToken info + | ValueSome info -> processToken info | _ -> () info.IsSome diff --git a/vsintegration/src/FSharp.LanguageService/Colorize.fs b/vsintegration/src/FSharp.LanguageService/Colorize.fs index 78bd2a66777..53ba15e2e18 100644 --- a/vsintegration/src/FSharp.LanguageService/Colorize.fs +++ b/vsintegration/src/FSharp.LanguageService/Colorize.fs @@ -124,17 +124,17 @@ type internal FSharpScanner_DEPRECATED(makeLineTokenizer : string -> FSharpLineT /// Scan a token from a line. This should only be used in cases where color information is irrelevant. /// Used by GetFullLineInfo (and only thus in a small workaround in GetDeclarations) and GetTokenInformationAt (thus GetF1KeywordString). member ws.ScanTokenWithDetails (lexState: _ ref) = - let colorInfoOption, newLexState = lineTokenizer.ScanToken(lexState.Value) + let struct (colorInfoOption, newLexState) = lineTokenizer.ScanToken(lexState.Value) lexState.Value <- newLexState colorInfoOption /// Scan a token from a line and write information about it into the tokeninfo object. member ws.ScanTokenAndProvideInfoAboutIt(_line, tokenInfo:TokenInfo, lexState: _ ref) = - let colorInfoOption, newLexState = lineTokenizer.ScanToken(!lexState) + let struct (colorInfoOption, newLexState) = lineTokenizer.ScanToken(!lexState) lexState.Value <- newLexState match colorInfoOption with - | None -> false - | Some colorInfo -> + | ValueNone -> false + | ValueSome colorInfo -> let color = colorInfo.ColorClass tokenInfo.Trigger <- enum (int32 colorInfo.FSharpTokenTriggerClass) // cast one enum to another tokenInfo.StartIndex <- colorInfo.LeftColumn @@ -259,10 +259,10 @@ type internal FSharpColorizer_DEPRECATED scanner.SetLineText lineText let rec tokens() = seq { match scanner.ScanTokenWithDetails(refState) with - | Some tok -> + | ValueSome tok -> yield tok yield! tokens() - | None -> () } + | ValueNone -> () } tokens() |> Array.ofSeq member private c.GetColorInfo(line,lineText,length,lastColorState) = @@ -342,8 +342,8 @@ type internal FSharpColorizer_DEPRECATED let rec searchForToken () = match scanner.ScanTokenWithDetails lexState with - | None -> None - | Some ti as result -> + | ValueNone -> ValueNone + | ValueSome ti as result -> if col >= ti.LeftColumn && col <= ti.RightColumn then result else diff --git a/vsintegration/src/FSharp.LanguageService/Intellisense.fs b/vsintegration/src/FSharp.LanguageService/Intellisense.fs index 8822b9a5ef1..6bedf5bf273 100644 --- a/vsintegration/src/FSharp.LanguageService/Intellisense.fs +++ b/vsintegration/src/FSharp.LanguageService/Intellisense.fs @@ -534,16 +534,16 @@ type internal FSharpIntellisenseInfo_DEPRECATED span.iStartIndex let textColorState = VsTextLines.TextColorState (VsTextView.Buffer view) match colorizer.Value.GetTokenInformationAt(textColorState,line,col) with - | Some token as original when col > 0 && shouldTryToFindIdentToTheLeft token -> + | ValueSome token as original when col > 0 && shouldTryToFindIdentToTheLeft token -> // try to step back one char match colorizer.Value.GetTokenInformationAt(textColorState,line,col-1) with - | Some token as newInfo when token.CharClass <> FSharpTokenCharKind.WhiteSpace -> newInfo, col - 1 + | ValueSome token as newInfo when token.CharClass <> FSharpTokenCharKind.WhiteSpace -> newInfo, col - 1 | _ -> original, col | otherwise -> otherwise, col match tokenInformation with - | None -> None - | Some token -> + | ValueNone -> None + | ValueSome token -> match token.CharClass, token.ColorClass with | FSharpTokenCharKind.Keyword, _ | FSharpTokenCharKind.Operator, _ From db09534c095b793e2d49ed66ff917d6cc895dd75 Mon Sep 17 00:00:00 2001 From: Andrii Chebukin Date: Mon, 17 Aug 2026 04:08:28 +0200 Subject: [PATCH 4/6] Add VS release notes entry for SourceTextData thread-safety fix (PR #20113) --- docs/release-notes/.VisualStudio/18.vNext.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/.VisualStudio/18.vNext.md b/docs/release-notes/.VisualStudio/18.vNext.md index ba03f663967..7ee2cd4b42b 100644 --- a/docs/release-notes/.VisualStudio/18.vNext.md +++ b/docs/release-notes/.VisualStudio/18.vNext.md @@ -15,6 +15,7 @@ * Fix doubled F# diagnostics in tooltips. ([Issue #16360](https://github.com/dotnet/fsharp/issues/16360)) * Fix `NotSupportedException` in the memory-mapped-file optimization when copying `ReadOnlyMemory` into `MemoryMappedFileViewStream`. ([Issue #20263](https://github.com/dotnet/fsharp/issues/20263)) * Reduce allocations in the VS project options reactor: the command-line options and project options caches and the mailbox reply payloads now hold struct tuples, and `IProjectSite.CompilationBinOutputPath` returns `string voption` picked with a new `Array.tryPickV`. ([PR #20413](https://github.com/dotnet/fsharp/pull/20413)) +* Fix a race condition in the editor's per-document token cache (`SourceTextData`) that could corrupt classification/tagging and symbol-lookup state under concurrent access, by backing it with a `ConcurrentDictionary`. Also removed redundant allocations on the tokenizer hot path (reused already-materialized line text in cache validation, dropped mutable indirection in token scanning). ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) ### Changed From a2ea410b38639fdfc1061e9d31778eb7b00d5c50 Mon Sep 17 00:00:00 2001 From: Andrii Chebukin Date: Wed, 9 Sep 2026 22:06:06 +0200 Subject: [PATCH 5/6] Add concurrent SourceTextData cache test; fix stale lex-state read it caught Address review comment: add a concurrent test against one shared SourceTextData, covering overlapping reads and invalidation. The test alternates classifySpans/getSymbolAtPosition calls across many threads on one document, interleaving two text versions of it (one with an unclosed block comment) so cache invalidation is exercised alongside concurrent reads. It reliably reproduced a real bug in getFromRefreshedTokenCache: after walking back to find the nearest valid cache entry at scanStartLine, the resume lex state was read from scanStartLine - 1 - one entry lower than the one just validated, and never checked itself. Under concurrent scans of different text on the same document, that neighboring entry can be stale, so classification would carry over a wrong lex state (typically "still inside a comment"), turning code after that point into one giant comment span, or occasionally throwing on a missing entry (silently swallowed by Assert.Exception, dropping the classification for that call). Fixed by reading LexStateAtStartOfLine off scanStartLine itself, the entry the preceding loop already confirmed valid. Co-Authored-By: Claude Sonnet 5 --- docs/release-notes/.VisualStudio/18.vNext.md | 2 +- .../LanguageService/Tokenizer.fs | 4 +- .../FSharp.Editor.Tests.fsproj | 1 + .../TokenizerCacheConcurrencyTests.fs | 155 ++++++++++++++++++ 4 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 vsintegration/tests/FSharp.Editor.Tests/TokenizerCacheConcurrencyTests.fs diff --git a/docs/release-notes/.VisualStudio/18.vNext.md b/docs/release-notes/.VisualStudio/18.vNext.md index 7ee2cd4b42b..72f79b2be9d 100644 --- a/docs/release-notes/.VisualStudio/18.vNext.md +++ b/docs/release-notes/.VisualStudio/18.vNext.md @@ -15,7 +15,7 @@ * Fix doubled F# diagnostics in tooltips. ([Issue #16360](https://github.com/dotnet/fsharp/issues/16360)) * Fix `NotSupportedException` in the memory-mapped-file optimization when copying `ReadOnlyMemory` into `MemoryMappedFileViewStream`. ([Issue #20263](https://github.com/dotnet/fsharp/issues/20263)) * Reduce allocations in the VS project options reactor: the command-line options and project options caches and the mailbox reply payloads now hold struct tuples, and `IProjectSite.CompilationBinOutputPath` returns `string voption` picked with a new `Array.tryPickV`. ([PR #20413](https://github.com/dotnet/fsharp/pull/20413)) -* Fix a race condition in the editor's per-document token cache (`SourceTextData`) that could corrupt classification/tagging and symbol-lookup state under concurrent access, by backing it with a `ConcurrentDictionary`. Also removed redundant allocations on the tokenizer hot path (reused already-materialized line text in cache validation, dropped mutable indirection in token scanning). ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) +* Fix a race condition in the editor's per-document token cache (`SourceTextData`) that could corrupt classification/tagging and symbol-lookup state under concurrent access, by backing it with a `ConcurrentDictionary`. Also fixed a pre-existing bug where resuming a cache scan read an unvalidated, potentially stale neighboring entry's lex state instead of the entry already confirmed valid, which could misclassify or drop tokens when concurrent edits interleaved; and removed redundant allocations on the tokenizer hot path (reused already-materialized line text in cache validation, dropped mutable indirection in token scanning). ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) ### Changed diff --git a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs index 9c80bf12a63..48fbbc59f1f 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs @@ -637,7 +637,9 @@ module internal Tokenizer = if scanStartLine = 0 then FSharpTokenizerLexState.Initial else - sourceTextDataCache.[scanStartLine - 1].Value.LexStateAtEndOfLine + // scanStartLine is the entry the loop above just proved valid; scanStartLine - 1 was + // never checked and can hold a stale entry from a concurrent scan of different text. + sourceTextDataCache.[scanStartLine].Value.LexStateAtStartOfLine for i = scanStartLine to endLine do ct.ThrowIfCancellationRequested() diff --git a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj index ecce1205b8c..100d4f1e8e7 100644 --- a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj +++ b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj @@ -85,6 +85,7 @@ + diff --git a/vsintegration/tests/FSharp.Editor.Tests/TokenizerCacheConcurrencyTests.fs b/vsintegration/tests/FSharp.Editor.Tests/TokenizerCacheConcurrencyTests.fs new file mode 100644 index 00000000000..2b09cb5746e --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Tests/TokenizerCacheConcurrencyTests.fs @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Editor.Tests + +open System +open System.Collections.Concurrent +open System.Threading +open System.Threading.Tasks +open Microsoft.CodeAnalysis +open Microsoft.CodeAnalysis.Classification +open Microsoft.CodeAnalysis.Text +open Microsoft.VisualStudio.FSharp.Editor +open Xunit + +/// Every editor operation on a document shares one token cache, keyed by document id and defines, so +/// classification and symbol lookup read and write the same entries from whatever thread they run on. +type TokenizerCacheConcurrencyTests() = + + let fileName = "C:\\test.fs" + let defines = [] + let langVersion = Some "preview" + + // Block comments and triple-quoted strings make a line's classification depend on the lex state + // threaded out of the line before it, so a torn cache misclassifies lines instead of losing them. + let source = + String.Join( + "\n", + [| + for i in 0..11 do + $"let value{i} = {i}" + $"// comment {i}" + "let block =" + " (* opening" + $" still inside {i} *)" + " 42" + $"let text{i} = \"\"\"triple" + $" quoted {i}\"\"\"" + $"let plain{i} = \"single line\"" + "type T() = class end" + |] + ) + + // Dropping every `*)` leaves the first block comment open to the end of the file: the lines keep + // their text and hash but change lex state, which is what drives cached entries out of the cache. + let sourceText = SourceText.From source + let commentedText = SourceText.From(source.Replace(" *)", "")) + + let newDocumentId () = + DocumentId.CreateNewId(ProjectId.CreateNewId()) + + let classify (documentId: DocumentId) (text: SourceText) (span: TextSpan) = + let spans = ResizeArray() + + Tokenizer.classifySpans (documentId, text, span, Some fileName, defines, langVersion, spans, CancellationToken.None) + + spans |> Seq.map (fun s -> s.ClassificationType, s.TextSpan) |> Array.ofSeq + + let lineSpan (text: SourceText) startLine endLine = + TextSpan.FromBounds(text.Lines.[startLine].Start, text.Lines.[endLine].End) + + /// What an uncontended scan of the document produces, one line at a time. + let scanEveryLine (text: SourceText) = + let documentId = newDocumentId () + + Array.init text.Lines.Count (fun i -> classify documentId text (lineSpan text i i)) + + let expectedRange (perLine: (string * TextSpan)[][]) startLine endLine = + Array.concat perLine.[startLine..endLine] + + let describe (spans: (string * TextSpan)[]) = + spans |> Seq.map (fun (kind, span) -> $"{kind}{span}") |> String.concat " " + + let runConcurrently workerCount (work: int -> unit) = + let workers = + Array.init workerCount (fun worker -> Task.Run(Action(fun () -> work worker))) + + Task.WaitAll workers + + let workerCount = max 8 Environment.ProcessorCount + + [] + member _.``Overlapping concurrent reads of one document's token cache agree with a single-threaded scan``() = + let perLine = scanEveryLine sourceText + let lineCount = sourceText.Lines.Count + let documentId = newDocumentId () + let failures = ConcurrentQueue() + + runConcurrently workerCount (fun worker -> + let random = Random(worker) + + for _ in 1..60 do + let startLine, endLine = + let a = random.Next lineCount + let b = random.Next lineCount + min a b, max a b + + let actual = classify documentId sourceText (lineSpan sourceText startLine endLine) + let expected = expectedRange perLine startLine endLine + + if actual <> expected then + failures.Enqueue $"lines {startLine}..{endLine}\n expected {describe expected}\n actual {describe actual}" + + // Symbol lookup reads the same cache, and does so a line at a time. + Tokenizer.getSymbolAtPosition ( + documentId, + sourceText, + random.Next sourceText.Length, + fileName, + defines, + SymbolLookupKind.Greedy, + false, + false, + langVersion, + CancellationToken.None + ) + |> ignore) + + let diverged = String.concat Environment.NewLine failures + Assert.True(failures.IsEmpty, $"Concurrent classification diverged:{Environment.NewLine}{diverged}") + + [] + member _.``Concurrent reads of two versions of a document keep invalidating the cache correctly``() = + let perLine = scanEveryLine sourceText + let commentedPerLine = scanEveryLine commentedText + let lineCount = sourceText.Lines.Count + + // Without this the test would pass on a cache that never invalidates anything. + Assert.NotEqual<(string * TextSpan)[]>(perLine.[lineCount - 1], commentedPerLine.[lineCount - 1]) + + let documentId = newDocumentId () + let failures = ConcurrentQueue() + + runConcurrently workerCount (fun worker -> + let random = Random(worker) + + for iteration in 1..60 do + let text, expectedPerLine = + if (worker + iteration) % 2 = 0 then + sourceText, perLine + else + commentedText, commentedPerLine + + let startLine, endLine = + let a = random.Next lineCount + let b = random.Next lineCount + min a b, max a b + + let actual = classify documentId text (lineSpan text startLine endLine) + let expected = expectedRange expectedPerLine startLine endLine + + if actual <> expected then + failures.Enqueue $"lines {startLine}..{endLine}\n expected {describe expected}\n actual {describe actual}") + + let diverged = String.concat Environment.NewLine failures + Assert.True(failures.IsEmpty, $"Concurrent classification diverged:{Environment.NewLine}{diverged}") From 1ab584a579722f0218ed2348505710648f3c4e62 Mon Sep 17 00:00:00 2001 From: Andrii Chebukin Date: Fri, 11 Sep 2026 18:12:30 +0200 Subject: [PATCH 6/6] Restore ScanToken's signature; add ScanTokenValue for the allocation-free path Address review comment: changing ScanToken to return struct (voption * lexState) broke source and binary compatibility for every existing caller of this public member. ScanToken now returns FSharpTokenInfo option * FSharpTokenizerLexState again, unchanged from before this PR. The allocation-free behavior is kept as a new member, ScanTokenValue, returning struct (FSharpTokenInfo voption * FSharpTokenizerLexState); ScanToken is a thin wrapper over it. The hot-path callers that motivated the original change (FSharpChecker.TokenizeLine, the editor Tokenizer, and the deprecated FSharp.LanguageService colorizer) now call ScanTokenValue directly, so they keep the allocation-free win. TokenizerTests.fs and docs/fcs/tokenizer.fsx go back to matching on Some/None against the restored ScanToken, exercising it (and ScanTokenValue transitively) unchanged from before this PR. Co-Authored-By: Claude Sonnet 5 --- docs/fcs/tokenizer.fsx | 6 +++--- docs/release-notes/.FSharp.Compiler.Service/11.0.100.md | 3 +-- src/Compiler/Service/ServiceLexing.fs | 9 +++++++-- src/Compiler/Service/ServiceLexing.fsi | 7 ++++++- src/Compiler/Service/service.fs | 2 +- ...FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl | 3 ++- tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs | 8 ++++---- .../src/FSharp.Editor/LanguageService/Tokenizer.fs | 2 +- vsintegration/src/FSharp.LanguageService/Colorize.fs | 4 ++-- 9 files changed, 27 insertions(+), 17 deletions(-) diff --git a/docs/fcs/tokenizer.fsx b/docs/fcs/tokenizer.fsx index afec54d9d19..690374e64ce 100644 --- a/docs/fcs/tokenizer.fsx +++ b/docs/fcs/tokenizer.fsx @@ -57,18 +57,18 @@ on the `FSharpSourceTokenizer` object that we created earlier: let tokenizer = sourceTok.CreateLineTokenizer("let answer=42") (** Now, we can write a simple recursive function that calls `ScanToken` on the `tokenizer` -until it returns `ValueNone` (indicating the end of line). When the function succeeds, it +until it returns `None` (indicating the end of line). When the function succeeds, it returns an `FSharpTokenInfo` object with all the interesting details: *) /// Tokenize a single line of F# code let rec tokenizeLine (tokenizer:FSharpLineTokenizer) state = match tokenizer.ScanToken(state) with - | ValueSome tok, state -> + | Some tok, state -> // Print token name printf "%s " tok.TokenName // Tokenize the rest, in the new state tokenizeLine tokenizer state - | ValueNone, state -> state + | None, state -> state (** The function returns the new state, which is needed if you need to tokenize multiple lines and an earlier line ends with a multi-line comment. As an initial state, we can use `0L`: diff --git a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md index 62262186353..a8aa47e0cab 100644 --- a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md +++ b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md @@ -183,7 +183,7 @@ ### Improved -* `FSharpLineTokenizer.ScanToken` no longer allocates an `option` box per token; it returns a `struct (FSharpTokenInfo voption * FSharpTokenizerLexState)` instead, removing per-token heap allocations on the hot tokenization path used by classification, brace matching, and the deprecated `FSharp.LanguageService` colorizer. ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) +* Add `FSharpLineTokenizer.ScanTokenValue`, an allocation-free counterpart of `ScanToken` returning `struct (FSharpTokenInfo voption * FSharpTokenizerLexState)`. `ScanToken` itself is unchanged; the hot tokenization path used by classification, brace matching, and the deprecated `FSharp.LanguageService` colorizer now calls `ScanTokenValue`, removing a per-token `option` allocation there. ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) * IL: cache C# extension methods per CCU ([PR #20256](https://github.com/dotnet/fsharp/pull/20256)) * Nullness warning FS3261 on dotted method or property access (e.g. `x.Member`) now underlines the receiver expression and includes the member name and (when known) the binding name in the message. ([Issue #19658](https://github.com/dotnet/fsharp/issues/19658), [PR #19814](https://github.com/dotnet/fsharp/pull/19814)) * Import: share assembly CCUs between projects ([PR #20296](https://github.com/dotnet/fsharp/pull/20296)) @@ -220,6 +220,5 @@ * `FSharp.Compiler.Syntax.SynInterpolatedStringPart.FillExpr` now carries a `SynInterpolationFormatting` value (separating .NET alignment/format from printf specifiers) instead of an `Ident option`. ([PR #19971](https://github.com/dotnet/fsharp/pull/19971)) * `FSharp.Compiler.Syntax.SynComponentInfo` now holds the type name as `synType: SynType option` instead of the previous `longId: LongIdent` field, so tuple-type extensions such as `type ('T1 * 'T2) with` can be represented. A `member LongIdent` compatibility property returns the long identifier for named types and an empty list for tuple or erroneous type names. AST consumers that pattern-matched on the `longId` field must switch to the `synType` field or the `LongIdent` member. ([PR #19602](https://github.com/dotnet/fsharp/pull/19602)) -* `FSharpLineTokenizer.ScanToken: lexState -> FSharpTokenInfo option * FSharpTokenizerLexState` now returns `struct (FSharpTokenInfo voption * FSharpTokenizerLexState)`. All in-tree callers (`FSharpChecker.TokenizeLine`, the editor `Tokenizer`, and the deprecated `FSharp.LanguageService` colorizer) have been updated accordingly. ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) * Optimizer: don't inline named functions in debug builds ([PR #19548](https://github.com/dotnet/fsharp/pull/19548) * LexFilter: drop non-strict mode ([PR #20106](https://github.com/dotnet/fsharp/pull/20106)) diff --git a/src/Compiler/Service/ServiceLexing.fs b/src/Compiler/Service/ServiceLexing.fs index 2ea04beb6f2..ae51bbb3359 100644 --- a/src/Compiler/Service/ServiceLexing.fs +++ b/src/Compiler/Service/ServiceLexing.fs @@ -1010,8 +1010,8 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi with _ -> false, (EOF LexerStateEncoding.revertToDefaultLexCont, 0, 0) - // Scan a token starting with the given lexer state - member x.ScanToken(lexState: FSharpTokenizerLexState) : struct (FSharpTokenInfo voption * FSharpTokenizerLexState) = + /// Scan a token starting with the given lexer state, without allocating an option for the result. + member x.ScanTokenValue(lexState: FSharpTokenizerLexState) : struct (FSharpTokenInfo voption * FSharpTokenizerLexState) = use _ = UseBuildPhase BuildPhase.Parse use _ = UseDiagnosticsLogger DiscardErrorsLogger @@ -1122,6 +1122,11 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi struct (tokenDataOption, lexintFinal) + // Scan a token starting with the given lexer state + member x.ScanToken(lexState: FSharpTokenizerLexState) : FSharpTokenInfo option * FSharpTokenizerLexState = + let struct (tokenDataOption, lexintFinal) = x.ScanTokenValue(lexState) + ValueOption.toOption tokenDataOption, lexintFinal + static member ColorStateOfLexState(lexState: FSharpTokenizerLexState) = LexerStateEncoding.colorStateOfLexState lexState diff --git a/src/Compiler/Service/ServiceLexing.fsi b/src/Compiler/Service/ServiceLexing.fsi index bcfc56c545a..766b81ab5a4 100755 --- a/src/Compiler/Service/ServiceLexing.fsi +++ b/src/Compiler/Service/ServiceLexing.fsi @@ -310,7 +310,12 @@ type FSharpTokenInfo = [] type FSharpLineTokenizer = /// Scan one token from the line - member ScanToken: lexState: FSharpTokenizerLexState -> struct (FSharpTokenInfo voption * FSharpTokenizerLexState) + member ScanToken: lexState: FSharpTokenizerLexState -> FSharpTokenInfo option * FSharpTokenizerLexState + + /// Scan one token from the line, without allocating an option for the result. Prefer this over + /// ScanToken on hot paths that tokenize every token of a file, such as classification or brace matching. + member ScanTokenValue: + lexState: FSharpTokenizerLexState -> struct (FSharpTokenInfo voption * FSharpTokenizerLexState) /// Get the color state from the lexer state static member ColorStateOfLexState: FSharpTokenizerLexState -> FSharpTokenizerColorState diff --git a/src/Compiler/Service/service.fs b/src/Compiler/Service/service.fs index 4bd65fb29ba..76146d1b8be 100644 --- a/src/Compiler/Service/service.fs +++ b/src/Compiler/Service/service.fs @@ -642,7 +642,7 @@ type FSharpChecker let mutable token = ValueNone let scanNext () = - let struct (t, s) = lineTokenizer.ScanToken(lexState) + let struct (t, s) = lineTokenizer.ScanTokenValue(lexState) token <- t lexState <- s token.IsSome diff --git a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl index 0e896db381f..2d0743249de 100644 --- a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl +++ b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl @@ -11505,7 +11505,8 @@ FSharp.Compiler.Tokenization.FSharpLexerFlags: FSharp.Compiler.Tokenization.FSha FSharp.Compiler.Tokenization.FSharpLexerFlags: Int32 value__ FSharp.Compiler.Tokenization.FSharpLineTokenizer: FSharp.Compiler.Tokenization.FSharpTokenizerColorState ColorStateOfLexState(FSharp.Compiler.Tokenization.FSharpTokenizerLexState) FSharp.Compiler.Tokenization.FSharpLineTokenizer: FSharp.Compiler.Tokenization.FSharpTokenizerLexState LexStateOfColorState(FSharp.Compiler.Tokenization.FSharpTokenizerColorState) -FSharp.Compiler.Tokenization.FSharpLineTokenizer: System.ValueTuple`2[Microsoft.FSharp.Core.FSharpValueOption`1[FSharp.Compiler.Tokenization.FSharpTokenInfo],FSharp.Compiler.Tokenization.FSharpTokenizerLexState] ScanToken(FSharp.Compiler.Tokenization.FSharpTokenizerLexState) +FSharp.Compiler.Tokenization.FSharpLineTokenizer: System.Tuple`2[Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Tokenization.FSharpTokenInfo],FSharp.Compiler.Tokenization.FSharpTokenizerLexState] ScanToken(FSharp.Compiler.Tokenization.FSharpTokenizerLexState) +FSharp.Compiler.Tokenization.FSharpLineTokenizer: System.ValueTuple`2[Microsoft.FSharp.Core.FSharpValueOption`1[FSharp.Compiler.Tokenization.FSharpTokenInfo],FSharp.Compiler.Tokenization.FSharpTokenizerLexState] ScanTokenValue(FSharp.Compiler.Tokenization.FSharpTokenizerLexState) FSharp.Compiler.Tokenization.FSharpSourceTokenizer: FSharp.Compiler.Tokenization.FSharpLineTokenizer CreateBufferTokenizer(Microsoft.FSharp.Core.FSharpFunc`2[System.Tuple`3[System.Char[],System.Int32,System.Int32],System.Int32]) FSharp.Compiler.Tokenization.FSharpSourceTokenizer: FSharp.Compiler.Tokenization.FSharpLineTokenizer CreateLineTokenizer(System.String) FSharp.Compiler.Tokenization.FSharpSourceTokenizer: Void .ctor(Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.String]) diff --git a/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs b/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs index 8bdbde460b3..805a159c8f4 100644 --- a/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs +++ b/tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs @@ -7,12 +7,12 @@ open Xunit let rec parseLine(line: string, state: FSharpTokenizerLexState ref, tokenizer: FSharpLineTokenizer) = seq { match tokenizer.ScanToken(state.Value) with - | ValueSome(tok), nstate -> + | Some(tok), nstate -> let str = line.Substring(tok.LeftColumn, tok.RightColumn - tok.LeftColumn + 1) yield str, tok state.Value <- nstate yield! parseLine(line, state, tokenizer) - | ValueNone, nstate -> + | None, nstate -> state.Value <- nstate } let tokenizeLines (lines:string[]) = @@ -30,8 +30,8 @@ let scanTokens (defines: string list) (source: string) = let tokenizer = sourceTok.CreateLineTokenizer(source) let rec loop (state: FSharpTokenizerLexState) acc = match tokenizer.ScanToken(state) with - | ValueSome tok, nstate -> loop nstate (tok :: acc) - | ValueNone, _ -> List.rev acc + | Some tok, nstate -> loop nstate (tok :: acc) + | None, _ -> List.rev acc loop FSharpTokenizerLexState.Initial [] [] diff --git a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs index 48fbbc59f1f..44578cfccc7 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs @@ -528,7 +528,7 @@ module internal Tokenizer = tokens.Add(SavedTokenInfo.Create token) let scanAndColorNextToken () = - let struct (info, nextLexState) = lineTokenizer.ScanToken(previousLexState) + let struct (info, nextLexState) = lineTokenizer.ScanTokenValue(previousLexState) previousLexState <- nextLexState // Apply some hacks to clean up the token stream (we apply more later) diff --git a/vsintegration/src/FSharp.LanguageService/Colorize.fs b/vsintegration/src/FSharp.LanguageService/Colorize.fs index 53ba15e2e18..a216765b01b 100644 --- a/vsintegration/src/FSharp.LanguageService/Colorize.fs +++ b/vsintegration/src/FSharp.LanguageService/Colorize.fs @@ -124,13 +124,13 @@ type internal FSharpScanner_DEPRECATED(makeLineTokenizer : string -> FSharpLineT /// Scan a token from a line. This should only be used in cases where color information is irrelevant. /// Used by GetFullLineInfo (and only thus in a small workaround in GetDeclarations) and GetTokenInformationAt (thus GetF1KeywordString). member ws.ScanTokenWithDetails (lexState: _ ref) = - let struct (colorInfoOption, newLexState) = lineTokenizer.ScanToken(lexState.Value) + let struct (colorInfoOption, newLexState) = lineTokenizer.ScanTokenValue(lexState.Value) lexState.Value <- newLexState colorInfoOption /// Scan a token from a line and write information about it into the tokeninfo object. member ws.ScanTokenAndProvideInfoAboutIt(_line, tokenInfo:TokenInfo, lexState: _ ref) = - let struct (colorInfoOption, newLexState) = lineTokenizer.ScanToken(!lexState) + let struct (colorInfoOption, newLexState) = lineTokenizer.ScanTokenValue(!lexState) lexState.Value <- newLexState match colorInfoOption with | ValueNone -> false