Make SourceTextData thread-safe with ConcurrentDictionary

docs/release-notes/.FSharp.Compiler.Service/11.0.100.md

@@ -183,6 +183,7 @@
 
 ### Improved
 
+* Add `FSharpLineTokenizer.ScanTokenValue`, an allocation-free counterpart of `ScanToken` returning `struct (FSharpTokenInfo voption * FSharpTokenizerLexState)`. `ScanToken` itself is unchanged; the hot tokenization path used by classification, brace matching, and the deprecated `FSharp.LanguageService` colorizer now calls `ScanTokenValue`, removing a per-token `option` allocation there. ([PR #20113](https://github.com/dotnet/fsharp/pull/20113))
 * IL: cache C# extension methods per CCU ([PR #20256](https://github.com/dotnet/fsharp/pull/20256))
 * Nullness warning FS3261 on dotted method or property access (e.g. `x.Member`) now underlines the receiver expression and includes the member name and (when known) the binding name in the message. ([Issue #19658](https://github.com/dotnet/fsharp/issues/19658), [PR #19814](https://github.com/dotnet/fsharp/pull/19814))
 * Import: share assembly CCUs between projects ([PR #20296](https://github.com/dotnet/fsharp/pull/20296))

docs/release-notes/.VisualStudio/18.vNext.md

@@ -15,6 +15,7 @@
 * Fix doubled F# diagnostics in tooltips. ([Issue #16360](https://github.com/dotnet/fsharp/issues/16360))
 * Fix `NotSupportedException` in the memory-mapped-file optimization when copying `ReadOnlyMemory` into `MemoryMappedFileViewStream`. ([Issue #20263](https://github.com/dotnet/fsharp/issues/20263))
 * Reduce allocations in the VS project options reactor: the command-line options and project options caches and the mailbox reply payloads now hold struct tuples, and `IProjectSite.CompilationBinOutputPath` returns `string voption` picked with a new `Array.tryPickV`. ([PR #20413](https://github.com/dotnet/fsharp/pull/20413))
+* Fix a race condition in the editor's per-document token cache (`SourceTextData`) that could corrupt classification/tagging and symbol-lookup state under concurrent access, by backing it with a `ConcurrentDictionary`. Also fixed a pre-existing bug where resuming a cache scan read an unvalidated, potentially stale neighboring entry's lex state instead of the entry already confirmed valid, which could misclassify or drop tokens when concurrent edits interleaved; and removed redundant allocations on the tokenizer hot path (reused already-materialized line text in cache validation, dropped mutable indirection in token scanning). ([PR #20113](https://github.com/dotnet/fsharp/pull/20113))
 
 ### Changed
 

src/Compiler/Service/ServiceLexing.fs

@@ -1010,8 +1010,8 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi
         with _ ->
             false, (EOF LexerStateEncoding.revertToDefaultLexCont, 0, 0)
 
-    // Scan a token starting with the given lexer state
-    member x.ScanToken(lexState: FSharpTokenizerLexState) : FSharpTokenInfo option * FSharpTokenizerLexState =
+    /// Scan a token starting with the given lexer state, without allocating an option for the result.
+    member x.ScanTokenValue(lexState: FSharpTokenizerLexState) : struct (FSharpTokenInfo voption * FSharpTokenizerLexState) =
 
         use _ = UseBuildPhase BuildPhase.Parse
         use _ = UseDiagnosticsLogger DiscardErrorsLogger
@@ -1022,12 +1022,12 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi
         let isCached, (token, leftc, rightc) = getTokenWithPosition lexcont
 
         // Check for end-of-string and failure
-        let tokenDataOption, lexcontFinal, tokenTag =
+        let struct (tokenDataOption, lexcontFinal, tokenTag) =
             match token with
             | EOF lexcont ->
                 // End of text! No more tokens.
-                None, lexcont, 0
-            | LEX_FAILURE _ -> None, LexerStateEncoding.revertToDefaultLexCont, 0
+                struct (ValueNone, lexcont, 0)
+            | LEX_FAILURE _ -> struct (ValueNone, LexerStateEncoding.revertToDefaultLexCont, 0)
             | _ ->
                 // Get the information about the token
                 let colorClass, charClass, triggerClass = TokenClassifications.tokenInfo token
@@ -1058,14 +1058,14 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi
                         FullMatchedLength = fullMatchedLength
                     }
 
-                Some tokenData, lexcontFinal, tokenTag
+                struct (ValueSome tokenData, lexcontFinal, tokenTag)
 
         // Check for patterns like #-IDENT and see if they look like meta commands for .fsx files. If they do then merge them into a single token.
-        let tokenDataOption, lexintFinal =
+        let struct (tokenDataOption, lexintFinal) =
             let lexintFinal = LexerStateEncoding.encodeLexInt lexcontFinal
 
             match tokenDataOption, singleLineTokenState, tokenTagToTokenId tokenTag with
-            | Some tokenData, SingleLineTokenState.BeforeHash, TOKEN_HASH ->
+            | ValueSome tokenData, SingleLineTokenState.BeforeHash, TOKEN_HASH ->
                 // Don't allow further matches.
                 singleLineTokenState <- SingleLineTokenState.NoFurtherMatchPossible
                 // Peek at the next token
@@ -1110,17 +1110,22 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi
 
                         let lexintFinal = LexerStateEncoding.encodeLexInt lexcontFinal
 
-                        Some tokenData, lexintFinal
-                    | _ -> tokenDataOption, lexintFinal
-                | _ -> tokenDataOption, lexintFinal
+                        struct (ValueSome tokenData, lexintFinal)
+                    | _ -> struct (tokenDataOption, lexintFinal)
+                | _ -> struct (tokenDataOption, lexintFinal)
             | _, SingleLineTokenState.BeforeHash, TOKEN_WHITESPACE ->
                 // Allow leading whitespace.
-                tokenDataOption, lexintFinal
+                struct (tokenDataOption, lexintFinal)
             | _ ->
                 singleLineTokenState <- SingleLineTokenState.NoFurtherMatchPossible
-                tokenDataOption, lexintFinal
+                struct (tokenDataOption, lexintFinal)
+
+        struct (tokenDataOption, lexintFinal)
 
-        tokenDataOption, lexintFinal
+    // Scan a token starting with the given lexer state
+    member x.ScanToken(lexState: FSharpTokenizerLexState) : FSharpTokenInfo option * FSharpTokenizerLexState =
+        let struct (tokenDataOption, lexintFinal) = x.ScanTokenValue(lexState)
+        ValueOption.toOption tokenDataOption, lexintFinal
 
     static member ColorStateOfLexState(lexState: FSharpTokenizerLexState) =
         LexerStateEncoding.colorStateOfLexState lexState

src/Compiler/Service/ServiceLexing.fsi

@@ -312,6 +312,11 @@ type FSharpLineTokenizer =
     /// Scan one token from the line
     member ScanToken: lexState: FSharpTokenizerLexState -> FSharpTokenInfo option * FSharpTokenizerLexState
 
+    /// Scan one token from the line, without allocating an option for the result. Prefer this over
+    /// ScanToken on hot paths that tokenize every token of a file, such as classification or brace matching.
+    member ScanTokenValue:
+        lexState: FSharpTokenizerLexState -> struct (FSharpTokenInfo voption * FSharpTokenizerLexState)
+
     /// Get the color state from the lexer state
     static member ColorStateOfLexState: FSharpTokenizerLexState -> FSharpTokenizerColorState
 

src/Compiler/Service/service.fs

@@ -638,16 +638,22 @@ type FSharpChecker
     member _.TokenizeLine(line: string, state: FSharpTokenizerLexState) =
         let tokenizer = FSharpSourceTokenizer([], None, None)
         let lineTokenizer = tokenizer.CreateLineTokenizer line
-        let mutable state = (None, state)
+        let mutable lexState = state
+        let mutable token = ValueNone
+
+        let scanNext () =
+            let struct (t, s) = lineTokenizer.ScanTokenValue(lexState)
+            token <- t
+            lexState <- s
+            token.IsSome
 
         let tokens =
             [|
-                while (state <- lineTokenizer.ScanToken(snd state)
-                       (fst state).IsSome) do
-                    yield (fst state).Value
+                while scanNext () do
+                    yield token.Value
             |]
 
-        tokens, snd state
+        tokens, lexState
 
     /// Tokenize an entire file, line by line
     member x.TokenizeFile(source: string) : FSharpTokenInfo[][] =

tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl

@@ -11506,6 +11506,7 @@ FSharp.Compiler.Tokenization.FSharpLexerFlags: Int32 value__
 FSharp.Compiler.Tokenization.FSharpLineTokenizer: FSharp.Compiler.Tokenization.FSharpTokenizerColorState ColorStateOfLexState(FSharp.Compiler.Tokenization.FSharpTokenizerLexState)
 FSharp.Compiler.Tokenization.FSharpLineTokenizer: FSharp.Compiler.Tokenization.FSharpTokenizerLexState LexStateOfColorState(FSharp.Compiler.Tokenization.FSharpTokenizerColorState)
 FSharp.Compiler.Tokenization.FSharpLineTokenizer: System.Tuple`2[Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Tokenization.FSharpTokenInfo],FSharp.Compiler.Tokenization.FSharpTokenizerLexState] ScanToken(FSharp.Compiler.Tokenization.FSharpTokenizerLexState)
+FSharp.Compiler.Tokenization.FSharpLineTokenizer: System.ValueTuple`2[Microsoft.FSharp.Core.FSharpValueOption`1[FSharp.Compiler.Tokenization.FSharpTokenInfo],FSharp.Compiler.Tokenization.FSharpTokenizerLexState] ScanTokenValue(FSharp.Compiler.Tokenization.FSharpTokenizerLexState)
 FSharp.Compiler.Tokenization.FSharpSourceTokenizer: FSharp.Compiler.Tokenization.FSharpLineTokenizer CreateBufferTokenizer(Microsoft.FSharp.Core.FSharpFunc`2[System.Tuple`3[System.Char[],System.Int32,System.Int32],System.Int32])
 FSharp.Compiler.Tokenization.FSharpSourceTokenizer: FSharp.Compiler.Tokenization.FSharpLineTokenizer CreateLineTokenizer(System.String)
 FSharp.Compiler.Tokenization.FSharpSourceTokenizer: Void .ctor(Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.String])