diff --git a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md index 27f2c0070d7..a8aa47e0cab 100644 --- a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md +++ b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md @@ -183,6 +183,7 @@ ### Improved +* Add `FSharpLineTokenizer.ScanTokenValue`, an allocation-free counterpart of `ScanToken` returning `struct (FSharpTokenInfo voption * FSharpTokenizerLexState)`. `ScanToken` itself is unchanged; the hot tokenization path used by classification, brace matching, and the deprecated `FSharp.LanguageService` colorizer now calls `ScanTokenValue`, removing a per-token `option` allocation there. ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) * IL: cache C# extension methods per CCU ([PR #20256](https://github.com/dotnet/fsharp/pull/20256)) * Nullness warning FS3261 on dotted method or property access (e.g. `x.Member`) now underlines the receiver expression and includes the member name and (when known) the binding name in the message. ([Issue #19658](https://github.com/dotnet/fsharp/issues/19658), [PR #19814](https://github.com/dotnet/fsharp/pull/19814)) * Import: share assembly CCUs between projects ([PR #20296](https://github.com/dotnet/fsharp/pull/20296)) diff --git a/docs/release-notes/.VisualStudio/18.vNext.md b/docs/release-notes/.VisualStudio/18.vNext.md index ba03f663967..72f79b2be9d 100644 --- a/docs/release-notes/.VisualStudio/18.vNext.md +++ b/docs/release-notes/.VisualStudio/18.vNext.md @@ -15,6 +15,7 @@ * Fix doubled F# diagnostics in tooltips. ([Issue #16360](https://github.com/dotnet/fsharp/issues/16360)) * Fix `NotSupportedException` in the memory-mapped-file optimization when copying `ReadOnlyMemory` into `MemoryMappedFileViewStream`. ([Issue #20263](https://github.com/dotnet/fsharp/issues/20263)) * Reduce allocations in the VS project options reactor: the command-line options and project options caches and the mailbox reply payloads now hold struct tuples, and `IProjectSite.CompilationBinOutputPath` returns `string voption` picked with a new `Array.tryPickV`. ([PR #20413](https://github.com/dotnet/fsharp/pull/20413)) +* Fix a race condition in the editor's per-document token cache (`SourceTextData`) that could corrupt classification/tagging and symbol-lookup state under concurrent access, by backing it with a `ConcurrentDictionary`. Also fixed a pre-existing bug where resuming a cache scan read an unvalidated, potentially stale neighboring entry's lex state instead of the entry already confirmed valid, which could misclassify or drop tokens when concurrent edits interleaved; and removed redundant allocations on the tokenizer hot path (reused already-materialized line text in cache validation, dropped mutable indirection in token scanning). ([PR #20113](https://github.com/dotnet/fsharp/pull/20113)) ### Changed diff --git a/src/Compiler/Service/ServiceLexing.fs b/src/Compiler/Service/ServiceLexing.fs index ce501ac7755..ae51bbb3359 100644 --- a/src/Compiler/Service/ServiceLexing.fs +++ b/src/Compiler/Service/ServiceLexing.fs @@ -1010,8 +1010,8 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi with _ -> false, (EOF LexerStateEncoding.revertToDefaultLexCont, 0, 0) - // Scan a token starting with the given lexer state - member x.ScanToken(lexState: FSharpTokenizerLexState) : FSharpTokenInfo option * FSharpTokenizerLexState = + /// Scan a token starting with the given lexer state, without allocating an option for the result. + member x.ScanTokenValue(lexState: FSharpTokenizerLexState) : struct (FSharpTokenInfo voption * FSharpTokenizerLexState) = use _ = UseBuildPhase BuildPhase.Parse use _ = UseDiagnosticsLogger DiscardErrorsLogger @@ -1022,12 +1022,12 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi let isCached, (token, leftc, rightc) = getTokenWithPosition lexcont // Check for end-of-string and failure - let tokenDataOption, lexcontFinal, tokenTag = + let struct (tokenDataOption, lexcontFinal, tokenTag) = match token with | EOF lexcont -> // End of text! No more tokens. - None, lexcont, 0 - | LEX_FAILURE _ -> None, LexerStateEncoding.revertToDefaultLexCont, 0 + struct (ValueNone, lexcont, 0) + | LEX_FAILURE _ -> struct (ValueNone, LexerStateEncoding.revertToDefaultLexCont, 0) | _ -> // Get the information about the token let colorClass, charClass, triggerClass = TokenClassifications.tokenInfo token @@ -1058,14 +1058,14 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi FullMatchedLength = fullMatchedLength } - Some tokenData, lexcontFinal, tokenTag + struct (ValueSome tokenData, lexcontFinal, tokenTag) // Check for patterns like #-IDENT and see if they look like meta commands for .fsx files. If they do then merge them into a single token. - let tokenDataOption, lexintFinal = + let struct (tokenDataOption, lexintFinal) = let lexintFinal = LexerStateEncoding.encodeLexInt lexcontFinal match tokenDataOption, singleLineTokenState, tokenTagToTokenId tokenTag with - | Some tokenData, SingleLineTokenState.BeforeHash, TOKEN_HASH -> + | ValueSome tokenData, SingleLineTokenState.BeforeHash, TOKEN_HASH -> // Don't allow further matches. singleLineTokenState <- SingleLineTokenState.NoFurtherMatchPossible // Peek at the next token @@ -1110,17 +1110,22 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi let lexintFinal = LexerStateEncoding.encodeLexInt lexcontFinal - Some tokenData, lexintFinal - | _ -> tokenDataOption, lexintFinal - | _ -> tokenDataOption, lexintFinal + struct (ValueSome tokenData, lexintFinal) + | _ -> struct (tokenDataOption, lexintFinal) + | _ -> struct (tokenDataOption, lexintFinal) | _, SingleLineTokenState.BeforeHash, TOKEN_WHITESPACE -> // Allow leading whitespace. - tokenDataOption, lexintFinal + struct (tokenDataOption, lexintFinal) | _ -> singleLineTokenState <- SingleLineTokenState.NoFurtherMatchPossible - tokenDataOption, lexintFinal + struct (tokenDataOption, lexintFinal) + + struct (tokenDataOption, lexintFinal) - tokenDataOption, lexintFinal + // Scan a token starting with the given lexer state + member x.ScanToken(lexState: FSharpTokenizerLexState) : FSharpTokenInfo option * FSharpTokenizerLexState = + let struct (tokenDataOption, lexintFinal) = x.ScanTokenValue(lexState) + ValueOption.toOption tokenDataOption, lexintFinal static member ColorStateOfLexState(lexState: FSharpTokenizerLexState) = LexerStateEncoding.colorStateOfLexState lexState diff --git a/src/Compiler/Service/ServiceLexing.fsi b/src/Compiler/Service/ServiceLexing.fsi index ea7d05b60fe..766b81ab5a4 100755 --- a/src/Compiler/Service/ServiceLexing.fsi +++ b/src/Compiler/Service/ServiceLexing.fsi @@ -312,6 +312,11 @@ type FSharpLineTokenizer = /// Scan one token from the line member ScanToken: lexState: FSharpTokenizerLexState -> FSharpTokenInfo option * FSharpTokenizerLexState + /// Scan one token from the line, without allocating an option for the result. Prefer this over + /// ScanToken on hot paths that tokenize every token of a file, such as classification or brace matching. + member ScanTokenValue: + lexState: FSharpTokenizerLexState -> struct (FSharpTokenInfo voption * FSharpTokenizerLexState) + /// Get the color state from the lexer state static member ColorStateOfLexState: FSharpTokenizerLexState -> FSharpTokenizerColorState diff --git a/src/Compiler/Service/service.fs b/src/Compiler/Service/service.fs index a07f5f90e4e..76146d1b8be 100644 --- a/src/Compiler/Service/service.fs +++ b/src/Compiler/Service/service.fs @@ -638,16 +638,22 @@ type FSharpChecker member _.TokenizeLine(line: string, state: FSharpTokenizerLexState) = let tokenizer = FSharpSourceTokenizer([], None, None) let lineTokenizer = tokenizer.CreateLineTokenizer line - let mutable state = (None, state) + let mutable lexState = state + let mutable token = ValueNone + + let scanNext () = + let struct (t, s) = lineTokenizer.ScanTokenValue(lexState) + token <- t + lexState <- s + token.IsSome let tokens = [| - while (state <- lineTokenizer.ScanToken(snd state) - (fst state).IsSome) do - yield (fst state).Value + while scanNext () do + yield token.Value |] - tokens, snd state + tokens, lexState /// Tokenize an entire file, line by line member x.TokenizeFile(source: string) : FSharpTokenInfo[][] = diff --git a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl index 5c9c346b613..2d0743249de 100644 --- a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl +++ b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.bsl @@ -11506,6 +11506,7 @@ FSharp.Compiler.Tokenization.FSharpLexerFlags: Int32 value__ FSharp.Compiler.Tokenization.FSharpLineTokenizer: FSharp.Compiler.Tokenization.FSharpTokenizerColorState ColorStateOfLexState(FSharp.Compiler.Tokenization.FSharpTokenizerLexState) FSharp.Compiler.Tokenization.FSharpLineTokenizer: FSharp.Compiler.Tokenization.FSharpTokenizerLexState LexStateOfColorState(FSharp.Compiler.Tokenization.FSharpTokenizerColorState) FSharp.Compiler.Tokenization.FSharpLineTokenizer: System.Tuple`2[Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Tokenization.FSharpTokenInfo],FSharp.Compiler.Tokenization.FSharpTokenizerLexState] ScanToken(FSharp.Compiler.Tokenization.FSharpTokenizerLexState) +FSharp.Compiler.Tokenization.FSharpLineTokenizer: System.ValueTuple`2[Microsoft.FSharp.Core.FSharpValueOption`1[FSharp.Compiler.Tokenization.FSharpTokenInfo],FSharp.Compiler.Tokenization.FSharpTokenizerLexState] ScanTokenValue(FSharp.Compiler.Tokenization.FSharpTokenizerLexState) FSharp.Compiler.Tokenization.FSharpSourceTokenizer: FSharp.Compiler.Tokenization.FSharpLineTokenizer CreateBufferTokenizer(Microsoft.FSharp.Core.FSharpFunc`2[System.Tuple`3[System.Char[],System.Int32,System.Int32],System.Int32]) FSharp.Compiler.Tokenization.FSharpSourceTokenizer: FSharp.Compiler.Tokenization.FSharpLineTokenizer CreateLineTokenizer(System.String) FSharp.Compiler.Tokenization.FSharpSourceTokenizer: Void .ctor(Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.String]) diff --git a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs index 6901ceb97b1..44578cfccc7 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs @@ -454,35 +454,39 @@ module internal Tokenizer = member val ClassifiedSpans = classifiedSpans member val SavedTokens = savedTokens + member data.IsValid(textLine: TextLine, lineContents: string) = + data.LineStart = textLine.Start && data.HashCode = lineContents.GetHashCode() + member data.IsValid(textLine: TextLine) = data.LineStart = textLine.Start && let lineContents = textLine.Text.ToString(textLine.Span) in data.HashCode = lineContents.GetHashCode() + // Shared by concurrent editor operations (classification and symbol lookup), so each entry access + // must be thread-safe. This only guarantees per-index atomicity, not a coherent snapshot across a + // range of lines: concurrent scans over overlapping line ranges can still interleave, but the cache + // self-heals on the next read via the IsValid/LexStateAtStartOfLine checks. type private SourceTextData(approxLines: int) = - let data = ResizeArray(approxLines) - - let extendTo i = - if i >= data.Count then - data.Capacity <- i + 1 - - for j in data.Count .. i do - data.Add(None) + let data = + ConcurrentDictionary(Environment.ProcessorCount, approxLines) member x.Item with get (i: int) = - extendTo i - data.[i] + match data.TryGetValue(i) with + | true, v -> ValueSome v + | _ -> ValueNone and set (i: int) v = - extendTo i - data.[i] <- v + match v with + | ValueSome v -> data.[i] <- v + | ValueNone -> data.TryRemove(i) |> ignore member x.ClearFrom(n) = let mutable i = n + let mutable cont = true - while i < data.Count && data.[i].IsSome do - data.[i] <- None - i <- i + 1 + while cont do + let removed, _ = data.TryRemove(i) + if removed then i <- i + 1 else cont <- false /// This saves the tokenization data for a file for as long as the DocumentId object is alive. /// This seems risky - if one single thing leaks a DocumentId (e.g. stores it in some global table of documents @@ -513,66 +517,54 @@ module internal Tokenizer = let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.Text let lineTokenizer = sourceTokenizer.CreateLineTokenizer(lineContents) let tokens = ResizeArray() - let mutable tokenInfoOption = None let mutable previousLexState = lexState - let processToken () = - let classificationType = - compilerTokenToRoslynToken (tokenInfoOption.Value.ColorClass) + let processToken token = + let classificationType = compilerTokenToRoslynToken token.ColorClass - for i = tokenInfoOption.Value.LeftColumn to tokenInfoOption.Value.RightColumn do + for i = token.LeftColumn to token.RightColumn do Array.set colorMap i classificationType - let token = tokenInfoOption.Value - let savedToken = SavedTokenInfo.Create token - - tokens.Add savedToken + tokens.Add(SavedTokenInfo.Create token) let scanAndColorNextToken () = - let info, nextLexState = lineTokenizer.ScanToken(previousLexState) - tokenInfoOption <- info + let struct (info, nextLexState) = lineTokenizer.ScanTokenValue(previousLexState) previousLexState <- nextLexState // Apply some hacks to clean up the token stream (we apply more later) match info with - | Some info when info.Tag = FSharpTokenTag.INT32_DOT_DOT -> - tokenInfoOption <- - Some - { - LeftColumn = info.LeftColumn - RightColumn = info.RightColumn - 2 - ColorClass = FSharpTokenColorKind.Number - CharClass = FSharpTokenCharKind.Literal - FSharpTokenTriggerClass = info.FSharpTokenTriggerClass - Tag = info.Tag - TokenName = "INT32" - FullMatchedLength = info.FullMatchedLength - 2 - } - - processToken () - - tokenInfoOption <- - Some - { - LeftColumn = info.RightColumn - 1 - RightColumn = info.RightColumn - ColorClass = FSharpTokenColorKind.Operator - CharClass = FSharpTokenCharKind.Operator - FSharpTokenTriggerClass = info.FSharpTokenTriggerClass - Tag = FSharpTokenTag.DOT_DOT - TokenName = "DOT_DOT" - FullMatchedLength = 2 - } - - processToken () - - | Some _ -> processToken () + | ValueSome info when info.Tag = FSharpTokenTag.INT32_DOT_DOT -> + processToken + { + LeftColumn = info.LeftColumn + RightColumn = info.RightColumn - 2 + ColorClass = FSharpTokenColorKind.Number + CharClass = FSharpTokenCharKind.Literal + FSharpTokenTriggerClass = info.FSharpTokenTriggerClass + Tag = info.Tag + TokenName = "INT32" + FullMatchedLength = info.FullMatchedLength - 2 + } + + processToken + { + LeftColumn = info.RightColumn - 1 + RightColumn = info.RightColumn + ColorClass = FSharpTokenColorKind.Operator + CharClass = FSharpTokenCharKind.Operator + FSharpTokenTriggerClass = info.FSharpTokenTriggerClass + Tag = FSharpTokenTag.DOT_DOT + TokenName = "DOT_DOT" + FullMatchedLength = 2 + } + + | ValueSome info -> processToken info | _ -> () - scanAndColorNextToken () + info.IsSome - while tokenInfoOption.IsSome do - scanAndColorNextToken () + while scanAndColorNextToken () do + () let mutable startPosition = 0 let mutable endPosition = startPosition @@ -635,8 +627,8 @@ module internal Tokenizer = while i > 0 && (match sourceTextDataCache.[i] with - | Some data -> not (data.IsValid(lines.[i])) - | None -> true) do + | ValueSome data -> not (data.IsValid(lines.[i])) + | ValueNone -> true) do i <- i - 1 i @@ -645,7 +637,9 @@ module internal Tokenizer = if scanStartLine = 0 then FSharpTokenizerLexState.Initial else - sourceTextDataCache.[scanStartLine - 1].Value.LexStateAtEndOfLine + // scanStartLine is the entry the loop above just proved valid; scanStartLine - 1 was + // never checked and can hold a stale entry from a concurrent scan of different text. + sourceTextDataCache.[scanStartLine].Value.LexStateAtStartOfLine for i = scanStartLine to endLine do ct.ThrowIfCancellationRequested() @@ -658,11 +652,15 @@ module internal Tokenizer = // 2. the hash codes match // 3. the start-of-line lex states are the same match sourceTextDataCache.[i] with - | Some data when data.IsValid(textLine) && data.LexStateAtStartOfLine.Equals(lexState) -> data + | ValueSome data when + data.IsValid(textLine, lineContents) + && data.LexStateAtStartOfLine.Equals(lexState) + -> + data | _ -> // Otherwise, we recompute let newData = scanSourceLine (sourceTokenizer, textLine, lineContents, lexState) - sourceTextDataCache.[i] <- Some newData + sourceTextDataCache.[i] <- ValueSome newData newData lexState <- lineData.LexStateAtEndOfLine @@ -673,10 +671,10 @@ module internal Tokenizer = // If necessary, invalidate all subsequent lines after endLine if endLine < lines.Count - 1 then match sourceTextDataCache.[endLine + 1] with - | Some data -> + | ValueSome data -> if not (data.LexStateAtStartOfLine.Equals(lexState)) then sourceTextDataCache.ClearFrom(endLine + 1) - | None -> () + | ValueNone -> () ] /// Generates a list of Classified Spans for tokens which undergo syntactic classification (i.e., are not typechecked). @@ -844,14 +842,10 @@ module internal Tokenizer = | SymbolLookupKind.Precise -> 0 | SymbolLookupKind.Greedy -> 1 - [ - for x in draftTokens do - if - x.LeftColumn <= linePos.Character - && (x.RightColumn + rightColumnCorrection) >= linePos.Character - then - yield x - ] + draftTokens + |> List.filter (fun x -> + x.LeftColumn <= linePos.Character + && (x.RightColumn + rightColumnCorrection) >= linePos.Character) // Select IDENT token. If failed, select OPERATOR token. let symbol = @@ -1024,6 +1018,9 @@ module internal Tokenizer = else false + let private forbiddenSymbolNameChars = + [| '.'; '+'; '$'; '&'; '['; ']'; '/'; '\\'; '*'; '"' |] + let isValidNameForSymbol (lexerSymbolKind: LexerSymbolKind, symbol: FSharpSymbol, name: string) : bool = let inline isIdentifier (ident: string) = @@ -1042,11 +1039,9 @@ module internal Tokenizer = not (String.IsNullOrEmpty s) && FSharpKeywords.NormalizeIdentifierBackticks s |> isIdentifier - let forbiddenChars = [| '.'; '+'; '$'; '&'; '['; ']'; '/'; '\\'; '*'; '\"' |] - let inline isTypeNameIdent (s: string) = not (String.IsNullOrEmpty s) - && s.IndexOfAny forbiddenChars = -1 + && s.IndexOfAny forbiddenSymbolNameChars = -1 && isFixableIdentifier s let inline isUnionCaseIdent (s: string) = diff --git a/vsintegration/src/FSharp.LanguageService/Colorize.fs b/vsintegration/src/FSharp.LanguageService/Colorize.fs index 78bd2a66777..a216765b01b 100644 --- a/vsintegration/src/FSharp.LanguageService/Colorize.fs +++ b/vsintegration/src/FSharp.LanguageService/Colorize.fs @@ -124,17 +124,17 @@ type internal FSharpScanner_DEPRECATED(makeLineTokenizer : string -> FSharpLineT /// Scan a token from a line. This should only be used in cases where color information is irrelevant. /// Used by GetFullLineInfo (and only thus in a small workaround in GetDeclarations) and GetTokenInformationAt (thus GetF1KeywordString). member ws.ScanTokenWithDetails (lexState: _ ref) = - let colorInfoOption, newLexState = lineTokenizer.ScanToken(lexState.Value) + let struct (colorInfoOption, newLexState) = lineTokenizer.ScanTokenValue(lexState.Value) lexState.Value <- newLexState colorInfoOption /// Scan a token from a line and write information about it into the tokeninfo object. member ws.ScanTokenAndProvideInfoAboutIt(_line, tokenInfo:TokenInfo, lexState: _ ref) = - let colorInfoOption, newLexState = lineTokenizer.ScanToken(!lexState) + let struct (colorInfoOption, newLexState) = lineTokenizer.ScanTokenValue(!lexState) lexState.Value <- newLexState match colorInfoOption with - | None -> false - | Some colorInfo -> + | ValueNone -> false + | ValueSome colorInfo -> let color = colorInfo.ColorClass tokenInfo.Trigger <- enum (int32 colorInfo.FSharpTokenTriggerClass) // cast one enum to another tokenInfo.StartIndex <- colorInfo.LeftColumn @@ -259,10 +259,10 @@ type internal FSharpColorizer_DEPRECATED scanner.SetLineText lineText let rec tokens() = seq { match scanner.ScanTokenWithDetails(refState) with - | Some tok -> + | ValueSome tok -> yield tok yield! tokens() - | None -> () } + | ValueNone -> () } tokens() |> Array.ofSeq member private c.GetColorInfo(line,lineText,length,lastColorState) = @@ -342,8 +342,8 @@ type internal FSharpColorizer_DEPRECATED let rec searchForToken () = match scanner.ScanTokenWithDetails lexState with - | None -> None - | Some ti as result -> + | ValueNone -> ValueNone + | ValueSome ti as result -> if col >= ti.LeftColumn && col <= ti.RightColumn then result else diff --git a/vsintegration/src/FSharp.LanguageService/Intellisense.fs b/vsintegration/src/FSharp.LanguageService/Intellisense.fs index 8822b9a5ef1..6bedf5bf273 100644 --- a/vsintegration/src/FSharp.LanguageService/Intellisense.fs +++ b/vsintegration/src/FSharp.LanguageService/Intellisense.fs @@ -534,16 +534,16 @@ type internal FSharpIntellisenseInfo_DEPRECATED span.iStartIndex let textColorState = VsTextLines.TextColorState (VsTextView.Buffer view) match colorizer.Value.GetTokenInformationAt(textColorState,line,col) with - | Some token as original when col > 0 && shouldTryToFindIdentToTheLeft token -> + | ValueSome token as original when col > 0 && shouldTryToFindIdentToTheLeft token -> // try to step back one char match colorizer.Value.GetTokenInformationAt(textColorState,line,col-1) with - | Some token as newInfo when token.CharClass <> FSharpTokenCharKind.WhiteSpace -> newInfo, col - 1 + | ValueSome token as newInfo when token.CharClass <> FSharpTokenCharKind.WhiteSpace -> newInfo, col - 1 | _ -> original, col | otherwise -> otherwise, col match tokenInformation with - | None -> None - | Some token -> + | ValueNone -> None + | ValueSome token -> match token.CharClass, token.ColorClass with | FSharpTokenCharKind.Keyword, _ | FSharpTokenCharKind.Operator, _ diff --git a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj index ecce1205b8c..100d4f1e8e7 100644 --- a/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj +++ b/vsintegration/tests/FSharp.Editor.Tests/FSharp.Editor.Tests.fsproj @@ -85,6 +85,7 @@ + diff --git a/vsintegration/tests/FSharp.Editor.Tests/TokenizerCacheConcurrencyTests.fs b/vsintegration/tests/FSharp.Editor.Tests/TokenizerCacheConcurrencyTests.fs new file mode 100644 index 00000000000..2b09cb5746e --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Tests/TokenizerCacheConcurrencyTests.fs @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Editor.Tests + +open System +open System.Collections.Concurrent +open System.Threading +open System.Threading.Tasks +open Microsoft.CodeAnalysis +open Microsoft.CodeAnalysis.Classification +open Microsoft.CodeAnalysis.Text +open Microsoft.VisualStudio.FSharp.Editor +open Xunit + +/// Every editor operation on a document shares one token cache, keyed by document id and defines, so +/// classification and symbol lookup read and write the same entries from whatever thread they run on. +type TokenizerCacheConcurrencyTests() = + + let fileName = "C:\\test.fs" + let defines = [] + let langVersion = Some "preview" + + // Block comments and triple-quoted strings make a line's classification depend on the lex state + // threaded out of the line before it, so a torn cache misclassifies lines instead of losing them. + let source = + String.Join( + "\n", + [| + for i in 0..11 do + $"let value{i} = {i}" + $"// comment {i}" + "let block =" + " (* opening" + $" still inside {i} *)" + " 42" + $"let text{i} = \"\"\"triple" + $" quoted {i}\"\"\"" + $"let plain{i} = \"single line\"" + "type T() = class end" + |] + ) + + // Dropping every `*)` leaves the first block comment open to the end of the file: the lines keep + // their text and hash but change lex state, which is what drives cached entries out of the cache. + let sourceText = SourceText.From source + let commentedText = SourceText.From(source.Replace(" *)", "")) + + let newDocumentId () = + DocumentId.CreateNewId(ProjectId.CreateNewId()) + + let classify (documentId: DocumentId) (text: SourceText) (span: TextSpan) = + let spans = ResizeArray() + + Tokenizer.classifySpans (documentId, text, span, Some fileName, defines, langVersion, spans, CancellationToken.None) + + spans |> Seq.map (fun s -> s.ClassificationType, s.TextSpan) |> Array.ofSeq + + let lineSpan (text: SourceText) startLine endLine = + TextSpan.FromBounds(text.Lines.[startLine].Start, text.Lines.[endLine].End) + + /// What an uncontended scan of the document produces, one line at a time. + let scanEveryLine (text: SourceText) = + let documentId = newDocumentId () + + Array.init text.Lines.Count (fun i -> classify documentId text (lineSpan text i i)) + + let expectedRange (perLine: (string * TextSpan)[][]) startLine endLine = + Array.concat perLine.[startLine..endLine] + + let describe (spans: (string * TextSpan)[]) = + spans |> Seq.map (fun (kind, span) -> $"{kind}{span}") |> String.concat " " + + let runConcurrently workerCount (work: int -> unit) = + let workers = + Array.init workerCount (fun worker -> Task.Run(Action(fun () -> work worker))) + + Task.WaitAll workers + + let workerCount = max 8 Environment.ProcessorCount + + [] + member _.``Overlapping concurrent reads of one document's token cache agree with a single-threaded scan``() = + let perLine = scanEveryLine sourceText + let lineCount = sourceText.Lines.Count + let documentId = newDocumentId () + let failures = ConcurrentQueue() + + runConcurrently workerCount (fun worker -> + let random = Random(worker) + + for _ in 1..60 do + let startLine, endLine = + let a = random.Next lineCount + let b = random.Next lineCount + min a b, max a b + + let actual = classify documentId sourceText (lineSpan sourceText startLine endLine) + let expected = expectedRange perLine startLine endLine + + if actual <> expected then + failures.Enqueue $"lines {startLine}..{endLine}\n expected {describe expected}\n actual {describe actual}" + + // Symbol lookup reads the same cache, and does so a line at a time. + Tokenizer.getSymbolAtPosition ( + documentId, + sourceText, + random.Next sourceText.Length, + fileName, + defines, + SymbolLookupKind.Greedy, + false, + false, + langVersion, + CancellationToken.None + ) + |> ignore) + + let diverged = String.concat Environment.NewLine failures + Assert.True(failures.IsEmpty, $"Concurrent classification diverged:{Environment.NewLine}{diverged}") + + [] + member _.``Concurrent reads of two versions of a document keep invalidating the cache correctly``() = + let perLine = scanEveryLine sourceText + let commentedPerLine = scanEveryLine commentedText + let lineCount = sourceText.Lines.Count + + // Without this the test would pass on a cache that never invalidates anything. + Assert.NotEqual<(string * TextSpan)[]>(perLine.[lineCount - 1], commentedPerLine.[lineCount - 1]) + + let documentId = newDocumentId () + let failures = ConcurrentQueue() + + runConcurrently workerCount (fun worker -> + let random = Random(worker) + + for iteration in 1..60 do + let text, expectedPerLine = + if (worker + iteration) % 2 = 0 then + sourceText, perLine + else + commentedText, commentedPerLine + + let startLine, endLine = + let a = random.Next lineCount + let b = random.Next lineCount + min a b, max a b + + let actual = classify documentId text (lineSpan text startLine endLine) + let expected = expectedRange expectedPerLine startLine endLine + + if actual <> expected then + failures.Enqueue $"lines {startLine}..{endLine}\n expected {describe expected}\n actual {describe actual}") + + let diverged = String.concat Environment.NewLine failures + Assert.True(failures.IsEmpty, $"Concurrent classification diverged:{Environment.NewLine}{diverged}")