From 2fa2421b572b5ecefa17d35d17455ce1290011a3 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Sep 2026 15:43:47 +0300 Subject: [PATCH 1/4] fix: snapshot the charts for the in-run results analysis - clone the charts under the chart lock before building the intermediate backtest result - the margin usage analysis enumerated the live chart while the algorithm thread sampled it - add a handler test that samples the margin chart while the in-run analysis runs --- Engine/Results/Analysis/ResultsAnalyzer.cs | 4 +- Engine/Results/BacktestingResultHandler.cs | 10 ++- .../Results/BacktestingResultHandlerTests.cs | 77 +++++++++++++++++++ 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index cae7a0e2c96a..ccf5edfc9a14 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -267,8 +267,8 @@ public static ResultsAnalyzer CreateForInRunAnalysis(QCAlgorithm algorithm, Lang /// The current intermediate backtest result. Its orders and order events /// are windows truncated to the most recent ones, so the in-run analyses can miss orders and /// events already evicted from them; the final analysis re-scans the complete data. Its charts - /// are the handler's live ones, read without synchronization: a torn read while the algorithm - /// thread updates them can fail a run, which the handler catches, and the next run retries. + /// are a copy the handler takes under its chart lock, so the analyses can enumerate them while + /// the algorithm thread keeps sampling the live ones. /// The full list of log lines produced so far; the analyzer analyzes the /// lines past the ones consumed by previous runs. /// The current total algorithm performance, for analyses that read diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index fff50479125e..fbd054500ca9 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -225,8 +225,16 @@ private void Update() const int maxOrders = 100; var orderCount = TransactionHandler.Orders.Count; + // The in-run analyses enumerate the charts while the algorithm thread keeps sampling them, + // which fails the enumeration, so hand them a copy taken under the chart lock + Dictionary charts; + lock (ChartLock) + { + charts = Charts.ToDictionary(x => x.Key, x => x.Value.Clone()); + } + var completeResult = new BacktestResult(new BacktestResultParameters( - Charts, + charts, orderCount > maxOrders ? TransactionHandler.Orders.Skip(orderCount - maxOrders).ToDictionary() : TransactionHandler.Orders.ToDictionary(), Algorithm.Transactions.TransactionRecord, new Dictionary(), diff --git a/Tests/Engine/Results/BacktestingResultHandlerTests.cs b/Tests/Engine/Results/BacktestingResultHandlerTests.cs index e62dd1551574..d74b14c8c186 100644 --- a/Tests/Engine/Results/BacktestingResultHandlerTests.cs +++ b/Tests/Engine/Results/BacktestingResultHandlerTests.cs @@ -22,10 +22,13 @@ using QuantConnect.Logging; using QuantConnect.Packets; using QuantConnect.Report; +using QuantConnect.Statistics; using QuantConnect.Tests.Engine.DataFeeds; using System; using System.Collections.Generic; using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace QuantConnect.Tests.Engine.Results { @@ -582,11 +585,85 @@ public void RecapturesStartingPortfolioValueAfterWarmup() } } + [Test] + public void InRunAnalysisRunsWhileTheAlgorithmSamplesTheCharts() + { + using var api = new Api.Api(); + using var messaging = new QuantConnect.Messaging.Messaging(); + var resultHandler = new TestableBacktestingResultHandler(); + resultHandler.Initialize(new(new BacktestNodePacket(), messaging, api, new BacktestingTransactionHandler(), null)); + + using var sampling = new CancellationTokenSource(); + try + { + var algorithm = new AlgorithmStub(); + resultHandler.SetAlgorithm(algorithm, 100000); + + // A margin chart large enough for the analysis to still be reading it when the next sample lands + var series = new Series("SPY", SeriesType.StackedArea, "%"); + var time = new DateTime(2024, 1, 1); + for (var i = 0; i < 100000; i++) + { + series.AddPoint(new ChartPoint(time.AddMinutes(i), 50)); + } + var chart = new Chart(BaseResultsHandler.PortfolioMarginKey); + chart.AddSeries(series); + resultHandler.Charts[chart.Name] = chart; + + // The handler stores its first result, running the in-run analysis on it, 5 seconds after its + // construction, and an update before that blocks the next ones until the backtest advances a day + Thread.Sleep(TimeSpan.FromSeconds(5.5)); + + // The algorithm thread samples under the chart lock. Sampling the last point again overwrites it, + // which keeps the chart bounded but still invalidates any enumeration of it in progress + var lastPoint = series.Values[^1]; + var sampler = Task.Run(() => + { + while (!sampling.IsCancellationRequested) + { + lock (resultHandler.ExposedChartLock) + { + series.AddPoint(new ChartPoint(lastPoint.Time, 50)); + } + } + }); + + // Locking the algorithm enables the handler updates + algorithm.SetLocked(); + Assert.IsTrue(resultHandler.InRunAnalysisRan.Wait(TimeSpan.FromSeconds(30)), "The in-run analysis did not run"); + + sampling.Cancel(); + sampler.Wait(); + Assert.IsNotNull(resultHandler.InRunAnalysisFindings, "The in-run analysis failed, see the logged error"); + } + finally + { + sampling.Cancel(); + resultHandler.Exit(); + } + } + private class TestableBacktestingResultHandler : BacktestingResultHandler { public decimal ExposedStartingPortfolioValue => StartingPortfolioValue; public decimal ExposedDailyPortfolioValue => DailyPortfolioValue; public decimal ExposedCumulativeMaxPortfolioValue => CumulativeMaxPortfolioValue; + public object ExposedChartLock => ChartLock; + + public ManualResetEventSlim InRunAnalysisRan { get; } = new(); + + /// + /// The findings of the first in-run analysis run, null when the analysis failed + /// + public IReadOnlyList InRunAnalysisFindings { get; private set; } + + protected override IReadOnlyList RunInRunResultsAnalysis(BacktestResult completeResult, + AlgorithmPerformance totalPerformance) + { + InRunAnalysisFindings = base.RunInRunResultsAnalysis(completeResult, totalPerformance); + InRunAnalysisRan.Set(); + return InRunAnalysisFindings; + } } } } From cf8b7293398e181b78a3b86e62f4d2c0143c2515 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Sep 2026 16:11:52 +0300 Subject: [PATCH 2/4] test: drop the 5 second wait from the in-run analysis test - reset the private first store time by reflection so the first update stores at once --- Tests/Engine/Results/BacktestingResultHandlerTests.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Tests/Engine/Results/BacktestingResultHandlerTests.cs b/Tests/Engine/Results/BacktestingResultHandlerTests.cs index d74b14c8c186..2bca3657e233 100644 --- a/Tests/Engine/Results/BacktestingResultHandlerTests.cs +++ b/Tests/Engine/Results/BacktestingResultHandlerTests.cs @@ -27,6 +27,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -611,8 +612,11 @@ public void InRunAnalysisRunsWhileTheAlgorithmSamplesTheCharts() resultHandler.Charts[chart.Name] = chart; // The handler stores its first result, running the in-run analysis on it, 5 seconds after its - // construction, and an update before that blocks the next ones until the backtest advances a day - Thread.Sleep(TimeSpan.FromSeconds(5.5)); + // construction, and an update before that blocks the next ones until the backtest advances a day, + // so make the first update store at once + typeof(BacktestingResultHandler) + .GetField("_nextS3Update", BindingFlags.Instance | BindingFlags.NonPublic) + .SetValue(resultHandler, DateTime.MinValue); // The algorithm thread samples under the chart lock. Sampling the last point again overwrites it, // which keeps the chart bounded but still invalidates any enumeration of it in progress From b994de982943608076a6fa42436ec25eeb533a46 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 30 Sep 2026 16:28:51 +0300 Subject: [PATCH 3/4] refactor: make the first result store delay overridable - add the protected virtual InitialResultStoreDelay property, 5 seconds by default - the in-run analysis test overrides it to zero instead of resetting the private field by reflection --- Engine/Results/BacktestingResultHandler.cs | 7 ++++++- Tests/Engine/Results/BacktestingResultHandlerTests.cs | 11 +++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index fbd054500ca9..1bb553df9ca1 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -74,6 +74,11 @@ public class BacktestingResultHandler : BaseResultsHandler, IResultHandler /// protected bool RunResultsAnalysis { get; set; } = true; + /// + /// The delay after the handler starts before the first result is stored, which also runs the first in-run analysis + /// + protected virtual TimeSpan InitialResultStoreDelay { get; } = TimeSpan.FromSeconds(5); + /// /// A dictionary containing summary statistics /// @@ -91,7 +96,7 @@ public BacktestingResultHandler() _chartSeriesCount = new(); // Delay uploading first packet - _nextS3Update = StartTime.AddSeconds(5); + _nextS3Update = StartTime.Add(InitialResultStoreDelay); } /// diff --git a/Tests/Engine/Results/BacktestingResultHandlerTests.cs b/Tests/Engine/Results/BacktestingResultHandlerTests.cs index 2bca3657e233..fe2474de238c 100644 --- a/Tests/Engine/Results/BacktestingResultHandlerTests.cs +++ b/Tests/Engine/Results/BacktestingResultHandlerTests.cs @@ -27,7 +27,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -611,13 +610,6 @@ public void InRunAnalysisRunsWhileTheAlgorithmSamplesTheCharts() chart.AddSeries(series); resultHandler.Charts[chart.Name] = chart; - // The handler stores its first result, running the in-run analysis on it, 5 seconds after its - // construction, and an update before that blocks the next ones until the backtest advances a day, - // so make the first update store at once - typeof(BacktestingResultHandler) - .GetField("_nextS3Update", BindingFlags.Instance | BindingFlags.NonPublic) - .SetValue(resultHandler, DateTime.MinValue); - // The algorithm thread samples under the chart lock. Sampling the last point again overwrites it, // which keeps the chart bounded but still invalidates any enumeration of it in progress var lastPoint = series.Values[^1]; @@ -654,6 +646,9 @@ private class TestableBacktestingResultHandler : BacktestingResultHandler public decimal ExposedCumulativeMaxPortfolioValue => CumulativeMaxPortfolioValue; public object ExposedChartLock => ChartLock; + // Store the first result, which runs the first in-run analysis, on the first update instead of 5 seconds in + protected override TimeSpan InitialResultStoreDelay => TimeSpan.Zero; + public ManualResetEventSlim InRunAnalysisRan { get; } = new(); /// From 39866d5d0305a6c08489cc4016f44aebbff182f3 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Wed, 30 Sep 2026 11:11:14 -0300 Subject: [PATCH 4/4] fix: store the chart snapshot without copying it again The in-run and final results hand StoreResult a copy of the charts taken under the chart lock, so it stores that copy instead of cloning the charts a second time. The final result takes its copy too, since the algorithm thread can still be sampling when it was stopped for exceeding a limit. The single point portfolio margin series are now removed from the stored result, like the live handler does, instead of from the charts the algorithm keeps sampling. Co-Authored-By: Claude Opus 5.5 (1M context) --- Engine/Results/BacktestingResultHandler.cs | 60 ++++++++++++---------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index 1bb553df9ca1..137fc806f2a7 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -232,14 +232,8 @@ private void Update() // The in-run analyses enumerate the charts while the algorithm thread keeps sampling them, // which fails the enumeration, so hand them a copy taken under the chart lock - Dictionary charts; - lock (ChartLock) - { - charts = Charts.ToDictionary(x => x.Key, x => x.Value.Clone()); - } - var completeResult = new BacktestResult(new BacktestResultParameters( - charts, + CloneCharts(), orderCount > maxOrders ? TransactionHandler.Orders.Skip(orderCount - maxOrders).ToDictionary() : TransactionHandler.Orders.ToDictionary(), Algorithm.Transactions.TransactionRecord, new Dictionary(), @@ -345,28 +339,26 @@ protected override void StoreResult(Packet packet) // Get Storage Location: var key = $"{AlgorithmId}.json"; - BacktestResult results; - lock (ChartLock) + // The charts are a snapshot taken under the chart lock, so they are cleaned up and stored without another copy + if (result.Results.Charts.TryGetValue(PortfolioMarginKey, out var marginChart)) { - results = new BacktestResult(new BacktestResultParameters( - result.Results.Charts.ToDictionary(x => x.Key, x => x.Value.Clone()), - result.Results.Orders, - result.Results.ProfitLoss, - result.Results.Statistics, - result.Results.RuntimeStatistics, - result.Results.RollingWindow, - null, // null order events, we store them separately - result.Results.TotalPerformance, - result.Results.AlgorithmConfiguration, - result.Results.State, - result.Results.Analysis, - result.Results.ServerStatistics)); - - if (result.Results.Charts.TryGetValue(PortfolioMarginKey, out var marginChart)) - { - PortfolioMarginChart.RemoveSinglePointSeries(marginChart); - } + PortfolioMarginChart.RemoveSinglePointSeries(marginChart); } + + var results = new BacktestResult(new BacktestResultParameters( + result.Results.Charts, + result.Results.Orders, + result.Results.ProfitLoss, + result.Results.Statistics, + result.Results.RuntimeStatistics, + result.Results.RollingWindow, + null, // null order events, we store them separately + result.Results.TotalPerformance, + result.Results.AlgorithmConfiguration, + result.Results.State, + result.Results.Analysis, + result.Results.ServerStatistics)); + // Save results SaveResults(key, results); @@ -397,7 +389,8 @@ protected void SendFinalResult() if (Algorithm != null) { //Convert local dictionary: - var charts = new Dictionary(Charts); + // The algorithm thread can still be sampling when it was stopped for exceeding a limit + var charts = CloneCharts(); var orders = new Dictionary(TransactionHandler.Orders); var profitLoss = new SortedDictionary(Algorithm.Transactions.TransactionRecord); var statisticsResults = GenerateStatisticsResults(charts, profitLoss, _capacityEstimate); @@ -532,6 +525,17 @@ private List CloneLogs() } } + /// + /// Takes a snapshot of the charts under the chart lock. + /// + private Dictionary CloneCharts() + { + lock (ChartLock) + { + return Charts.ToDictionary(x => x.Key, x => x.Value.Clone()); + } + } + /// /// Sends the in-run analysis findings to the browser in their own packet. ///