From aae740fb4f1ca4c1fff5e4a3b67302cec51b529b Mon Sep 17 00:00:00 2001 From: Builder Date: Sun, 13 Sep 2026 12:24:34 +0800 Subject: [PATCH] fix(backtest): report walk-forward OOS over the test window only (BUG-21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine is handed `train_start - 252 .. test_end` so its first rebalance has a warm-up, and `result["daily_returns"]` covers that whole range. It was appended as the fold's OOS in full: oos_ret = result["daily_returns"] oos_returns_list.append(oos_ret) m = all_metrics(oos_ret, oos_bench) So a fold with a 63-day test window reported 315 days of "OOS" -- 252 of which were in-sample. Every fold's metrics, the concatenated OOS series, the aggregate metrics and the stability analysis were all computed over train+test. Measured on 900 dates x 30 assets (train 252, test 63): before fold 0 oos_days = 315 after fold 0 oos_days = 63 (and 63 for every full fold; the last partial window reports 18, correctly) **This is a reporting-window defect, not signal leakage.** The fold signal is recomputed from train-only ICs and that part is correct -- both the audit agent and I verified it independently. Recording it as a contract violation of 第四条 would have overstated it and hidden where the real defect is. The docstring of NO_LOOKAHEAD_CONTRACT 第四条 is about signal construction, and signal construction was never the problem. Also fixes the same confusion one line away: `fold_details["oos_days"]` was `len(fold_signal)` -- the signal length, i.e. train+test -- rather than the length of the OOS returns actually being reported. Two regression tests added, both checked against the old implementation and both failing there: - test_each_fold_reports_only_its_test_window - test_concatenated_oos_is_not_the_whole_sample 1320 passed, 1 skipped. Co-Authored-By: Claude Opus 4.8 (1M context) --- backtest/walkforward.py | 15 ++++- tests/test_backtest/test_walkforward.py | 75 +++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/backtest/walkforward.py b/backtest/walkforward.py index 7ef5ea9..b087802 100644 --- a/backtest/walkforward.py +++ b/backtest/walkforward.py @@ -179,8 +179,21 @@ def run( financials=financials, ) + # The engine is handed extra history so its first rebalance has + # a warm-up, but only the test window is out of sample. Taking + # `result["daily_returns"]` whole made every fold's reported + # metrics cover train + test -- a 63-day test window was being + # reported over 315 days, so the "OOS" numbers were mostly + # in-sample performance. + test_start_ts = prices.index[test_sl.start] + oos_ret = result["daily_returns"] + if len(oos_ret) > 0: + oos_ret = oos_ret.loc[oos_ret.index >= test_start_ts] + oos_bench = result.get("benchmark_returns", test_benchmark) + if hasattr(oos_bench, "index") and len(oos_bench) > 0: + oos_bench = oos_bench.loc[oos_bench.index >= test_start_ts] oos_returns_list.append(oos_ret) oos_benchmark_list.append(oos_bench) @@ -199,7 +212,7 @@ def run( "fold": fi, "train": f"{fold_info['train_dates'][0]} -> {fold_info['train_dates'][1]}", "test": f"{fold_info['test_dates'][0]} -> {fold_info['test_dates'][1]}", - "oos_days": len(fold_signal), + "oos_days": len(oos_ret), "sharpe": fold_metrics[-1].get("sharpe_ratio", 0) if fold_metrics else 0, }) diff --git a/tests/test_backtest/test_walkforward.py b/tests/test_backtest/test_walkforward.py index 647cfa2..6e67434 100644 --- a/tests/test_backtest/test_walkforward.py +++ b/tests/test_backtest/test_walkforward.py @@ -30,3 +30,78 @@ def test_insufficient_data_raises(self): validator = WalkForwardValidator(train_period=504, test_period=126) with pytest.raises(ValueError, match="Not enough data"): validator.run(short_signal, short_prices, short_returns, benchmark, sector_map) + + +def _panel(n_dates=900, n_assets=30, seed=0): + np.random.seed(seed) + dates = pd.bdate_range("2020-01-01", periods=n_dates) + assets = [f"S{i}" for i in range(n_assets)] + ret = pd.DataFrame(np.random.randn(n_dates, n_assets) * 0.015, index=dates, columns=assets) + prices = (1 + ret).cumprod() * 100 + factors = {"f": pd.DataFrame(np.random.randn(n_dates, n_assets), index=dates, columns=assets)} + signal = pd.DataFrame(0.0, index=dates, columns=assets) + return signal, prices, ret, ret.mean(axis=1), pd.Series("Tech", index=assets), factors + + +class TestOOSWindow: + """Regression for BUG-21. + + The engine is handed extra history so its first rebalance has a warm-up, + and `result["daily_returns"]` covers that whole range. Reporting it whole + meant each fold's "OOS" metrics spanned train + test: a 63-day test window + was reported over 315 days, so the headline OOS numbers were mostly + in-sample performance. + + Note this is a *reporting window* defect, not signal leakage -- the fold + signal is recomputed from train-only ICs and that part is correct. + """ + + def test_each_fold_reports_only_its_test_window(self): + signal, prices, ret, bench, sector_map, factors = _panel() + validator = WalkForwardValidator( + train_period=252, test_period=63, step_size=63, mode="rolling" + ) + result = validator.run( + signal=signal, prices=prices, returns=ret, benchmark_returns=bench, + sector_map=sector_map, factors=factors, + ) + + folds = result["fold_metrics"] + assert folds, "expected folds" + + for fm in folds: + assert fm["oos_days"] <= 63, ( + f"fold {fm['fold']} reported {fm['oos_days']} OOS days for a " + f"63-day test window" + ) + + def test_concatenated_oos_is_not_the_whole_sample(self): + """Every fold's OOS used to span train+test, so the concatenated series + covered nearly the entire sample instead of the union of the test + windows.""" + signal, prices, ret, bench, sector_map, factors = _panel() + validator = WalkForwardValidator( + train_period=252, test_period=63, step_size=63, mode="rolling" + ) + result = validator.run( + signal=signal, prices=prices, returns=ret, benchmark_returns=bench, + sector_map=sector_map, factors=factors, + ) + + oos = result["oos_returns"] + n_folds = len(result["fold_metrics"]) + n_dates = len(prices) + + # Test windows tile forward without overlapping, so the union is about + # n_folds * test_period. Before the fix each fold returned train+test, + # which would put this near the full sample length. + assert len(oos) <= n_folds * 63 + 63, ( + f"concatenated OOS covers {len(oos)} days for {n_folds} folds of 63 " + f"days -- it is spanning the training periods again" + ) + assert len(oos) < n_dates, "OOS covers the whole sample" + + # And it must start at the first fold's test window, not at the start + # of that fold's warm-up. + first_test_start = pd.Timestamp(result["fold_metrics"][0]["test_start"]) + assert pd.Timestamp(oos.index[0]) >= first_test_start