From db82fb73915f1f4a3279cf1c5621906088e1f72f Mon Sep 17 00:00:00 2001 From: Builder Date: Sun, 13 Sep 2026 10:32:20 +0800 Subject: [PATCH] fix(data): stop fundamentals being visible before their publish_date (BUG-22) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daily financial panel was forward-filled from the *fiscal period end* and then backward-filled: df_full = df.reindex(full_idx) # df still keyed on qdate df_full = df_full.groupby("asset").ffill() df_full = df_full.groupby("asset").bfill() # fill initial NaN Since A-share reports land 40-50 days after the period closes, this published figures weeks before anyone could have seen them, and the bfill pushed a company's first report back to the first date of the sample. Measured on the provider's own output (seed 7, 000001.SH): fiscal_period_end 2021-03-31, publish_date 2021-05-20, lag 50 days 2021-04-10 (40d before) -> visible <- should be unknown 2021-05-10 (10d before) -> visible <- should be unknown 2021-05-20 (publish day) -> visible ok Fix: re-index each report to its publish_date before forward-filling, so a value first appears on the day it became public, and drop the bfill -- before a company's first report there is genuinely nothing to know, and filling it backwards is itself a look-ahead. The fiscal period is preserved in a new `fiscal_period_end` column rather than overloading the index, which now carries the visibility date. This matters because NO_LOOKAHEAD_CONTRACT 第三条 requires exactly this (`mask = financials['publish_date'] <= signal_date`). The rule and the point-in-time accessors both already existed; neither was wired to the panel the factors actually read. Every fundamental-factor result computed through the daily panel is affected, so prior fundamental ICs need re-running. After the fix: before publish_date -> nan (unknown) on/after -> the published value before the first report -> nan Two regression tests added, both checked against the old implementation: test_no_value_is_visible_before_its_publish_date and test_nothing_is_backfilled_before_the_first_report -- both fail there. The pre-existing test_publish_date_after_report_date asserted the old model (that index dates which are quarter-ends are the original report rows). That assumption no longer holds, so it now compares publish_date against the new fiscal_period_end column, which checks the same property against the new shape. 1315 passed, 1 skipped. Co-Authored-By: Claude Opus 4.8 (1M context) --- data/providers/synthetic.py | 23 ++++++-- tests/test_data/test_pipeline_pit.py | 88 ++++++++++++++++++++-------- 2 files changed, 82 insertions(+), 29 deletions(-) diff --git a/data/providers/synthetic.py b/data/providers/synthetic.py index 0415fb0..5088719 100644 --- a/data/providers/synthetic.py +++ b/data/providers/synthetic.py @@ -509,15 +509,30 @@ def _generate_financials(self) -> pd.DataFrame: ]) df = df.set_index(["date", "asset"]).sort_index() + # Point-in-time: a quarter's figures become visible on their + # *publish_date*, not on the fiscal period end. A-share reports land + # 40-50 days after the period closes, so re-indexing each report to its + # publication date before forward-filling is what stops the daily panel + # from showing a figure before it was public. + # + # The previous version forward-filled from the fiscal period end + # (`df` was still keyed on `qdate`) and then bfilled, which made a + # Q2 report visible 33 trading days early and pushed the first report + # back to the start of the sample. + pit = df.reset_index() + pit["fiscal_period_end"] = pit["date"] + pit["date"] = pit["publish_date"] + pit = pit.set_index(["date", "asset"]).sort_index() + # Forward-fill to daily frequency by joining with all dates - # Create empty frame with all date-asset combos full_idx = pd.MultiIndex.from_product( [all_dates, self._assets], names=["date", "asset"] ) - df_full = df.reindex(full_idx) - # Forward-fill financial values but keep publish_date from the original quarter + df_full = pit.reindex(full_idx) df_full = df_full.groupby("asset").ffill() - df_full = df_full.groupby("asset").bfill() # Fill initial NaN + # Deliberately no bfill: before a company's first report is published + # there is genuinely nothing to know, and filling it backwards is + # itself a look-ahead. return df_full diff --git a/tests/test_data/test_pipeline_pit.py b/tests/test_data/test_pipeline_pit.py index c8f2203..ae4471f 100644 --- a/tests/test_data/test_pipeline_pit.py +++ b/tests/test_data/test_pipeline_pit.py @@ -150,12 +150,13 @@ def test_financials_have_publish_date(self): assert "publish_date" in fin.columns def test_publish_date_after_report_date(self): - """publish_date should be after the original quarter-end report date. + """publish_date must not precede the fiscal period it describes. - After forward-fill to daily frequency, the index 'date' is a daily - timestamp that may exceed the publish_date. We verify the relationship - on the original quarterly rows by checking unique publish_date values - against the quarter-end dates they were generated from. + After the PIT fix, the index 'date' is the *visibility* date -- the day + a value became usable -- not the fiscal period end. Reports are + re-indexed to their publish_date before forward-filling, so the period + the figures describe now lives in the `fiscal_period_end` column and + that is what this compares against. """ provider = SyntheticDataProvider(n_stocks=10, seed=42) provider._generate_all() @@ -164,24 +165,61 @@ def test_publish_date_after_report_date(self): if "publish_date" not in fin.columns: pytest.skip("No publish_date column") + if "fiscal_period_end" not in fin.columns: + pytest.skip("No fiscal_period_end column") + + rows = fin.dropna(subset=["publish_date", "fiscal_period_end"]).reset_index() + rows = rows.drop_duplicates(subset=["fiscal_period_end", "publish_date"]) + assert not rows.empty + + for _, row in rows.iterrows(): + assert row["publish_date"] >= row["fiscal_period_end"], ( + f"publish_date {row['publish_date']} < fiscal_period_end " + f"{row['fiscal_period_end']}" + ) + + def test_no_value_is_visible_before_its_publish_date(self): + """Regression for BUG-22. + + The daily panel used to be forward-filled from the fiscal period end + and then bfilled, so a Q2 report was visible ~33 trading days before + publication and the first report was pushed back to the start of the + sample. A report's figures must first appear on their publish_date. + """ + provider = SyntheticDataProvider(n_stocks=5, seed=7) + provider._generate_all() + fin = provider.get_financials("2020-01-01", "2023-12-31").reset_index() + + rows = fin.dropna(subset=["publish_date"]) + assert not rows.empty + + checked = 0 + for (_, pub), sub in rows.groupby(["asset", "publish_date"]): + first_visible = pd.Timestamp(sub["date"].min()) + assert first_visible >= pd.Timestamp(pub), ( + f"a value published {pd.Timestamp(pub).date()} is already " + f"visible on {first_visible.date()}" + ) + checked += 1 + assert checked > 0, "no (asset, publish_date) groups were checked" + + def test_nothing_is_backfilled_before_the_first_report(self): + """Regression for BUG-22, second half. + + `bfill()` used to push the first report back to the first date of the + sample, so fundamentals were "known" months before anything was + published. Before a company's first report there is genuinely nothing + to know. + """ + provider = SyntheticDataProvider(n_stocks=5, seed=7) + provider._generate_all() + fin = provider.get_financials("2020-01-01", "2023-12-31") + + earliest_publish = fin["publish_date"].min() + assert pd.notna(earliest_publish) - # Get unique publish_dates and their associated quarter-end dates - valid = fin.dropna(subset=["publish_date"])[["publish_date"]] - unique = valid.reset_index().drop_duplicates(subset=["date", "publish_date"]) - - # After forward-fill, the original quarter-end dates are preserved in - # the 'date' index. But forward-fill means later daily dates share the - # same publish_date. We only check rows where date is a quarter-end - # (the original report dates). - quarter_ends = unique[unique["date"].dt.is_quarter_end] - if quarter_ends.empty: - # If no quarter-end rows in range, just verify publish_dates are - # reasonable (within 60 days of any date in the row) - for _, row in unique.head(5).iterrows(): - delta = (row["publish_date"] - row["date"]).days - assert delta < 60, f"publish_date too far from date: {delta} days" - else: - for _, row in quarter_ends.iterrows(): - assert row["publish_date"] >= row["date"], ( - f"publish_date {row['publish_date']} < report_date {row['date']}" - ) + before = fin.loc[fin.index.get_level_values("date") < earliest_publish] + assert before["market_cap"].isna().all(), ( + "fundamental values exist before the first publish_date, which can " + "only come from a backward fill" + )