From fcd1804fc8fc8b7b29960a12f683e6cbc5bb64ad Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 11:26:05 +0200 Subject: [PATCH] fix(priceforecast): bucket local hours, not UTC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hour-of-week and month indexing used t.UTC(), so the baked Nordic evening peak (17–20 local) landed 1–2 hours late in CET/CEST. Fit and Predict now share the Europe/Stockholm civil clock. Closes #1161 Signed-off-by: Fredrik Ahlgren --- .changeset/priceforecast-local-hours.md | 7 ++ go/internal/priceforecast/forecast.go | 71 ++++++++----- go/internal/priceforecast/forecast_test.go | 113 ++++++++++++++------- 3 files changed, 131 insertions(+), 60 deletions(-) create mode 100644 .changeset/priceforecast-local-hours.md diff --git a/.changeset/priceforecast-local-hours.md b/.changeset/priceforecast-local-hours.md new file mode 100644 index 000000000..44e3e2e30 --- /dev/null +++ b/.changeset/priceforecast-local-hours.md @@ -0,0 +1,7 @@ +--- +"ftw": patch +--- + +Price-forecast hour-of-week buckets use Europe/Stockholm civil hours, so +Nordic evening peaks land in the evening prior instead of 1–2 hours late +under UTC indexing. diff --git a/go/internal/priceforecast/forecast.go b/go/internal/priceforecast/forecast.go index 6024d089b..34347d1b5 100644 --- a/go/internal/priceforecast/forecast.go +++ b/go/internal/priceforecast/forecast.go @@ -22,9 +22,9 @@ // The model is zone-aware: each bidding zone trains independently // because SE3 and SE4 behave very differently at peak hours. // -// Confidence: we track sample count per bucket + global MAE. The MPC -// can downweight these estimates vs. real day-ahead prices by looking -// at the confidence flag on each forecasted slot. +// Counts and MAE are diagnostics for refit logs and Model() snapshots. +// Predict returns the blended climatology; the MPC blends toward it +// with a time e-folding, not via these fields. package priceforecast import ( @@ -42,6 +42,10 @@ import ( "sync" "time" + // Embedded zoneinfo so hour-of-week buckets cannot silently fall + // back to UTC when the host has no tzdata (tests, stripped images). + _ "time/tzdata" + "github.com/srcfl/ftw/go/internal/state" ) @@ -56,14 +60,14 @@ const MinTrustSamples = 4 // Derived from the ZoneModel at refit time — NOT persisted as separate // state, recomputed from bucket data. type ZoneModel struct { - Zone string `json:"zone"` - Bucket [Buckets]float64 `json:"bucket"` // EMA öre/kWh (raw spot) - Counts [Buckets]int64 `json:"counts"` - Month [12]float64 `json:"month"` // monthly multiplier (normalized) - Samples int64 `json:"samples"` - MAE float64 `json:"mae"` // EMA of |actual − predicted| - Alpha float64 `json:"alpha"` // EMA coefficient - FittedAt int64 `json:"fitted_at"` + Zone string `json:"zone"` + Bucket [Buckets]float64 `json:"bucket"` // EMA öre/kWh (raw spot) + Counts [Buckets]int64 `json:"counts"` + Month [12]float64 `json:"month"` // monthly multiplier (normalized) + Samples int64 `json:"samples"` + MAE float64 `json:"mae"` // EMA of |actual − predicted| + Alpha float64 `json:"alpha"` // EMA coefficient + FittedAt int64 `json:"fitted_at"` } // bakedPrior returns the typical-Nordic hour-of-week prior shape for a @@ -135,13 +139,13 @@ func NewZoneModel(zone string) *ZoneModel { // value is already prior-blended via FitFromHistory, so we just apply // the monthly seasonality. // -// Coerces t to UTC so hour-of-week + month indexing is stable across -// DST transitions. FitFromHistory does the same (see line 183), so Fit -// and Predict agree on bucket addressing. +// Indexes hour-of-week and month on the Europe/Stockholm civil clock +// so the baked Nordic prior (evening 17–20 local) lines up with CET/CEST +// peaks. FitFromHistory uses the same conversion, and the same instant +// always hits the same bucket regardless of t's attached Location. func (m ZoneModel) Predict(t time.Time) float64 { - u := t.UTC() - idx := hourOfWeek(u) - return m.Bucket[idx] * m.Month[int(u.Month())-1] + c := civil(t) + return m.Bucket[hourOfWeek(t)] * m.Month[int(c.Month())-1] } // overallMean across buckets weighted by counts. @@ -185,7 +189,7 @@ func (m *ZoneModel) FitFromHistory(pts []state.PricePoint) { var monthSum [12]float64 var monthCnt [12]int64 for _, p := range pts { - t := time.UnixMilli(p.SlotTsMs).UTC() + t := civil(time.UnixMilli(p.SlotTsMs)) idx := hourOfWeek(t) sum[idx] += p.SpotOreKwh cnt[idx]++ @@ -229,7 +233,7 @@ func (m *ZoneModel) FitFromHistory(pts []state.PricePoint) { // MAE: fit quality on history itself. var abserr float64 for _, p := range pts { - t := time.UnixMilli(p.SlotTsMs).UTC() + t := time.UnixMilli(p.SlotTsMs) abserr += math.Abs(p.SpotOreKwh - m.Predict(t)) } m.MAE = abserr / float64(len(pts)) @@ -237,14 +241,29 @@ func (m *ZoneModel) FitFromHistory(pts []state.PricePoint) { m.FittedAt = time.Now().UnixMilli() } -// hourOfWeek: Mon=0..Sun=6 × 24. Coerces to UTC so the bucket index is -// deterministic across DST transitions — without this, a wall-clock -// 19:00 call returns a different bucket in summer than in winter, -// silently misaligning the learned EMA against Fit's UTC-indexed data. +// bucketTZ is the civil clock for hour-of-week and month buckets. +// The baked prior is a Nordic local-hour shape; UTC indexing put CEST +// evening peaks two hours late (19:00 CEST = 17:00 UTC). Stockholm is +// CET/CEST, matching SE1–SE4 / DK / NO / DE. +var bucketTZ = mustLoadTZ("Europe/Stockholm") + +func mustLoadTZ(name string) *time.Location { + loc, err := time.LoadLocation(name) + if err != nil { + return time.UTC + } + return loc +} + +func civil(t time.Time) time.Time { return t.In(bucketTZ) } + +// hourOfWeek: Mon=0..Sun=6 × 24 on the Europe/Stockholm civil clock. +// 19:00 CET and 19:00 CEST share a bucket, matching the baked evening +// peak. The same instant presented as UTC or local still agrees. func hourOfWeek(t time.Time) int { - u := t.UTC() - wd := (int(u.Weekday()) + 6) % 7 - return wd*24 + u.Hour() + c := civil(t) + wd := (int(c.Weekday()) + 6) % 7 + return wd*24 + c.Hour() } // ---- Service ---- diff --git a/go/internal/priceforecast/forecast_test.go b/go/internal/priceforecast/forecast_test.go index 315e4b4bd..f19872e39 100644 --- a/go/internal/priceforecast/forecast_test.go +++ b/go/internal/priceforecast/forecast_test.go @@ -10,13 +10,24 @@ import ( "github.com/srcfl/ftw/go/internal/state" ) +func stockholm(t *testing.T) *time.Location { + t.Helper() + loc, err := time.LoadLocation("Europe/Stockholm") + if err != nil { + t.Fatalf("Europe/Stockholm tzdata unavailable: %v", err) + } + return loc +} + func TestFreshModelHasSensibleCurve(t *testing.T) { // Untrained model returns baked-in typical Nordic pattern: // midday trough, morning + evening peaks. Tests shape, not exact values. + // Civil times are Europe/Stockholm — the clock the prior is drawn in. + loc := stockholm(t) m := NewZoneModel("SE3") - midday := time.Date(2026, 6, 15, 13, 0, 0, 0, time.UTC) - evening := time.Date(2026, 6, 15, 19, 0, 0, 0, time.UTC) - overnight := time.Date(2026, 6, 15, 3, 0, 0, 0, time.UTC) + midday := time.Date(2026, 6, 15, 13, 0, 0, 0, loc) + evening := time.Date(2026, 6, 15, 19, 0, 0, 0, loc) + overnight := time.Date(2026, 6, 15, 3, 0, 0, 0, loc) pm := m.Predict(midday) pe := m.Predict(evening) po := m.Predict(overnight) @@ -27,8 +38,8 @@ func TestFreshModelHasSensibleCurve(t *testing.T) { t.Errorf("midday (%.1f) should be below overnight (%.1f) due to solar flood", pm, po) } // Winter vs summer seasonality - wintr := time.Date(2026, 1, 15, 19, 0, 0, 0, time.UTC) - smrEv := time.Date(2026, 7, 15, 19, 0, 0, 0, time.UTC) + wintr := time.Date(2026, 1, 15, 19, 0, 0, 0, loc) + smrEv := time.Date(2026, 7, 15, 19, 0, 0, 0, loc) if !(m.Predict(wintr) > m.Predict(smrEv)) { t.Errorf("winter (%.1f) should exceed summer (%.1f)", m.Predict(wintr), m.Predict(smrEv)) } @@ -36,13 +47,16 @@ func TestFreshModelHasSensibleCurve(t *testing.T) { func TestFitsHourOfWeekPattern(t *testing.T) { // Synthetic: SE3 prices with morning peak 150, midday trough 30, - // evening peak 200. Two years of data so the Bayesian prior (weight - // ≈ 8) is swamped by ~100 samples per hour-of-week bucket. + // evening peak 200, keyed to Stockholm local hours. Two years of + // data so the Bayesian prior (weight ≈ 8) is swamped by ~100 + // samples per hour-of-week bucket. + loc := stockholm(t) var pts []state.PricePoint - start := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + start := time.Date(2024, 1, 1, 0, 0, 0, 0, loc) for d := 0; d < 730; d++ { // 2 years + day := start.AddDate(0, 0, d) for h := 0; h < 24; h++ { - ts := start.Add(time.Duration(d*24+h) * time.Hour) + ts := time.Date(day.Year(), day.Month(), day.Day(), h, 0, 0, 0, loc) var price float64 switch { case h >= 7 && h <= 9: @@ -68,15 +82,15 @@ func TestFitsHourOfWeekPattern(t *testing.T) { // With ~100+ samples per bucket, fit should be very close to data. // Tolerance generous because month multipliers still apply some // seasonal scaling. - mornMon := time.Date(2026, 3, 2, 8, 0, 0, 0, time.UTC) + mornMon := time.Date(2026, 3, 2, 8, 0, 0, 0, loc) if got := m.Predict(mornMon); math.Abs(got-150) > 20 { t.Errorf("Mon 08:00 peak: got %f, want ~150 (±20)", got) } - trough := time.Date(2026, 3, 4, 13, 0, 0, 0, time.UTC) + trough := time.Date(2026, 3, 4, 13, 0, 0, 0, loc) if got := m.Predict(trough); math.Abs(got-30) > 20 { t.Errorf("Wed 13:00 trough: got %f, want ~30 (±20)", got) } - eve := time.Date(2026, 3, 6, 19, 0, 0, 0, time.UTC) + eve := time.Date(2026, 3, 6, 19, 0, 0, 0, loc) if got := m.Predict(eve); math.Abs(got-200) > 20 { t.Errorf("Fri 19:00 peak: got %f, want ~200 (±20)", got) } @@ -87,11 +101,13 @@ func TestSparseHistoryFallsBackToPriorShape(t *testing.T) { // so the predictions should still show the baked hour-of-week // shape — morning + evening peaks, midday trough — even if the // short training sample happened to be uniform. + loc := stockholm(t) var pts []state.PricePoint - start := time.Date(2026, 1, 5, 0, 0, 0, 0, time.UTC) + start := time.Date(2026, 1, 5, 0, 0, 0, 0, loc) for d := 0; d < 3; d++ { + day := start.AddDate(0, 0, d) for h := 0; h < 24; h++ { - ts := start.Add(time.Duration(d*24+h) * time.Hour) + ts := time.Date(day.Year(), day.Month(), day.Day(), h, 0, 0, 0, loc) pts = append(pts, state.PricePoint{ Zone: "SE3", SlotTsMs: ts.UnixMilli(), SlotLenMin: 60, SpotOreKwh: 100, // totally flat — unusual @@ -102,9 +118,9 @@ func TestSparseHistoryFallsBackToPriorShape(t *testing.T) { m.FitFromHistory(pts) // Even though training data was flat, shape persists from prior. - morn := time.Date(2026, 3, 2, 8, 0, 0, 0, time.UTC) - midday := time.Date(2026, 3, 2, 13, 0, 0, 0, time.UTC) - eve := time.Date(2026, 3, 2, 19, 0, 0, 0, time.UTC) + morn := time.Date(2026, 3, 2, 8, 0, 0, 0, loc) + midday := time.Date(2026, 3, 2, 13, 0, 0, 0, loc) + eve := time.Date(2026, 3, 2, 19, 0, 0, 0, loc) if !(m.Predict(morn) > m.Predict(midday)) { t.Errorf("morning (%f) should beat midday (%f) — prior shape lost", m.Predict(morn), m.Predict(midday)) @@ -150,15 +166,10 @@ SE4,1735689600000,60,90.0 // TestPredictStableAcrossDST ensures Predict returns the same value for // the same absolute instant regardless of the timezone the caller has -// attached to the time.Time struct. Before the UTC coercion in -// hourOfWeek + Predict, passing a local-zone time around DST boundaries -// produced a different bucket (and thus price) than passing the UTC -// equivalent — Erik's 21:00 bug was on this exact code path. +// attached to the time.Time struct. hourOfWeek converts to Stockholm +// first, so UTC and local presentations of one instant agree. func TestPredictStableAcrossDST(t *testing.T) { - stockholm, err := time.LoadLocation("Europe/Stockholm") - if err != nil { - t.Skipf("Europe/Stockholm tzdata unavailable: %v", err) - } + loc := stockholm(t) m := NewZoneModel("SE3") // Several points over the year — including both DST transitions. cases := []struct { @@ -182,7 +193,7 @@ func TestPredictStableAcrossDST(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { utc := tc.inst - local := utc.In(stockholm) + local := utc.In(loc) if !utc.Equal(local) { t.Fatalf("instants must be equal — test bug") } @@ -201,20 +212,17 @@ func TestPredictStableAcrossDST(t *testing.T) { // bucket index itself must not change when the same instant is // represented in a different timezone. func TestHourOfWeekStableAcrossDST(t *testing.T) { - stockholm, err := time.LoadLocation("Europe/Stockholm") - if err != nil { - t.Skipf("Europe/Stockholm tzdata unavailable: %v", err) - } + loc := stockholm(t) // Pick a few instants across DST boundaries. instants := []time.Time{ - time.Date(2026, 3, 29, 1, 0, 0, 0, time.UTC), // spring forward - time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC), // fall back - time.Date(2026, 7, 15, 17, 0, 0, 0, time.UTC), // summer + time.Date(2026, 3, 29, 1, 0, 0, 0, time.UTC), // spring forward + time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC), // fall back + time.Date(2026, 7, 15, 17, 0, 0, 0, time.UTC), // summer time.Date(2026, 12, 15, 20, 0, 0, 0, time.UTC), // winter } for _, inst := range instants { utc := inst - local := inst.In(stockholm) + local := inst.In(loc) if hourOfWeek(utc) != hourOfWeek(local) { t.Errorf("hourOfWeek differs: utc=%d local=%d (inst=%v)", hourOfWeek(utc), hourOfWeek(local), inst) @@ -222,6 +230,43 @@ func TestHourOfWeekStableAcrossDST(t *testing.T) { } } +// TestSE3CESTEveningPeakLandsInEveningPrior is the #1161 regression: +// 19:00 CEST must use the baked 17–20 local evening bucket, not the +// UTC hour (17) that the same instant occupies. +func TestSE3CESTEveningPeakLandsInEveningPrior(t *testing.T) { + loc := stockholm(t) + m := NewZoneModel("SE3") + + // Wednesday 15 Jul 2026 19:00 CEST = 17:00 UTC. + evening := time.Date(2026, 7, 15, 19, 0, 0, 0, loc) + if evening.UTC().Hour() != 17 { + t.Fatalf("test bug: 19:00 CEST should be 17:00 UTC, got %02d", evening.UTC().Hour()) + } + + idx := hourOfWeek(evening) + want := 2*24 + 19 // Wed 19:00 local + utcIdx := 2*24 + 17 + if idx != want { + t.Errorf("19:00 CEST bucket = %d, want %d (UTC 17:00 would be %d)", idx, want, utcIdx) + } + + peak := m.Predict(evening) + after := m.Predict(time.Date(2026, 7, 15, 21, 0, 0, 0, loc)) + if !(peak > after) { + t.Errorf("19:00 CEST (%.1f) should exceed 21:00 CEST (%.1f) — peak must follow local evening, not UTC", + peak, after) + } + + // Passing the instant as UTC must still hit the local-19 bucket. + asUTC := time.Date(2026, 7, 15, 17, 0, 0, 0, time.UTC) + if hourOfWeek(asUTC) != want { + t.Errorf("17:00 UTC (19:00 CEST) bucket = %d, want %d", hourOfWeek(asUTC), want) + } + if m.Predict(asUTC) != peak { + t.Errorf("Predict(17:00 UTC)=%.1f, Predict(19:00 CEST)=%.1f — same instant", m.Predict(asUTC), peak) + } +} + func TestSeedFromCSVRejectsMissingColumns(t *testing.T) { st, _ := state.Open(filepath.Join(t.TempDir(), "t.db")) defer st.Close()