fix(history): keep the chart running when optional PV never reports

.changeset/history-optional-pv.md

@@ -0,0 +1,5 @@
+---
+"ftw": patch
+---
+
+Keep the history chart and site energy running when a device can report PV but does not, such as a Zap P1 meter with PV reading off, or while a device identity is still unconfirmed. Before, adding such a device stopped history until it was removed, even across restarts. Forecast learning keeps its stricter checks.

go/cmd/ftw/forecast_measurements.go

@@ -14,25 +14,41 @@ import (
 // forecastMeasurementOptions declares known significant sources. FlowIDs remain
 // unset: driver names and inverter groups do not identify physical measurements.
 func forecastMeasurementOptions(cfg *config.Config, catalog []drivers.CatalogEntry) telemetry.ForecastOptions {
-	opts, _ := forecastMeasurementTopology(cfg, catalog)
+	opts, _, _ := forecastMeasurementTopology(cfg, catalog)
+	return opts
+}
+
+// historyMeasurementOptions qualifies the stored chart and site energy. They
+// need a measured balance, not the learner's proof that an optional source is
+// absent: a source that reports PV is still checked, but one that never
+// reports it does not stop history (#1441). Identity checks stay with learning.
+func historyMeasurementOptions(cfg *config.Config, catalog []drivers.CatalogEntry) telemetry.ForecastOptions {
+	opts, _, optional := forecastMeasurementTopology(cfg, catalog)
+	flows := opts.ExpectedFlows[:0:0]
+	for _, flow := range opts.ExpectedFlows {
+		if !optional[flow] {
+			flows = append(flows, flow)
+		}
+	}
+	opts.ExpectedFlows = flows
 	return opts
 }
 
 // forecastMeasurementTopologyUnknown names sources whose presence config cannot
 // establish. Callers must require fresh observations of these optional flows
 // before qualifying a complete household balance; this is not only a log hint.
 func forecastMeasurementTopologyUnknown(cfg *config.Config, catalog []drivers.CatalogEntry) []string {
-	_, unknown := forecastMeasurementTopology(cfg, catalog)
+	_, unknown, _ := forecastMeasurementTopology(cfg, catalog)
 	return unknown
 }
 
-func forecastMeasurementTopology(cfg *config.Config, catalog []drivers.CatalogEntry) (telemetry.ForecastOptions, []string) {
-	opts := telemetry.ForecastOptions{}
+// optional holds PV flows that config does not declare; see historyMeasurementOptions.
+func forecastMeasurementTopology(cfg *config.Config, catalog []drivers.CatalogEntry) (opts telemetry.ForecastOptions, unknown []string, optional map[telemetry.ForecastFlow]bool) {
+	optional = make(map[telemetry.ForecastFlow]bool)
 	if cfg == nil {
 		opts.HouseholdInvalidReason = "missing_config"
-		return opts, []string{"missing_config"}
+		return opts, []string{"missing_config"}, optional
 	}
-	var unknown []string
 	type configured struct {
 		driver config.Driver
 		entry  drivers.CatalogEntry
@@ -97,6 +113,7 @@ func forecastMeasurementTopology(cfg *config.Config, catalog []drivers.CatalogEn
 			add(d, telemetry.DerPV)
 			if !declared {
 				unknown = append(unknown, d.Name+":optional_pv_requires_measurement")
+				optional[telemetry.ForecastFlow{Driver: d.Name, DerType: telemetry.DerPV}] = true
 			}
 		}
 	}
@@ -117,7 +134,7 @@ func forecastMeasurementTopology(cfg *config.Config, catalog []drivers.CatalogEn
 		opts.HouseholdInvalidReason = "configured_pv_without_measurement_source"
 		unknown = append(unknown, "site:configured_pv_without_measurement_source")
 	}
-	return opts, unknown
+	return opts, unknown, optional
 }
 
 func forecastCapability(e drivers.CatalogEntry, kind string) bool {

go/cmd/ftw/forecast_site.go

@@ -76,11 +76,12 @@ func (s *forecastSiteConfig) Snapshot() forecastSite {
 	defer s.mu.RUnlock()
 	v := s.value
 	v.Options.ExpectedFlows = append([]telemetry.ForecastFlow(nil), v.Options.ExpectedFlows...)
+	v.HistoryOptions.ExpectedFlows = append([]telemetry.ForecastFlow(nil), v.HistoryOptions.ExpectedFlows...)
 	return v
 }
 
 func (s *forecastSiteConfig) Configure(cfg *config.Config, catalog []drivers.CatalogEntry) bool {
-	v := forecastSite{Meter: cfg.SiteMeterDriver(), Timezone: forecastTimezone(), Options: forecastMeasurementOptions(cfg, catalog)}
+	v := forecastSite{Meter: cfg.SiteMeterDriver(), Timezone: forecastTimezone(), Options: forecastMeasurementOptions(cfg, catalog), HistoryOptions: historyMeasurementOptions(cfg, catalog)}
 	if cfg.Weather != nil && cfg.Weather.Provider != "" && cfg.Weather.Provider != "none" {
 		v.Latitude, v.Longitude = cfg.Weather.Latitude, cfg.Weather.Longitude
 		v.HasLocation = !math.IsNaN(v.Latitude) && !math.IsNaN(v.Longitude) && math.Abs(v.Latitude) <= 90 && math.Abs(v.Longitude) <= 180

go/cmd/ftw/forecast_tracking.go

@@ -35,6 +35,7 @@ type forecastSite struct {
 	HasPVScale            bool
 	Timezone              string
 	Options               telemetry.ForecastOptions
+	HistoryOptions        telemetry.ForecastOptions // chart and site energy; outside every revision
 	PVLearningStartedMS   int64
 	LoadLearningStartedMS int64
 }

go/cmd/ftw/history_options_test.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/srcfl/ftw/go/internal/config"
+	"github.com/srcfl/ftw/go/internal/drivers"
+	"github.com/srcfl/ftw/go/internal/state"
+	"github.com/srcfl/ftw/go/internal/telemetry"
+)
+
+// Issue #1441: adding a Zap P1 meter (catalog meter+pv+battery, PV reading
+// off by default) froze the chart for good. The Zap never reports PV, so the
+// forecast topology kept the household balance invalid on every tick and
+// after every restart. The chart needs a measured balance, not proof that an
+// optional source is absent or that device identities are confirmed.
+func TestHistoryKeepsRunningWithOptionalPVAndUnconfirmedIdentity(t *testing.T) {
+	st, err := state.Open(filepath.Join(t.TempDir(), "state.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer st.Close()
+	catalog := []drivers.CatalogEntry{
+		{Path: "drivers/zap.lua", Filename: "zap.lua", Capabilities: []string{"meter", "pv", "battery"}},
+		{Path: "drivers/sma.lua", Filename: "sma.lua", Capabilities: []string{"pv"}},
+		{Path: "drivers/pixii.lua", Filename: "pixii.lua", Capabilities: []string{"battery"}},
+	}
+	cfg := &config.Config{Drivers: []config.Driver{
+		{Name: "sourceful-zap", Lua: "drivers/zap.lua", IsSiteMeter: true, Config: map[string]any{"host": "zap.local"}},
+		{Name: "sma", Lua: "drivers/sma.lua"},
+		{Name: "pixii", Lua: "drivers/pixii.lua", BatteryCapacityWh: 10000},
+	}}
+	ids := map[string]string{"sourceful-zap": "zap:sn", "sma": "sma:sn"} // pixii identity not yet known
+	site := newForecastSiteConfig(st)
+	site.identity = func(name string) (string, bool) { id := ids[name]; return id, id != "" }
+	site.Configure(cfg, catalog)
+
+	tel := telemetry.NewStore()
+	for name, kind := range map[string]telemetry.DerType{"sourceful-zap": telemetry.DerMeter, "sma": telemetry.DerPV, "pixii": telemetry.DerBattery} {
+		tel.EnsureDriverHealth(name)
+		w := map[telemetry.DerType]float64{telemetry.DerMeter: 1500, telemetry.DerPV: -2000, telemetry.DerBattery: 500}[kind]
+		tel.Update(name, kind, w, nil, nil)
+		tel.RecordDriverSuccess(name)
+	}
+	snap := site.Snapshot()
+	if r := tel.ForecastMeasurementNow("sourceful-zap", snap.Options); r.Valid {
+		t.Fatalf("forecast learning must stay strict: %+v", r)
+	}
+	point, ok := buildHistoryPoint(tel, tickPersistControl{SiteMeterDriver: "sourceful-zap"},
+		time.Now().UnixMilli(), time.Minute, snap.HistoryOptions)
+	if !ok || point.LoadW != 3000 || point.PVW != -2000 || point.BatW != 500 {
+		t.Fatalf("history stopped: point=%+v available=%v", point, ok)
+	}
+
+	// A known source that goes silent still stops history; only optional,
+	// never-reported PV is treated as absent.
+	tel.DriverHealthMut("pixii").SetOffline()
+	if _, ok := buildHistoryPoint(tel, tickPersistControl{SiteMeterDriver: "sourceful-zap"},
+		time.Now().UnixMilli(), time.Minute, snap.HistoryOptions); ok {
+		t.Fatal("offline battery treated as zero in history")
+	}
+	tel.Update("pixii", telemetry.DerBattery, 500, nil, nil)
+	tel.RecordDriverSuccess("pixii")
+	// Once the Zap does report PV, a stale PV reading stops history again.
+	tel.Update("sourceful-zap", telemetry.DerPV, -100, nil, nil)
+	tel.Get("sourceful-zap", telemetry.DerPV).UpdatedAt = time.Now().Add(-2 * time.Minute)
+	if _, ok := buildHistoryPoint(tel, tickPersistControl{SiteMeterDriver: "sourceful-zap"},
+		time.Now().UnixMilli(), time.Minute, snap.HistoryOptions); ok {
+		t.Fatal("stale reported PV treated as zero in history")
+	}
+}

go/cmd/ftw/main.go

@@ -2954,7 +2954,7 @@ func main() {
 				// ctrl, so the stored tick has to show the hold already
 				// released rather than one the blocked tick never executed.
 				clearBatteryManualHoldForDispatchBlock(ctrl, ctrlMu)
-				sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot().Options)
+				sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot().HistoryOptions)
 				if err != nil {
 					slog.Warn("tick persistence failed", "samples", sampleCount, "err", err)
 				}
@@ -3207,7 +3207,7 @@ func main() {
 			// ---- Persist the tick: history snapshot + flushed metrics ----
 			// One transaction for both — separate commits doubled the WAL
 			// commit rate for no isolation benefit (SD-card wear).
-			sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot().Options)
+			sampleCount, err := persistTelemetryTick(st, tel, snapshotTickPersistControl(ctrl, ctrlMu), nowMs, watchdogTimeout, energyIdentity, forecastSettings.Snapshot().HistoryOptions)
 			if err != nil {
 				slog.Warn("tick persistence failed", "samples", sampleCount, "err", err)
 			}