MetricSQLite / VictoriaMetrics / Prometheus as timeseries backend

default.nix

@@ -1,6 +1,6 @@
 { lib, buildPythonPackage, fetchPypi, setuptools, setuptools-scm, entsoe-apy
 , fastapi, jinja2, numpy, pandas, pydantic, pymodbus, pyomo, pyyaml, uvicorn
-, xsdata, httpx }:
+, xsdata, httpx, urllib3, python-snappy, metricsqlite }:
 
 buildPythonPackage {
   pname = "open-ess";
@@ -11,6 +11,7 @@ buildPythonPackage {
 
   nativeBuildInputs = [ setuptools setuptools-scm ];
   propagatedBuildInputs = [
+    metricsqlite
     entsoe-apy
     fastapi
     jinja2
@@ -21,6 +22,9 @@ buildPythonPackage {
     pyomo
     pyyaml
     uvicorn
+    # Victoriametrics client
+    urllib3
+    python-snappy
   ];
 
   meta = with lib; {

open_ess/battery_system/__init__.py

@@ -1,4 +1,23 @@
-from .battery_system import BatterySystem, VictronBatterySystem
+from .battery_system import (
+    BatteryQueries,
+    BatterySystem,
+    EnergyQueries,
+    EnergyQueryDef,
+    EnergyQuerySet,
+    PowerQueries,
+    PowerQueryDef,
+    VictronBatterySystem,
+)
 from .config import BatterySystemConfig
 
-__all__ = ["BatterySystem", "BatterySystemConfig", "VictronBatterySystem"]
+__all__ = [
+    "BatteryQueries",
+    "BatterySystem",
+    "BatterySystemConfig",
+    "EnergyQueries",
+    "EnergyQueryDef",
+    "EnergyQuerySet",
+    "PowerQueries",
+    "PowerQueryDef",
+    "VictronBatterySystem",
+]

open_ess/battery_system/battery_system.py

@@ -1,5 +1,6 @@
 import logging
 from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
 from datetime import UTC, datetime, timedelta
 
 from open_ess.victron_modbus import VictronClient
@@ -9,6 +10,49 @@
 logger = logging.getLogger(__name__)
 
 
+@dataclass
+class EnergyQueryDef:
+    query: str
+    label: str
+
+
+@dataclass
+class EnergyQueries:
+    queries: list[EnergyQueryDef] = field(default_factory=list)
+
+
+@dataclass
+class EnergyQuerySet:
+    """Structured energy queries for a battery system."""
+
+    energy_to_charger: str  # AC input to charger
+    energy_from_inverter: str  # AC output from inverter
+    energy_to_battery: str  # DC energy into battery
+    energy_from_battery: str  # DC energy from battery
+    energy_loss_to_battery: str  # Charge losses (AC - DC)
+    energy_loss_from_battery: str  # Discharge losses (DC - AC)
+
+
+@dataclass
+class PowerQueryDef:
+    query: str
+    label: str
+    is_total: bool | None = None  # True=total only, False=phases only, None=both
+
+
+@dataclass
+class PowerQueries:
+    queries: list[PowerQueryDef] = field(default_factory=list)
+    phases: list[str] = field(default_factory=list)
+
+
+@dataclass
+class BatteryQueries:
+    soc_query: str
+    voltage_query: str
+    schedule_soc_query: str
+
+
 class BatterySystem(ABC):
     def __init__(self, config: BatterySystemConfig):
         self._config = config
@@ -25,9 +69,33 @@ def name(self) -> str | None:
     @abstractmethod
     def id(self) -> str | None: ...
 
+    @property
+    @abstractmethod
+    def device_serial(self) -> str | None:
+        """Device serial number used for metrics labeling."""
+        ...
+
     @abstractmethod
     def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: ...
 
+    @abstractmethod
+    def get_soc(self) -> float | None: ...
+
+    @abstractmethod
+    def get_energy_queries(self) -> EnergyQuerySet:
+        """Return structured energy queries for this battery system."""
+        ...
+
+    @abstractmethod
+    def get_power_queries(self, phases: list[str] | None = None) -> PowerQueries:
+        """Return power chart queries for this battery system."""
+        ...
+
+    @abstractmethod
+    def get_battery_queries(self) -> BatteryQueries:
+        """Return battery chart queries (SOC, voltage, schedule)."""
+        ...
+
 
 class VictronBatterySystem(BatterySystem):
     def __init__(self, config: BatterySystemConfig, control: VictronClient):
@@ -37,12 +105,115 @@ def __init__(self, config: BatterySystemConfig, control: VictronClient):
 
     @property
     def id(self) -> str | None:
-        if self._victron_client.serial is None:
-            return None
-        return f"victron/{self._victron_client.serial}"
+        return self._victron_client.serial
+
+    @property
+    def device_serial(self) -> str | None:
+        return self._victron_client.serial
 
     def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None:
         if until is None:
             until = datetime.now(tz=UTC) + timedelta(hours=1)
         logger.info(f"{self.name}: Set setpoint to {power} W")
         self._victron_client.set_ess_setpoint(power, until)
+
+    def get_soc(self) -> float | None:
+        return self._victron_client.current_soc
+
+    def get_energy_queries(self) -> EnergyQuerySet:
+        device = self.device_serial
+
+        # AC side energy (charger input / inverter output)
+        energy_to_charger = f'increase(openess_energy_kwh{{from="ac_in", to="system", device="{device}"}}[$step])'
+        energy_from_inverter = f'increase(openess_energy_kwh{{from="system", to="ac_out", device="{device}"}}[$step])'
+
+        # DC side energy (battery charge/discharge)
+        energy_to_battery = f'increase(openess_energy_kwh{{from="system", to="battery", device="{device}"}}[$step])'
+        energy_from_battery = f'increase(openess_energy_kwh{{from="battery", to="system", device="{device}"}}[$step])'
+
+        # Losses = AC - DC (positive means energy lost as heat)
+        energy_loss_to_battery = f"({energy_to_charger}) - ({energy_to_battery})"
+        energy_loss_from_battery = f"({energy_from_battery}) - ({energy_from_inverter})"
+
+        return EnergyQuerySet(
+            energy_to_charger=energy_to_charger,
+            energy_from_inverter=energy_from_inverter,
+            energy_to_battery=energy_to_battery,
+            energy_from_battery=energy_from_battery,
+            energy_loss_to_battery=energy_loss_to_battery,
+            energy_loss_from_battery=energy_loss_from_battery,
+        )
+
+    def get_power_queries(self, phases: list[str] | None = None) -> PowerQueries:
+        device = self.device_serial or "unknown"
+        bs_name = self.config.name or self.id
+        queries: list[PowerQueryDef] = []
+
+        if phases and len(phases) > 1:
+            # Multi-phase: add total and per-phase queries
+            queries.append(
+                PowerQueryDef(
+                    query=f"""
+                      sum by (device) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[$step]))
+                      - on(device)
+                      sum by (device) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[$step]))
+                    """,
+                    label=f"{bs_name} AC",
+                    is_total=True,
+                )
+            )
+            for phase in phases:
+                queries.append(
+                    PowerQueryDef(
+                        query=f"""
+                          sum by (device, phase) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}", phase="{phase}"}}[$step]))
+                          - on(device, phase)
+                          sum by (device, phase) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}", phase="{phase}"}}[$step]))
+                        """,
+                        label=f"{bs_name} AC {phase}",
+                        is_total=False,
+                    )
+                )
+        else:
+            # Single phase or unknown
+            queries.append(
+                PowerQueryDef(
+                    query=f"""
+                      sum by (device) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[$step]))
+                      - on(device)
+                      sum by (device) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[$step]))
+                    """,
+                    label=f"{bs_name} AC",
+                )
+            )
+
+        # Battery DC power (works for both single and multi-phase)
+        queries.append(
+            PowerQueryDef(
+                query=f"""
+                  avg_over_time(openess_power_watts{{from="system", to="battery", unit="battery", device="{device}"}}[$step])
+                  or
+                  avg_over_time(openess_power_watts{{from="system", to="battery", unit="vebus", device="{device}"}}[$step])
+                """,
+                label=f"{bs_name} Battery",
+            )
+        )
+
+        return PowerQueries(queries=queries, phases=phases or [])
+
+    def get_battery_queries(self) -> BatteryQueries:
+        device = self.device_serial or "unknown"
+
+        return BatteryQueries(
+            soc_query=f"""
+              openess_soc_ratio{{device="{device}", node="battery", unit="battery"}} * 100
+              or
+              openess_soc_ratio{{device="{device}", node="battery", unit="vebus"}} * 100
+            """,
+            voltage_query=f"""
+              openess_voltage_volts{{device="{device}", node="battery", unit="battery"}}
+              or
+              openess_voltage_volts{{device="{device}", node="battery", unit="vebus"}}
+            """,
+            schedule_soc_query=f'first_over_time(openess_scheduled_soc_ratio{{device="{device}"}}) * 100',
+        )

open_ess/battery_system/config.py

@@ -1,6 +1,6 @@
 from typing import Annotated, Literal
 
-from pydantic import BaseModel, Field, computed_field, model_validator
+from pydantic import BaseModel, Field, model_validator
 
 from open_ess.victron_modbus import VictronConfig
 
@@ -9,24 +9,9 @@ class MqttControl(BaseModel):
     type: Literal["mqtt"] = "mqtt"
     topic: str
 
-    @property
-    def metrics_prefix(self) -> str:
-        return f"mqtt/{self.topic}"
-
-
-class MetricsConfig(BaseModel):
-    battery_soc: str | list[str] | None = None
-    battery_voltage: str | list[str] | None = None
-    power_to_system: str | list[str] | None = None
-    power_to_battery: str | list[str] | None = None
-    energy_to_system: str | list[str] | None = None
-    energy_from_system: str | list[str] | None = None
-    energy_to_battery: str | list[str] | None = None
-    energy_from_battery: str | list[str] | None = None
-
 
 class BatterySystemConfig(BaseModel):
-    name: str | None = None  # Is set to self.id if not provided.
+    name: str | None = None
     monitor_only: bool = False
     phases: int = 1
     capacity_kwh: float | None = None
@@ -37,15 +22,6 @@ class BatterySystemConfig(BaseModel):
     max_soc: int = 100
 
     control: Annotated[VictronConfig | MqttControl, Field(discriminator="type")]
-    metrics: MetricsConfig = MetricsConfig()
-
-    @computed_field  # type: ignore[prop-decorator]
-    @property
-    def id(self) -> str:
-        if isinstance(self.control, VictronConfig):
-            return f"victron/vebus/{self.control.vebus_id}"
-        else:
-            return f"mqtt/{self.control.topic}"
 
     @property
     def is_victron(self) -> bool:
@@ -63,56 +39,3 @@ def check_power_limits(self) -> "BatterySystemConfig":
                     "max_invert_power_kw is not configured. Either set a value or set monitor_only to True."
                 )
         return self
-
-    @model_validator(mode="after")
-    def set_defaults(self) -> "BatterySystemConfig":
-        if self.name is None:
-            self.name = self.id
-
-        if isinstance(self.control, VictronConfig):
-            vebus_prefix = self.control.vebus_prefix
-            bms_prefix = self.control.battery_prefix
-
-            if self.metrics.battery_soc is None:
-                if bms_prefix:
-                    self.metrics.battery_soc = [f"{bms_prefix}/soc", f"{vebus_prefix}/soc"]
-                else:
-                    self.metrics.battery_soc = f"{vebus_prefix}/soc"
-            if self.metrics.battery_voltage is None:
-                if bms_prefix:
-                    self.metrics.battery_voltage = [f"{bms_prefix}/voltage/battery", f"{vebus_prefix}/voltage/battery"]
-                else:
-                    self.metrics.battery_voltage = f"{vebus_prefix}/voltage/battery"
-            if self.metrics.power_to_system is None:
-                self.metrics.power_to_system = f"{vebus_prefix}/power/ac_in/l1"
-            if self.metrics.power_to_battery is None:
-                if bms_prefix:
-                    self.metrics.power_to_battery = [f"{bms_prefix}/power/battery", f"{vebus_prefix}/power/battery"]
-                else:
-                    self.metrics.power_to_battery = f"{vebus_prefix}/power/battery"
-            if self.metrics.energy_to_system is None:
-                self.metrics.energy_to_system = f"{vebus_prefix}/energy/ac_in_import"  # TODO + ac_out_import
-            if self.metrics.energy_from_system is None:
-                self.metrics.energy_from_system = f"{vebus_prefix}/energy/ac_in_export"  # TODO + ac_out_export
-            if self.metrics.energy_to_battery is None:
-                if bms_prefix:
-                    self.metrics.energy_to_battery = [
-                        f"{bms_prefix}/energy/charged_energy",
-                        f"{bms_prefix}/power/battery",  # integrate power to obtain energy
-                        f"{vebus_prefix}/power/battery",  # integrate power to obtain energy
-                    ]
-                else:
-                    self.metrics.energy_to_battery = f"{vebus_prefix}/power/battery"
-            if self.metrics.energy_from_battery is None:
-                if bms_prefix:
-                    self.metrics.energy_from_battery = [
-                        f"{bms_prefix}/energy/discharged_energy",
-                        f"-{bms_prefix}/power/battery",  # integrate power to obtain energy
-                        f"-{vebus_prefix}/power/battery",  # integrate power to obtain energy
-                    ]
-                else:
-                    self.metrics.energy_from_battery = f"-{vebus_prefix}/power/battery"
-        else:
-            pass  # TODO
-
-        return self

open_ess/config.py

@@ -4,17 +4,18 @@
 from pydantic import BaseModel
 
 from open_ess.battery_system import BatterySystemConfig
-from open_ess.database import DatabaseConfig
 from open_ess.frontend import FrontendConfig
 from open_ess.pricing import PriceConfig
+from open_ess.timeseries import TimeseriesConfig
+from open_ess.timeseries.metricsqlite.config import MetricSQLiteConfig
 
 # TODO: Validate config. If a battery defines mqtt control, require mqtt config.
 
 
 class Config(BaseModel):
-    database: DatabaseConfig
     frontend: FrontendConfig
     prices: PriceConfig
+    timeseries: TimeseriesConfig = MetricSQLiteConfig()
     battery_system: BatterySystemConfig | list[BatterySystemConfig]
 
     @property