diff --git a/default.nix b/default.nix index 26b0725..f464bdf 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,6 @@ { lib, buildPythonPackage, fetchPypi, setuptools, setuptools-scm, entsoe-apy , fastapi, jinja2, numpy, pandas, pydantic, pymodbus, pyomo, pyyaml, uvicorn -, xsdata, httpx }: +, xsdata, httpx, urllib3, python-snappy, metricsqlite }: buildPythonPackage { pname = "open-ess"; @@ -11,6 +11,7 @@ buildPythonPackage { nativeBuildInputs = [ setuptools setuptools-scm ]; propagatedBuildInputs = [ + metricsqlite entsoe-apy fastapi jinja2 @@ -21,6 +22,9 @@ buildPythonPackage { pyomo pyyaml uvicorn + # Victoriametrics client + urllib3 + python-snappy ]; meta = with lib; { diff --git a/open_ess/battery_system/__init__.py b/open_ess/battery_system/__init__.py index 48377e4..1ce04ef 100644 --- a/open_ess/battery_system/__init__.py +++ b/open_ess/battery_system/__init__.py @@ -1,4 +1,23 @@ -from .battery_system import BatterySystem, VictronBatterySystem +from .battery_system import ( + BatteryQueries, + BatterySystem, + EnergyQueries, + EnergyQueryDef, + EnergyQuerySet, + PowerQueries, + PowerQueryDef, + VictronBatterySystem, +) from .config import BatterySystemConfig -__all__ = ["BatterySystem", "BatterySystemConfig", "VictronBatterySystem"] +__all__ = [ + "BatteryQueries", + "BatterySystem", + "BatterySystemConfig", + "EnergyQueries", + "EnergyQueryDef", + "EnergyQuerySet", + "PowerQueries", + "PowerQueryDef", + "VictronBatterySystem", +] diff --git a/open_ess/battery_system/battery_system.py b/open_ess/battery_system/battery_system.py index 4bfc80c..0758b49 100644 --- a/open_ess/battery_system/battery_system.py +++ b/open_ess/battery_system/battery_system.py @@ -1,5 +1,6 @@ import logging from abc import ABC, abstractmethod +from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta from open_ess.victron_modbus import VictronClient @@ -9,6 +10,49 @@ logger = logging.getLogger(__name__) +@dataclass +class EnergyQueryDef: + query: str + label: str + + +@dataclass +class EnergyQueries: + queries: list[EnergyQueryDef] = field(default_factory=list) + + +@dataclass +class EnergyQuerySet: + """Structured energy queries for a battery system.""" + + energy_to_charger: str # AC input to charger + energy_from_inverter: str # AC output from inverter + energy_to_battery: str # DC energy into battery + energy_from_battery: str # DC energy from battery + energy_loss_to_battery: str # Charge losses (AC - DC) + energy_loss_from_battery: str # Discharge losses (DC - AC) + + +@dataclass +class PowerQueryDef: + query: str + label: str + is_total: bool | None = None # True=total only, False=phases only, None=both + + +@dataclass +class PowerQueries: + queries: list[PowerQueryDef] = field(default_factory=list) + phases: list[str] = field(default_factory=list) + + +@dataclass +class BatteryQueries: + soc_query: str + voltage_query: str + schedule_soc_query: str + + class BatterySystem(ABC): def __init__(self, config: BatterySystemConfig): self._config = config @@ -25,9 +69,33 @@ def name(self) -> str | None: @abstractmethod def id(self) -> str | None: ... + @property + @abstractmethod + def device_serial(self) -> str | None: + """Device serial number used for metrics labeling.""" + ... + @abstractmethod def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: ... + @abstractmethod + def get_soc(self) -> float | None: ... + + @abstractmethod + def get_energy_queries(self) -> EnergyQuerySet: + """Return structured energy queries for this battery system.""" + ... + + @abstractmethod + def get_power_queries(self, phases: list[str] | None = None) -> PowerQueries: + """Return power chart queries for this battery system.""" + ... + + @abstractmethod + def get_battery_queries(self) -> BatteryQueries: + """Return battery chart queries (SOC, voltage, schedule).""" + ... + class VictronBatterySystem(BatterySystem): def __init__(self, config: BatterySystemConfig, control: VictronClient): @@ -37,12 +105,115 @@ def __init__(self, config: BatterySystemConfig, control: VictronClient): @property def id(self) -> str | None: - if self._victron_client.serial is None: - return None - return f"victron/{self._victron_client.serial}" + return self._victron_client.serial + + @property + def device_serial(self) -> str | None: + return self._victron_client.serial def set_ess_setpoint(self, power: float, until: datetime | None = None) -> None: if until is None: until = datetime.now(tz=UTC) + timedelta(hours=1) logger.info(f"{self.name}: Set setpoint to {power} W") self._victron_client.set_ess_setpoint(power, until) + + def get_soc(self) -> float | None: + return self._victron_client.current_soc + + def get_energy_queries(self) -> EnergyQuerySet: + device = self.device_serial + + # AC side energy (charger input / inverter output) + energy_to_charger = f'increase(openess_energy_kwh{{from="ac_in", to="system", device="{device}"}}[$step])' + energy_from_inverter = f'increase(openess_energy_kwh{{from="system", to="ac_out", device="{device}"}}[$step])' + + # DC side energy (battery charge/discharge) + energy_to_battery = f'increase(openess_energy_kwh{{from="system", to="battery", device="{device}"}}[$step])' + energy_from_battery = f'increase(openess_energy_kwh{{from="battery", to="system", device="{device}"}}[$step])' + + # Losses = AC - DC (positive means energy lost as heat) + energy_loss_to_battery = f"({energy_to_charger}) - ({energy_to_battery})" + energy_loss_from_battery = f"({energy_from_battery}) - ({energy_from_inverter})" + + return EnergyQuerySet( + energy_to_charger=energy_to_charger, + energy_from_inverter=energy_from_inverter, + energy_to_battery=energy_to_battery, + energy_from_battery=energy_from_battery, + energy_loss_to_battery=energy_loss_to_battery, + energy_loss_from_battery=energy_loss_from_battery, + ) + + def get_power_queries(self, phases: list[str] | None = None) -> PowerQueries: + device = self.device_serial or "unknown" + bs_name = self.config.name or self.id + queries: list[PowerQueryDef] = [] + + if phases and len(phases) > 1: + # Multi-phase: add total and per-phase queries + queries.append( + PowerQueryDef( + query=f""" + sum by (device) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[$step])) + - on(device) + sum by (device) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[$step])) + """, + label=f"{bs_name} AC", + is_total=True, + ) + ) + for phase in phases: + queries.append( + PowerQueryDef( + query=f""" + sum by (device, phase) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}", phase="{phase}"}}[$step])) + - on(device, phase) + sum by (device, phase) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}", phase="{phase}"}}[$step])) + """, + label=f"{bs_name} AC {phase}", + is_total=False, + ) + ) + else: + # Single phase or unknown + queries.append( + PowerQueryDef( + query=f""" + sum by (device) (avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[$step])) + - on(device) + sum by (device) (avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[$step])) + """, + label=f"{bs_name} AC", + ) + ) + + # Battery DC power (works for both single and multi-phase) + queries.append( + PowerQueryDef( + query=f""" + avg_over_time(openess_power_watts{{from="system", to="battery", unit="battery", device="{device}"}}[$step]) + or + avg_over_time(openess_power_watts{{from="system", to="battery", unit="vebus", device="{device}"}}[$step]) + """, + label=f"{bs_name} Battery", + ) + ) + + return PowerQueries(queries=queries, phases=phases or []) + + def get_battery_queries(self) -> BatteryQueries: + device = self.device_serial or "unknown" + + return BatteryQueries( + soc_query=f""" + openess_soc_ratio{{device="{device}", node="battery", unit="battery"}} * 100 + or + openess_soc_ratio{{device="{device}", node="battery", unit="vebus"}} * 100 + """, + voltage_query=f""" + openess_voltage_volts{{device="{device}", node="battery", unit="battery"}} + or + openess_voltage_volts{{device="{device}", node="battery", unit="vebus"}} + """, + schedule_soc_query=f'first_over_time(openess_scheduled_soc_ratio{{device="{device}"}}) * 100', + ) diff --git a/open_ess/battery_system/config.py b/open_ess/battery_system/config.py index 144efa0..9f45038 100644 --- a/open_ess/battery_system/config.py +++ b/open_ess/battery_system/config.py @@ -1,6 +1,6 @@ from typing import Annotated, Literal -from pydantic import BaseModel, Field, computed_field, model_validator +from pydantic import BaseModel, Field, model_validator from open_ess.victron_modbus import VictronConfig @@ -9,24 +9,9 @@ class MqttControl(BaseModel): type: Literal["mqtt"] = "mqtt" topic: str - @property - def metrics_prefix(self) -> str: - return f"mqtt/{self.topic}" - - -class MetricsConfig(BaseModel): - battery_soc: str | list[str] | None = None - battery_voltage: str | list[str] | None = None - power_to_system: str | list[str] | None = None - power_to_battery: str | list[str] | None = None - energy_to_system: str | list[str] | None = None - energy_from_system: str | list[str] | None = None - energy_to_battery: str | list[str] | None = None - energy_from_battery: str | list[str] | None = None - class BatterySystemConfig(BaseModel): - name: str | None = None # Is set to self.id if not provided. + name: str | None = None monitor_only: bool = False phases: int = 1 capacity_kwh: float | None = None @@ -37,15 +22,6 @@ class BatterySystemConfig(BaseModel): max_soc: int = 100 control: Annotated[VictronConfig | MqttControl, Field(discriminator="type")] - metrics: MetricsConfig = MetricsConfig() - - @computed_field # type: ignore[prop-decorator] - @property - def id(self) -> str: - if isinstance(self.control, VictronConfig): - return f"victron/vebus/{self.control.vebus_id}" - else: - return f"mqtt/{self.control.topic}" @property def is_victron(self) -> bool: @@ -63,56 +39,3 @@ def check_power_limits(self) -> "BatterySystemConfig": "max_invert_power_kw is not configured. Either set a value or set monitor_only to True." ) return self - - @model_validator(mode="after") - def set_defaults(self) -> "BatterySystemConfig": - if self.name is None: - self.name = self.id - - if isinstance(self.control, VictronConfig): - vebus_prefix = self.control.vebus_prefix - bms_prefix = self.control.battery_prefix - - if self.metrics.battery_soc is None: - if bms_prefix: - self.metrics.battery_soc = [f"{bms_prefix}/soc", f"{vebus_prefix}/soc"] - else: - self.metrics.battery_soc = f"{vebus_prefix}/soc" - if self.metrics.battery_voltage is None: - if bms_prefix: - self.metrics.battery_voltage = [f"{bms_prefix}/voltage/battery", f"{vebus_prefix}/voltage/battery"] - else: - self.metrics.battery_voltage = f"{vebus_prefix}/voltage/battery" - if self.metrics.power_to_system is None: - self.metrics.power_to_system = f"{vebus_prefix}/power/ac_in/l1" - if self.metrics.power_to_battery is None: - if bms_prefix: - self.metrics.power_to_battery = [f"{bms_prefix}/power/battery", f"{vebus_prefix}/power/battery"] - else: - self.metrics.power_to_battery = f"{vebus_prefix}/power/battery" - if self.metrics.energy_to_system is None: - self.metrics.energy_to_system = f"{vebus_prefix}/energy/ac_in_import" # TODO + ac_out_import - if self.metrics.energy_from_system is None: - self.metrics.energy_from_system = f"{vebus_prefix}/energy/ac_in_export" # TODO + ac_out_export - if self.metrics.energy_to_battery is None: - if bms_prefix: - self.metrics.energy_to_battery = [ - f"{bms_prefix}/energy/charged_energy", - f"{bms_prefix}/power/battery", # integrate power to obtain energy - f"{vebus_prefix}/power/battery", # integrate power to obtain energy - ] - else: - self.metrics.energy_to_battery = f"{vebus_prefix}/power/battery" - if self.metrics.energy_from_battery is None: - if bms_prefix: - self.metrics.energy_from_battery = [ - f"{bms_prefix}/energy/discharged_energy", - f"-{bms_prefix}/power/battery", # integrate power to obtain energy - f"-{vebus_prefix}/power/battery", # integrate power to obtain energy - ] - else: - self.metrics.energy_from_battery = f"-{vebus_prefix}/power/battery" - else: - pass # TODO - - return self diff --git a/open_ess/config.py b/open_ess/config.py index 66bdc8f..1fbd0f8 100644 --- a/open_ess/config.py +++ b/open_ess/config.py @@ -4,17 +4,18 @@ from pydantic import BaseModel from open_ess.battery_system import BatterySystemConfig -from open_ess.database import DatabaseConfig from open_ess.frontend import FrontendConfig from open_ess.pricing import PriceConfig +from open_ess.timeseries import TimeseriesConfig +from open_ess.timeseries.metricsqlite.config import MetricSQLiteConfig # TODO: Validate config. If a battery defines mqtt control, require mqtt config. class Config(BaseModel): - database: DatabaseConfig frontend: FrontendConfig prices: PriceConfig + timeseries: TimeseriesConfig = MetricSQLiteConfig() battery_system: BatterySystemConfig | list[BatterySystemConfig] @property diff --git a/open_ess/database/__init__.py b/open_ess/database/__init__.py deleted file mode 100644 index 1947c7a..0000000 --- a/open_ess/database/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from .config import DatabaseConfig -from .database import Database, DatabaseConnection -from .service import DatabaseService -from .util import dt_to_ms, ms_to_dt - -__all__ = [ - "Database", - "DatabaseConnection", - "DatabaseConfig", - "DatabaseService", - "ms_to_dt", - "dt_to_ms", -] diff --git a/open_ess/database/config.py b/open_ess/database/config.py deleted file mode 100644 index 678ac5f..0000000 --- a/open_ess/database/config.py +++ /dev/null @@ -1,13 +0,0 @@ -from pathlib import Path - -from pydantic import BaseModel - - -class DatabaseCompressionConfig(BaseModel): - enable: bool = True - bucket_seconds: int = 60 - - -class DatabaseConfig(BaseModel): - path: Path - compression: DatabaseCompressionConfig = DatabaseCompressionConfig() diff --git a/open_ess/database/database.py b/open_ess/database/database.py deleted file mode 100644 index 146795f..0000000 --- a/open_ess/database/database.py +++ /dev/null @@ -1,518 +0,0 @@ -import logging -import sqlite3 -from datetime import datetime, timedelta -from pathlib import Path - -from .config import DatabaseConfig -from .migration_runner import run_migrations -from .util import base_conditions, dt_to_ms, ms_to_dt - -logger = logging.getLogger(__name__) - - -class Database: - def __init__(self, config: DatabaseConfig): - self._config = config - config.path.parent.mkdir(parents=True, exist_ok=True) - with self.connect() as conn: - conn.execute("PRAGMA journal_mode=WAL") - # ^ WAL mode allows concurrent reads/writes without blocking - conn.execute("PRAGMA busy_timeout = 30000") - # ^ Wait up to 30 seconds for locks instead of failing immediately - - @property - def config(self) -> DatabaseConfig: - return self._config - - def connect(self) -> "DatabaseConnection": - return DatabaseConnection(self._config.path) - - def run_migrations(self) -> None: - with self.connect() as conn: - run_migrations(conn) - - -class DatabaseConnection: - def __init__(self, path: Path): - self._conn = sqlite3.connect(path) - self._conn.row_factory = sqlite3.Row - # ^ Makes column access by name possible - - def close(self) -> None: - self._conn.close() - - def __enter__(self) -> "DatabaseConnection": - return self - - def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: - self.close() - - def execute(self, sql: str, parameters: list | tuple | None = None) -> sqlite3.Cursor: - if parameters is None: - parameters = [] - return self._conn.execute(sql, parameters) - - def commit(self) -> None: - self._conn.commit() - - def vacuum(self) -> None: - self._conn.execute("PRAGMA incremental_vacuum") - - def _get_labels( - self, table_name: str, timestamp_name: str, start: datetime | None = None, end: datetime | None = None - ) -> list[str]: - conditions = [] - params = [] - if start is not None: - conditions.append(f"{timestamp_name} >= ?") - params.append(dt_to_ms(start)) - if end is not None: - conditions.append(f"{timestamp_name} < ?") - params.append(dt_to_ms(end)) - - where_clause = "WHERE " + " AND ".join(conditions) if conditions else "" - query = f""" - SELECT DISTINCT label - FROM {table_name} - {where_clause} - """ - cursor = self._conn.execute(query, params) - return [row[0] for row in cursor.fetchall()] - - # ------------------------------------------------------------------------- - # Power - # ------------------------------------------------------------------------- - - def insert_power(self, label: str, timestamp: datetime, power: float | None) -> None: - if power is None: - return - self._conn.execute( - "INSERT INTO power (label, start_time, sample_count, value) VALUES (?, ?, 1, ?)", - (label, dt_to_ms(timestamp), power), - ) - self._conn.commit() - - def get_power( - self, - label: str, - start: datetime | None = None, - end: datetime | None = None, - bucket_seconds: float | None = 60, - limit: int | None = None, - ) -> list[tuple[datetime, float]]: - if isinstance(label, list): - label = label[0] - - conditions, params = base_conditions(label, start, end, timestamp_name="start_time") - - if bucket_seconds is not None: - bucket_ms = round(bucket_seconds * 1000) - select_clause = "(start_time / ?) * ? as bucket, AVG(value) as avg_value" - params = [bucket_ms, bucket_ms, *params] - group_by = "GROUP BY bucket" - order_by = "bucket" - else: - select_clause = "start_time, value" - group_by = "" - order_by = "start_time" - - limit_clause = "" - if limit: - limit_clause = "LIMIT ?" - params.append(limit) - - where_clause = " AND ".join(conditions) - query = f""" - SELECT {select_clause} - FROM power - WHERE {where_clause} - {group_by} - ORDER BY {order_by} - {limit_clause} - """ - cursor = self._conn.execute(query, params) - return [(ms_to_dt(row[0]), row[1]) for row in cursor.fetchall()] - - def get_power_labels(self, start: datetime | None = None, end: datetime | None = None) -> list[str]: - return self._get_labels("power", "start_time", start, end) - - def get_all_power( - self, start: datetime, end: datetime | None = None, bucket_seconds: float | None = None - ) -> dict[str, list[tuple[datetime, float]]]: - power_series = {} - for label in self.get_power_labels(start, end): - power_series[label] = self.get_power(label, start, end, bucket_seconds) - return power_series - - def compress_power(self, older_than: datetime, bucket_seconds: float) -> tuple[int, int]: - older_than = older_than.replace(second=0, microsecond=0) - bucket_ms = round(bucket_seconds * 1000) - cutoff_ms = dt_to_ms(older_than) - # ^ cutoff alignment with bucket size enforces that we never cross bucket boundaries. This - # makes calculating average power much easier since we don't need to work with weighted - # averages and take duration of semi-bucket into account. - - bucket_query = """ - SELECT - label_id, - (start_time / ?) * ? AS bucket, - SUM(sample_count) AS total_samples, - COUNT(*) AS row_count - FROM _power - WHERE start_time < ? - GROUP BY label_id, bucket - HAVING row_count > 1 - ORDER BY bucket - """ - cursor = self._conn.execute(bucket_query, (bucket_ms, bucket_ms, cutoff_ms)) - buckets = cursor.fetchall() - - total_sample_count = 0 - total_bucket_count = 0 - for row in buckets: - label_id = row["label_id"] - bucket_start = row["bucket"] - bucket_end = bucket_start + bucket_ms - total_samples = row["total_samples"] - - cursor = self._conn.execute( - """ - SELECT start_time, end_time, sample_count, value - FROM _power - WHERE label_id = ? AND start_time >= ? AND start_time < ? - ORDER BY start_time - """, - (label_id, bucket_start, bucket_end), - ) - samples = cursor.fetchall() - - sample_count = 0 - total_power = 0.0 - for sample in samples: - sample_count += sample["sample_count"] - total_power += sample["value"] - average_power = total_power / sample_count - - self._conn.execute( - "DELETE FROM _power WHERE label_id = ? AND start_time >= ? AND start_time < ?", - (label_id, bucket_start, bucket_end), - ) - self._conn.execute( - "INSERT INTO _power (label_id, start_time, end_time, sample_count, value) VALUES (?, ?, ?, ?, ?)", - (label_id, bucket_start, bucket_end, total_samples, average_power), - ) - - total_sample_count += sample_count - total_bucket_count += 1 - - self._conn.commit() - return total_sample_count, total_bucket_count - - # "Abuse" power table for voltages because the compression algorithm also works perfectly fine for voltages. - insert_voltage = insert_power - get_voltage = get_power - - # ------------------------------------------------------------------------- - # Energy - # ------------------------------------------------------------------------- - - def insert_energy( - self, - label: str, - timestamp: datetime, - energy: float | None, - ) -> None: - if energy is None: - return - self._conn.execute( - """ - INSERT INTO energy (label, timestamp, value) - SELECT ?, ?, ? - WHERE ? != COALESCE( - (SELECT value FROM energy - WHERE label = ? - ORDER BY timestamp DESC LIMIT 1), - -1 - ) - """, - (label, dt_to_ms(timestamp), energy, energy, label), - ) - self._conn.commit() - - def get_energy( - self, - label: str, - start: datetime | None, - end: datetime | None, - normalize: bool = False, - ) -> list[tuple[datetime, float]]: - conditions, params = base_conditions(label, start, end) - where_clause = "WHERE " + " AND ".join(conditions) - query = f""" - SELECT timestamp, value - FROM energy - {where_clause} - ORDER BY timestamp - """ - cursor = self._conn.execute(query, params) - - result = [(row[0], row[1]) for row in cursor.fetchall()] - if normalize and result: - start_energy = result[0][1] - result = [(t, v - start_energy) for t, v in result] - return result - - def get_energy_aggregated( - self, - label: str, - aggregation_seconds: float, - start: datetime | None, - end: datetime | None, - center_buckets: bool = False, - ) -> list[tuple[datetime, float]]: - if start: - start -= timedelta(seconds=aggregation_seconds) - if end: - end += timedelta(seconds=aggregation_seconds) - - agg_ms = int(aggregation_seconds * 1000) - conditions, params = base_conditions(label, start, end) - where_clause = "WHERE " + " AND ".join(conditions) - query = f""" - SELECT - (timestamp / ?) * ? AS bucket, - SUM(delta) AS energy_sum - FROM ( - SELECT - timestamp, - CASE - WHEN prev IS NULL THEN 0 -- first value - WHEN value < prev THEN value -- time series was reset to zero - ELSE value - prev - END AS delta - FROM ( - SELECT - timestamp, - value, - LAG(value) OVER (ORDER BY timestamp) AS prev - FROM energy - {where_clause} - ) - ) - GROUP BY bucket - ORDER BY bucket - """ - cursor = self._conn.execute(query, [agg_ms, agg_ms, *params]) - - center_offset = agg_ms // 2 if center_buckets else 0 - return [(ms_to_dt(r[0] + center_offset), round(r[1], 3)) for r in cursor.fetchall()] - - def get_energy_labels(self, start: datetime | None, end: datetime | None = None) -> list[str]: - return self._get_labels("energy", "timestamp", start, end) - - def get_all_energy( - self, start: datetime, end: datetime | None = None, normalize: bool = False - ) -> dict[str, list[tuple[datetime, float]]]: - energy_series = {} - for label in self.get_energy_labels(start, end): - energy_series[label] = self.get_energy(label, start, end, normalize) - return energy_series - - def get_grid_energy_total( - self, start: datetime | None, end: datetime | None = None, normalize: bool = False - ) -> dict[str, list[tuple[datetime, float]]]: - # TODO: per phase and total - return { - "from_net_total": self.get_energy("from_net_total", start, end, normalize), - "to_net_total": self.get_energy("to_net_total", start, end, normalize), - } - - def integrate_power( - self, label: str, start: datetime, end: datetime, bucket_seconds: int = 60 - ) -> list[tuple[datetime, float]]: - power_series = self.get_power(label, start, end, bucket_seconds=bucket_seconds) - if not power_series: - return [] - - energy_series = [(power_series[0][0] - timedelta(seconds=bucket_seconds), 0.0)] - for ts, v in power_series: - energy_series.append((ts, energy_series[-1][-1] + v / 1000 / (3600 / bucket_seconds))) - return energy_series - - # ------------------------------------------------------------------------- - # Day-ahead prices - # ------------------------------------------------------------------------- - - def insert_price(self, area: str, start_time: datetime, end_time: datetime, price: float) -> None: - self._conn.execute( - """ - INSERT INTO day_ahead_prices (area, start_time, end_time, price) - VALUES (?, ?, ?, ?) - ON CONFLICT (area, start_time) DO UPDATE SET - end_time = excluded.end_time, price = excluded.price - """, - (area, dt_to_ms(start_time), dt_to_ms(end_time), price), - ) - self._conn.commit() - - def insert_prices(self, area: str, prices: list[tuple[datetime, datetime, float]]) -> None: - self._conn.executemany( - """ - INSERT INTO day_ahead_prices (area, start_time, end_time, price) - VALUES (?, ?, ?, ?) - ON CONFLICT (area, start_time) DO UPDATE SET - end_time = excluded.end_time, price = excluded.price - """, - [(area, dt_to_ms(start), dt_to_ms(end), price) for start, end, price in prices], - ) - self._conn.commit() - logger.debug(f"Inserted {len(prices)} price records") - - def get_prices( - self, area: str, start: datetime, end: datetime | None = None, aggregate_minutes: float | None = None - ) -> list[tuple[datetime, float]]: - conditions, params = base_conditions(area, start, end, label_name="area", timestamp_name="start_time") - - if aggregate_minutes is not None: - timestamp_column = "bucket" - value_column = "avg_value" - select_clause = f"(start_time / ?) * ? as {timestamp_column}, AVG(price) as {value_column}" - group_by = f"GROUP BY {timestamp_column}" - agg_ms = round(aggregate_minutes * 60000) - params = [agg_ms, agg_ms, *params] - else: - timestamp_column = "start_time" - group_by = "" - value_column = "price" - select_clause = f"{timestamp_column}, {value_column}" - - where_clause = "WHERE " + " AND ".join(conditions) - cursor = self._conn.execute( - f""" - SELECT {select_clause} - FROM day_ahead_prices - {where_clause} - {group_by} - ORDER BY {timestamp_column} - """, - params, - ) - # TODO: store prices in €/kWh instead of €/MWh - return [(ms_to_dt(row[timestamp_column]), row[value_column] / 1000) for row in cursor.fetchall()] - - def get_hourly_prices( - self, area: str, start: datetime, end: datetime | None = None - ) -> list[tuple[datetime, float]]: - """Get hourly aggregated prices. Returns list of (hour_start, price_eur_per_kwh).""" - params = [area, dt_to_ms(start)] - if end is not None: - params.append(dt_to_ms(end)) - cursor = self._conn.execute( - f""" - SELECT (start_time / 3600000) * 3600000 AS hour, AVG(price) / 1000.0 AS price - FROM day_ahead_prices - WHERE area = ? AND start_time >= ?{" AND start_time < ?" if end is not None else ""} - GROUP BY hour ORDER BY hour - """, - params, - ) - return [(ms_to_dt(row["hour"]), row["price"]) for row in cursor.fetchall()] - - def get_latest_price_time(self, area: str) -> datetime | None: - cursor = self._conn.execute( - "SELECT MAX(end_time) as latest FROM day_ahead_prices WHERE area = ?", - (area,), - ) - row = cursor.fetchone() - if row and row["latest"]: - return ms_to_dt(row["latest"]) - return None - - # ------------------------------------------------------------------------- - # Battery SOC - # ------------------------------------------------------------------------- - - def insert_soc(self, label: str, timestamp: datetime, soc: int) -> None: - # TODO: also insert if last update was more than 5 minutes ago - self._conn.execute( - """ - INSERT INTO battery_soc (label, timestamp, value) - SELECT ?, ?, ? - WHERE ? != COALESCE( - (SELECT value FROM battery_soc WHERE label = ? ORDER BY timestamp DESC LIMIT 1), - -1 - ) - """, - (label, dt_to_ms(timestamp), soc, soc, label), - ) - self._conn.commit() - - def get_battery_soc(self, label: str, start: datetime, end: datetime) -> list[tuple[datetime, float]]: - if isinstance(label, list): - label = label[0] - cursor = self._conn.execute( - "SELECT timestamp, value FROM battery_soc WHERE label = ? AND timestamp >= ? AND timestamp < ? ORDER BY timestamp", - [label, dt_to_ms(start), dt_to_ms(end)], - ) - return [(ms_to_dt(row["timestamp"]), row["value"]) for row in cursor.fetchall()] - - def get_current_soc(self) -> int | None: - cursor = self._conn.execute("SELECT value FROM battery_soc ORDER BY timestamp DESC LIMIT 1") - row = cursor.fetchone() - return row["value"] if row else None - - def get_soc_at(self, timestamp: datetime) -> int | None: - """Get battery SOC reading at or after timestamp. - Before would seemingly make more sense but might return a very out of data SoC value after a - cold-start which would mess with the optimizer.""" - ts_ms = dt_to_ms(timestamp) - cursor = self._conn.execute( - "SELECT value FROM battery_soc WHERE timestamp >= ? ORDER BY timestamp ASC LIMIT 1", - [ts_ms], - ) - row = cursor.fetchone() - if not row: - cursor = self._conn.execute( - "SELECT value FROM battery_soc WHERE timestamp <= ? ORDER BY timestamp DESC LIMIT 1", - [ts_ms], - ) - row = cursor.fetchone() - return row["value"] if row else None - - # ------------------------------------------------------------------------- - # Charge schedule - # ------------------------------------------------------------------------- - - def set_schedule(self, battery_id: str, entries: list[tuple[datetime, datetime, int, float]]) -> None: - # The view on the actual table already handles OR REPLACE and add OR REPLACE to this query messes up the - # insertion... The table_id is then recreated on every conflict. - if not entries: - return - - entries = sorted(entries, key=lambda row: row[0]) - first_ts, _, _, _ = entries[0] - - self._conn.execute( - "DELETE FROM charge_schedule WHERE label = ? AND start_time >= ?", [battery_id, dt_to_ms(first_ts)] - ) - self._conn.executemany( - "INSERT INTO charge_schedule (label, start_time, end_time, power, expected_soc) VALUES (?, ?, ?, ?, ?)", - [(battery_id, dt_to_ms(start), dt_to_ms(end), power, soc) for start, end, power, soc in entries], - ) - self._conn.commit() - - def get_schedule( - self, battery_id: str, start: datetime | None = None, end: datetime | None = None - ) -> list[tuple[datetime, datetime, int, int]]: - conditions, params = base_conditions(battery_id, start, end, timestamp_name="start_time") - where_clause = "WHERE " + " AND ".join(conditions) - cursor = self._conn.execute( - f""" - SELECT start_time, end_time, power, expected_soc - FROM charge_schedule - {where_clause} - ORDER BY start_time - """, - params, - ) - return [(ms_to_dt(row[0]), ms_to_dt(row[1]), row[2], row[3]) for row in cursor.fetchall()] diff --git a/open_ess/database/migration_runner.py b/open_ess/database/migration_runner.py deleted file mode 100644 index 406de11..0000000 --- a/open_ess/database/migration_runner.py +++ /dev/null @@ -1,68 +0,0 @@ -import logging -from datetime import UTC, datetime -from importlib import import_module -from pathlib import Path -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .database import DatabaseConnection - -MIGRATIONS_DIR = Path(__file__).parent / "migrations" - -logger = logging.getLogger(__name__) - - -def get_migrations() -> list[tuple[int, str]]: - """Discover all migration files in the migrations directory. - - Returns: - List of (version, module_name) tuples, sorted by version. - """ - migrations = [] - for file in MIGRATIONS_DIR.glob("*.py"): - if file.name.startswith("_"): - continue - # Parse version from filename like "001_initial.py" - parts = file.stem.split("_", 1) - if len(parts) >= 1 and parts[0].isdigit(): - version = int(parts[0]) - module_name = f"open_ess.database.migrations.{file.stem}" - migrations.append((version, module_name)) - return sorted(migrations) - - -def run_migration(version: int, module_name: str, conn: "DatabaseConnection") -> None: - """Run a single migration. - - Args: - version: Migration version number - module_name: Full module name to import - conn: SQLite connection - """ - module = import_module(module_name) - module.upgrade(conn) - - -def run_migrations(conn: "DatabaseConnection") -> None: - conn.execute(""" - CREATE TABLE IF NOT EXISTS schema_version ( - version INTEGER PRIMARY KEY, - applied_at TEXT NOT NULL - ) - """) - conn.commit() - - cursor = conn.execute("SELECT MAX(version) as version FROM schema_version") - row = cursor.fetchone() - current_version = row["version"] or 0 - - for version, module_name in get_migrations(): - if version > current_version: - logger.info(f"Running migration {version}: {module_name}") - run_migration(version, module_name, conn) - conn.execute( - "INSERT INTO schema_version (version, applied_at) VALUES (?, ?)", - (version, datetime.now(UTC)), - ) - conn.commit() - logger.info(f"Migration {version} complete") diff --git a/open_ess/database/migrations/001_initial.py b/open_ess/database/migrations/001_initial.py deleted file mode 100644 index c5721b7..0000000 --- a/open_ess/database/migrations/001_initial.py +++ /dev/null @@ -1,202 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from ..database import DatabaseConnection - - -def upgrade(conn: "DatabaseConnection") -> None: - conn.execute("PRAGMA foreign_keys = ON") - - # ------------- - # Labels - # ------------- - - conn.execute(""" - CREATE TABLE labels ( - label_id INTEGER PRIMARY KEY, - label TEXT UNIQUE NOT NULL - ) - """) - - # ------------- - # Day-ahead prices - # ------------- - - conn.execute(""" - CREATE TABLE day_ahead_prices ( - area TEXT NOT NULL, - start_time INTEGER NOT NULL, - end_time INTEGER NOT NULL, - price REAL NOT NULL, - PRIMARY KEY (area, start_time) - ) - """) - - # ------------- - # Charge schedule - # ------------- - - conn.execute(""" - CREATE TABLE _charge_schedule ( - label_id INTEGER NOT NULL, - start_time INTEGER NOT NULL, - end_time INTEGER NOT NULL, - power INTEGER NOT NULL, - expected_soc REAL NOT NULL, - PRIMARY KEY (label_id, start_time), - FOREIGN KEY (label_id) REFERENCES labels(label_id) - ) - """) - conn.execute(""" - CREATE VIEW charge_schedule AS - SELECT l.label, cs.start_time, cs.end_time, cs.power, cs.expected_soc - FROM _charge_schedule AS cs - JOIN labels AS l USING (label_id) - """) - conn.execute(""" - CREATE TRIGGER charge_schedule_insert - INSTEAD OF INSERT ON charge_schedule - BEGIN - INSERT OR IGNORE INTO labels(label) VALUES (NEW.label); - INSERT OR REPLACE INTO _charge_schedule(label_id, start_time, end_time, power, expected_soc) - VALUES ( - (SELECT label_id FROM labels WHERE label = NEW.label), - NEW.start_time, - NEW.end_time, - NEW.power, - NEW.expected_soc - ); - END - """) - conn.execute(""" - CREATE TRIGGER charge_schedule_delete - INSTEAD OF DELETE ON charge_schedule - BEGIN - DELETE FROM _charge_schedule - WHERE label_id = (SELECT label_id FROM labels WHERE label = OLD.label) - AND start_time = OLD.start_time; - END - """) - - # ------------- - # Power - # ------------- - - conn.execute(""" - CREATE TABLE _power ( - label_id INTEGER NOT NULL, - start_time INTEGER NOT NULL, - end_time INTEGER, - sample_count INTEGER, - value REAL NOT NULL, - PRIMARY KEY (label_id, start_time), - FOREIGN KEY (label_id) REFERENCES labels(label_id) - ) - """) - conn.execute(""" - CREATE VIEW power AS - SELECT l.label, p.start_time, p.end_time, p.sample_count, p.value - FROM _power AS p - JOIN labels AS l USING (label_id) - """) - conn.execute(""" - CREATE TRIGGER power_insert - INSTEAD OF INSERT ON power - BEGIN - INSERT OR IGNORE INTO labels(label) VALUES (NEW.label); - INSERT INTO _power(label_id, start_time, end_time, sample_count, value) - VALUES ( - (SELECT label_id FROM labels WHERE label = NEW.label), - NEW.start_time, - NEW.end_time, - NEW.sample_count, - NEW.value - ); - END - """) - conn.execute(""" - CREATE TRIGGER power_delete - INSTEAD OF DELETE ON power - BEGIN - DELETE FROM _power - WHERE label_id = (SELECT label_id FROM labels WHERE label = OLD.label) - AND start_time = OLD.start_time; - END - """) - - # ------------- - # Energy - # ------------- - - conn.execute(""" - CREATE TABLE _energy ( - label_id INTEGER NOT NULL, - timestamp INTEGER NOT NULL, - value REAL NOT NULL, - PRIMARY KEY (label_id, timestamp), - FOREIGN KEY (label_id) REFERENCES labels(label_id) - ) - """) - conn.execute(""" - CREATE VIEW energy AS - SELECT l.label, e.timestamp, e.value - FROM _energy AS e - JOIN labels AS l USING (label_id) - """) - conn.execute(""" - CREATE TRIGGER energy_insert - INSTEAD OF INSERT ON energy - BEGIN - INSERT OR IGNORE INTO labels(label) VALUES (NEW.label); - INSERT INTO _energy(label_id, timestamp, value) - VALUES ( - (SELECT label_id FROM labels WHERE label = NEW.label), - NEW.timestamp, - NEW.value - ); - END - """) - conn.execute(""" - CREATE TRIGGER energy_delete - INSTEAD OF DELETE ON energy - BEGIN - DELETE FROM _energy - WHERE label_id = (SELECT label_id FROM labels WHERE label = OLD.label) - AND timestamp = OLD.timestamp; - END - """) - - # ------------- - # Battery SoC - # ------------- - - conn.execute(""" - CREATE TABLE _battery_soc ( - label_id INTEGER NOT NULL, - timestamp INTEGER NOT NULL, - value REAL NOT NULL, - PRIMARY KEY (label_id, timestamp), - FOREIGN KEY (label_id) REFERENCES labels(label_id) - ) - """) - conn.execute(""" - CREATE VIEW battery_soc AS - SELECT l.label, e.timestamp, e.value - FROM _battery_soc AS e - JOIN labels AS l USING (label_id) - """) - conn.execute(""" - CREATE TRIGGER battery_soc_insert - INSTEAD OF INSERT ON battery_soc - BEGIN - INSERT OR IGNORE INTO labels(label) VALUES (NEW.label); - INSERT INTO _battery_soc(label_id, timestamp, value) - VALUES ( - (SELECT label_id FROM labels WHERE label = NEW.label), - NEW.timestamp, - NEW.value - ); - END - """) - - conn.commit() diff --git a/open_ess/database/migrations/__init__.py b/open_ess/database/migrations/__init__.py deleted file mode 100644 index ea0d785..0000000 --- a/open_ess/database/migrations/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Migration scripts.""" diff --git a/open_ess/database/service.py b/open_ess/database/service.py deleted file mode 100644 index b88ee5b..0000000 --- a/open_ess/database/service.py +++ /dev/null @@ -1,41 +0,0 @@ -import logging -from datetime import UTC, datetime, timedelta - -from open_ess.service import Service - -from .database import Database, DatabaseConnection - -logger = logging.getLogger(__name__) - - -class DatabaseService(Service): - def __init__(self, database: Database): - super().__init__("DatabaseService") - self._database = database - self._config = database.config - self._db_conn: DatabaseConnection | None = None - - def on_start(self) -> None: - self._db_conn = self._database.connect() - logger.info("DatabaseService started") - - def tick(self) -> None: - self._run_compression() - - def _run_compression(self) -> None: - if self._db_conn is None: - return None - if self._config.compression.enable: - n_samples, _n_buckets = self._db_conn.compress_power( - datetime.now(UTC), self._config.compression.bucket_seconds - ) - if n_samples > 0: - self._db_conn.vacuum() - - def wait_until_next(self) -> None: - now = datetime.now(UTC) - next_run = now.replace(second=0, microsecond=0) + timedelta(minutes=1, seconds=10) - # ^ Run next compression 10 seconds after a new minute starts. This ensures that all new metrics - # have been written to the database. - - self.wait_seconds((next_run - now).total_seconds()) diff --git a/open_ess/database/util.py b/open_ess/database/util.py deleted file mode 100644 index ac8d692..0000000 --- a/open_ess/database/util.py +++ /dev/null @@ -1,29 +0,0 @@ -from datetime import UTC, datetime - - -def dt_to_ms(dt: datetime) -> int: - """UTC datetime to Unix milliseconds.""" - return int(dt.timestamp() * 1000) - - -def ms_to_dt(ms: int) -> datetime: - """Unix milliseconds to UTC datetime.""" - return datetime.fromtimestamp(ms / 1000, tz=UTC) - - -def base_conditions( - label: str, - start: datetime | None, - end: datetime | None, - label_name: str = "label", - timestamp_name: str = "timestamp", -) -> tuple[list[str], list[str | int]]: - conditions = [f"{label_name} = ?"] - params: list = [label] - if start is not None: - conditions.append(f"{timestamp_name} >= ?") - params.append(dt_to_ms(start)) - if end is not None: - conditions.append(f"{timestamp_name} < ?") - params.append(dt_to_ms(end)) - return conditions, params diff --git a/open_ess/frontend/README.md b/open_ess/frontend/README.md index 9893ef8..2906ffe 100644 --- a/open_ess/frontend/README.md +++ b/open_ess/frontend/README.md @@ -5,25 +5,5 @@ It's a bit of a mess now so if you read this and think you can do better, feel f contact me on GitHub or create a pull request! In short, the frontend uses FastAPI and Pydantic models to define the api endpoints. `main.py` -runs the FastAPI stuff with `uvicorn`. Typescript definitions for the api responses and -api endpoints are generated to `src/types.ts` and `npm` compiles the `.ts` files to `.js` -using `esbuild`. - -And here's the long version: - -`routes/api.py` defines the api using FastAPI and Pydantic. These can be used to generate -`src/types.ts` using the `generate-types` command. The script can be found at -`open_ess/scripts/generate_types.py`. - -The pages themselves are also written in typescript and can also be found in -`open_ess/frontend/src`. These are then compiled to javascript and stored in -`open_ess/frontend/static`. This can be done with either; - - `esbuild open_ess/frontend/src/*.ts --outdir=open_ess/frontend/static --bundle --minify` - -or - - `npm run build` - -The `datatables` library is pretty small and is bundled with the `.js` files. `plotly` is very -big (5MB) and is not bundled. `plotly` is in `static/vendor` and is loaded via `base.html`. +runs the FastAPI stuff with `uvicorn`. Javascript definitions for the api responses and +api endpoints are generated to `src/api.js` and `npm`. diff --git a/open_ess/frontend/app.py b/open_ess/frontend/app.py index 64aacef..8cbc009 100644 --- a/open_ess/frontend/app.py +++ b/open_ess/frontend/app.py @@ -7,9 +7,10 @@ from fastapi.staticfiles import StaticFiles from open_ess.battery_system import BatterySystem -from open_ess.database import Database +from open_ess.timeseries import TimeseriesBackend +from open_ess.timeseries.metricsqlite.backend import MetricSQLiteBackend -from .routes import api_router, pages_router +from .routes import api_router, pages_router, timeseries_router if TYPE_CHECKING: from open_ess.config import Config @@ -18,17 +19,17 @@ def create_app( - database: Database, config: "Config", battery_systems: list[BatterySystem], + mql_client: TimeseriesBackend | None = None, ) -> FastAPI: @asynccontextmanager async def lifespan(_app: FastAPI) -> AsyncGenerator[None]: - _app.state.database = database.connect() _app.state.price_config = config.prices _app.state.battery_systems = battery_systems + _app.state.mql_client = mql_client yield - _app.state.database.close() + _app.state.mql_client.close() app = FastAPI( title="OpenESS", @@ -39,4 +40,11 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None]: app.include_router(pages_router) app.include_router(api_router, prefix="/api") + # Mount /query and /range_query + if mql_client is not None: + if isinstance(mql_client, MetricSQLiteBackend): + app.include_router(mql_client.create_fastapi_router(), prefix="/api/v1") + else: + app.include_router(timeseries_router, prefix="/api/v1") + return app diff --git a/open_ess/frontend/cli.py b/open_ess/frontend/cli.py deleted file mode 100644 index 63e18a4..0000000 --- a/open_ess/frontend/cli.py +++ /dev/null @@ -1,36 +0,0 @@ -import logging - -import uvicorn - -from open_ess.config import Config -from open_ess.database import Database -from open_ess.frontend.app import create_app -from open_ess.util import parse_args, setup_logging - -setup_logging() -logger = logging.getLogger(__name__) - - -def main() -> None: - args = parse_args("Open Energy Storage System web dashboard") - - config = Config.from_file(args.config) - if not config.frontend.enable: - logger.info("Frontend is not enabled. Exiting...") - return - - database = Database(config.database) - - logger.info(f"Starting web server on http://{config.frontend.host}:{config.frontend.port}") - - app = create_app(database, config, battery_systems=[]) - uvicorn.run( - app, - host=config.frontend.host, - port=config.frontend.port, - log_level="info", - ) - - -if __name__ == "__main__": - main() diff --git a/open_ess/frontend/dependencies.py b/open_ess/frontend/dependencies.py index 8b870c9..3a66eae 100644 --- a/open_ess/frontend/dependencies.py +++ b/open_ess/frontend/dependencies.py @@ -3,12 +3,8 @@ from fastapi import Depends, Request from open_ess.battery_system import BatterySystem -from open_ess.database import DatabaseConnection from open_ess.pricing import PriceConfig - - -def get_database(request: Request) -> DatabaseConnection: - return request.app.state.database # type: ignore[no-any-return] +from open_ess.timeseries import TimeseriesBackend def get_price_config(request: Request) -> PriceConfig: @@ -19,7 +15,11 @@ def get_battery_systems(request: Request) -> list[BatterySystem]: return request.app.state.battery_systems # type: ignore[no-any-return] +def get_mql_client(request: Request) -> TimeseriesBackend: + return request.app.state.mql_client # type: ignore[no-any-return] + + # Type aliases for cleaner route signatures -Database = Annotated[DatabaseConnection, Depends(get_database)] PriceConfigDep = Annotated[PriceConfig, Depends(get_price_config)] BatterySystemsDep = Annotated[list[BatterySystem], Depends(get_battery_systems)] +MqlClientDep = Annotated[TimeseriesBackend, Depends(get_mql_client)] diff --git a/open_ess/frontend/routes/__init__.py b/open_ess/frontend/routes/__init__.py index df385c6..2a8a778 100644 --- a/open_ess/frontend/routes/__init__.py +++ b/open_ess/frontend/routes/__init__.py @@ -1,4 +1,5 @@ from .api import router as api_router from .pages import router as pages_router +from .timeseries import router as timeseries_router -__all__ = ["api_router", "pages_router"] +__all__ = ["api_router", "pages_router", "timeseries_router"] diff --git a/open_ess/frontend/routes/api.py b/open_ess/frontend/routes/api.py index ec31e11..4ae87a2 100644 --- a/open_ess/frontend/routes/api.py +++ b/open_ess/frontend/routes/api.py @@ -1,25 +1,24 @@ import logging from datetime import UTC, datetime, timedelta from enum import StrEnum +from typing import TYPE_CHECKING, Literal from fastapi import APIRouter, HTTPException, Query from pydantic import BaseModel -from open_ess.frontend.dependencies import BatterySystemsDep, Database, PriceConfigDep - -from .util import TimeSeries, data_to_timeseries, find_full_battery_cycles +from open_ess.frontend.dependencies import BatterySystemsDep, MqlClientDep, PriceConfigDep +from open_ess.timeseries import TimeseriesBackend, VectorResult +if TYPE_CHECKING: + pass logger = logging.getLogger(__name__) router = APIRouter(tags=["api"]) -class PowerResponse(BaseModel): - series: dict[str, TimeSeries] - - -class EnergyResponse(BaseModel): - series: dict[str, TimeSeries] +# --------- # +# /health # +# --------- # class HealthResponse(BaseModel): @@ -29,12 +28,12 @@ class HealthResponse(BaseModel): @router.get("/health", response_model=HealthResponse) -async def health_check(db: Database) -> HealthResponse: +async def health_check() -> HealthResponse: try: - # TODO: - cursor = db.execute("SELECT name FROM sqlite_master WHERE type='table'") - tables = [row["name"] for row in cursor.fetchall()] - return HealthResponse(status="ok", database="connected", tables=tables) + # TODO + # cursor = db.execute("SELECT name FROM sqlite_master WHERE type='table'") + # tables = [row["name"] for row in cursor.fetchall()] + return HealthResponse(status="ok", database="connected", tables=[]) except Exception as e: logger.exception("Health check failed") raise HTTPException(status_code=500, detail=str(e)) from e @@ -50,20 +49,36 @@ class BatterySystemInfo(BaseModel): name: str -class SystemLayoutData(BaseModel): - phases: list[int] - # TODO: grid_labels: list[str] # ["L1", "L2", "L3"] - has_solar: bool +class SolarInverterInfo(BaseModel): + id: str + name: str + + +class SystemLayoutResponse(BaseModel): + grid_phases: list[str] battery_systems: list[BatterySystemInfo] + solar_inverters: list[SolarInverterInfo] -@router.get("/system-layout", response_model=SystemLayoutData) -async def get_system_layout(battery_systems: BatterySystemsDep) -> SystemLayoutData: - return SystemLayoutData( - phases=[1, 2, 3], - # grid_labels=["L1", "L2", "L3"], - has_solar=True, # TODO - battery_systems=[BatterySystemInfo(id=b.id, name=b.name) for b in battery_systems], +@router.get("/system-layout", response_model=SystemLayoutResponse) +async def get_system_layout( + mql_client: MqlClientDep, + battery_systems: BatterySystemsDep, +) -> SystemLayoutResponse: + phases = ["L1", "L2", "L3"] + if mql_client: + result: VectorResult = mql_client.query('openess_power_watts{from="grid"}') + phase_labels: set[str] = set() + for series in result.series: + phase_label = series.metric.get("phase", "") + phase_labels.add(phase_label) + if phase_labels: + phases = sorted(phase_labels) + + return SystemLayoutResponse( + grid_phases=phases, + battery_systems=[BatterySystemInfo(id=b.id, name=b.name or b.id) for b in battery_systems], + solar_inverters=[], # TODO ) @@ -72,63 +87,94 @@ class BatteryPowerValues(BaseModel): inverter: float | None battery: float | None losses: float | None + soc: float | None class PowerFlowData(BaseModel): grid: dict[str, float | None] solar: float | None - consumption: dict[str, float] # e.g. {"L1": 800, "L2": 300, "L3": 200} + consumption: dict[str, float] batteries: dict[str, BatteryPowerValues] +def _get_instant_value(result) -> float | None: + """Extract the value from an instant query result.""" + if hasattr(result, "series") and result.series: + return result.series[0].value + return None + + +def _get_instant_values_by_label(result, label_key: str) -> dict[str, float]: + """Extract values from an instant query result, keyed by a label.""" + values: dict[str, float] = {} + if hasattr(result, "series"): + for series in result.series: + key = series.metric.get(label_key, "") + if key: + values[key] = series.value + return values + + @router.get("/power-flow", response_model=PowerFlowData) -async def get_power_flow(db: Database, battery_systems: BatterySystemsDep) -> PowerFlowData: - start = datetime.now(UTC) - timedelta(seconds=10) - - grid_power = {} - for i in (1, 2, 3): - power = None - result = db.get_power(f"grid/power/l{i}", start=start, bucket_seconds=None) - if result: - _, power = result[-1] - grid_power[f"L{i}"] = power - - solar_power = None - result = db.get_power("victron/pvinverter/31/power/l1", start=start, bucket_seconds=None) - if result: - _, solar_power = result[-1] - - batteries = {} +async def get_power_flow( + mql_client: MqlClientDep, + battery_systems: BatterySystemsDep, +) -> PowerFlowData: + if mql_client is None: + raise HTTPException(503, "Timeseries backend not configured") + + # Grid power per phase + grid_result = mql_client.query('openess_power_watts{from="grid"}') + grid_power = _get_instant_values_by_label(grid_result, "phase") + + # Solar power (sum all PV inverter phases) + solar_result = mql_client.query('sum(openess_power_watts{from="pvinverter"})') + solar_power = _get_instant_value(solar_result) + + # Battery power for each system + batteries: dict[str, BatteryPowerValues] = {} for battery_system in battery_systems: - charger = 0 - inverter = 0 - battery = 0 - losses = 0 - system = 0 - result = db.get_power(battery_system.config.metrics.power_to_system, start=start, bucket_seconds=None) - if result: - _, system = result[-1] - if system < 0: - charger = -system - if system > 0: - inverter = system - - result = db.get_power(battery_system.config.metrics.power_to_battery, start=start, bucket_seconds=None) - if result: - _, battery = result[-1] - losses = battery - system + device = battery_system.device_serial + + # AC power: charger input - inverter output + ac_in_result = mql_client.query(f'sum(openess_power_watts{{from="ac_in", to="system", device="{device}"}})') + ac_out_result = mql_client.query(f'sum(openess_power_watts{{from="system", to="ac_out", device="{device}"}})') + ac_in = _get_instant_value(ac_in_result) or 0 + ac_out = _get_instant_value(ac_out_result) or 0 + + charger = ac_in + inverter = ac_out + + # DC battery power + battery_result = mql_client.query(f'openess_power_watts{{from="system", to="battery", device="{device}"}}') + battery_power = _get_instant_value(battery_result) or 0 + + # SOC + soc_result = mql_client.query(f'openess_soc_ratio{{device="{device}", node="battery"}} * 100') + soc = _get_instant_value(soc_result) + + # Losses = AC net - DC + ac_net = ac_in - ac_out + losses = ac_net - battery_power batteries[battery_system.id] = BatteryPowerValues( charger=charger, inverter=inverter, - battery=battery, + battery=battery_power, losses=losses, + soc=soc, ) + # Consumption = grid + solar + battery discharge - battery charge + # For now, return zeros (would need more complex calculation) + consumption: dict[str, float] = {} + for phase, grid_val in grid_power.items(): + consumption[phase] = grid_val or 0.0 + return PowerFlowData( grid=grid_power, solar=solar_power, - consumption={"L1": 0.0, "L2": 0.0, "L3": 0.0}, + consumption=consumption, batteries=batteries, ) @@ -145,8 +191,6 @@ class Status(StrEnum): class ServiceMessage(BaseModel): - timestamp: datetime - status: Status message: str @@ -168,7 +212,7 @@ async def services_status() -> ServicesStatusResponse: optimizer=ServiceStatus(status=Status.OK, messages=[]), ) except Exception as e: - logger.exception("Health check failed") + logger.exception("Services status check failed") raise HTTPException(status_code=500, detail=str(e)) from e @@ -186,269 +230,244 @@ async def get_battery_ids(battery_systems: BatterySystemsDep) -> list[str]: # ------------------------ # -class BatteryEnergySeries(BaseModel): - energy_to_charger: list[float | None] = [] - energy_from_inverter: list[float | None] = [] - energy_to_battery: list[float | None] = [] - energy_from_battery: list[float | None] = [] - energy_loss_to_battery: list[float | None] = [] - energy_loss_from_battery: list[float | None] = [] +class EnergyQueryDef(BaseModel): + query: str + label: str + color: str + negate: bool = False -class EnergyGraphResponse(BaseModel): - timestamps: list[datetime] +class EnergyViewConfig(BaseModel): + """Configuration for a single energy chart view.""" - grid_import: dict[str, list[float | None]] - grid_export: dict[str, list[float | None]] + id: str + name: str + queries: list[EnergyQueryDef] - battery_systems: dict[str, BatteryEnergySeries] - solar: list[float | None] = [] - to_consumption: list[float | None] = [] - from_consumption: list[float | None] = [] +class EnergyQueriesResponse(BaseModel): + """Response containing all energy chart view configurations.""" + views: list[EnergyViewConfig] -@router.get("/energy-graph", response_model=EnergyGraphResponse) -async def get_energy_flow_endpoint( - db: Database, - battery_systems: BatterySystemsDep, - battery_id: str | None = Query(default=None), - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), - bucket_minutes: int = Query(default=60), -) -> EnergyGraphResponse: + +@router.get("/charts/energy-queries", response_model=EnergyQueriesResponse) +async def get_energy_queries(battery_systems: BatterySystemsDep) -> EnergyQueriesResponse: try: - battery_system = None - if battery_id: - for bs in battery_systems: - if bs.id == battery_id: - battery_system = bs - break - elif len(battery_systems) == 1: - battery_system = battery_systems[0] + views: list[EnergyViewConfig] = [] - if battery_system is None: - if battery_id: - raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") - else: - raise HTTPException(status_code=400, detail="Please provide a battery_id") + # Collect grid queries across all battery systems + grid_import_parts: list[str] = [] + grid_export_parts: list[str] = [] + consumption_parts: list[str] = [] - now = datetime.now(UTC) - if start is None: - start = now - timedelta(hours=24) - if end is None: - end = now + for battery_system in battery_systems: + device = battery_system.device_serial + bs_name = battery_system.name or battery_system.id + queries = battery_system.get_energy_queries() + + # Battery system view: shows charge/discharge from battery's perspective + views.append( + EnergyViewConfig( + id=battery_system.id, + name=bs_name, + queries=[ + EnergyQueryDef( + query=queries.energy_to_charger, + label="Charge", + color="#3498db", + negate=True, + ), + EnergyQueryDef( + query=queries.energy_from_inverter, + label="Discharge", + color="#f39c12", + ), + EnergyQueryDef( + query=queries.energy_loss_to_battery, + label="Charge Losses", + color="#e74c3c", + negate=True, + ), + EnergyQueryDef( + query=queries.energy_loss_from_battery, + label="Discharge Losses", + color="#c0392b", + negate=True, + ), + ], + ) + ) - series = { - "grid_import": db.get_energy_aggregated( - "grid/energy/import/total", bucket_minutes * 60, start, end, center_buckets=True - ), - "grid_export": db.get_energy_aggregated( - "grid/energy/export/total", bucket_minutes * 60, start, end, center_buckets=True - ), - "vebus_228_import": db.get_energy_aggregated( - battery_system.config.metrics.energy_to_system, bucket_minutes * 60, start, end, center_buckets=True - ), - "vebus_228_export": db.get_energy_aggregated( - battery_system.config.metrics.energy_from_system, bucket_minutes * 60, start, end, center_buckets=True - ), - } - - timestamps = set() - series_as_dict: dict[str, dict[datetime, float]] = {name: {} for name in series} - for name, s in series.items(): - for ts, v in s: - timestamps.add(ts) - series_as_dict[name][ts] = v - timestamps = list(sorted(timestamps)) - - grid_exports = { - "From MP": [], - } - grid_imports = { - "Consumption": [], - "To MP": [], - } - battery_stats = BatteryEnergySeries() - for ts in timestamps: - # Grid export - from_mp = series_as_dict["vebus_228_export"].get(ts) - grid_exports["From MP"].append(from_mp) - unaccounted_export = series_as_dict["grid_export"].get(ts, 0) - (from_mp or 0) - - # Grid import - to_mp = series_as_dict["vebus_228_import"].get(ts) - grid_imports["To MP"].append(to_mp) - grid_import = series_as_dict["grid_import"].get(ts) - if grid_import is not None: - grid_import -= (to_mp or 0) - unaccounted_export - grid_imports["Consumption"].append(grid_import) - - # Battery stats - battery_stats.energy_to_charger.append(to_mp) - battery_stats.energy_from_inverter.append(from_mp) - - return EnergyGraphResponse( - timestamps=timestamps, - grid_export=grid_exports, - grid_import=grid_imports, - battery_systems={"MultiPlus": battery_stats}, - ) - except Exception as e: - logger.exception("Failed to get energy flow") - raise HTTPException(status_code=500, detail=str(e)) from e + # Collect grid queries per device + grid_import_parts.append( + f'increase(openess_energy_kwh{{from="grid", to="system", device="{device}"}}[$step])' + ) + grid_export_parts.append( + f'increase(openess_energy_kwh{{from="system", to="grid", device="{device}"}}[$step])' + ) + # Consumption = AC out from inverter + consumption_parts.append( + f'increase(openess_energy_kwh{{from="system", to="ac_out", device="{device}"}}[$step])' + ) -@router.get("/power-graph", response_model=PowerResponse) -async def get_power_graph( - db: Database, - battery_systems: BatterySystemsDep, - battery_id: str | None = Query(default=None), - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), - aggregate_minutes: int = Query(default=1), -) -> PowerResponse: - try: - battery_system = None - if battery_id: - for bs in battery_systems: - if bs.id == battery_id: - battery_system = bs - break - elif len(battery_systems) == 1: - battery_system = battery_systems[0] + # Grid view: shows grid import/export + grid_import_query = " + ".join(f"({q})" for q in grid_import_parts) if grid_import_parts else "" + grid_export_query = " + ".join(f"({q})" for q in grid_export_parts) if grid_export_parts else "" - if battery_system is None: - if battery_id: - raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") - else: - raise HTTPException(status_code=400, detail="Please provide a battery_id") + grid_queries: list[EnergyQueryDef] = [] + if grid_import_query: + grid_queries.append(EnergyQueryDef(query=grid_import_query, label="Import", color="#e74c3c", negate=True)) + if grid_export_query: + grid_queries.append(EnergyQueryDef(query=grid_export_query, label="Export", color="#2ecc71")) + # TODO: Add solar query when solar support is implemented - now = datetime.now(UTC) - if start is None: - start = now - timedelta(hours=24) - if end is None: - end = now + views.append(EnergyViewConfig(id="grid", name="Grid", queries=grid_queries)) - bucket_seconds = aggregate_minutes * 60 + # Consumption view: shows energy consumed (AC out) + consumption_query = " + ".join(f"({q})" for q in consumption_parts) if consumption_parts else "" + consumption_queries: list[EnergyQueryDef] = [] + if consumption_query: + consumption_queries.append(EnergyQueryDef(query=consumption_query, label="Consumption", color="#9b59b6")) + if grid_import_query: + consumption_queries.append(EnergyQueryDef(query=grid_import_query, label="From Grid", color="#e74c3c")) + # TODO: Add solar contribution - series = {f"Grid L{i}": db.get_power(f"grid/power/l{i}", start, end, bucket_seconds) for i in (1, 2, 3)} + views.append(EnergyViewConfig(id="consumption", name="Consumption", queries=consumption_queries)) - series["To MP"] = db.get_power(battery_system.config.metrics.power_to_system, start, end, bucket_seconds) + return EnergyQueriesResponse(views=views) + except Exception as e: + logger.exception("Failed to get energy queries") + raise HTTPException(status_code=500, detail=str(e)) from e - series["Battery"] = db.get_power(battery_system.config.metrics.power_to_battery, start, end, bucket_seconds) - series["Solar"] = [ - (t, -p) for t, p in db.get_power("victron/pvinverter/31/power/l1", start, end, bucket_seconds) - ] +class PowerQueryDef(BaseModel): + label: str + query: str + is_total: bool | None = None - series["Schedule"] = [] - print(type(battery_system)) - print(type(battery_system.config)) +class ChartsPowerResponse(BaseModel): + queries: list[PowerQueryDef] + phases: list[str] - for ts_start, ts_end, v, _ in db.get_schedule(battery_system.config.id, start): - series["Schedule"].extend([(ts_start, v), (ts_end, v)]) - return PowerResponse(series={k: data_to_timeseries(v) for k, v in series.items()}) - except Exception as e: - logger.exception("Failed to get power data") - raise HTTPException(status_code=500, detail=str(e)) from e +def _discover_phases(mql_client, query: str) -> list[str]: + if mql_client is None: + return [] + result = mql_client.query(query) + phase_set: set[str] = set() + if hasattr(result, "series"): + for series in result.series: + phase_label = series.metric.get("phase") + if phase_label: + phase_set.add(phase_label) + return sorted(phase_set) -class PricePoint(BaseModel): - time: datetime - market: float | None - buy: float | None - sell: float | None +@router.get("/charts/power-queries", response_model=ChartsPowerResponse) +async def get_power_queries( + mql_client: MqlClientDep, + battery_systems: BatterySystemsDep, +) -> ChartsPowerResponse: + queries: list[PowerQueryDef] = [] + + # Discover grid phases + phases = _discover_phases(mql_client, 'openess_power_watts{from="grid"}') + + # Grid power queries + if len(phases) > 1: + queries.append( + PowerQueryDef( + query='sum(avg_over_time(openess_power_watts{from="grid"}[$step]))', + label="Grid", + is_total=True, + ) + ) + for phase in phases: + queries.append( + PowerQueryDef( + query=f'avg_over_time(openess_power_watts{{from="grid", phase="{phase}"}}[$step])', + label=f"Grid {phase}", + is_total=False, + ) + ) + else: + queries.append( + PowerQueryDef( + query='avg_over_time(openess_power_watts{from="grid"}[$step])', + label="Grid", + ) + ) + + # Battery system queries + for bs in battery_systems: + device = bs.device_serial or "unknown" + bs_phases = _discover_phases(mql_client, f'openess_power_watts{{to="system", device="{device}"}}') + bs_queries = bs.get_power_queries(bs_phases) + for q in bs_queries.queries: + queries.append(PowerQueryDef(query=q.query, label=q.label, is_total=q.is_total)) + + return ChartsPowerResponse( + queries=queries, + phases=phases, + ) -class PricesResponse(BaseModel): - area: str - aggregate_minutes: int - unit: str = "€/kWh" # TODO: based on area - timeseries: list[PricePoint] +class PriceQueriesResponse(BaseModel): + market_query: str + buy_query: str + sell_query: str + step: Literal["15m", "1h"] + currency: str = "€" # TODO: based on area -@router.get("/prices", response_model=PricesResponse) -async def get_price_data( - db: Database, +@router.get("/charts/price-queries", response_model=PriceQueriesResponse) +async def get_price_queries( price_config: PriceConfigDep, area: str | None = Query(default=None), - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), - aggregate_minutes: int | None = Query(default=None), -) -> PricesResponse: +) -> PriceQueriesResponse: try: - if area is None: + if not area: area = price_config.area - now = datetime.now(UTC) - if start is None: - start = now - timedelta(days=7) - if end is None: - end = now + timedelta(days=2) - if aggregate_minutes is None: - aggregate_minutes = price_config.aggregate_minutes - - timeseries = [] - for timestamp, price in db.get_prices(area, start, end, aggregate_minutes=aggregate_minutes): - timeseries.append( - PricePoint( - time=timestamp, - market=round(price, 4), - buy=round(price_config.buy_price(price), 4), - sell=round(price_config.sell_price(price), 4), - ) - ) + # TODO: validate area value - return PricesResponse( - area=area, - aggregate_minutes=aggregate_minutes, - timeseries=timeseries, + step: Literal["15m", "1h"] = "1h" if price_config.hourly_average else "15m" + + return PriceQueriesResponse( + market_query=f'avg_over_time(openess_prices{{area="{area}", price="market"}}[{step}])', + buy_query=f'avg_over_time(openess_prices{{area="{area}", price="buy"}}[{step}])', + sell_query=f'avg_over_time(openess_prices{{area="{area}", price="sell"}}[{step}])', + step=step, ) except Exception as e: logger.exception("Failed to get prices") raise HTTPException(status_code=500, detail=str(e)) from e -class BatteryGraphResponse(BaseModel): - soc: TimeSeries - schedule: TimeSeries # Scheduled (past and future) SoC - voltage: TimeSeries +class BatteryQueriesResponse(BaseModel): + soc_query: str + schedule_soc_query: str + voltage_query: str -@router.get("/battery-graph", response_model=dict[str, BatteryGraphResponse]) -async def get_battery_graph( - db: Database, +@router.get("/charts/battery-queries", response_model=dict[str, BatteryQueriesResponse]) +async def get_battery_queries( battery_systems: BatterySystemsDep, - battery_id: str | None = Query(default=None), - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), -) -> dict[str, BatteryGraphResponse]: +) -> dict[str, BatteryQueriesResponse]: try: - now = datetime.now(UTC) - if start is None: - start = now - timedelta(hours=48) - if end is None: - end = now + timedelta(hours=24) - result = {} - for battery_system in battery_systems: - if battery_id is not None and battery_system.config.id != battery_id: - continue - - soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) - scheduled = [(t, soc) for _, t, _, soc in db.get_schedule(battery_system.config.id, start)] - voltage = db.get_voltage(battery_system.config.metrics.battery_voltage, start, end, bucket_seconds=60) - - result[battery_system.config.name] = BatteryGraphResponse( - soc=data_to_timeseries(soc, rounding=1), - schedule=data_to_timeseries(scheduled, rounding=1), - voltage=data_to_timeseries(voltage, rounding=2), + for bs in battery_systems: + queries = bs.get_battery_queries() + result[bs.config.name] = BatteryQueriesResponse( + soc_query=queries.soc_query, + schedule_soc_query=queries.schedule_soc_query, + voltage_query=queries.voltage_query, ) return result except Exception as e: - logger.exception("Failed to get battery SOC") + logger.exception("Failed to get battery queries") raise HTTPException(status_code=500, detail=str(e)) from e @@ -469,58 +488,124 @@ class EfficiencyScatterPoint(BaseModel): @router.get("/efficiency-scatter", response_model=list[EfficiencyScatterPoint]) async def get_efficiency_scatter( - db: Database, - limit: int = Query(default=2000), + mql_client: MqlClientDep, + battery_systems: BatterySystemsDep, + battery_id: str | None = Query(default=None), + start: datetime | None = Query(default=None), + end: datetime | None = Query(default=None), aggregate_minutes: int = Query(default=10), idle_threshold: int = Query(default=5), + limit: int = Query(default=2000), ) -> list[EfficiencyScatterPoint]: try: - ac_in = db.get_power("victron/vebus/228/power/ac_in/l1", bucket_seconds=aggregate_minutes * 60, limit=limit) - ac_out = db.get_power("victron/vebus/228/power/ac_out/l1", bucket_seconds=aggregate_minutes * 60, limit=limit) - dc = db.get_power("victron/vebus/228/power/battery", bucket_seconds=aggregate_minutes * 60, limit=limit) - # dc = db.get_power("victron/battery/225/power/battery", bucket_seconds=aggregate_minutes * 60, limit=limit) - - data = {ts: [v_in - v_out, None] for (ts, v_in), (_, v_out) in zip(ac_in, ac_out, strict=False)} - for ts, v in dc: - if ts in data: - data[ts][1] = v - - points = [] - for ts, (ac, dc) in data.items(): - if abs(dc) < idle_threshold: - category = "idling" - # elif dc > 0 and soc == 100 and abs(ac) < balancing_threshold: - # category = "balancing" - elif dc > 0: - category = "charging" + if mql_client is None: + raise HTTPException(503, "Timeseries backend not configured") + + battery_system = None + if battery_id: + for bs in battery_systems: + if bs.id == battery_id: + battery_system = bs + break + elif len(battery_systems) == 1: + battery_system = battery_systems[0] + + if battery_system is None: + if battery_id: + raise HTTPException(status_code=400, detail=f"No battery system with id '{battery_id}'") else: - category = "discharging" - - losses = ac - dc - efficiency = None - if category == "charging" and ac > 0: - efficiency = (dc / ac) * 100 - elif category == "discharging" and dc < 0: - efficiency = (ac / dc) * 100 - - points.append( - EfficiencyScatterPoint( - time=ts, - battery_power=round(abs(dc), 1), - inverter_charger_power=round(ac, 1), - losses=round(losses, 1), - efficiency=round(efficiency, 1) if efficiency is not None else None, - soc=None, - category=category, - ) - ) + raise HTTPException(status_code=400, detail="Please provide a battery_id") + + now = datetime.now(UTC) + if start is None: + start = now - timedelta(days=7) + if end is None: + end = now - return points + return _get_efficiency_scatter(mql_client, battery_system, start, end, aggregate_minutes, idle_threshold, limit) + except HTTPException: + raise except Exception as e: logger.exception("Failed to get efficiency scatter data") raise HTTPException(status_code=500, detail=str(e)) from e +def _get_efficiency_scatter( + mql_client: TimeseriesBackend, + battery_system, + start: datetime, + end: datetime, + aggregate_minutes: int, + idle_threshold: int, + limit: int, +) -> list[EfficiencyScatterPoint]: + """Get efficiency scatter data from timeseries backend.""" + device = battery_system.device_serial + step = f"{aggregate_minutes}m" + + # Query AC in, AC out, and battery DC power + ac_in_query = f'sum(avg_over_time(openess_power_watts{{from="ac_in", to="system", device="{device}"}}[{step}]))' + ac_out_query = f'sum(avg_over_time(openess_power_watts{{from="system", to="ac_out", device="{device}"}}[{step}]))' + dc_query = f'avg_over_time(openess_power_watts{{from="system", to="battery", device="{device}"}}[{step}])' + + ac_in_result = mql_client.query_range(ac_in_query, start, end, step) + ac_out_result = mql_client.query_range(ac_out_query, start, end, step) + dc_result = mql_client.query_range(dc_query, start, end, step) + + # Convert to dicts + def result_to_dict(result) -> dict[datetime, float]: + if not result.series or not result.series[0].values: + return {} + return {ts: val for ts, val in result.series[0].values} + + ac_in_data = result_to_dict(ac_in_result) + ac_out_data = result_to_dict(ac_out_result) + dc_data = result_to_dict(dc_result) + + # Merge data by timestamp + all_timestamps = set(ac_in_data.keys()) & set(dc_data.keys()) + if ac_out_data: + all_timestamps &= set(ac_out_data.keys()) + + points = [] + for ts in sorted(all_timestamps): + ac_in = ac_in_data.get(ts, 0) + ac_out = ac_out_data.get(ts, 0) + ac = ac_in - ac_out # Net AC power (positive = charging) + dc = dc_data[ts] + + if abs(dc) < idle_threshold: + category = "idling" + elif dc > 0: + category = "charging" + else: + category = "discharging" + + losses = abs(ac) - abs(dc) + efficiency = None + if category == "charging" and ac > 0: + efficiency = (dc / ac) * 100 + elif category == "discharging" and dc < 0 and ac_out > 0: + efficiency = (ac_out / abs(dc)) * 100 + + points.append( + EfficiencyScatterPoint( + time=ts, + battery_power=round(abs(dc), 1), + inverter_charger_power=round(ac, 1), + losses=round(losses, 1), + efficiency=round(efficiency, 1) if efficiency is not None else None, + soc=None, + category=category, + ) + ) + + if len(points) >= limit: + break + + return points + + class BatteryCycle(BaseModel): start_time: datetime end_time: datetime @@ -528,8 +613,8 @@ class BatteryCycle(BaseModel): min_soc: float ac_energy_in: float | None ac_energy_out: float | None - dc_energy_in: float - dc_energy_out: float + dc_energy_in: float | None + dc_energy_out: float | None system_efficiency: float | None battery_efficiency: float | None charger_efficiency: float | None @@ -540,7 +625,7 @@ class BatteryCycle(BaseModel): @router.get("/cycles", response_model=list[BatteryCycle]) async def get_battery_cycles( - db: Database, + mql_client: MqlClientDep, battery_systems: BatterySystemsDep, price_config: PriceConfigDep, battery_id: str | None = Query(default=None), @@ -549,6 +634,9 @@ async def get_battery_cycles( min_soc_swing: int = Query(default=10), ) -> list[BatteryCycle]: try: + if mql_client is None: + raise HTTPException(503, "Timeseries backend not configured") + battery_system = None if battery_id: for bs in battery_systems: @@ -570,143 +658,157 @@ async def get_battery_cycles( if end is None: end = now - battery_soc = db.get_battery_soc(battery_system.config.metrics.battery_soc, start, end) - raw_cycles = find_full_battery_cycles(battery_soc, full_threshold=90, min_soc_swing=min_soc_swing) - - cycles = [] - for cycle_start, cycle_end, min_soc in raw_cycles: - duration = (cycle_end - cycle_start).total_seconds() / 3600.0 - - # TODO: this is cursed AF and should be fixed - dc_energy_in = 0.0 - dc_energy_out = 0.0 - for _, p in db.get_power(battery_system.config.metrics.power_to_battery[0], cycle_start, cycle_end): - # for _, p in db.get_power("vebus_228_battery", cycle_start, cycle_end): - if p > 0: - dc_energy_in += p - else: - dc_energy_out += -p - dc_energy_in /= 60000 - dc_energy_out /= 60000 - - ac_in_import = db.get_energy( - battery_system.config.metrics.energy_to_system, cycle_start, cycle_end, normalize=True - ) - # ac_out_import = db.get_energy("vebus_228_ac_out_import", cycle_start, cycle_end, normalize=True) - ac_in_export = db.get_energy( - battery_system.config.metrics.energy_from_system, cycle_start, cycle_end, normalize=True - ) - # ac_out_export = db.get_energy("vebus_228_ac_out_export", cycle_start, cycle_end, normalize=True) - - ac_energy_in = 0.0 - if ac_in_import: - ac_energy_in += ac_in_import[-1][1] - # if ac_out_import: - # ac_energy_in += ac_out_import[-1][1] - ac_energy_out = 0.0 - if ac_in_export: - ac_energy_out += ac_in_export[-1][1] - # if ac_out_export: - # ac_energy_out += ac_out_export[-1][1] - - profit = 0.0 - scheduled_profit = 0.0 - e_in = { - ts: v - for ts, v in db.get_energy_aggregated( - battery_system.config.metrics.energy_to_system, 3600, cycle_start, cycle_end - ) - } - e_out = { - ts: v - for ts, v in db.get_energy_aggregated( - battery_system.config.metrics.energy_from_system, 3600, cycle_start, cycle_end - ) - } - scheduled = {ts: v for ts, _, v, _ in db.get_schedule(battery_system.config.id, cycle_start)} - for ts, v in db.get_prices(price_config.area, cycle_start, cycle_end, aggregate_minutes=60): - profit -= price_config.buy_price(v) * e_in.get(ts, 0) - profit += price_config.sell_price(v) * e_out.get(ts, 0) - scheduled_power = scheduled.get(ts, 0) - if scheduled_power > 0: - scheduled_profit -= price_config.buy_price(v) * scheduled_power / 1000 - if scheduled_power < 0: - scheduled_profit += price_config.sell_price(v) * -scheduled_power / 1000 - - cycles.append( - BatteryCycle( - start_time=cycle_start, - end_time=cycle_end, - duration_hours=round(duration, 2), - min_soc=round(min_soc, 1), - ac_energy_in=round(ac_energy_in, 2) if ac_energy_in else None, - ac_energy_out=round(ac_energy_out, 2) if ac_energy_out else None, - dc_energy_in=round(dc_energy_in, 2), - dc_energy_out=round(dc_energy_out, 2), - system_efficiency=round(ac_energy_out / ac_energy_in * 100, 1) if ac_energy_in else None, - battery_efficiency=round(dc_energy_out / dc_energy_in * 100, 1) if dc_energy_in else None, - charger_efficiency=round(dc_energy_in / ac_energy_in * 100, 1) if ac_energy_in else None, - inverter_efficiency=round(ac_energy_out / dc_energy_out * 100, 1) if dc_energy_out else None, - profit=round(profit, 2), - scheduled_profit=round(scheduled_profit, 2), - ) - ) - - return cycles + return _get_battery_cycles(mql_client, battery_system, price_config, start, end, min_soc_swing) + except HTTPException: + raise except Exception as e: logger.exception("Failed to get battery cycles") raise HTTPException(status_code=500, detail=str(e)) from e -# -------------------------# -# Generalized endpoints # -# -------------------------# +def _get_battery_cycles( + mql_client: TimeseriesBackend, + battery_system, + price_config, + start: datetime, + end: datetime, + min_soc_swing: int, +) -> list[BatteryCycle]: + """Get battery cycles from timeseries backend.""" + from .util import find_full_battery_cycles + + device = battery_system.device_serial + + # Get SOC data at 1-minute resolution for cycle detection + soc_query = f'openess_soc_ratio{{device="{device}", node="battery"}} * 100' + soc_result = mql_client.query_range(soc_query, start, end, step="1m") + + if not soc_result.series or not soc_result.series[0].values: + return [] + + battery_soc = [(ts, val) for ts, val in soc_result.series[0].values] + + # Find cycles using the existing algorithm + raw_cycles = find_full_battery_cycles(battery_soc, full_threshold=90, min_soc_swing=min_soc_swing) + + cycles = [] + for cycle_start, cycle_end, min_soc in raw_cycles: + duration = (cycle_end - cycle_start).total_seconds() / 3600.0 + + # Query energy for this cycle + # AC energy in (charger input) + ac_in_query = f'increase(openess_energy_kwh{{from="ac_in", to="system", device="{device}"}}[1h])' + ac_in_result = mql_client.query_range(ac_in_query, cycle_start, cycle_end, step="1h") + ac_energy_in = _sum_series_values(ac_in_result) + + # AC energy out (inverter output) + ac_out_query = f'increase(openess_energy_kwh{{from="system", to="ac_out", device="{device}"}}[1h])' + ac_out_result = mql_client.query_range(ac_out_query, cycle_start, cycle_end, step="1h") + ac_energy_out = _sum_series_values(ac_out_result) + + # DC energy in (battery charge) + dc_in_query = f'increase(openess_energy_kwh{{from="system", to="battery", device="{device}"}}[1h])' + dc_in_result = mql_client.query_range(dc_in_query, cycle_start, cycle_end, step="1h") + dc_energy_in = _sum_series_values(dc_in_result) + + # DC energy out (battery discharge) + dc_out_query = f'increase(openess_energy_kwh{{from="battery", to="system", device="{device}"}}[1h])' + dc_out_result = mql_client.query_range(dc_out_query, cycle_start, cycle_end, step="1h") + dc_energy_out = _sum_series_values(dc_out_result) + + # Calculate efficiencies + system_eff = None + if ac_energy_in and ac_energy_in > 0: + system_eff = round((ac_energy_out or 0) / ac_energy_in * 100, 1) + + battery_eff = None + if dc_energy_in and dc_energy_in > 0: + battery_eff = round((dc_energy_out or 0) / dc_energy_in * 100, 1) + + charger_eff = None + if ac_energy_in and ac_energy_in > 0: + charger_eff = round((dc_energy_in or 0) / ac_energy_in * 100, 1) + + inverter_eff = None + if dc_energy_out and dc_energy_out > 0: + inverter_eff = round((ac_energy_out or 0) / dc_energy_out * 100, 1) + + # Calculate profit using hourly prices + profit = _calculate_cycle_profit(mql_client, price_config, device, cycle_start, cycle_end) + + cycles.append( + BatteryCycle( + start_time=cycle_start, + end_time=cycle_end, + duration_hours=round(duration, 2), + min_soc=round(min_soc, 1), + ac_energy_in=round(ac_energy_in, 3) if ac_energy_in else None, + ac_energy_out=round(ac_energy_out, 3) if ac_energy_out else None, + dc_energy_in=round(dc_energy_in, 3) if dc_energy_in else None, + dc_energy_out=round(dc_energy_out, 3) if dc_energy_out else None, + system_efficiency=system_eff, + battery_efficiency=battery_eff, + charger_efficiency=charger_eff, + inverter_efficiency=inverter_eff, + profit=profit, + scheduled_profit=None, # TODO: implement scheduled profit + ) + ) + return cycles -# TODO: add parameter to select subset of series -@router.get("/power", response_model=PowerResponse) -async def get_power( - db: Database, - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), - aggregate_minutes: int = Query(default=1), -) -> PowerResponse: - try: - now = datetime.now(UTC) - if start is None: - start = now - timedelta(hours=24) - if end is None: - end = now - series = db.get_all_power(start, end, aggregate_minutes * 60) - return PowerResponse(series={k: data_to_timeseries(v) for k, v in series.items()}) - except Exception as e: - logger.exception("Failed to get debug power flows") - raise HTTPException(status_code=500, detail=str(e)) from e +def _sum_series_values(result) -> float | None: + """Sum all values in a range query result.""" + if not result.series or not result.series[0].values: + return None + total = sum(val for _, val in result.series[0].values) + return total if total > 0 else None -# TODO: add parameter to select subset of series -# TODO: add normalize parameter -@router.get("/energy", response_model=EnergyResponse) -async def get_energy( - db: Database, - start: datetime | None = Query(default=None), - end: datetime | None = Query(default=None), -) -> EnergyResponse: - try: - now = datetime.now(UTC) - if start is None: - start = now - timedelta(hours=24) - if end is None: - end = now - # Get counter-based energy flows - series = db.get_all_energy(start, end, normalize=True) +def _calculate_cycle_profit( + mql_client: TimeseriesBackend, + price_config, + device: str, + cycle_start: datetime, + cycle_end: datetime, +) -> float | None: + """Calculate profit for a battery cycle based on hourly prices.""" + area = price_config.area - # Get integrated power flows - for label in db.get_power_labels(start, end): - series[f"{label} [integrated]"] = db.integrate_power(label, start, end) + # Get hourly prices + price_query = f'avg_over_time(openess_prices{{area="{area}", price="market"}}[1h])' + price_result = mql_client.query_range(price_query, cycle_start, cycle_end, step="1h") - return EnergyResponse(series={k: data_to_timeseries(v) for k, v in series.items()}) - except Exception as e: - logger.exception("Failed to get debug energy flows") - raise HTTPException(status_code=500, detail=str(e)) from e + if not price_result.series or not price_result.series[0].values: + return None + + # Get hourly energy in/out + ac_in_query = f'increase(openess_energy_kwh{{from="ac_in", to="system", device="{device}"}}[1h])' + ac_out_query = f'increase(openess_energy_kwh{{from="system", to="ac_out", device="{device}"}}[1h])' + + ac_in_result = mql_client.query_range(ac_in_query, cycle_start, cycle_end, step="1h") + ac_out_result = mql_client.query_range(ac_out_query, cycle_start, cycle_end, step="1h") + + # Build timestamp -> value dicts + prices = {ts: val for ts, val in price_result.series[0].values} + energy_in = {} + energy_out = {} + + if ac_in_result.series and ac_in_result.series[0].values: + energy_in = {ts: val for ts, val in ac_in_result.series[0].values} + if ac_out_result.series and ac_out_result.series[0].values: + energy_out = {ts: val for ts, val in ac_out_result.series[0].values} + + profit = 0.0 + for ts, market_price in prices.items(): + e_in = energy_in.get(ts, 0) + e_out = energy_out.get(ts, 0) + + buy_price = price_config.buy_price(market_price) + sell_price = price_config.sell_price(market_price) + + profit -= buy_price * e_in + profit += sell_price * e_out + + return round(profit, 2) if profit != 0 else None diff --git a/open_ess/frontend/routes/timeseries.py b/open_ess/frontend/routes/timeseries.py new file mode 100644 index 0000000..b04f8fa --- /dev/null +++ b/open_ess/frontend/routes/timeseries.py @@ -0,0 +1,99 @@ +"""Timeseries query proxy routes. + +These routes proxy queries to the timeseries backend (VictoriaMetrics). +For MetricSQLite, the native metricsqlite.fastapi routes are used instead. +""" + +from datetime import UTC, datetime + +from fastapi import APIRouter, HTTPException + +from open_ess.timeseries import ScalarResult, VectorResult + +from ..dependencies import MqlClientDep + +router = APIRouter() + + +def _parse_timestamp(value: float | str) -> datetime: + """Parse a timestamp from float (unix seconds) or ISO string.""" + if isinstance(value, str): + # Try parsing as ISO format + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + # Try parsing as float string + return datetime.fromtimestamp(float(value), tz=UTC) + return datetime.fromtimestamp(value, tz=UTC) + + +@router.get("/query") +async def get_query( + timeseries: MqlClientDep, + query: str, + time: float | str | None = None, +) -> dict: + """Execute an instant query against the timeseries backend.""" + if timeseries is None: + raise HTTPException(503, "Timeseries backend not configured") + + eval_time = _parse_timestamp(time) if time is not None else None + result = timeseries.query(query, eval_time) + + if isinstance(result, ScalarResult): + return { + "status": "success", + "data": { + "resultType": "scalar", + "result": [result.timestamp.timestamp(), str(result.value)], + }, + } + + if isinstance(result, VectorResult): + return { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": series.metric, + "value": [series.timestamp.timestamp(), str(series.value)], + } + for series in result.series + ], + }, + } + + raise HTTPException(500, f"Unexpected result type: {type(result)}") + + +@router.get("/query_range") +async def get_query_range( + timeseries: MqlClientDep, + query: str, + start: float | str, + end: float | str, + step: str = "1m", +) -> dict: + """Execute a range query against the timeseries backend.""" + if timeseries is None: + raise HTTPException(503, "Timeseries backend not configured") + + start_time = _parse_timestamp(start) + end_time = _parse_timestamp(end) + + result = timeseries.query_range(query, start_time, end_time, step) + + return { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": series.metric, + "values": [[ts.timestamp(), str(val)] for ts, val in series.values], + } + for series in result.series + ], + }, + } diff --git a/open_ess/frontend/routes/util.py b/open_ess/frontend/routes/util.py index 204e038..3adce21 100644 --- a/open_ess/frontend/routes/util.py +++ b/open_ess/frontend/routes/util.py @@ -1,14 +1,53 @@ from collections.abc import Iterable from datetime import datetime +from typing import TYPE_CHECKING from pydantic import BaseModel +if TYPE_CHECKING: + from open_ess.timeseries import InstantQueryResult, RangeQueryResult + class TimeSeries(BaseModel): timestamps: list[datetime] values: list[float] +def range_result_to_timeseries(result: "RangeQueryResult", rounding: int | None = None) -> TimeSeries: + """Convert a RangeQueryResult to TimeSeries format. + + Takes the first series if multiple are returned. + """ + timestamps: list[datetime] = [] + values: list[float] = [] + + if result.series: + series = result.series[0] + for ts, val in series.values: + timestamps.append(ts) + if rounding is not None: + values.append(round(val, rounding)) + else: + values.append(val) + + return TimeSeries(timestamps=timestamps, values=values) + + +def instant_result_to_value(result: "InstantQueryResult") -> float | None: + """Extract the value from an instant query result. + + For ScalarResult, returns the scalar value. + For VectorResult, returns the first series' value. + """ + from open_ess.timeseries import ScalarResult, VectorResult + + if isinstance(result, ScalarResult): + return result.value + if isinstance(result, VectorResult) and result.series: + return result.series[0].value + return None + + def data_to_timeseries(data: Iterable[tuple[datetime, float]], rounding: int | None = None) -> TimeSeries: timestamps = [] values = [] diff --git a/open_ess/frontend/static/api.js b/open_ess/frontend/static/api.js index 28e0ce8..363bf0b 100644 --- a/open_ess/frontend/static/api.js +++ b/open_ess/frontend/static/api.js @@ -23,8 +23,8 @@ * @property {number} [min_soc] * @property {(number | null)} [ac_energy_in] * @property {(number | null)} [ac_energy_out] - * @property {number} [dc_energy_in] - * @property {number} [dc_energy_out] + * @property {(number | null)} [dc_energy_in] + * @property {(number | null)} [dc_energy_out] * @property {(number | null)} [system_efficiency] * @property {(number | null)} [battery_efficiency] * @property {(number | null)} [charger_efficiency] @@ -33,29 +33,20 @@ * @property {(number | null)} [scheduled_profit] */ -/** - * @typedef {Object} BatteryEnergySeries - * @property {(number | null)[]} [energy_to_charger] - * @property {(number | null)[]} [energy_from_inverter] - * @property {(number | null)[]} [energy_to_battery] - * @property {(number | null)[]} [energy_from_battery] - * @property {(number | null)[]} [energy_loss_to_battery] - * @property {(number | null)[]} [energy_loss_from_battery] - */ - -/** - * @typedef {Object} BatteryGraphResponse - * @property {TimeSeries} [soc] - * @property {TimeSeries} [schedule] - * @property {TimeSeries} [voltage] - */ - /** * @typedef {Object} BatteryPowerValues * @property {(number | null)} [charger] * @property {(number | null)} [inverter] * @property {(number | null)} [battery] * @property {(number | null)} [losses] + * @property {(number | null)} [soc] + */ + +/** + * @typedef {Object} BatteryQueriesResponse + * @property {string} [soc_query] + * @property {string} [schedule_soc_query] + * @property {string} [voltage_query] */ /** @@ -64,6 +55,12 @@ * @property {string} [name] */ +/** + * @typedef {Object} ChartsPowerResponse + * @property {PowerQueryDef[]} [queries] + * @property {string[]} [phases] + */ + /** * @typedef {Object} EfficiencyScatterPoint * @property {string} [time] @@ -76,19 +73,23 @@ */ /** - * @typedef {Object} EnergyGraphResponse - * @property {string[]} [timestamps] - * @property {Object.} [grid_import] - * @property {Object.} [grid_export] - * @property {Object.} [battery_systems] - * @property {(number | null)[]} [solar] - * @property {(number | null)[]} [to_consumption] - * @property {(number | null)[]} [from_consumption] + * @typedef {Object} EnergyQueriesResponse + * @property {EnergyViewConfig[]} [views] */ /** - * @typedef {Object} EnergyResponse - * @property {Object.} [series] + * @typedef {Object} EnergyQueryDef + * @property {string} [query] + * @property {string} [label] + * @property {string} [color] + * @property {boolean} [negate] + */ + +/** + * @typedef {Object} EnergyViewConfig + * @property {string} [id] + * @property {string} [name] + * @property {EnergyQueryDef[]} [queries] */ /** @@ -107,30 +108,23 @@ */ /** - * @typedef {Object} PowerResponse - * @property {Object.} [series] + * @typedef {Object} PowerQueryDef + * @property {string} [label] + * @property {string} [query] + * @property {(boolean | null)} is_total */ /** - * @typedef {Object} PricePoint - * @property {string} [time] - * @property {(number | null)} [market] - * @property {(number | null)} [buy] - * @property {(number | null)} [sell] - */ - -/** - * @typedef {Object} PricesResponse - * @property {string} [area] - * @property {number} [aggregate_minutes] - * @property {string} [unit] - * @property {PricePoint[]} [timeseries] + * @typedef {Object} PriceQueriesResponse + * @property {string} [market_query] + * @property {string} [buy_query] + * @property {string} [sell_query] + * @property {Literal} [step] + * @property {string} [currency] */ /** * @typedef {Object} ServiceMessage - * @property {string} [timestamp] - * @property {Status} [status] * @property {string} [message] */ @@ -147,16 +141,16 @@ */ /** - * @typedef {Object} SystemLayoutData - * @property {number[]} [phases] - * @property {boolean} [has_solar] - * @property {BatterySystemInfo[]} [battery_systems] + * @typedef {Object} SolarInverterInfo + * @property {string} [id] + * @property {string} [name] */ /** - * @typedef {Object} TimeSeries - * @property {string[]} [timestamps] - * @property {number[]} [values] + * @typedef {Object} SystemLayoutData + * @property {string[]} [grid_phases] + * @property {BatterySystemInfo[]} [battery_systems] + * @property {SolarInverterInfo[]} [solar_inverters] */ // =================== @@ -223,20 +217,10 @@ }, /** - * @param {(string | null)} [params.battery_id] - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @param {number} [params.bucket_minutes] - * @returns {Promise} + * @returns {Promise} */ - energyGraph: async function(params) { - var searchParams = new URLSearchParams(); - if (params.battery_id !== undefined) searchParams.set('battery_id', String(params.battery_id)); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - if (params.bucket_minutes !== undefined) searchParams.set('bucket_minutes', String(params.bucket_minutes)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/energy-graph' + query); + chartsEnergyQueries: async function() { + var response = await fetch('/api/charts/energy-queries'); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -244,20 +228,10 @@ }, /** - * @param {(string | null)} [params.battery_id] - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @param {number} [params.aggregate_minutes] - * @returns {Promise} + * @returns {Promise} */ - powerGraph: async function(params) { - var searchParams = new URLSearchParams(); - if (params.battery_id !== undefined) searchParams.set('battery_id', String(params.battery_id)); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - if (params.aggregate_minutes !== undefined) searchParams.set('aggregate_minutes', String(params.aggregate_minutes)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/power-graph' + query); + chartsPowerQueries: async function() { + var response = await fetch('/api/charts/power-queries'); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -266,19 +240,13 @@ /** * @param {(string | null)} [params.area] - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @param {(number | null)} [params.aggregate_minutes] - * @returns {Promise} + * @returns {Promise} */ - prices: async function(params) { + chartsPriceQueries: async function(params) { var searchParams = new URLSearchParams(); if (params.area !== undefined) searchParams.set('area', String(params.area)); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - if (params.aggregate_minutes !== undefined) searchParams.set('aggregate_minutes', String(params.aggregate_minutes)); var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/prices' + query); + var response = await fetch('/api/charts/price-queries' + query); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -286,18 +254,10 @@ }, /** - * @param {(string | null)} [params.battery_id] - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @returns {Promise>} + * @returns {Promise>} */ - batteryGraph: async function(params) { - var searchParams = new URLSearchParams(); - if (params.battery_id !== undefined) searchParams.set('battery_id', String(params.battery_id)); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/battery-graph' + query); + chartsBatteryQueries: async function() { + var response = await fetch('/api/charts/battery-queries'); if (!response.ok) { throw new Error('HTTP ' + response.status); } @@ -305,16 +265,22 @@ }, /** - * @param {number} [params.limit] + * @param {(string | null)} [params.battery_id] + * @param {(string | null)} [params.start] + * @param {(string | null)} [params.end] * @param {number} [params.aggregate_minutes] * @param {number} [params.idle_threshold] + * @param {number} [params.limit] * @returns {Promise} */ efficiencyScatter: async function(params) { var searchParams = new URLSearchParams(); - if (params.limit !== undefined) searchParams.set('limit', String(params.limit)); + if (params.battery_id !== undefined) searchParams.set('battery_id', String(params.battery_id)); + if (params.start !== undefined) searchParams.set('start', String(params.start)); + if (params.end !== undefined) searchParams.set('end', String(params.end)); if (params.aggregate_minutes !== undefined) searchParams.set('aggregate_minutes', String(params.aggregate_minutes)); if (params.idle_threshold !== undefined) searchParams.set('idle_threshold', String(params.idle_threshold)); + if (params.limit !== undefined) searchParams.set('limit', String(params.limit)); var query = searchParams.toString() ? '?' + searchParams.toString() : ''; var response = await fetch('/api/efficiency-scatter' + query); if (!response.ok) { @@ -342,42 +308,6 @@ throw new Error('HTTP ' + response.status); } return response.json(); - }, - - /** - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @param {number} [params.aggregate_minutes] - * @returns {Promise} - */ - power: async function(params) { - var searchParams = new URLSearchParams(); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - if (params.aggregate_minutes !== undefined) searchParams.set('aggregate_minutes', String(params.aggregate_minutes)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/power' + query); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); - }, - - /** - * @param {(string | null)} [params.start] - * @param {(string | null)} [params.end] - * @returns {Promise} - */ - energy: async function(params) { - var searchParams = new URLSearchParams(); - if (params.start !== undefined) searchParams.set('start', String(params.start)); - if (params.end !== undefined) searchParams.set('end', String(params.end)); - var query = searchParams.toString() ? '?' + searchParams.toString() : ''; - var response = await fetch('/api/energy' + query); - if (!response.ok) { - throw new Error('HTTP ' + response.status); - } - return response.json(); } }; diff --git a/open_ess/frontend/static/cycles.js b/open_ess/frontend/static/cycles.js index 86b3d77..947dedb 100644 --- a/open_ess/frontend/static/cycles.js +++ b/open_ess/frontend/static/cycles.js @@ -3,6 +3,8 @@ 'use strict'; var cyclesTable = null; + var currentScatterMode = 'losses'; // 'losses' or 'efficiency' + var cachedScatterData = null; function getEfficiencyClass(efficiency) { if (efficiency == null) return ''; @@ -34,102 +36,128 @@ limit: parseInt(limit), }); - if (data.length === 0) { - document.getElementById(elementId).innerHTML = '
No data available
'; - return; - } + cachedScatterData = data; + renderScatterChart(data); - var isDark = Utils.isDarkTheme(); - var settings = Settings.load(); - var useKw = settings.powerUnit === 'kw'; - var divisor = useKw ? 1000 : 1; - var powerUnit = useKw ? 'kW' : 'W'; - - var categories = { - charging: { data: [], color: 'rgba(52, 152, 219, 0.5)', name: 'Charging' }, - discharging: { data: [], color: 'rgba(231, 76, 60, 0.5)', name: 'Discharging' }, - idling: { data: [], color: 'rgba(149, 165, 166, 0.5)', name: 'Idling' }, - balancing: { data: [], color: 'rgba(155, 89, 182, 0.5)', name: 'Balancing' }, - }; + } catch (error) { + console.error('Error loading scatter chart:', error); + document.getElementById(elementId).innerHTML = '
Failed to load scatter chart
'; + } + } + + function renderScatterChart(data) { + var elementId = 'scatter-chart'; - for (var i = 0; i < data.length; i++) { - var d = data[i]; - if (d.category && categories[d.category]) { + if (!data || data.length === 0) { + document.getElementById(elementId).innerHTML = '
No data available
'; + return; + } + + var isDark = Utils.isDarkTheme(); + var settings = Settings.load(); + var useKw = settings.powerUnit === 'kw'; + var divisor = useKw ? 1000 : 1; + var powerUnit = useKw ? 'kW' : 'W'; + var isEfficiencyMode = currentScatterMode === 'efficiency'; + + var categories = { + charging: { data: [], color: 'rgba(52, 152, 219, 0.5)', name: 'Charging' }, + discharging: { data: [], color: 'rgba(231, 76, 60, 0.5)', name: 'Discharging' }, + idling: { data: [], color: 'rgba(149, 165, 166, 0.5)', name: 'Idling' }, + balancing: { data: [], color: 'rgba(155, 89, 182, 0.5)', name: 'Balancing' }, + }; + + for (var i = 0; i < data.length; i++) { + var d = data[i]; + if (d.category && categories[d.category]) { + // For efficiency mode, only include points with valid efficiency + if (!isEfficiencyMode || d.efficiency != null) { categories[d.category].data.push(d); } } + } - function fmtPower(w) { - return useKw ? (w / 1000).toFixed(2) : Math.round(w).toString(); - } + function fmtPower(w) { + return useKw ? (w / 1000).toFixed(2) : Math.round(w).toString(); + } - function buildHoverText(d) { - var eff = d.efficiency != null ? d.efficiency.toFixed(1) + '%' : 'N/A'; - var soc = d.soc != null ? d.soc + '%' : 'N/A'; - var time = formatScatterTime(d.time || ''); - - switch (d.category) { - case 'charging': - return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Charger: ' + fmtPower(d.inverter_charger_power || 0) + ' ' + powerUnit + '
Losses: ' + fmtPower(d.losses || 0) + ' ' + powerUnit + '
Efficiency: ' + eff; - case 'discharging': - return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Inverter: ' + fmtPower(Math.abs(d.inverter_charger_power || 0)) + ' ' + powerUnit + '
Losses: ' + fmtPower(d.losses || 0) + ' ' + powerUnit + '
Efficiency: ' + eff; - case 'balancing': - return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Balancing power'; - case 'idling': - return 'Time: ' + time + '
SOC: ' + soc + '
Idle consumption: ' + fmtPower(d.losses || 0) + ' ' + powerUnit; - default: - return 'Time: ' + time; - } + function buildHoverText(d) { + var eff = d.efficiency != null ? d.efficiency.toFixed(1) + '%' : 'N/A'; + var soc = d.soc != null ? d.soc + '%' : 'N/A'; + var time = formatScatterTime(d.time || ''); + + switch (d.category) { + case 'charging': + return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Charger: ' + fmtPower(d.inverter_charger_power || 0) + ' ' + powerUnit + '
Losses: ' + fmtPower(d.losses || 0) + ' ' + powerUnit + '
Efficiency: ' + eff; + case 'discharging': + return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Inverter: ' + fmtPower(Math.abs(d.inverter_charger_power || 0)) + ' ' + powerUnit + '
Losses: ' + fmtPower(d.losses || 0) + ' ' + powerUnit + '
Efficiency: ' + eff; + case 'balancing': + return 'Time: ' + time + '
SOC: ' + soc + '
Battery: ' + fmtPower(d.battery_power || 0) + ' ' + powerUnit + '
Balancing power'; + case 'idling': + return 'Time: ' + time + '
SOC: ' + soc + '
Idle consumption: ' + fmtPower(d.losses || 0) + ' ' + powerUnit; + default: + return 'Time: ' + time; } + } - var traces = Object.keys(categories).map(function(key) { - var cat = categories[key]; - return { - x: cat.data.map(function(d) { return (d.battery_power || 0) / divisor; }), - y: cat.data.map(function(d) { return (d.losses || 0) / divisor; }), - type: 'scatter', - mode: 'markers', - name: cat.name, - marker: { color: cat.color, size: 8 }, - text: cat.data.map(buildHoverText), - hoverinfo: 'text', - }; - }); - - var layout = { - margin: { t: 30, r: 30, b: 60, l: 60 }, - paper_bgcolor: 'transparent', - plot_bgcolor: 'transparent', - font: { - family: '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif', - color: isDark ? '#e4e4e4' : '#333333', - }, - xaxis: { - title: 'Battery Power (' + powerUnit + ')', - gridcolor: isDark ? '#2a2a4a' : '#eeeeee', - linecolor: isDark ? '#3a3a5a' : '#dddddd', - rangemode: 'tozero', - }, - yaxis: { - title: 'Losses (' + powerUnit + ')', - gridcolor: isDark ? '#2a2a4a' : '#eeeeee', - linecolor: isDark ? '#3a3a5a' : '#dddddd', - rangemode: 'tozero', - }, - legend: { - orientation: 'h', - y: -0.15, - font: { color: isDark ? '#e4e4e4' : '#333333' }, - }, - hovermode: 'closest', + var traces = Object.keys(categories).map(function(key) { + var cat = categories[key]; + return { + x: cat.data.map(function(d) { return (d.battery_power || 0) / divisor; }), + y: cat.data.map(function(d) { + if (isEfficiencyMode) { + return d.efficiency || 0; + } + return (d.losses || 0) / divisor; + }), + type: 'scatter', + mode: 'markers', + name: cat.name, + marker: { color: cat.color, size: 8 }, + text: cat.data.map(buildHoverText), + hoverinfo: 'text', }; + }); - document.getElementById(elementId).innerHTML = ''; - Plotly.newPlot(elementId, traces, layout, { responsive: true, displayModeBar: false }); + var yAxisTitle = isEfficiencyMode ? 'Efficiency (%)' : 'Losses (' + powerUnit + ')'; + var yAxisRange = isEfficiencyMode ? [50, 100] : undefined; - } catch (error) { - console.error('Error loading scatter chart:', error); - document.getElementById(elementId).innerHTML = '
Failed to load scatter chart
'; + var layout = { + margin: { t: 30, r: 30, b: 60, l: 60 }, + paper_bgcolor: 'transparent', + plot_bgcolor: 'transparent', + font: { + family: '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif', + color: isDark ? '#e4e4e4' : '#333333', + }, + xaxis: { + title: 'Battery Power (' + powerUnit + ')', + gridcolor: isDark ? '#2a2a4a' : '#eeeeee', + linecolor: isDark ? '#3a3a5a' : '#dddddd', + rangemode: 'tozero', + }, + yaxis: { + title: yAxisTitle, + gridcolor: isDark ? '#2a2a4a' : '#eeeeee', + linecolor: isDark ? '#3a3a5a' : '#dddddd', + rangemode: isEfficiencyMode ? undefined : 'tozero', + range: yAxisRange, + }, + legend: { + orientation: 'h', + y: -0.15, + font: { color: isDark ? '#e4e4e4' : '#333333' }, + }, + hovermode: 'closest', + }; + + document.getElementById(elementId).innerHTML = ''; + Plotly.newPlot(elementId, traces, layout, { responsive: true, displayModeBar: false }); + } + + function reloadScatterFromCache() { + if (cachedScatterData) { + renderScatterChart(cachedScatterData); } } @@ -242,6 +270,24 @@ document.getElementById('days-select').value = Settings.loadPagePref('cycles', 'days', '30'); document.getElementById('swing-select').value = Settings.loadPagePref('cycles', 'swing', '10'); + // Initialize scatter mode from saved preference + currentScatterMode = Settings.loadPagePref('cycles', 'scatterMode', 'losses'); + var scatterModeButtons = document.querySelectorAll('#scatter-mode-buttons .btn-toggle'); + scatterModeButtons.forEach(function(btn) { + btn.classList.toggle('active', btn.dataset.value === currentScatterMode); + }); + + // Scatter mode toggle handlers + scatterModeButtons.forEach(function(btn) { + btn.addEventListener('click', function() { + document.querySelectorAll('#scatter-mode-buttons .btn-toggle').forEach(function(b) { b.classList.remove('active'); }); + btn.classList.add('active'); + currentScatterMode = btn.dataset.value || 'losses'; + Settings.savePagePref('cycles', 'scatterMode', currentScatterMode); + reloadScatterFromCache(); + }); + }); + document.getElementById('scatter-aggregate-select').addEventListener('change', function(e) { Settings.savePagePref('cycles', 'aggregate', e.target.value); loadScatterChart(); diff --git a/open_ess/frontend/static/dashboard.js b/open_ess/frontend/static/dashboard.js index 0f4a158..eaba6a8 100644 --- a/open_ess/frontend/static/dashboard.js +++ b/open_ess/frontend/static/dashboard.js @@ -15,6 +15,7 @@ function renderPowerFlowDiagram(container, layout) { var batteryCount = layout.battery_systems.length; + var solarCount = layout.solar_inverters.length; var html = '
' + '' + @@ -29,12 +30,12 @@ '
' + '
Grid
' + '
' + - layout.phases.map(function(p) { return '
L' + p + ': -- W
'; }).join('') + + layout.grid_phases.map(function(p) { return '
' + p + ': -- W
'; }).join('') + '
' + '
-- W
' + ''; - if (layout.has_solar) { + if (solarCount >= 1) { html += '
' + '
' + '' + @@ -54,7 +55,7 @@ '
' + '
Consumption
' + '
' + - layout.phases.map(function(p) { return '
L' + p + ': -- W
'; }).join('') + + layout.grid_phases.map(function(p) { return '
' + p + ': -- W
'; }).join('') + '
' + '
-- W
' + '
'; @@ -71,6 +72,7 @@ '' + '
' + battery.name + '
' + '
' + + '
SOC: --%
' + '
Charger:
' + '
Inverter:
' + '
Battery:
' + @@ -123,7 +125,7 @@ paths += ''; } - if (layout.has_solar) { + if (layout.solar_inverters.length >= 1) { var solarBlock = document.getElementById('block-solar'); if (solarBlock) { var solarRect = solarBlock.getBoundingClientRect(); @@ -149,12 +151,12 @@ function updatePowerFlowData(layout, data) { var gridTotal = 0; - for (var i = 0; i < layout.phases.length; i++) { - var phase = layout.phases[i]; - var value = data.grid['L' + phase] || 0; + for (var i = 0; i < layout.grid_phases.length; i++) { + var phase = layout.grid_phases[i]; + var value = data.grid[phase] || 0; gridTotal += value; - var el = document.getElementById('grid-L' + phase); - if (el) el.textContent = 'L' + phase + ': ' + formatPower(value); + var el = document.getElementById('grid-' + phase); + if (el) el.textContent = phase + ': ' + formatPower(value); } var gridTotalEl = document.getElementById('grid-total'); if (gridTotalEl) { @@ -163,28 +165,32 @@ } var consTotal = 0; - for (var j = 0; j < layout.phases.length; j++) { - var p = layout.phases[j]; - var v = data.consumption['L' + p] || 0; + for (var j = 0; j < layout.grid_phases.length; j++) { + var p = layout.grid_phases[j]; + var v = data.consumption[p] || 0; consTotal += v; - var cel = document.getElementById('consumption-L' + p); - if (cel) cel.textContent = 'L' + p + ': ' + formatPower(v); + var cel = document.getElementById('consumption-' + p); + if (cel) cel.textContent = p + ': ' + formatPower(v); } var consTotalEl = document.getElementById('consumption-total'); if (consTotalEl) consTotalEl.textContent = formatPower(consTotal); - if (layout.has_solar && data.solar !== null) { + if (layout.solar_inverters.length >= 1 && data.solar !== null) { var solarEl = document.getElementById('solar-total'); if (solarEl) solarEl.textContent = formatPower(data.solar); } for (var k = 0; k < layout.battery_systems.length; k++) { var battery = layout.battery_systems[k]; - var battData = data.batteries[battery.id]; + var battData = data.batteries[battery.id] || {}; var chargerPwr = battData.charger || 0; var inverterPwr = battData.inverter || 0; var batteryPwr = battData.battery || 0; var lossesPwr = battData.losses || 0; + var soc = battData.soc; + + var socEl = document.getElementById(battery.id + '-soc'); + if (socEl) socEl.textContent = 'SOC: ' + (soc != null ? Math.round(soc) + '%' : '--%'); var chargerEl = document.getElementById(battery.id + '-charger-power'); if (chargerEl) chargerEl.textContent = 'Charger: ' + formatPower(chargerPwr); @@ -222,7 +228,7 @@ consLine.classList.toggle('flow-active', Math.abs(consTotal) > 50); } - if (layout.has_solar && data.solar !== null) { + if (layout.solar_inverters.length >= 1 && data.solar !== null) { var solarLine = document.getElementById('line-solar'); if (solarLine) { solarLine.classList.toggle('flow-generating', data.solar > 50); diff --git a/open_ess/frontend/static/debug.js b/open_ess/frontend/static/debug.js index 3c96896..34fbce3 100644 --- a/open_ess/frontend/static/debug.js +++ b/open_ess/frontend/static/debug.js @@ -1,4 +1,4 @@ -// Debug page - Power and Energy charts +// Debug page - Raw Power and Energy metrics (function() { 'use strict'; @@ -10,55 +10,81 @@ return document.getElementById('aggregate-select'); } + /** + * Execute a query and convert to Plotly traces. + * @param {string} query - MetricsQL query + * @param {Date} start - Start time + * @param {Date} end - End time + * @param {string} step - Step string + * @returns {Promise} Array of Plotly traces + */ + async function queryToTraces(query, start, end, step) { + try { + var result = await Timeseries.queryRangeRaw(query, start, end, step); + return Timeseries.toPlotlyTraces(result); + } catch (e) { + console.error('Query failed:', query, e); + return []; + } + } + async function loadPowerChart() { var elementId = 'power-chart'; Utils.showLoading(elementId); var hours = parseInt(getHoursSelect().value); var aggregateMinutes = parseInt(getAggregateSelect().value); + var step = aggregateMinutes + 'm'; var now = new Date(); var start = new Date(now.getTime() - hours * 60 * 60 * 1000); try { - var data = await Api.power({ - start: Utils.formatDate(start), - end: Utils.formatDate(now), - aggregate_minutes: aggregateMinutes, - }); + // Query raw power metrics + var powerQuery = 'avg_over_time(openess_power_watts[' + step + '])'; + var scheduledQuery = 'avg_over_time(openess_scheduled_power_watts[' + step + '])'; - if (!data.series || Object.keys(data.series).length === 0) { - Utils.showError(elementId, 'No power flow data available'); + var [powerTraces, scheduledTraces] = await Promise.all([ + queryToTraces(powerQuery, start, now, step), + queryToTraces(scheduledQuery, start, now, step), + ]); + + var traces = powerTraces.concat(scheduledTraces); + + if (traces.length === 0) { + Utils.showError(elementId, 'No power data available'); return; } + // Style scheduled traces differently + traces.forEach(function(trace) { + if (trace.name && trace.name.indexOf('scheduled') !== -1) { + trace.line = { width: 1.5, dash: 'dot' }; + } else { + trace.line = { width: 1.5 }; + } + }); + var settings = Settings.load(); var useKw = settings.powerUnit === 'kw'; var powerUnit = useKw ? 'kW' : 'W'; + var divisor = useKw ? 1000 : 1; - var traces = []; - var sortedKeys = Object.keys(data.series).sort(); - - for (var i = 0; i < sortedKeys.length; i++) { - var key = sortedKeys[i]; - var series = data.series[key]; - if (!series.timestamps || !series.values) continue; - - traces.push({ - x: series.timestamps.map(function(t) { return new Date(t); }), - y: series.values, - type: 'scatter', - mode: 'lines', - name: key, - line: { width: 1.5 }, - connectgaps: false, - hovertemplate: '%{y:.1f} ' + powerUnit + '' + key + '', + if (useKw) { + traces.forEach(function(trace) { + trace.y = trace.y.map(function(v) { return v / divisor; }); }); } + traces.forEach(function(trace) { + trace.hovertemplate = '%{y:.1f} ' + powerUnit + '' + trace.name + ''; + }); + var layout = Utils.getDefaultLayout(); Utils.layoutSetXRange(layout, start, now); layout.hovermode = 'x unified'; + layout.yaxis = layout.yaxis || {}; + layout.yaxis.title = { text: powerUnit }; Utils.makePlot(elementId, traces, layout); } catch (error) { console.error('Error loading power flows:', error); @@ -71,56 +97,49 @@ Utils.showLoading(elementId); var hours = parseInt(getHoursSelect().value); + var aggregateMinutes = parseInt(getAggregateSelect().value); + var step = aggregateMinutes + 'm'; var now = new Date(); var start = new Date(now.getTime() - hours * 60 * 60 * 1000); try { - var data = await Api.energy({ - start: Utils.formatDate(start), - end: Utils.formatDate(now), + // Query raw energy counter and integrated power + var energyQuery = 'openess_energy_kwh'; + var integratedQuery = 'integrate(openess_power_watts) / 3600000'; // Convert Ws to kWh + + var [energyTraces, integratedTraces] = await Promise.all([ + queryToTraces(energyQuery, start, now, step), + queryToTraces(integratedQuery, start, now, step), + ]); + + // Mark integrated traces + integratedTraces.forEach(function(trace) { + trace.name = trace.name + ' [integrated]'; + trace.line = { width: 1.5, dash: 'dot' }; }); - if (!data.series || Object.keys(data.series).length === 0) { - Utils.showError(elementId, 'No energy flow data available'); + var traces = energyTraces.concat(integratedTraces); + + if (traces.length === 0) { + Utils.showError(elementId, 'No energy data available'); return; } - var settings = Settings.load(); - var useKw = settings.powerUnit === 'kw'; - var energyUnit = useKw ? 'kWh' : 'Wh'; - - var traces = []; - var sortedKeys = Object.keys(data.series).sort(); - - for (var i = 0; i < sortedKeys.length; i++) { - var key = sortedKeys[i]; - var series = data.series[key]; - if (!series.timestamps || !series.values) continue; - - var timestamps = series.timestamps.map(function(t) { return new Date(t); }); - timestamps.push(new Date()); - var lastValue = series.values[series.values.length - 1]; - var values = series.values.concat([lastValue]); - - var isIntegrated = key.includes('[integrated]'); - traces.push({ - x: timestamps, - y: values, - type: 'scatter', - mode: 'lines', - name: key, - line: { - width: isIntegrated ? 1.5 : 2, - dash: isIntegrated ? 'dot' : 'solid', - }, - hovertemplate: '%{y:.2f} ' + energyUnit + '' + key + '', - }); - } + // Style energy traces + energyTraces.forEach(function(trace) { + trace.line = { width: 2 }; + }); + + traces.forEach(function(trace) { + trace.hovertemplate = '%{y:.3f} kWh' + trace.name + ''; + }); var layout = Utils.getDefaultLayout(); Utils.layoutSetXRange(layout, start, now); layout.hovermode = 'x unified'; + layout.yaxis = layout.yaxis || {}; + layout.yaxis.title = { text: 'kWh' }; Utils.makePlot(elementId, traces, layout); } catch (error) { console.error('Error loading energy flows:', error); diff --git a/open_ess/frontend/static/metrics.js b/open_ess/frontend/static/metrics.js index 5667b61..2cb80f5 100644 --- a/open_ess/frontend/static/metrics.js +++ b/open_ess/frontend/static/metrics.js @@ -4,8 +4,10 @@ var dashboardStart = null; var dashboardEnd = null; - var currentFoR = 'multiplus'; - var cachedEnergyData = null; + var currentEnergyView = null; // Current energy view ID + var currentPowerMode = 'total'; // 'total' or 'phases' + var cachedPowerConfig = null; // Cached power chart config + var cachedEnergyConfig = null; // Cached energy chart config var rangeOffset = 0; var isRelayoutInProgress = false; @@ -88,89 +90,182 @@ }); } - function renderGridEnergyChart(elementId, data, start, end) { - var settings = Settings.load(); - var useKw = settings.powerUnit === 'kw'; - var toDisplay = useKw ? function(wh) { return wh ? wh / 1000 : 0; } : function(wh) { return wh || 0; }; - - var timestamps = (data.timestamps || []).map(function(t) { return new Date(t); }); - - var gridExport = data.grid_export || {}; - var gridImport = data.grid_import || {}; - - var traces = [ - { - x: timestamps, - y: (gridExport["From MP"] || []).map(function(v) { return toDisplay(v); }), - type: 'bar', - name: 'From MP', - marker: { color: '#278e60' }, - textposition: 'none', - }, - { - x: timestamps, - y: (gridImport["Consumption"] || []).map(function(v) { return -toDisplay(v); }), - type: 'bar', - name: 'Consumption', - marker: { color: '#3498db' }, - textposition: 'none', - }, - { - x: timestamps, - y: (gridImport["To MP"] || []).map(function(v) { return -toDisplay(v); }), - type: 'bar', - name: 'To MP', - marker: { color: '#3498ab' }, - textposition: 'none', - }, - ]; - - var layout = Utils.getDefaultLayout(); - Utils.layoutSetXRange(layout, start, end); - Utils.layoutAddNowLine(layout, start, end); - Utils.makePlot(elementId, traces, layout); + /** + * Execute a MetricsQL query and return a Plotly trace. + * @param {string} query - MetricsQL query with $step placeholder + * @param {string} label - Trace label + * @param {Date} start - Start time + * @param {Date} end - End time + * @param {string} step - Step string like '60m' + * @param {Object} opts - Trace options (color, negate) + * @returns {Promise} Plotly trace or null + */ + async function executeEnergyQuery(query, label, start, end, step, opts) { + if (!query) return null; + opts = opts || {}; + + try { + var resolvedQuery = query.replace(/\$step/g, step); + var result = await Timeseries.queryRangeRaw(resolvedQuery, start, end, step); + var plotlyTraces = Timeseries.toPlotlyTraces(result); + + if (plotlyTraces[0]) { + var trace = plotlyTraces[0]; + trace.name = label; + trace.type = 'bar'; + delete trace.mode; + delete trace.line; + trace.marker = { color: opts.color || '#95a5a6' }; + + if (opts.negate) { + trace.y = trace.y.map(function(v) { return -v; }); + } + + return trace; + } + } catch (e) { + console.error('Query failed for', label, ':', e); + } + return null; + } + + /** + * Initialize energy view buttons based on config. + */ + function initEnergyViewButtons() { + if (!cachedEnergyConfig || !cachedEnergyConfig.views) return; + + var container = document.getElementById('for-buttons'); + if (!container) return; + + // Clear existing buttons + container.innerHTML = ''; + + // Create buttons for each view + cachedEnergyConfig.views.forEach(function(view, index) { + var btn = document.createElement('button'); + btn.className = 'btn-toggle'; + btn.dataset.value = view.id; + btn.textContent = view.name; + + // Set first button or saved preference as active + if (currentEnergyView === view.id || (!currentEnergyView && index === 0)) { + btn.classList.add('active'); + currentEnergyView = view.id; + } + + btn.addEventListener('click', function() { + container.querySelectorAll('.btn-toggle').forEach(function(b) { + b.classList.remove('active'); + }); + btn.classList.add('active'); + currentEnergyView = view.id; + Settings.savePagePref('dashboard', 'for', currentEnergyView); + reloadEnergyChart(); + }); + + container.appendChild(btn); + }); + } + + /** + * Get the current view config. + * @returns {Object|null} Current view config or null + */ + function getCurrentEnergyView() { + if (!cachedEnergyConfig || !cachedEnergyConfig.views) return null; + + for (var i = 0; i < cachedEnergyConfig.views.length; i++) { + if (cachedEnergyConfig.views[i].id === currentEnergyView) { + return cachedEnergyConfig.views[i]; + } + } + // Fallback to first view + return cachedEnergyConfig.views[0] || null; } - function renderBatteryEnergyChart(elementId, data, start, end) { - var settings = Settings.load(); - var useKw = settings.powerUnit === 'kw'; - var toDisplay = useKw ? function(wh) { return wh ? wh / 1000 : 0; } : function(wh) { return wh || 0; }; - - var timestamps = (data.timestamps || []).map(function(t) { return new Date(t); }); - var mpData = (data.battery_systems || {})["MultiPlus"] || {}; - - var traces = [ - { - x: timestamps, - y: (mpData.energy_from_inverter || []).map(function(v) { return toDisplay(v); }), - type: 'bar', - name: 'Inverter Output', - marker: { color: '#f39c12' }, - textposition: 'none', - }, - { - x: timestamps, - y: (mpData.energy_to_charger || []).map(function(v) { return -toDisplay(v); }), - type: 'bar', - name: 'Charger Input', - marker: { color: '#3498db' }, - textposition: 'none', - }, - ]; - - var layout = Utils.getDefaultLayout(); - Utils.layoutSetXRange(layout, start, end); - Utils.layoutAddNowLine(layout, start, end); - Utils.makePlot(elementId, traces, layout); + async function loadEnergyChart(elementId, start, end, bucketMinutes) { + Utils.showLoading(elementId); + + try { + // Fetch query definitions (cached after first call) + if (!cachedEnergyConfig) { + cachedEnergyConfig = await Api.chartsEnergyQueries(); + initEnergyViewButtons(); + } + + var view = getCurrentEnergyView(); + if (!view || !view.queries) { + Utils.showError(elementId, 'No energy view configured'); + return; + } + + var step = bucketMinutes + 'm'; + var promises = []; + + // Execute all queries for the current view + view.queries.forEach(function(queryDef) { + promises.push(executeEnergyQuery( + queryDef.query, + queryDef.label, + start, end, step, + { color: queryDef.color, negate: queryDef.negate } + )); + }); + + var results = await Promise.all(promises); + var traces = results.filter(function(t) { return t !== null; }); + + var layout = Utils.getDefaultLayout(); + layout.barmode = 'relative'; + Utils.layoutSetXRange(layout, start, end); + Utils.layoutAddNowLine(layout, start, end); + layout.yaxis = layout.yaxis || {}; + layout.yaxis.title = { text: 'kWh' }; + Utils.makePlot(elementId, traces, layout); + } catch (error) { + console.error('Error loading energy data:', error); + Utils.showError(elementId, 'Failed to load energy data'); + } } - function renderEnergyFlowChart(elementId, data, start, end, frameOfReference) { - frameOfReference = frameOfReference || 'multiplus'; - if (frameOfReference === 'grid') { - renderGridEnergyChart(elementId, data, start, end); - } else { - renderBatteryEnergyChart(elementId, data, start, end); + /** + * Fetch power chart configuration from backend. + * @returns {Promise} Chart config with queries, phases, has_phase_toggle + */ + async function fetchPowerChartConfig() { + if (cachedPowerConfig) { + return cachedPowerConfig; + } + cachedPowerConfig = await Api.chartsPowerQueries({}); + + // Show/hide phase toggle button based on phases + var toggleContainer = document.getElementById('power-phase-buttons'); + if (toggleContainer) { + toggleContainer.style.display = cachedPowerConfig.phases.length > 1 ? '' : 'none'; } + + return cachedPowerConfig; + } + + /** + * Filter queries based on current phase mode. + * @param {Array} queries - All query definitions + * @param {string} mode - 'total' or 'phases' + * @returns {Array} Filtered queries + */ + function filterQueriesByMode(queries, mode) { + return queries.filter(function(q) { + // null means show in both modes + if (q.is_total === null) { + return true; + } + if (mode === 'total') { + return q.is_total === true; + } else { + return q.is_total === false; + } + }); } async function loadPowerChart(elementId, start, end, aggregateMinutes) { @@ -178,108 +273,136 @@ Utils.showLoading(elementId); try { - var data = await Api.powerGraph({ - start: Utils.formatDate(start), - end: Utils.formatDate(end), - aggregate_minutes: aggregateMinutes, + var config = await fetchPowerChartConfig(); + var step = aggregateMinutes + 'm'; + + // Filter queries based on current mode + var activeQueries = filterQueriesByMode(config.queries, currentPowerMode); + + // Execute all queries in parallel using Timeseries helper + var tracePromises = activeQueries.map(async function(q) { + var query = q.query.replace(/\$step/g, step); + try { + var result = await Timeseries.queryRangeRaw(query, start, end, step); + var traces = Timeseries.toPlotlyTraces(result); + if (traces[0]) { + traces[0].name = q.label; + } + return traces[0] || null; + } catch (e) { + console.error('Query failed for', q.label, ':', e); + return null; + } }); + var traces = await Promise.all(tracePromises); + // Filter out null results and apply settings var settings = Settings.load(); var useKw = settings.powerUnit === 'kw'; var unit = useKw ? 'kW' : 'W'; + var divisor = useKw ? 1000 : 1; - var traces = []; - var series = data.series || {}; - var sortedKeys = Object.keys(series).sort(); - - for (var i = 0; i < sortedKeys.length; i++) { - var key = sortedKeys[i]; - var s = series[key]; - if (!s.timestamps || !s.values) continue; - - traces.push({ - x: s.timestamps.map(function(t) { return new Date(t); }), - y: s.values, - type: 'scatter', - mode: 'lines', - name: key, - line: { width: 1.5 }, - connectgaps: false, - hovertemplate: '%{y:.1f} ' + unit + '' + key + '', - }); - } + var validTraces = traces.filter(function(t) { return t !== null; }); + validTraces.forEach(function(trace) { + if (useKw) { + trace.y = trace.y.map(function(v) { return v / divisor; }); + } + trace.hovertemplate = '%{y:.1f} ' + unit + '' + trace.name + ''; + }); var layout = Utils.getDefaultLayout(); Utils.layoutSetXRange(layout, start, end); Utils.layoutAddNowLine(layout, start, end); - Utils.makePlot(elementId, traces, layout); + Utils.makePlot(elementId, validTraces, layout); } catch (error) { console.error('Error loading power data:', error); Utils.showError(elementId, 'Failed to load power data'); } } - async function loadPriceChart(elementId, start, end) { + /** + * Re-render power chart with current settings (called when toggle changes). + */ + function reloadPowerChart() { + if (dashboardStart && dashboardEnd) { + var hours = parseInt(document.getElementById('range-select').value); + var aggregateMinutes = getAggregateMinutes(hours); + loadPowerChart('power-chart', dashboardStart, dashboardEnd, aggregateMinutes); + } + } + + /** + * Extend trace data with one extra point for step-function display. + * @param {Object} trace - Plotly trace with x and y arrays + * @param {string} step - Step string like '1h' or '15m' + */ + function extendTraceForStepFunction(trace, step) { + if (trace.x.length > 0) { + var lastTime = trace.x[trace.x.length - 1]; + var stepMs = step === '1h' ? 3600000 : 900000; + trace.x.push(new Date(lastTime.getTime() + stepMs)); + trace.y.push(trace.y[trace.y.length - 1]); + } + } + + async function loadPriceChart(elementId, start, end, aggregateMinutes) { Utils.showLoading(elementId); + // Extend end to show future prices var extendedEnd = new Date(end.getTime() + 2 * 24 * 60 * 60 * 1000); try { - var data = await Api.prices({ - start: Utils.formatDate(start), - end: Utils.formatDate(extendedEnd), - }); + // Fetch query definitions from backend + var config = await Api.chartsPriceQueries({}); + + // Execute all price queries in parallel + var [marketResult, buyResult, sellResult] = await Promise.all([ + Timeseries.queryRangeRaw(config.market_query, start, extendedEnd, config.step).catch(function() { return null; }), + Timeseries.queryRangeRaw(config.buy_query, start, extendedEnd, config.step).catch(function() { return null; }), + Timeseries.queryRangeRaw(config.sell_query, start, extendedEnd, config.step).catch(function() { return null; }), + ]); - if (!data.timeseries || data.timeseries.length === 0) { + var marketTraces = Timeseries.toPlotlyTraces(marketResult, { name: 'Market' }); + var buyTraces = Timeseries.toPlotlyTraces(buyResult, { name: 'Buy' }); + var sellTraces = Timeseries.toPlotlyTraces(sellResult, { name: 'Sell' }); + + if (marketTraces.length === 0 && buyTraces.length === 0 && sellTraces.length === 0) { Utils.showError(elementId, 'No price data available'); return; } var settings = Settings.load(); var priceMultiplier = settings.priceUnit === 'cent' ? 100 : 1; - var priceLabel = settings.priceUnit === 'cent' ? 'ct/kWh' : (data.unit || '€/kWh'); - - var timestamps = data.timeseries.map(function(d) { return new Date(d.time); }); - var marketPrices = data.timeseries.map(function(d) { return (d.market || 0) * priceMultiplier; }); - var buyPrices = data.timeseries.map(function(d) { return (d.buy || 0) * priceMultiplier; }); - var sellPrices = data.timeseries.map(function(d) { return (d.sell || 0) * priceMultiplier; }); - - var lastTime = timestamps[timestamps.length - 1]; - var extendedTime = new Date(lastTime.getTime() + (data.aggregate_minutes || 60) * 60 * 1000); - timestamps.push(extendedTime); - marketPrices.push(marketPrices[marketPrices.length - 1]); - buyPrices.push(buyPrices[buyPrices.length - 1]); - sellPrices.push(sellPrices[sellPrices.length - 1]); - - var traces = [ - { - name: 'Market', - x: timestamps, - y: marketPrices, - type: 'scatter', - mode: 'lines', - line: { shape: 'hv', color: '#95a5a6', width: 1 }, - hovertemplate: 'Market: %{y:.2f} ' + priceLabel + '', - }, - { - name: 'Buy', - x: timestamps, - y: buyPrices, - type: 'scatter', - mode: 'lines', - line: { shape: 'hv', color: '#e74c3c', width: 1.5 }, - hovertemplate: 'Buy: %{y:.2f} ' + priceLabel + '', - }, - { - name: 'Sell', - x: timestamps, - y: sellPrices, - type: 'scatter', - mode: 'lines', - line: { shape: 'hv', color: '#2ecc71', width: 1.5 }, - hovertemplate: 'Sell: %{y:.2f} ' + priceLabel + '', - }, - ]; + var priceLabel = settings.priceUnit === 'cent' ? 'ct/kWh' : (config.currency + '/kWh'); + + var traces = []; + + if (marketTraces[0]) { + var marketTrace = marketTraces[0]; + marketTrace.y = marketTrace.y.map(function(v) { return v * priceMultiplier; }); + marketTrace.line = { shape: 'hv', color: '#95a5a6', width: 1 }; + marketTrace.hovertemplate = 'Market: %{y:.2f} ' + priceLabel + ''; + extendTraceForStepFunction(marketTrace, config.step); + traces.push(marketTrace); + } + + if (buyTraces[0]) { + var buyTrace = buyTraces[0]; + buyTrace.y = buyTrace.y.map(function(v) { return v * priceMultiplier; }); + buyTrace.line = { shape: 'hv', color: '#e74c3c', width: 1.5 }; + buyTrace.hovertemplate = 'Buy: %{y:.2f} ' + priceLabel + ''; + extendTraceForStepFunction(buyTrace, config.step); + traces.push(buyTrace); + } + + if (sellTraces[0]) { + var sellTrace = sellTraces[0]; + sellTrace.y = sellTrace.y.map(function(v) { return v * priceMultiplier; }); + sellTrace.line = { shape: 'hv', color: '#2ecc71', width: 1.5 }; + sellTrace.hovertemplate = 'Sell: %{y:.2f} ' + priceLabel + ''; + extendTraceForStepFunction(sellTrace, config.step); + traces.push(sellTrace); + } var layout = Utils.getDefaultLayout(); Utils.layoutSetXRange(layout, start, end); @@ -293,50 +416,77 @@ } } - async function loadSocChart(elementId, start, end) { + async function loadSocChart(elementId, start, end, aggregateMinutes) { Utils.showLoading(elementId); try { - var data = await Api.batteryGraph({ - start: Utils.formatDate(start), - end: Utils.formatDate(end), - }); - - var keys = Object.keys(data); - var multipleSystems = keys.length > 1; + // Fetch query definitions from backend + var step = aggregateMinutes + 'm'; + var config = await Api.chartsBatteryQueries(); + var batteryNames = Object.keys(config); + var multipleSystems = batteryNames.length > 1; var traces = []; - for (var i = 0; i < keys.length; i++) { - var name = keys[i]; - var battery = data[name]; + // Query all battery systems in parallel + var queryPromises = batteryNames.map(async function(name) { + var queries = config[name]; + var [socResult, scheduleResult, voltageResult] = await Promise.all([ + Timeseries.queryRangeRaw(queries.soc_query, start, end, step).catch(function() { return null; }), + Timeseries.queryRangeRaw(queries.schedule_soc_query, start, end, step).catch(function() { return null; }), + Timeseries.queryRangeRaw(queries.voltage_query, start, end, step).catch(function() { return null; }), + ]); + return { name: name, socResult: socResult, scheduleResult: scheduleResult, voltageResult: voltageResult }; + }); - var socTrace = Utils.makeTrace('SoC', Utils.timeseriesExtendToNow(battery.soc || { timestamps: [], values: [] })); - socTrace.line = { color: '#3498db', width: 2 }; - socTrace.hovertemplate = '%{y}%SoC'; - if (multipleSystems) { - socTrace.legendgroup = name; - socTrace.legendgrouptitle = { text: name }; + var results = await Promise.all(queryPromises); + + for (var i = 0; i < results.length; i++) { + var result = results[i]; + var batteryName = result.name; + var prefix = multipleSystems ? batteryName + ' ' : ''; + + // SOC trace + var socTraces = Timeseries.toPlotlyTraces(result.socResult); + if (socTraces[0]) { + var socTrace = socTraces[0]; + socTrace.name = prefix + 'SoC'; + socTrace.line = { color: '#3498db', width: 2 }; + socTrace.hovertemplate = '%{y:.1f}%' + socTrace.name + ''; + if (multipleSystems) { + socTrace.legendgroup = batteryName; + socTrace.legendgrouptitle = { text: batteryName }; + } + traces.push(socTrace); } - traces.push(socTrace); - - var schedTrace = Utils.makeTrace('Scheduled', battery.schedule || { timestamps: [], values: [] }); - schedTrace.line = { color: '#2ecc71', width: 2, dash: 'dot' }; - schedTrace.hovertemplate = '%{y}%Scheduled'; - if (multipleSystems) { - schedTrace.legendgroup = name; - schedTrace.legendgrouptitle = { text: name }; + + // Scheduled SOC trace + var schedTraces = Timeseries.toPlotlyTraces(result.scheduleResult); + if (schedTraces[0]) { + var schedTrace = schedTraces[0]; + schedTrace.name = prefix + 'Scheduled'; + schedTrace.line = { color: '#2ecc71', width: 2, dash: 'dot' }; + schedTrace.hovertemplate = '%{y:.1f}%' + schedTrace.name + ''; + if (multipleSystems) { + schedTrace.legendgroup = batteryName; + schedTrace.legendgrouptitle = { text: batteryName }; + } + traces.push(schedTrace); } - traces.push(schedTrace); - - var voltTrace = Utils.makeTrace('Voltage', battery.voltage || { timestamps: [], values: [] }); - voltTrace.line = { color: '#ff7171', width: 2 }; - voltTrace.hovertemplate = '%{y}VVoltage'; - voltTrace.yaxis = 'y2'; - if (multipleSystems) { - voltTrace.legendgroup = name; - voltTrace.legendgrouptitle = { text: name }; + + // Voltage trace (secondary y-axis) + var voltTraces = Timeseries.toPlotlyTraces(result.voltageResult); + if (voltTraces[0]) { + var voltTrace = voltTraces[0]; + voltTrace.name = prefix + 'Voltage'; + voltTrace.line = { color: '#ff7171', width: 2 }; + voltTrace.hovertemplate = '%{y:.1f}V' + voltTrace.name + ''; + voltTrace.yaxis = 'y2'; + if (multipleSystems) { + voltTrace.legendgroup = batteryName; + voltTrace.legendgrouptitle = { text: batteryName }; + } + traces.push(voltTrace); } - traces.push(voltTrace); } var layout = Utils.getDefaultLayout(); @@ -362,19 +512,6 @@ } } - async function loadAndCacheEnergyData(start, end, bucketMinutes) { - try { - cachedEnergyData = await Api.energyGraph({ - start: Utils.formatDate(start), - end: Utils.formatDate(end), - bucket_minutes: bucketMinutes, - }); - } catch (error) { - console.error('Error fetching energy data:', error); - cachedEnergyData = null; - } - } - async function loadDashboard() { var hours = parseInt(document.getElementById('range-select').value); var aggregateMinutes = getAggregateMinutes(hours); @@ -386,38 +523,37 @@ updateRangeLabel(); - cachedEnergyData = null; - await Promise.all([ - loadAndCacheEnergyData(dashboardStart, dashboardEnd, bucketMinutes), + loadEnergyChart('energy-chart', dashboardStart, dashboardEnd, bucketMinutes), loadPowerChart('power-chart', dashboardStart, dashboardEnd, aggregateMinutes), - loadPriceChart('prices-chart', dashboardStart, dashboardEnd), - loadSocChart('soc-chart', dashboardStart, dashboardEnd) + loadPriceChart('prices-chart', dashboardStart, dashboardEnd, aggregateMinutes), + loadSocChart('soc-chart', dashboardStart, dashboardEnd, aggregateMinutes) ]); - if (cachedEnergyData) { - renderEnergyFlowChart('energy-chart', cachedEnergyData, dashboardStart, dashboardEnd, currentFoR); - } - setupZoomSync(); } - function renderEnergyOnly() { - if (dashboardStart && dashboardEnd && cachedEnergyData) { - renderEnergyFlowChart('energy-chart', cachedEnergyData, dashboardStart, dashboardEnd, currentFoR); + function reloadEnergyChart() { + if (dashboardStart && dashboardEnd) { + var hours = parseInt(document.getElementById('range-select').value); + var bucketMinutes = getBucketMinutes(hours); + loadEnergyChart('energy-chart', dashboardStart, dashboardEnd, bucketMinutes); } } document.addEventListener('DOMContentLoaded', function() { var savedRange = Settings.loadPagePref('dashboard', 'range', '24'); - var savedFoR = Settings.loadPagePref('dashboard', 'for', 'multiplus'); + var savedEnergyView = Settings.loadPagePref('dashboard', 'for', null); + var savedPowerMode = Settings.loadPagePref('dashboard', 'powerMode', 'total'); document.getElementById('range-select').value = savedRange; - currentFoR = savedFoR; + currentEnergyView = savedEnergyView; // Will be validated when config loads + currentPowerMode = savedPowerMode; - var forButtons = document.querySelectorAll('#for-buttons .btn-toggle'); - forButtons.forEach(function(btn) { - btn.classList.toggle('active', btn.dataset.value === savedFoR); + // Power phase toggle buttons + var powerPhaseButtons = document.querySelectorAll('#power-phase-buttons .btn-toggle'); + powerPhaseButtons.forEach(function(btn) { + btn.classList.toggle('active', btn.dataset.value === savedPowerMode); }); document.getElementById('range-select').addEventListener('change', function(e) { @@ -438,13 +574,14 @@ } }); - forButtons.forEach(function(btn) { + // Power phase toggle handlers + powerPhaseButtons.forEach(function(btn) { btn.addEventListener('click', function() { - document.querySelectorAll('#for-buttons .btn-toggle').forEach(function(b) { b.classList.remove('active'); }); + document.querySelectorAll('#power-phase-buttons .btn-toggle').forEach(function(b) { b.classList.remove('active'); }); btn.classList.add('active'); - currentFoR = btn.dataset.value || 'multiplus'; - Settings.savePagePref('dashboard', 'for', currentFoR); - renderEnergyOnly(); + currentPowerMode = btn.dataset.value || 'total'; + Settings.savePagePref('dashboard', 'powerMode', currentPowerMode); + reloadPowerChart(); }); }); diff --git a/open_ess/frontend/static/timeseries.js b/open_ess/frontend/static/timeseries.js new file mode 100644 index 0000000..09fafc0 --- /dev/null +++ b/open_ess/frontend/static/timeseries.js @@ -0,0 +1,262 @@ +/** + * Timeseries query helper for frontend visualization. + * + * This module provides a unified interface for querying the timeseries backend + * (VictoriaMetrics or MetricSQLite) using MetricsQL queries defined in the + * BatterySystemConfig. + * + * Usage: + * await Timeseries.init(batteryId); + * var result = await Timeseries.queryRange('power_grid', start, end); + * var traces = Timeseries.toPlotlyTraces(result, { name: 'Grid Power' }); + */ +var Timeseries = (function () { + /** @type {Object} */ + var queries = {}; + + /** @type {string|null} */ + var currentBatteryId = null; + + /** + * Initialize the timeseries helper for a battery system. + * Fetches the resolved MetricsQL queries from the backend. + * + * @param {string} batteryId - The battery system ID + * @returns {Promise} + */ + async function init(batteryId) { + if (currentBatteryId === batteryId && Object.keys(queries).length > 0) { + return; // Already initialized for this battery + } + + var response = await fetch("/api/queries/" + encodeURIComponent(batteryId)); + if (!response.ok) { + throw new Error("Failed to fetch queries: HTTP " + response.status); + } + queries = await response.json(); + currentBatteryId = batteryId; + } + + /** + * Calculate appropriate step based on time range. + * + * @param {Date} start - Start time + * @param {Date} end - End time + * @returns {string} Step string (e.g., "1m", "5m", "1h") + */ + function calculateStep(start, end) { + var durationMs = end.getTime() - start.getTime(); + var hour = 3600000; + + if (durationMs < hour) return "1m"; + if (durationMs < 6 * hour) return "5m"; + if (durationMs < 24 * hour) return "15m"; + if (durationMs < 7 * 24 * hour) return "1h"; + return "6h"; + } + + /** + * Execute a range query against the timeseries backend. + * + * @param {string} queryName - Name of the query (e.g., "power_grid", "soc") + * @param {Date} start - Start time + * @param {Date} end - End time + * @param {string} [step] - Optional step override (e.g., "5m") + * @returns {Promise} Query result in Prometheus/VM format + */ + async function queryRange(queryName, start, end, step) { + var query = queries[queryName]; + if (!query) { + throw new Error("Unknown query: " + queryName + ". Available: " + Object.keys(queries).join(", ")); + } + + var params = new URLSearchParams({ + query: query, + start: (start.getTime() / 1000).toString(), + end: (end.getTime() / 1000).toString(), + step: step || calculateStep(start, end), + }); + + var response = await fetch("/api/v1/query_range?" + params); + if (!response.ok) { + throw new Error("Query failed: HTTP " + response.status); + } + return response.json(); + } + + /** + * Execute a raw MetricsQL query (not from the predefined queries). + * + * @param {string} query - MetricsQL query string + * @param {Date} start - Start time + * @param {Date} end - End time + * @param {string} [step] - Optional step override + * @returns {Promise} Query result + */ + async function queryRangeRaw(query, start, end, step) { + var params = new URLSearchParams({ + query: query, + start: (start.getTime() / 1000).toString(), + end: (end.getTime() / 1000).toString(), + step: step || calculateStep(start, end), + }); + + var response = await fetch("/api/v1/query_range?" + params); + if (!response.ok) { + throw new Error("Query failed: HTTP " + response.status); + } + return response.json(); + } + + /** + * Execute an instant query against the timeseries backend. + * + * @param {string} query - MetricsQL query string + * @param {Date} [time] - Optional evaluation time (defaults to now) + * @returns {Promise} Query result in Prometheus/VM format + */ + async function queryInstant(query, time) { + var params = new URLSearchParams({ query: query }); + if (time) { + params.set("time", (time.getTime() / 1000).toString()); + } + + var response = await fetch("/api/v1/query?" + params); + if (!response.ok) { + throw new Error("Query failed: HTTP " + response.status); + } + return response.json(); + } + + /** + * Extract a single value from an instant query result. + * + * @param {Object} result - Query result from queryInstant + * @returns {number|null} The value or null if not found + */ + function getInstantValue(result) { + if (!result || !result.data || !result.data.result || result.data.result.length === 0) { + return null; + } + var first = result.data.result[0]; + if (first.value && first.value.length === 2) { + return parseFloat(first.value[1]); + } + return null; + } + + /** + * Extract all values from an instant query result as a map. + * + * @param {Object} result - Query result from queryInstant + * @param {string} labelKey - Label key to use as map key + * @returns {Object} Map of label value to metric value + */ + function getInstantValues(result, labelKey) { + var values = {}; + if (!result || !result.data || !result.data.result) { + return values; + } + result.data.result.forEach(function(series) { + var key = series.metric[labelKey] || "unknown"; + if (series.value && series.value.length === 2) { + values[key] = parseFloat(series.value[1]); + } + }); + return values; + } + + /** + * Format metric labels into a readable string. + * + * @param {Object} metric - Metric labels object + * @returns {string} Formatted label string + */ + function formatLabels(metric) { + var name = metric.__name__ || "unknown"; + var labels = Object.entries(metric) + .filter(function (e) { + return e[0] !== "__name__"; + }) + .map(function (e) { + return e[0] + "=" + e[1]; + }) + .join(", "); + return labels ? name + "{" + labels + "}" : name; + } + + /** + * Convert a query result to Plotly traces. + * + * @param {Object} result - Query result from queryRange + * @param {Object} [options] - Options + * @param {string} [options.name] - Override trace name (for single series) + * @param {function} [options.nameFormatter] - Function to format series name from labels + * @returns {Array} Array of Plotly trace objects + */ + function toPlotlyTraces(result, options) { + options = options || {}; + + if (!result || !result.data || !result.data.result) { + console.warn("Invalid query result:", result); + return []; + } + + return result.data.result.map(function (series, index) { + var name; + if (options.name && result.data.result.length === 1) { + name = options.name; + } else if (options.nameFormatter) { + name = options.nameFormatter(series.metric); + } else { + name = formatLabels(series.metric); + } + + return { + x: series.values.map(function (v) { + return new Date(v[0] * 1000); + }), + y: series.values.map(function (v) { + return parseFloat(v[1]); + }), + type: "scatter", + mode: "lines", + name: name, + line: { width: 1.5 }, + }; + }); + } + + /** + * Get the list of available query names. + * + * @returns {string[]} Array of query names + */ + function getQueryNames() { + return Object.keys(queries); + } + + /** + * Get a specific query string. + * + * @param {string} name - Query name + * @returns {string|undefined} The query string or undefined + */ + function getQuery(name) { + return queries[name]; + } + + return { + init: init, + queryRange: queryRange, + queryRangeRaw: queryRangeRaw, + queryInstant: queryInstant, + getInstantValue: getInstantValue, + getInstantValues: getInstantValues, + toPlotlyTraces: toPlotlyTraces, + calculateStep: calculateStep, + formatLabels: formatLabels, + getQueryNames: getQueryNames, + getQuery: getQuery, + }; +})(); diff --git a/open_ess/frontend/templates/base.html b/open_ess/frontend/templates/base.html index c435c8a..1264520 100644 --- a/open_ess/frontend/templates/base.html +++ b/open_ess/frontend/templates/base.html @@ -36,6 +36,7 @@ + {% block scripts %}{% endblock %} diff --git a/open_ess/frontend/templates/cycles.html b/open_ess/frontend/templates/cycles.html index a68fdde..b99fc28 100644 --- a/open_ess/frontend/templates/cycles.html +++ b/open_ess/frontend/templates/cycles.html @@ -24,6 +24,10 @@

Power vs Losses

+
+ + +
diff --git a/open_ess/frontend/templates/metrics.html b/open_ess/frontend/templates/metrics.html index 0595b68..983de55 100644 --- a/open_ess/frontend/templates/metrics.html +++ b/open_ess/frontend/templates/metrics.html @@ -25,16 +25,20 @@

Energy

- - - +
-

Power

+
+

Power

+ +
diff --git a/open_ess/main.py b/open_ess/main.py index 353beb6..78a160b 100644 --- a/open_ess/main.py +++ b/open_ess/main.py @@ -5,11 +5,11 @@ from open_ess.battery_system import BatterySystem, VictronBatterySystem from open_ess.config import Config -from open_ess.database import Database, DatabaseService from open_ess.frontend import create_app from open_ess.optimizer import OptimizerService from open_ess.pricing import EntsoeService from open_ess.service import ServiceManager +from open_ess.timeseries import TimeseriesBackend, create_backend from open_ess.util import EndpointFilter, parse_args, setup_logging from open_ess.victron_modbus import VictronService @@ -21,26 +21,22 @@ def main() -> None: args = parse_args("Open Energy Storage System - optimize charging based on day-ahead prices") config = Config.from_file(args.config) - database = Database(config.database) - database.run_migrations() + # Create MetricsQL client (either MetricSQLite or VictoriaMetrics/Prometheus). + mql_client: TimeseriesBackend = create_backend(config.timeseries) + logger.info(f"Using mql_client backend: {config.timeseries.backend}") # Create services service_manager = ServiceManager() - service_manager.register_service(DatabaseService(database)) - service_manager.register_service(EntsoeService(database, config.prices)) + service_manager.register_service(EntsoeService(mql_client, config.prices)) battery_systems: list[BatterySystem] = [] for battery_config in config.battery_systems: if battery_config.is_victron: - victron_service = VictronService(database, battery_config) + victron_service = VictronService(battery_config, mql_client) service_manager.register_service(victron_service) battery_system = VictronBatterySystem(battery_config, victron_service.client) battery_systems.append(battery_system) service_manager.register_service( - OptimizerService( - database, - battery_system=battery_system, - price_config=config.prices, - ), + OptimizerService(battery_system, config.prices, mql_client), requires=victron_service, ) @@ -48,6 +44,7 @@ def main() -> None: def shutdown(signum: int, frame: object) -> None: logger.info("Shutting down...") service_manager.stop() + mql_client.close() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) @@ -60,7 +57,7 @@ def shutdown(signum: int, frame: object) -> None: logging.getLogger("uvicorn.access").addFilter(EndpointFilter(["/api/power-flow"])) - app = create_app(database, config, battery_systems) + app = create_app(config, battery_systems, mql_client) uvicorn.run( app, host=config.frontend.host, diff --git a/open_ess/optimizer/optimizer.py b/open_ess/optimizer/optimizer.py index ce9195b..a167668 100644 --- a/open_ess/optimizer/optimizer.py +++ b/open_ess/optimizer/optimizer.py @@ -8,9 +8,9 @@ import pyomo.environ as pyo from pyomo.opt import SolverFactory -from open_ess.battery_system import BatterySystemConfig -from open_ess.database import DatabaseConnection +from open_ess.battery_system import BatterySystem, BatterySystemConfig from open_ess.pricing import PriceConfig +from open_ess.timeseries import TimeseriesBackend logger = logging.getLogger(__name__) @@ -26,15 +26,15 @@ class Optimizer: the need for a separate binary variable. """ - def __init__(self, db: DatabaseConnection, price_config: PriceConfig, battery_config: BatterySystemConfig): - self._database = db + def __init__(self, price_config: PriceConfig, battery_system: BatterySystem, mql_client: TimeseriesBackend): self._price_config = price_config - self._battery_config = battery_config + self._battery_system = battery_system + self._mql_client = mql_client # TODO: check for cbc @property def battery_config(self) -> BatterySystemConfig: - return self._battery_config + return self._battery_system.config def optimize(self) -> list[tuple[datetime, datetime, int, float]]: """Generate optimal charge schedule using mixed-integer linear programming. @@ -48,20 +48,22 @@ def optimize(self) -> list[tuple[datetime, datetime, int, float]]: # Get hourly prices for the planning horizon now = datetime.now(UTC) start_hour = now.replace(minute=0, second=0, microsecond=0) - prices = self._database.get_prices( + prices = self._mql_client.get_prices( self._price_config.area, start=start_hour - timedelta(weeks=6), - aggregate_minutes=self._price_config.aggregate_minutes, + end=start_hour + timedelta(days=2), + hourly=self._price_config.hourly_average, ) # Get current SOC - current_soc = self._database.get_current_soc() + current_soc = self._battery_system.get_soc() + logger.info(current_soc) if current_soc is None: logger.error("No SoC data available") return [] - current_soc = min(max(current_soc, self._battery_config.min_soc), self._battery_config.max_soc) + current_soc = min(max(current_soc, self.battery_config.min_soc), self.battery_config.max_soc) # ^ current_soc may be outside the allowed boundaries. Clamp it between the bounds or pyomo will fail. - # TODO: actually fix the issue by allowing soc to be out of bound but don't allow it to go out of bounds. + # TODO: actually fix the issue by allowing soc to BE out of bound but don't allow it to GO out of bounds. if not prices: logger.warning("No price data available") @@ -80,13 +82,13 @@ def optimize(self) -> list[tuple[datetime, datetime, int, float]]: return [] # Build piecewise linear breakpoints for loss functions - assert self._battery_config.max_charge_power_kw is not None - assert self._battery_config.max_invert_power_kw is not None + assert self.battery_config.max_charge_power_kw is not None + assert self.battery_config.max_invert_power_kw is not None charger_bp, charger_loss_vals = build_piecewise_loss_points( - self._battery_config.max_charge_power_kw, charger_loss + self.battery_config.max_charge_power_kw, charger_loss ) inverter_bp, inverter_loss_vals = build_piecewise_loss_points( - self._battery_config.max_invert_power_kw, inverter_loss + self.battery_config.max_invert_power_kw, inverter_loss ) model = pyo.ConcreteModel() @@ -99,9 +101,9 @@ def optimize(self) -> list[tuple[datetime, datetime, int, float]]: model.market_price = pyo.Param(model.T, initialize=price_dict) # Decision variables - model.charge_power = pyo.Var(model.T, bounds=(0, self._battery_config.max_charge_power_kw)) - model.discharge_power = pyo.Var(model.T, bounds=(0, self._battery_config.max_invert_power_kw)) - model.soc = pyo.Var(model.T, bounds=(self._battery_config.min_soc, self._battery_config.max_soc)) + model.charge_power = pyo.Var(model.T, bounds=(0, self.battery_config.max_charge_power_kw)) + model.discharge_power = pyo.Var(model.T, bounds=(0, self.battery_config.max_invert_power_kw)) + model.soc = pyo.Var(model.T, bounds=(self.battery_config.min_soc, self.battery_config.max_soc)) # Auxiliary variables for piecewise linear losses max_charger_loss = charger_loss_vals[-1] @@ -140,12 +142,11 @@ def soc_balance_rule(model: pyomo.core.Model, t: int) -> Any: * self._price_config.aggregate_minutes / 60 ) - soc_change = 100 * net_energy / self._battery_config.capacity_kwh + soc_change = 100 * net_energy / self.battery_config.capacity_kwh return model.soc[t] == prev_soc + soc_change model.soc_balance = pyo.Constraint(model.T, rule=soc_balance_rule) model.final_soc = pyo.Constraint(expr=model.soc[model_length - 1] == current_soc) - # ^ Final SoC should equal starting SOC (energy neutral over horizon) # Objective: minimize cost (buy cost - sell revenue) diff --git a/open_ess/optimizer/service.py b/open_ess/optimizer/service.py index 136bdc3..bf49a5c 100644 --- a/open_ess/optimizer/service.py +++ b/open_ess/optimizer/service.py @@ -2,9 +2,9 @@ from datetime import UTC, datetime, timedelta from open_ess.battery_system import BatterySystem -from open_ess.database import Database, DatabaseConnection from open_ess.pricing import PriceConfig from open_ess.service import Service +from open_ess.timeseries import Sample, TimeseriesBackend from .optimizer import Optimizer @@ -14,26 +14,22 @@ class OptimizerService(Service): def __init__( self, - db: Database, battery_system: BatterySystem, price_config: PriceConfig, + mql_client: TimeseriesBackend, ): super().__init__("OptimizerService") - self._db = db self._battery_system = battery_system self._price_config = price_config + self._mql_client = mql_client - self._db_conn: DatabaseConnection | None = None self._optimizer: Optimizer | None = None def on_start(self) -> None: - self._db_conn = self._db.connect() - self._optimizer = Optimizer( - self._db_conn, price_config=self._price_config, battery_config=self._battery_system.config - ) + self._optimizer = Optimizer(self._price_config, self._battery_system, self._mql_client) def tick(self) -> None: - if self._optimizer is None or self._db_conn is None: + if self._optimizer is None: return logger.debug("Running charge optimizer(s)") @@ -41,7 +37,7 @@ def tick(self) -> None: if schedule: _, _, power, _ = schedule[0] self._battery_system.set_ess_setpoint(power) - self._db_conn.set_schedule(self._battery_system.id, schedule) # type: ignore[arg-type] + self._upsert_schedule(schedule) logger.debug(f"Updated schedule with {len(schedule)} entries") else: logger.warning("Optimizer returned empty schedule") @@ -55,3 +51,41 @@ def wait_until_next(self) -> None: microsecond=0, ) + timedelta(minutes=self._price_config.aggregate_minutes) self.wait_seconds((next_run - now).total_seconds()) + + def _upsert_schedule(self, schedule: list[tuple[datetime, datetime, int, float]]) -> None: + """ + Schedules are stored in a bit of an insane way in the timeseries backend... + The timestamp of each inserted sample is increased by a tiny bit, proportional to how far in + the future the sample is. This way, a first_over_time() query will return the most recently + generated sample for a given timestamp. + """ + + device_id = self._battery_system.id + if device_id is None: + logger.warning("Cannot upsert schedule: battery system has no device ID") + return + + samples: list[Sample] = [] + + now = datetime.now(UTC) + for ts_start, ts_end, power, soc in schedule: + delta = timedelta(milliseconds=1 + (ts_start - now).total_seconds() // 60) + + samples.append( + Sample( + metric="openess_scheduled_power_watts", + timestamp=ts_start + delta, + value=power, + labels={"device": device_id}, + ) + ) + samples.append( + Sample( + metric="openess_scheduled_soc_ratio", + timestamp=ts_end + delta, + value=soc / 100, + labels={"device": device_id}, + ) + ) + + self._mql_client.write(samples) diff --git a/open_ess/pricing/__init__.py b/open_ess/pricing/__init__.py index 921e29e..d29b1aa 100644 --- a/open_ess/pricing/__init__.py +++ b/open_ess/pricing/__init__.py @@ -1,5 +1,6 @@ +from .areas import AREAS from .client import EntsoeClient from .config import PriceConfig from .service import EntsoeService -__all__ = ["EntsoeClient", "EntsoeService", "PriceConfig"] +__all__ = ["AREAS", "EntsoeClient", "EntsoeService", "PriceConfig"] diff --git a/open_ess/pricing/client.py b/open_ess/pricing/client.py index edf0031..83b1e64 100644 --- a/open_ess/pricing/client.py +++ b/open_ess/pricing/client.py @@ -8,7 +8,8 @@ from entsoe.utils import add_timestamps, extract_records from pandas import DataFrame -from open_ess.database import DatabaseConnection +from open_ess.timeseries import Sample, TimeseriesBackend, VectorResult +from open_ess.util import ms_to_dt from .areas import AREAS from .config import PriceConfig @@ -17,21 +18,19 @@ class EntsoeClient: - def __init__(self, config: PriceConfig, db: DatabaseConnection): + def __init__(self, config: PriceConfig, mql_client: TimeseriesBackend): if config.area not in AREAS: raise ValueError(f"Unknown area code: '{config.area}'") self._config = config - self._db = db - - self._eic_code, tz_name = AREAS[config.area] - self._tz = ZoneInfo(tz_name) + self._mql_client = mql_client if config.entsoe_api_key: set_config(security_token=config.entsoe_api_key) + @staticmethod def fetch_day_ahead_prices( - self, + area: str, start: datetime, end: datetime, ) -> list[tuple[datetime, datetime, float]]: @@ -39,29 +38,33 @@ def fetch_day_ahead_prices( Fetch day-ahead prices from ENTSO-E for a given area and time range. Args: + area: start: Start datetime (UTC) end: End datetime (UTC) Returns: List of (start_time, end_time, price) tuples with prices in EUR/MWh """ + eic_code, tz_name = AREAS[area] + tz = ZoneInfo(tz_name) # ENTSO-E expects timestamps formatted as YYYYMMDDhhmm in the area's local timezone - start_local = start.astimezone(self._tz) - end_local = end.astimezone(self._tz) + start_local = start.astimezone(tz) + end_local = end.astimezone(tz) period_start = int(start_local.strftime("%Y%m%d%H%M")) period_end = int(end_local.strftime("%Y%m%d%H%M")) try: result = EnergyPrices( - in_domain=self._eic_code, - out_domain=self._eic_code, + in_domain=eic_code, + out_domain=eic_code, period_start=period_start, period_end=period_end, ).query_api() except ET.ParseError: - # On a 404, entsoe-apy still tries to parse the result which fails. Other errors such as 503 and timeouts are retried + # On a 404, entsoe-apy still tries to parse the result which fails. + # Other errors such as 503 and timeouts are retried. return [] records = extract_records(result) @@ -75,27 +78,57 @@ def fetch_day_ahead_prices( row_start = datetime.fromisoformat(row["timestamp"]) interval_minutes = _parse_resolution(row["time_series.period.resolution"]) row_end = row_start + timedelta(minutes=interval_minutes) - price = float(row["time_series.period.point.price_amount"]) + price = float(row["time_series.period.point.price_amount"]) / 1000 prices.append((row_start, row_end, price)) return prices - def fetch_missing_prices(self) -> None: + def fetch_missing_prices(self, area: str) -> None: + if area not in AREAS: + raise ValueError(f"Unknown area code: '{area}'") + now = datetime.now(UTC) end_of_tomorrow = (now + timedelta(days=2)).replace(hour=0, minute=0, second=0, microsecond=0) - latest = self._db.get_latest_price_time(self._config.area) - if latest is None: + query_result = self._mql_client.query( + f'timestamp(openess_prices{{area="{area}", price="market"}}[8w])', time=end_of_tomorrow + ) + assert isinstance(query_result, VectorResult) + result = query_result + if len(result.series) == 0: fetch_start = now.replace(hour=0, minute=0, second=0, microsecond=0) - fetch_start -= timedelta(days=14) - elif latest >= end_of_tomorrow: - return + fetch_start -= timedelta(weeks=8) else: - fetch_start = latest + latest_ms = int(result.series[0].value) * 1000 + latest = ms_to_dt(latest_ms) + if latest >= end_of_tomorrow: + return + else: + fetch_start = latest logger.info(f"Fetching prices for {self._config.area} from {fetch_start} to {end_of_tomorrow}") - prices = self.fetch_day_ahead_prices(fetch_start, end_of_tomorrow) + prices = self.fetch_day_ahead_prices(area, fetch_start, end_of_tomorrow) if prices: - self._db.insert_prices(self._config.area, prices) + self._upsert_prices(area, prices) + + def _upsert_prices(self, area: str, prices: list[tuple[datetime, datetime, float]]) -> None: + def make_sample(_ts: datetime, _price_type: str, _price: float) -> Sample: + return Sample( + metric="openess_prices", + labels={ + "area": area, + "price": _price_type, + }, + timestamp=_ts, + value=_price, + ) + + samples: list[Sample] = [] + for ts_start, ts_end, price in prices: + ts = ts_start + (ts_end - ts_start) / 2 + samples.append(make_sample(ts, "market", price)) + samples.append(make_sample(ts, "buy", self._config.buy_price(price))) + samples.append(make_sample(ts, "sell", self._config.sell_price(price))) + self._mql_client.write(samples) def _parse_resolution(resolution: str) -> int: diff --git a/open_ess/pricing/service.py b/open_ess/pricing/service.py index bcb9b10..2ec37b5 100644 --- a/open_ess/pricing/service.py +++ b/open_ess/pricing/service.py @@ -1,7 +1,7 @@ import logging -from open_ess.database import Database, DatabaseConnection from open_ess.service import Service +from open_ess.timeseries import TimeseriesBackend from .client import EntsoeClient from .config import PriceConfig @@ -10,20 +10,15 @@ class EntsoeService(Service): - """Fetches day-ahead prices from ENTSO-E at regular intervals.""" - - def __init__(self, db: Database, config: PriceConfig): + def __init__(self, mql_client: TimeseriesBackend, config: PriceConfig): super().__init__("EntsoeService") - self._db = db + self._mql_client = mql_client self._config = config self._check_interval = 3600 self._client: EntsoeClient | None = None - self._db_conn: DatabaseConnection | None = None def on_start(self) -> None: - self._db_conn = self._db.connect() - self._client = EntsoeClient(self._config, self._db_conn) - self._fetch_prices() + self._client = EntsoeClient(self._config, self._mql_client) def tick(self) -> None: self._fetch_prices() @@ -32,7 +27,7 @@ def _fetch_prices(self) -> None: if self._client is None: return None try: - self._client.fetch_missing_prices() + self._client.fetch_missing_prices(self._config.area) except Exception as e: logger.error(f"Failed to fetch ENTSO-E prices: {e}") diff --git a/open_ess/scripts/generate_types.py b/open_ess/scripts/generate_types.py index 93c4833..98cdb4e 100644 --- a/open_ess/scripts/generate_types.py +++ b/open_ess/scripts/generate_types.py @@ -14,8 +14,9 @@ from enum import Enum from pathlib import Path from types import NoneType, UnionType -from typing import Any, TypedDict, get_args, get_origin +from typing import Annotated, Any, TypedDict, get_args, get_origin +from fastapi.params import Depends from fastapi.routing import APIRoute from pydantic import BaseModel @@ -29,6 +30,17 @@ class _ParamInfo(TypedDict): logger = logging.getLogger(__name__) +def is_dependency_injection(annotation: Any) -> bool: + """Check if an annotation is a FastAPI dependency (Annotated[X, Depends(...)]).""" + if get_origin(annotation) is Annotated: + args = get_args(annotation) + # args[0] is the base type, args[1:] are the metadata + for metadata in args[1:]: + if isinstance(metadata, Depends): + return True + return False + + def python_type_to_jsdoc(python_type: Any, models: dict[str, type]) -> str: """Convert a Python type annotation to JSDoc type.""" origin = get_origin(python_type) @@ -166,11 +178,11 @@ def generate_api_function(route: APIRoute, models_dict: dict[str, type]) -> tupl sig = inspect.signature(endpoint) for param_name, param in sig.parameters.items(): - # Skip dependency injection parameters - if param_name in ("db", "battery_configs", "price_config", "battery_systems"): - continue - annotation = param.annotation + + # Skip dependency injection parameters (Annotated[X, Depends(...)]) + if is_dependency_injection(annotation): + continue if annotation is inspect.Parameter.empty: continue diff --git a/open_ess/timeseries/__init__.py b/open_ess/timeseries/__init__.py new file mode 100644 index 0000000..7b23084 --- /dev/null +++ b/open_ess/timeseries/__init__.py @@ -0,0 +1,50 @@ +from .base import ( + InstantQueryResult, + InstantSeries, + RangeQueryResult, + RangeSeries, + Sample, + ScalarResult, + TimeseriesBackend, + VectorResult, +) +from .config import TimeseriesConfig +from .metricsqlite.config import MetricSQLiteConfig +from .victoriametrics.config import VictoriaMetricsConfig + + +def create_backend( + config: TimeseriesConfig, +) -> TimeseriesBackend: + """Create a timeseries backend from config. + + Args: + config: Timeseries configuration (MetricSQLiteConfig or VictoriaMetricsConfig). + + Returns: + Configured backend instance. + """ + if isinstance(config, VictoriaMetricsConfig): + from .victoriametrics.backend import VictoriaMetricsBackend + + return VictoriaMetricsBackend(config) + elif isinstance(config, MetricSQLiteConfig): + from .metricsqlite.backend import MetricSQLiteBackend + + return MetricSQLiteBackend(config) + else: + raise ValueError(f"Unknown timeseries config type: {type(config)}") + + +__all__ = [ + "InstantQueryResult", + "InstantSeries", + "RangeQueryResult", + "RangeSeries", + "Sample", + "ScalarResult", + "TimeseriesBackend", + "TimeseriesConfig", + "VectorResult", + "create_backend", +] diff --git a/open_ess/timeseries/base.py b/open_ess/timeseries/base.py new file mode 100644 index 0000000..ac8cb5f --- /dev/null +++ b/open_ess/timeseries/base.py @@ -0,0 +1,155 @@ +"""Base interface for timeseries backends.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime +from typing import TYPE_CHECKING, Literal + +if TYPE_CHECKING: + pass + + +@dataclass +class Sample: + """A single metric sample for writing.""" + + metric: str + value: float + timestamp: datetime + labels: dict[str, str] = field(default_factory=dict) + + +# --- Instant Query Result Types --- + + +@dataclass +class ScalarResult: + """Result of an instant query returning a scalar value.""" + + result_type: Literal["scalar"] = field(default="scalar", repr=False) + timestamp: datetime = field(default_factory=datetime.now) + value: float = 0.0 + + +@dataclass +class InstantSeries: + """A series with a single value (instant vector element).""" + + metric: dict[str, str] + timestamp: datetime + value: float + + +@dataclass +class VectorResult: + """Result of an instant query returning an instant vector.""" + + result_type: Literal["vector"] = field(default="vector", repr=False) + series: list[InstantSeries] = field(default_factory=list) + + +@dataclass +class RangeSeries: + """A series with multiple values over time (range vector/matrix element).""" + + metric: dict[str, str] + values: list[tuple[datetime, float]] # (timestamp, value) pairs + + +InstantQueryResult = ScalarResult | VectorResult + + +# --- Range Query Result Type --- + + +@dataclass +class RangeQueryResult: + """Result of a range query (always returns a matrix).""" + + series: list[RangeSeries] = field(default_factory=list) + + +class TimeseriesBackend(ABC): + """Abstract base class for timeseries backends. + + Provides a unified interface for writing and querying metrics, + supporting both VictoriaMetrics and MetricSQLite backends. + """ + + @abstractmethod + def write(self, samples: list[Sample]) -> None: + """Write a batch of samples. + + Args: + samples: List of samples to write. + """ + ... + + @abstractmethod + def query(self, query: str, time: datetime | None = None) -> InstantQueryResult: + """Execute an instant query. + + Args: + query: MetricsQL/PromQL query string. + time: Evaluation timestamp. Defaults to now. + + Returns: + ScalarResult, VectorResult, or MatrixResult depending on query. + """ + ... + + @abstractmethod + def query_range( + self, + query: str, + start: datetime, + end: datetime, + step: str = "1m", + ) -> RangeQueryResult: + """Execute a range query. + + Args: + query: MetricsQL/PromQL query string. + start: Start of time range. + end: End of time range. + step: Query resolution (e.g., "1m", "5m", "1h"). + + Returns: + RangeQueryResult containing series with values at each step. + """ + ... + + @abstractmethod + def close(self) -> None: + """Close any connections.""" + ... + + def get_prices( + self, + area: str, + start: datetime, + end: datetime, + hourly: bool = False, + price: Literal["market", "buy", "sell"] = "market", + ) -> list[tuple[datetime, float]]: + """Prices are returned in currency per Kwh (usually €/kWh).""" + # Lazy import to avoid circular dependency + from open_ess.pricing import AREAS + + # Validate area and price to prevent MetricsQL injection. + if area not in AREAS: + raise ValueError(f"Unknown area code: '{area}'") + if price not in ("market", "buy", "sell"): + raise ValueError(f"Unknown price type: '{price}'") + + if hourly: + query = f'avg_over_time(openess_prices{{area="{area}", price="{price}"}}[1h])' + step = "1h" + else: + query = f'openess_prices{{area="{area}", price="{price}"}}' + step = "15m" + result = self.query_range(query, start, end, step) + + if not result.series: + return [] + return list(result.series[0].values) diff --git a/open_ess/timeseries/config.py b/open_ess/timeseries/config.py new file mode 100644 index 0000000..804803d --- /dev/null +++ b/open_ess/timeseries/config.py @@ -0,0 +1,11 @@ +from typing import Annotated + +from pydantic import Field + +from .metricsqlite.config import MetricSQLiteConfig +from .victoriametrics.config import VictoriaMetricsConfig + +TimeseriesConfig = Annotated[ + MetricSQLiteConfig | VictoriaMetricsConfig, + Field(discriminator="backend"), +] diff --git a/open_ess/timeseries/metricsqlite/__init__.py b/open_ess/timeseries/metricsqlite/__init__.py new file mode 100644 index 0000000..2f5d3e5 --- /dev/null +++ b/open_ess/timeseries/metricsqlite/__init__.py @@ -0,0 +1,6 @@ +"""MetricSQLite timeseries backend.""" + +from .backend import MetricSQLiteBackend +from .config import MetricSQLiteConfig + +__all__ = ["MetricSQLiteBackend", "MetricSQLiteConfig"] diff --git a/open_ess/timeseries/metricsqlite/backend.py b/open_ess/timeseries/metricsqlite/backend.py new file mode 100644 index 0000000..88ab86d --- /dev/null +++ b/open_ess/timeseries/metricsqlite/backend.py @@ -0,0 +1,116 @@ +import logging +from datetime import UTC, datetime + +import fastapi +from metricsqlite import MetricsQLiteClient +from metricsqlite.engine import InstantVector, MatrixResult, RangeVectorResult, ScalarResult +from metricsqlite.fastapi import create_router + +from ..base import ( + InstantQueryResult, + InstantSeries, + RangeQueryResult, + RangeSeries, + Sample, + TimeseriesBackend, + VectorResult, +) +from ..base import ScalarResult as BaseScalarResult +from .config import MetricSQLiteConfig + +logger = logging.getLogger(__name__) + + +class MetricSQLiteBackend(TimeseriesBackend): + def __init__(self, config: MetricSQLiteConfig): + self.config = config + self._client = MetricsQLiteClient(config.db_path, enable_wal=True) + self._client.connect() + self._client.create_tables() + + def write(self, samples: list[Sample]) -> None: + """Write samples as gauge metrics.""" + for sample in samples: + timestamp_ms = int(sample.timestamp.timestamp() * 1000) + self._client.insert_gauge( + name=sample.metric, + value=sample.value, + timestamp=timestamp_ms, + labels=sample.labels if sample.labels else None, + ) + + def query(self, query: str, time: datetime | None = None) -> InstantQueryResult: + """Execute an instant query.""" + eval_time: float | None = None + if time is not None: + eval_time = time.timestamp() * 1000 + + result = self._client.query(query, time=eval_time) + return self._convert_instant_result(result) + + def query_range( + self, + query: str, + start: datetime, + end: datetime, + step: str = "1m", + ) -> RangeQueryResult: + """Execute a range query.""" + start_ms = start.timestamp() * 1000 + end_ms = end.timestamp() * 1000 + + result = self._client.query_range(query, start=start_ms, end=end_ms, step=step) + return self._convert_range_result(result) + + def _convert_instant_result(self, result: InstantVector | RangeVectorResult | ScalarResult) -> InstantQueryResult: + """Convert metricsqlite instant query result.""" + if isinstance(result, ScalarResult): + return BaseScalarResult( + timestamp=datetime.fromtimestamp(result.timestamp / 1000, tz=UTC), + value=result.value, + ) + + if isinstance(result, RangeVectorResult): + # Range vector from instant query (e.g., metric[5m]) + # Convert to VectorResult by taking the last value + logger.warning("Instant query returned range vector, taking last value") + series = [] + for labels, samples in result.series: + if samples: + last = samples[-1] + series.append( + InstantSeries( + metric=labels, + timestamp=datetime.fromtimestamp(last.timestamp / 1000, tz=UTC), + value=last.value, + ) + ) + return VectorResult(series=series) + + # InstantVector + series = [] + for labels, sample in result.series: + series.append( + InstantSeries( + metric=labels, + timestamp=datetime.fromtimestamp(sample.timestamp / 1000, tz=UTC), + value=sample.value, + ) + ) + return VectorResult(series=series) + + def _convert_range_result(self, result: MatrixResult) -> RangeQueryResult: + """Convert metricsqlite range query result.""" + series = [] + for labels, samples in result.series: + values = [(datetime.fromtimestamp(sample.timestamp / 1000, tz=UTC), sample.value) for sample in samples] + series.append(RangeSeries(metric=labels, values=values)) + return RangeQueryResult(series=series) + + def create_fastapi_router(self) -> fastapi.APIRouter: + router: fastapi.APIRouter = create_router(self._client) + return router + + def close(self) -> None: + """Close the database connection.""" + self._client.close() diff --git a/open_ess/timeseries/metricsqlite/config.py b/open_ess/timeseries/metricsqlite/config.py new file mode 100644 index 0000000..20ad67a --- /dev/null +++ b/open_ess/timeseries/metricsqlite/config.py @@ -0,0 +1,9 @@ +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel + + +class MetricSQLiteConfig(BaseModel): + backend: Literal["metricsqlite"] = "metricsqlite" + db_path: Path = Path("openess.db") diff --git a/open_ess/timeseries/victoriametrics/__init__.py b/open_ess/timeseries/victoriametrics/__init__.py new file mode 100644 index 0000000..e5dcf37 --- /dev/null +++ b/open_ess/timeseries/victoriametrics/__init__.py @@ -0,0 +1,14 @@ +"""VictoriaMetrics timeseries backend.""" + +from .backend import VictoriaMetricsBackend +from .client import RemoteWriteClient, RemoteWriteError, Sample, timestamp_ms +from .config import VictoriaMetricsConfig + +__all__ = [ + "RemoteWriteClient", + "RemoteWriteError", + "Sample", + "VictoriaMetricsBackend", + "VictoriaMetricsConfig", + "timestamp_ms", +] diff --git a/open_ess/timeseries/victoriametrics/backend.py b/open_ess/timeseries/victoriametrics/backend.py new file mode 100644 index 0000000..07a4f60 --- /dev/null +++ b/open_ess/timeseries/victoriametrics/backend.py @@ -0,0 +1,205 @@ +"""VictoriaMetrics timeseries backend implementation.""" + +import json +import logging +from datetime import UTC, datetime + +from urllib3 import HTTPConnectionPool, HTTPSConnectionPool +from urllib3.util import parse_url + +from ..base import ( + InstantQueryResult, + InstantSeries, + RangeQueryResult, + RangeSeries, + Sample, + ScalarResult, + TimeseriesBackend, + VectorResult, +) +from .client import RemoteWriteClient +from .client import Sample as RemoteWriteSample +from .config import VictoriaMetricsConfig + +logger = logging.getLogger(__name__) + + +class VictoriaMetricsBackend(TimeseriesBackend): + """VictoriaMetrics backend using remote write protocol for writes and HTTP API for queries.""" + + def __init__(self, config: VictoriaMetricsConfig): + self.config = config + + # Parse URL to get components + parsed = parse_url(config.url) + self._host = parsed.host + self._port = parsed.port + self._scheme = parsed.scheme or "http" + + # Build base path (strip trailing slash) + self._base_path = (parsed.path or "").rstrip("/") + + # Set up connection pool for queries + pool_cls = HTTPSConnectionPool if self._scheme == "https" else HTTPConnectionPool + pool_kwargs: dict = { + "host": self._host, + "port": self._port, + "timeout": config.timeout, + "maxsize": 2, + "block": True, + } + if config.username and config.password: + import base64 + + credentials = f"{config.username}:{config.password}".encode() + auth = base64.b64encode(credentials).decode("ascii") + pool_kwargs["headers"] = {"Authorization": f"Basic {auth}"} + self._pool = pool_cls(**pool_kwargs) + + # Set up remote write client + write_url = f"{self._scheme}://{self._host}" + if self._port: + write_url += f":{self._port}" + write_url += f"{self._base_path}/api/v1/write" + + self._write_client = RemoteWriteClient( + url=write_url, + username=config.username, + password=config.password, + timeout=config.timeout, + ) + + def write(self, samples: list[Sample]) -> None: + """Write samples using Prometheus remote write protocol.""" + if not samples: + return + + remote_samples = [ + RemoteWriteSample( + metric=s.metric, + value=s.value, + timestamp_ms=int(s.timestamp.timestamp() * 1000), + labels=s.labels, + ) + for s in samples + ] + self._write_client.write(remote_samples) + + def query(self, query: str, time: datetime | None = None) -> InstantQueryResult: + """Execute an instant query.""" + params = {"query": query} + if time is not None: + params["time"] = str(int(time.timestamp())) + + response = self._pool.request( + "GET", + f"{self._base_path}/api/v1/query", + fields=params, + ) + + if response.status != 200: + raise RuntimeError(f"Query failed: {response.status} {response.data.decode('utf-8', errors='replace')}") + + data = json.loads(response.data.decode("utf-8")) + return self._parse_instant_response(data) + + def query_range( + self, + query: str, + start: datetime, + end: datetime, + step: str = "1m", + ) -> RangeQueryResult: + """Execute a range query.""" + params = { + "query": query, + "start": str(int(start.timestamp())), + "end": str(int(end.timestamp())), + "step": step, + } + + response = self._pool.request( + "GET", + f"{self._base_path}/api/v1/query_range", + fields=params, + ) + + if response.status != 200: + raise RuntimeError(f"Query failed: {response.status} {response.data.decode('utf-8', errors='replace')}") + + data = json.loads(response.data.decode("utf-8")) + return self._parse_range_response(data) + + def _parse_instant_response(self, data: dict) -> InstantQueryResult: + """Parse VictoriaMetrics/Prometheus instant query response.""" + if data.get("status") != "success": + error = data.get("error", "Unknown error") + raise RuntimeError(f"Query error: {error}") + + result_type = data.get("data", {}).get("resultType", "vector") + result = data.get("data", {}).get("result", []) + + if result_type == "scalar": + # Scalar: [timestamp, value] + ts, val = result + return ScalarResult( + timestamp=datetime.fromtimestamp(float(ts), tz=UTC), + value=float(val), + ) + + if result_type == "vector": + # Vector: list of {metric, value: [timestamp, value]} + series = [] + for item in result: + metric = item.get("metric", {}) + ts, val = item["value"] + series.append( + InstantSeries( + metric=metric, + timestamp=datetime.fromtimestamp(float(ts), tz=UTC), + value=float(val), + ) + ) + return VectorResult(series=series) + + if result_type == "matrix": + # Matrix from instant query (range selector like [5m]) + # Convert to VectorResult by taking the last value from each series + logger.warning("Instant query returned matrix (range selector?), taking last value") + series = [] + for item in result: + metric = item.get("metric", {}) + values = item.get("values", []) + if values: + ts, val = values[-1] + series.append( + InstantSeries( + metric=metric, + timestamp=datetime.fromtimestamp(float(ts), tz=UTC), + value=float(val), + ) + ) + return VectorResult(series=series) + + raise RuntimeError(f"Unknown result type: {result_type}") + + def _parse_range_response(self, data: dict) -> RangeQueryResult: + """Parse VictoriaMetrics/Prometheus range query response.""" + if data.get("status") != "success": + error = data.get("error", "Unknown error") + raise RuntimeError(f"Query error: {error}") + + result = data.get("data", {}).get("result", []) + series = [] + + for item in result: + metric = item.get("metric", {}) + values = [(datetime.fromtimestamp(float(ts), tz=UTC), float(val)) for ts, val in item.get("values", [])] + series.append(RangeSeries(metric=metric, values=values)) + + return RangeQueryResult(series=series) + + def close(self) -> None: + """Close connections.""" + self._write_client.close() + self._pool.close() diff --git a/open_ess/timeseries/victoriametrics/client.py b/open_ess/timeseries/victoriametrics/client.py new file mode 100644 index 0000000..561ae28 --- /dev/null +++ b/open_ess/timeseries/victoriametrics/client.py @@ -0,0 +1,149 @@ +"""Prometheus remote write client for VictoriaMetrics and Prometheus.""" + +import logging +import time +from dataclasses import dataclass, field + +import snappy +from urllib3 import HTTPConnectionPool, HTTPSConnectionPool +from urllib3.util import parse_url + +from .protobuf import encode_timeseries, encode_write_request + +logger = logging.getLogger(__name__) + + +@dataclass +class Sample: + """A single metric sample.""" + + metric: str + value: float + timestamp_ms: int + labels: dict[str, str] = field(default_factory=dict) + + def to_labels(self) -> list[tuple[str, str]]: + """Convert to list of label tuples including __name__.""" + return [("__name__", self.metric), *self.labels.items()] + + +class RemoteWriteError(Exception): + """Error during remote write.""" + + pass + + +class RemoteWriteClient: + """Client for Prometheus remote write protocol. + + Sends metrics to VictoriaMetrics or Prometheus using the remote write + protocol (protobuf + snappy compression). + """ + + def __init__( + self, + url: str, + *, + username: str | None = None, + password: str | None = None, + timeout: float = 30.0, + ): + """Initialize the remote write client. + + Args: + url: Remote write endpoint URL (e.g., "http://localhost:8428/api/v1/write") + username: Optional username for basic auth + password: Optional password for basic auth + timeout: Request timeout in seconds + """ + self.url = url + self.timeout = timeout + + parsed = parse_url(url) + self._path = parsed.path or "/api/v1/write" + self._host = parsed.host + self._port = parsed.port + + # Set up connection pool + pool_cls = HTTPSConnectionPool if parsed.scheme == "https" else HTTPConnectionPool + pool_kwargs: dict = { + "host": self._host, + "port": self._port, + "timeout": timeout, + "maxsize": 1, + "block": True, + } + if username and password: + pool_kwargs["headers"] = {"Authorization": f"Basic {self._encode_basic_auth(username, password)}"} + self._pool = pool_cls(**pool_kwargs) + + @staticmethod + def _encode_basic_auth(username: str, password: str) -> str: + """Encode credentials for basic auth header.""" + import base64 + + credentials = f"{username}:{password}".encode() + return base64.b64encode(credentials).decode("ascii") + + def write(self, samples: list[Sample]) -> None: + """Write a batch of samples. + + Args: + samples: List of samples to write. + + Raises: + RemoteWriteError: If the write fails. + """ + if not samples: + return + + # Group samples by metric+labels to create timeseries + timeseries_map: dict[tuple, list[tuple[float, int]]] = {} + for sample in samples: + key = tuple(sorted(sample.to_labels())) + if key not in timeseries_map: + timeseries_map[key] = [] + timeseries_map[key].append((sample.value, sample.timestamp_ms)) + + # Encode timeseries + encoded_timeseries = [] + for labels_tuple, sample_list in timeseries_map.items(): + labels = list(labels_tuple) + # Sort samples by timestamp as required by spec + sample_list.sort(key=lambda x: x[1]) + encoded_timeseries.append(encode_timeseries(labels, sample_list)) + + # Encode write request and compress + write_request = encode_write_request(encoded_timeseries) + compressed = snappy.compress(write_request) + + # Send request + headers = { + "Content-Type": "application/x-protobuf", + "Content-Encoding": "snappy", + "X-Prometheus-Remote-Write-Version": "0.1.0", + "User-Agent": "OpenESS/1.0", + } + + response = self._pool.urlopen( + "POST", + self._path, + body=compressed, + headers=headers, + ) + + if response.status >= 400: + raise RemoteWriteError( + f"Remote write failed: {response.status} {response.data.decode('utf-8', errors='replace')}" + ) + + logger.debug(f"Wrote {len(samples)} samples to {self.url}") + + def close(self) -> None: + """Close the connection pool.""" + self._pool.close() + + +def timestamp_ms() -> int: + """Get current timestamp in milliseconds.""" + return int(time.time() * 1000) diff --git a/open_ess/timeseries/victoriametrics/config.py b/open_ess/timeseries/victoriametrics/config.py new file mode 100644 index 0000000..3271bd9 --- /dev/null +++ b/open_ess/timeseries/victoriametrics/config.py @@ -0,0 +1,11 @@ +from typing import Literal + +from pydantic import BaseModel + + +class VictoriaMetricsConfig(BaseModel): + backend: Literal["victoriametrics"] + url: str + username: str | None = None + password: str | None = None + timeout: float = 30.0 diff --git a/open_ess/timeseries/victoriametrics/protobuf.py b/open_ess/timeseries/victoriametrics/protobuf.py new file mode 100644 index 0000000..8883840 --- /dev/null +++ b/open_ess/timeseries/victoriametrics/protobuf.py @@ -0,0 +1,118 @@ +"""Minimal protobuf encoder for Prometheus remote write protocol. + +Encodes WriteRequest, TimeSeries, Label, and Sample messages without +requiring protoc or the protobuf library. Only supports the specific +wire format needed for remote write. + +Protobuf wire format reference: +- Varint: variable-length integer encoding +- Wire type 0: varint (int64) +- Wire type 1: 64-bit fixed (double) +- Wire type 2: length-delimited (string, bytes, embedded message) +""" + +import struct + + +def _encode_varint(value: int) -> bytes: + """Encode an unsigned integer as a varint.""" + parts = [] + while value > 127: + parts.append((value & 0x7F) | 0x80) + value >>= 7 + parts.append(value) + return bytes(parts) + + +def _encode_signed_varint(value: int) -> bytes: + """Encode a signed integer as a varint (two's complement for negatives).""" + if value < 0: + value = value + (1 << 64) + return _encode_varint(value) + + +def _encode_field(field_number: int, wire_type: int, data: bytes) -> bytes: + """Encode a field with its tag.""" + tag = (field_number << 3) | wire_type + return _encode_varint(tag) + data + + +def _encode_string(field_number: int, value: str) -> bytes: + """Encode a string field (wire type 2).""" + encoded = value.encode("utf-8") + return _encode_field(field_number, 2, _encode_varint(len(encoded)) + encoded) + + +def _encode_double(field_number: int, value: float) -> bytes: + """Encode a double field (wire type 1).""" + return _encode_field(field_number, 1, struct.pack(" bytes: + """Encode an int64 field (wire type 0).""" + return _encode_field(field_number, 0, _encode_signed_varint(value)) + + +def _encode_message(field_number: int, data: bytes) -> bytes: + """Encode an embedded message field (wire type 2).""" + return _encode_field(field_number, 2, _encode_varint(len(data)) + data) + + +def encode_label(name: str, value: str) -> bytes: + """Encode a Label message. + + message Label { + string name = 1; + string value = 2; + } + """ + return _encode_string(1, name) + _encode_string(2, value) + + +def encode_sample(value: float, timestamp_ms: int) -> bytes: + """Encode a Sample message. + + message Sample { + double value = 1; + int64 timestamp = 2; + } + """ + return _encode_double(1, value) + _encode_int64(2, timestamp_ms) + + +def encode_timeseries(labels: list[tuple[str, str]], samples: list[tuple[float, int]]) -> bytes: + """Encode a TimeSeries message. + + message TimeSeries { + repeated Label labels = 1; + repeated Sample samples = 2; + } + + Args: + labels: List of (name, value) tuples. Must include ("__name__", metric_name). + Will be sorted by name as required by the spec. + samples: List of (value, timestamp_ms) tuples. + """ + data = b"" + # Labels must be sorted by name + for label_name, label_value in sorted(labels, key=lambda x: x[0]): + data += _encode_message(1, encode_label(label_name, label_value)) + for sample_value, timestamp_ms in samples: + data += _encode_message(2, encode_sample(sample_value, timestamp_ms)) + return data + + +def encode_write_request(timeseries: list[bytes]) -> bytes: + """Encode a WriteRequest message. + + message WriteRequest { + repeated TimeSeries timeseries = 1; + } + + Args: + timeseries: List of pre-encoded TimeSeries messages. + """ + data = b"" + for ts in timeseries: + data += _encode_message(1, ts) + return data diff --git a/open_ess/util.py b/open_ess/util.py index 72b34f2..d463f26 100644 --- a/open_ess/util.py +++ b/open_ess/util.py @@ -1,5 +1,6 @@ import argparse import logging +from datetime import UTC, datetime from pathlib import Path from typing import ClassVar @@ -70,3 +71,13 @@ def parse_args(description: str) -> argparse.Namespace: help="Path to config file (YAML)", ) return parser.parse_args() + + +def dt_to_ms(dt: datetime) -> int: + """UTC datetime to Unix milliseconds.""" + return int(dt.timestamp() * 1000) + + +def ms_to_dt(ms: int) -> datetime: + """Unix milliseconds to UTC datetime.""" + return datetime.fromtimestamp(ms / 1000, tz=UTC) diff --git a/open_ess/victron_modbus/client.py b/open_ess/victron_modbus/client.py index bfaf303..ea04360 100644 --- a/open_ess/victron_modbus/client.py +++ b/open_ess/victron_modbus/client.py @@ -3,7 +3,7 @@ from threading import Lock from typing import TYPE_CHECKING -from open_ess.database import Database, DatabaseConnection +from open_ess.timeseries import Sample, TimeseriesBackend from .config import VictronConfig from .modbus_client import VictronModbusClient @@ -14,6 +14,11 @@ logger = logging.getLogger(__name__) +POWER_METRIC = "openess_power_watts" +ENERGY_METRIC = "openess_energy_kwh" +SOC_METRIC = "openess_soc_ratio" +VOLTAGE_METRIC = "openess_voltage_volts" + def _get_float(values: dict[Register, float | bytes | None], key: Register) -> float | None: """Extract a float value from read_many results, filtering out bytes and None.""" @@ -22,24 +27,27 @@ def _get_float(values: dict[Register, float | bytes | None], key: Register) -> f class VictronClient: - def __init__(self, database: Database, config: "BatterySystemConfig"): + def __init__( + self, + config: "BatterySystemConfig", + mql_client: TimeseriesBackend | None = None, + ): if not isinstance(config.control, VictronConfig): raise TypeError(f"VictronClient requires VictronConfig, got {type(config.control).__name__}") - self._db = database self._config = config self._control: VictronConfig = config.control self._client = VictronModbusClient(self._control) + self._mql_client = mql_client - self._db_conn: DatabaseConnection | None = None self._serial: str | None = None + self._current_soc: float | None = None self._setpoint: float = 0.0 # In Watt self._setpoint_expiration: datetime | None = None self._lock = Lock() def initialize(self) -> bool: - self._db_conn = self._db.connect() if not self.connect(): return False @@ -51,6 +59,9 @@ def initialize(self) -> bool: if not self._config.monitor_only: self.write(self.system_id, System.ESS_MODE, 3) + soc = self.read(self.system_id, System.BATTERY_SOC) + self._current_soc = soc if isinstance(soc, (int, float)) else None + return True @property @@ -85,15 +96,16 @@ def pvinverter_id(self) -> int | None: def need_mode_3(self) -> bool: return not self._config.monitor_only + @property + def current_soc(self) -> float | None: + return self._current_soc + def set_ess_setpoint(self, power: float, until: datetime) -> None: with self._lock: self._setpoint = power self._setpoint_expiration = until def write_setpoints(self) -> None: - if self._db_conn is None: - return - if self._config.monitor_only: return @@ -111,7 +123,7 @@ def write_setpoints(self) -> None: return idle_threshold = self._config.idle_threshold_w / 1000 - if (self._db_conn.get_current_soc() or 0) >= 99 and self._setpoint >= -idle_threshold: + if (self._current_soc or 0) >= 99 and self._setpoint >= -idle_threshold: # Keep putting power into the battery to allow balancing of the cells by the BMS. # TODO: implement balancing limits? self.write(self.vebus_id, VEBus.ESS_SETPOINT_L1, int((self._config.max_charge_power_kw or 0) * 1000)) @@ -128,148 +140,126 @@ def write_setpoints(self) -> None: if self._control.disable_inverter_when_idle: self.write(self.vebus_id, VEBus.ESS_DISABLE_FEEDBACK, 1) - def collect_and_store_measurements(self) -> None: - if self._db_conn is None: + def scrape_metrics(self) -> None: + if self._mql_client is None: return timestamp = datetime.now(UTC) + samples: list[Sample] = [] + + def add(metric: str, value: float | None, labels: dict[str, str]) -> None: + if self.serial is not None: + labels["device"] = self.serial + if value is not None: + samples.append(Sample(metric, value, timestamp, labels)) - # Read System registers + # Grid power system_regs = [System.GRID_L1, System.GRID_L2, System.GRID_L3] system_values = self.read_many(self.system_id, system_regs) - self._db_conn.insert_power("grid/power/l1", timestamp, _get_float(system_values, System.GRID_L1)) - self._db_conn.insert_power("grid/power/l2", timestamp, _get_float(system_values, System.GRID_L2)) - self._db_conn.insert_power("grid/power/l3", timestamp, _get_float(system_values, System.GRID_L3)) + add(POWER_METRIC, _get_float(system_values, System.GRID_L1), {"from": "grid", "phase": "L1"}) + add(POWER_METRIC, _get_float(system_values, System.GRID_L2), {"from": "grid", "phase": "L2"}) + add(POWER_METRIC, _get_float(system_values, System.GRID_L3), {"from": "grid", "phase": "L3"}) + # Grid energy if self.grid_id: - # TODO: check if grid meter delivers data per phase or not grid_values = self.read_many( - self.grid_id, - [GridMeter.ENERGY_TO_NET_TOTAL, GridMeter.ENERGY_FROM_NET_TOTAL], + self.grid_id, [GridMeter.ENERGY_TO_GRID_TOTAL, GridMeter.ENERGY_FROM_GRID_TOTAL] ) - self._db_conn.insert_energy( - "grid/energy/import/total", timestamp, _get_float(grid_values, GridMeter.ENERGY_FROM_NET_TOTAL) + add( + ENERGY_METRIC, + _get_float(grid_values, GridMeter.ENERGY_FROM_GRID_TOTAL), + {"from": "grid", "phase": "total"}, ) - self._db_conn.insert_energy( - "grid/energy/export/total", timestamp, _get_float(grid_values, GridMeter.ENERGY_TO_NET_TOTAL) + add( + ENERGY_METRIC, _get_float(grid_values, GridMeter.ENERGY_TO_GRID_TOTAL), {"to": "grid", "phase": "total"} ) + # PV inverter if self.pvinverter_id: - pvinverter_values = self.read_many( + pv_values = self.read_many( self.pvinverter_id, - [ - SolarInverter.ENERGY_L1, - SolarInverter.POWER_L1, - ], - ) - self._db_conn.insert_energy( - f"victron/pvinverter/{self.pvinverter_id}/energy/l1", - timestamp, - _get_float(pvinverter_values, SolarInverter.ENERGY_L1), - ) - self._db_conn.insert_power( - f"victron/pvinverter/{self.pvinverter_id}/power/l1", - timestamp, - _get_float(pvinverter_values, SolarInverter.POWER_L1), + [SolarInverter.ENERGY_L1, SolarInverter.POWER_L1], ) + pv_labels = {"from": "pvinverter", "unit_id": str(self.pvinverter_id), "phase": "L1"} + add(POWER_METRIC, _get_float(pv_values, SolarInverter.POWER_L1), pv_labels) + add(ENERGY_METRIC, _get_float(pv_values, SolarInverter.ENERGY_L1), pv_labels) - # VEBus registers for each device + # VEBus (inverter/charger) vebus_regs = [ VEBus.AC_INPUT_POWER_L1, - # VEBus.AC_INPUT_POWER_L2, - # VEBus.AC_INPUT_POWER_L3, VEBus.AC_OUTPUT_POWER_L1, - # VEBus.AC_OUTPUT_POWER_L2, - # VEBus.AC_OUTPUT_POWER_L3, VEBus.DC_CURRENT, VEBus.DC_VOLTAGE, VEBus.SOC, - # Energy counters VEBus.ENERGY_AC_IN1_TO_AC_OUT, VEBus.ENERGY_AC_IN1_TO_BATTERY, - # VEBus.ENERGY_AC_IN2_TO_AC_OUT, - # VEBus.ENERGY_AC_IN2_TO_BATTERY, VEBus.ENERGY_AC_OUT_TO_AC_IN1, - # VEBus.ENERGY_AC_OUT_TO_AC_IN2, VEBus.ENERGY_BATTERY_TO_AC_IN1, - # VEBus.ENERGY_BATTERY_TO_AC_IN2, VEBus.ENERGY_BATTERY_TO_AC_OUT, VEBus.ENERGY_AC_OUT_TO_BATTERY, ] - - vebus_prefix = self._control.vebus_prefix - vebus_values = self.read_many(self.vebus_id, vebus_regs) - self._db_conn.insert_power( - f"{vebus_prefix}/power/ac_in/l1", timestamp, _get_float(vebus_values, VEBus.AC_INPUT_POWER_L1) + # VEBus power + add( + POWER_METRIC, + _get_float(vebus_values, VEBus.AC_INPUT_POWER_L1), + {"from": "ac_in", "to": "system", "phase": "L1"}, ) - self._db_conn.insert_power( - f"{vebus_prefix}/power/ac_out/l1", timestamp, _get_float(vebus_values, VEBus.AC_OUTPUT_POWER_L1) + add( + POWER_METRIC, + _get_float(vebus_values, VEBus.AC_OUTPUT_POWER_L1), + {"from": "system", "to": "ac_out", "phase": "L1"}, ) - soc = _get_float(vebus_values, VEBus.SOC) - if soc is not None: - self._db_conn.insert_soc(f"{vebus_prefix}/soc", timestamp, int(soc)) - - dc_current = _get_float(vebus_values, VEBus.DC_CURRENT) - dc_voltage = _get_float(vebus_values, VEBus.DC_VOLTAGE) - if dc_voltage is not None: - self._db_conn.insert_voltage(f"{vebus_prefix}/voltage/battery", timestamp, dc_voltage) - if dc_current is not None: - dc_power = dc_current * dc_voltage - self._db_conn.insert_power(f"{vebus_prefix}/power/battery", timestamp, dc_power) - - # Energy flows - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_in_to_ac_out", - timestamp, - _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_AC_OUT), - ) - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_in_import", timestamp, _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_BATTERY) - ) - # self._database.insert_energy("", timestamp, _get_float(vebus_values,VEBus.ENERGY_AC_IN2_TO_AC_OUT)) - # self._database.insert_energy("", timestamp, _get_float(vebus_values,VEBus.ENERGY_AC_IN2_TO_BATTERY)) - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_out_to_ac_in", - timestamp, - _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_AC_IN1), - ) - # self._database.insert_energy("", timestamp, _get_float(vebus_values,VEBus.ENERGY_AC_OUT_TO_AC_IN2)) - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_in_export", timestamp, _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_IN1) - ) - # self._database.insert_energy("", timestamp, _get_float(vebus_values,VEBus.ENERGY_BATTERY_TO_AC_IN2)) - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_out_export", timestamp, _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_OUT) + # VEBus battery + vebus_dc_current = _get_float(vebus_values, VEBus.DC_CURRENT) + vebus_dc_voltage = _get_float(vebus_values, VEBus.DC_VOLTAGE) + vebus_battery_power = ( + vebus_dc_current * vebus_dc_voltage + if vebus_dc_current is not None and vebus_dc_voltage is not None + else None ) - self._db_conn.insert_energy( - f"{vebus_prefix}/energy/ac_out_import", timestamp, _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_BATTERY) - ) - + add(POWER_METRIC, vebus_battery_power, {"from": "system", "to": "battery", "unit": "vebus"}) + add(VOLTAGE_METRIC, vebus_dc_voltage, {"node": "battery", "unit": "vebus"}) + vebus_soc = _get_float(vebus_values, VEBus.SOC) + if vebus_soc is not None: + add(SOC_METRIC, vebus_soc / 100, {"node": "battery", "unit": "vebus"}) + + # VEBus energy + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_AC_OUT), {"from": "ac_in", "to": "ac_out"}) + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_IN1_TO_BATTERY), {"from": "ac_in", "to": "system"}) + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_AC_IN1), {"from": "ac_out", "to": "ac_in"}) + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_IN1), {"from": "system", "to": "ac_in"}) + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_BATTERY_TO_AC_OUT), {"from": "system", "to": "ac_out"}) + add(ENERGY_METRIC, _get_float(vebus_values, VEBus.ENERGY_AC_OUT_TO_BATTERY), {"from": "ac_out", "to": "system"}) + + # BMS + bms_soc = None if self.battery_id is not None: - bms_prefix = self._control.battery_prefix - bms_values = self.read_many( self.battery_id, - [ - Battery.DC_VOLTAGE, - Battery.DC_POWER, - Battery.SOC, - # Battery.CHARGED_ENERGY, - # Battery.DISCHARGED_ENERGY, - ], - ) - - self._db_conn.insert_power( - f"{bms_prefix}/power/battery", timestamp, _get_float(bms_values, Battery.DC_POWER) + [Battery.DC_VOLTAGE, Battery.DC_POWER, Battery.SOC], ) - self._db_conn.insert_voltage( - f"{bms_prefix}/voltage/battery", timestamp, _get_float(bms_values, Battery.DC_VOLTAGE) + add( + POWER_METRIC, + _get_float(bms_values, Battery.DC_POWER), + {"from": "system", "to": "battery", "unit": "battery"}, ) + add(VOLTAGE_METRIC, _get_float(bms_values, Battery.DC_VOLTAGE), {"node": "battery", "unit": "battery"}) bms_soc = _get_float(bms_values, Battery.SOC) if bms_soc is not None: - self._db_conn.insert_soc(f"{bms_prefix}/soc", timestamp, round(bms_soc)) + add(SOC_METRIC, bms_soc / 100, {"node": "battery", "unit": "battery"}) + + if samples: + try: + self._mql_client.write(samples) + except Exception as e: + logger.exception(f"Failed to write samples to timeseries backend: {e}") + + if bms_soc is not None: + self._current_soc = bms_soc + elif vebus_soc is not None: + self._current_soc = vebus_soc # --------------------------------# # VictronModbusClient bindings # diff --git a/open_ess/victron_modbus/config.py b/open_ess/victron_modbus/config.py index 23d6334..f66d7a0 100644 --- a/open_ess/victron_modbus/config.py +++ b/open_ess/victron_modbus/config.py @@ -18,11 +18,3 @@ class VictronConfig(BaseModel): disable_charger_when_idle: bool = False disable_inverter_when_idle: bool = False - - @property - def vebus_prefix(self) -> str: - return f"victron/vebus/{self.vebus_id}" - - @property - def battery_prefix(self) -> str | None: - return f"victron/battery/{self.battery_id}" if self.battery_id else None diff --git a/open_ess/victron_modbus/registers.py b/open_ess/victron_modbus/registers.py index e9c2d5e..0be6801 100644 --- a/open_ess/victron_modbus/registers.py +++ b/open_ess/victron_modbus/registers.py @@ -272,5 +272,5 @@ class GridMeter: ENERGY_FROM_NET_L3 = Register("Energy from net L3", 2630, DataType.UINT32, scale=100) ENERGY_TO_NET_L3 = Register("Energy to net L3", 2632, DataType.UINT32, scale=100) - ENERGY_FROM_NET_TOTAL = Register("Energy from net", 2634, DataType.UINT32, scale=100) - ENERGY_TO_NET_TOTAL = Register("Energy to net", 2636, DataType.UINT32, scale=100) + ENERGY_FROM_GRID_TOTAL = Register("Energy from net", 2634, DataType.UINT32, scale=100) + ENERGY_TO_GRID_TOTAL = Register("Energy to net", 2636, DataType.UINT32, scale=100) diff --git a/open_ess/victron_modbus/service.py b/open_ess/victron_modbus/service.py index 2dd48c0..f3e78f8 100644 --- a/open_ess/victron_modbus/service.py +++ b/open_ess/victron_modbus/service.py @@ -2,8 +2,8 @@ import time from typing import TYPE_CHECKING -from open_ess.database import Database from open_ess.service import Service +from open_ess.timeseries import TimeseriesBackend from .client import VictronClient @@ -14,12 +14,13 @@ class VictronService(Service): - """Collects measurements from Victron GX every second.""" - - def __init__(self, db: Database, config: "BatterySystemConfig"): + def __init__( + self, + config: "BatterySystemConfig", + mql_client: TimeseriesBackend | None = None, + ): super().__init__("VictronService") - self._config = config - self._client = VictronClient(db, config) + self._client = VictronClient(config, mql_client) @property def client(self) -> VictronClient: @@ -32,7 +33,7 @@ def on_start(self) -> None: def tick(self) -> None: self._client.write_setpoints() - self._client.collect_and_store_measurements() + self._client.scrape_metrics() def wait_until_next(self) -> None: # Sleep until the start of the next second diff --git a/pyproject.toml b/pyproject.toml index 1b94c86..dea5a4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,15 @@ warn_unused_ignores = true disallow_untyped_defs = true [[tool.mypy.overrides]] -module = ["entsoe", "entsoe.*", "pyomo", "pyomo.*"] +module = [ + "entsoe", "entsoe.*", + "metricsqlite", "metricsqlite.*", + "pandas", "pandas.*", + "pyomo", "pyomo.*", + "snappy", + "urllib3", "urllib3.*", + "yaml", +] ignore_missing_imports = true [[tool.mypy.overrides]] @@ -63,6 +71,7 @@ version = "0.0.0" authors = [{ name = "David van 't Wout", email = "david@vtwout.com" }] requires-python = ">=3.11" # Because of entsoe-apy dependencies = [ + "metricsqlite", "entsoe-apy", "fastapi", "jinja2", @@ -71,8 +80,11 @@ dependencies = [ "pydantic", "pymodbus", "pyomo", + "python-snappy", "pyyaml", "uvicorn", + "urllib3", # victoriametrics client + "python-snappy", # victoriametrics client ] [project.optional-dependencies] diff --git a/shell.nix b/shell.nix index ac12a0e..50f2d1a 100644 --- a/shell.nix +++ b/shell.nix @@ -3,7 +3,21 @@ # Note: ruff is dynamically linked and the version installed by pip won't work on NixOS. # This can be fixed by adding `programs.nix-ld.enable = true;` to your NixOS config. -let open-ess = pkgs.python3.pkgs.callPackage ./default.nix { }; +let + metricsqlite = pkgs.python3.pkgs.buildPythonPackage { + pname = "metricsqlite"; + version = "0.0.0"; + format = "pyproject"; + src = pkgs.fetchFromGitHub { + owner = "DavidvtWout"; + repo = "MetricSQLite"; + rev = "9758b9f"; + hash = "sha256-k0dsp/Ycaq/tqOEoOalH7d7RYpP+N8n7HI8NvxbOEu0="; + }; + nativeBuildInputs = with pkgs.python3.pkgs; [ setuptools setuptools-scm ]; + propagatedBuildInputs = with pkgs.python3.pkgs; [ pydantic ]; + }; + open-ess = pkgs.python3.pkgs.callPackage ./default.nix { inherit metricsqlite; }; in pkgs.mkShell { packages = with pkgs; [ (python3.withPackages (pp: