From ac1754aae7914f61b8f3002de2ea16b1c4172cef Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Wed, 7 Oct 2026 17:47:53 +0000 Subject: [PATCH 01/12] Update total annualized dist MC for CT --- context/README.md | 24 +++---- .../ct_eversource_dist_mc_methodology.md | 65 ++++++++++++++----- .../dist_mc_definition_choice.md | 36 +++++----- rate_design/hp_rates/ct/Justfile | 3 +- .../marginal_costs/ct_marginal_costs_2025.csv | 2 +- tests/test_ct_dist_mc.py | 20 ++++-- 6 files changed, 98 insertions(+), 52 deletions(-) diff --git a/context/README.md b/context/README.md index 79e71ac8..f3f1cb84 100644 --- a/context/README.md +++ b/context/README.md @@ -77,18 +77,18 @@ Cost-reflective TOU design: theory and window optimization. How we derive BAT marginal-cost inputs from regulatory studies. -| File | Purpose | -| ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| ny_dist_mc_bat_methodology.md | Shared methodology for deriving sub-TX and distribution BAT marginal cost inputs from the seven NY MCOS workbooks: pipeline overview, key conceptual choices (incremental not cumulative, levelized 7-year window, no discounting, 2026 dollar year), per-utility deviations and harmonization, final values and sanity check, research appendices. | -| ny_mcos_studies_comparison.md | Cross-utility comparison of the seven NY 2025 MCOS studies: what MCOS numbers mean (undiluted/diluted, grid topology), methodology comparison, annualization mechanics (workbook-verified step-by-step), system-wide diluted/undiluted tables, component taxonomy mapping (physical hierarchy → cost-center labels), bulk vs local TX scrub, reassessment of bulk TX MC gap. | -| ny_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for NY BAT: the gap, data sources (OATT ATRR, CLCPA project costs, FERC Order 1920), five options with tradeoffs, recommended approach (OATT as upper-bound proxy), hourly PoP allocation. | -| ri_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for RI BAT: data sources (RNS rate, AESC 2024 avoided PTF cost, ISO-NE 2050 study), four options with tradeoffs, recommended approach (AESC primary, RNS upper bound), hourly PoP allocation, comparison with NY. | -| ct_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for CT BAT: same AESC 2024 $69/kW-year and top-100 exceedance as RI, plus `--allocation-load` flag to toggle between NE system aggregate and CT zone load for peak-hour identification; utilities `ct_eversource` and `ct_ui`; implementation files and Just commands. | -| capacity_market_comparison_nyiso_isone.md | Comparison of NYISO ICAP (Strip/Monthly/Spot) vs ISO-NE FCM (FCA/ARA/MRA) capacity market structures: structural analogy table, forward horizons, price volatility differences, implications for BAT marginal cost signal (Spot vs FCA vs MRA), current approach in NY vs RI, trade-offs. | -| dist_mc_definition_choice.md | Cross-cutting: which LRMC definition the `sub_tx_and_dist` BAT input uses, and why it differs by state. Taxonomy (Turvey / AIC-avoided / FLIC) and why AIC and FLIC differ ~10-50x, source-number table (NY FLIC $1-8, RI AESC $80, BGE Brattle $32, BGE E3 $203-258, OPC $25-34, BAT paper $54.46), what is implemented per state (NY=FLIC, RI/MD=avoided), the platform tension (NY-vs-RI/MD; methodology-doc §3-vs-§7), rationale, and BGE sensitivities. | -| md_bge_dist_mc_methodology.md | BGE (MD) sub-TX + distribution MC implementation: adopts Brattle avoided distribution cost $32/kW-yr (2022$), RI-style single-scalar config + CPI handling, why avoided cost is consistent with BGE's own peak-driven COS, the ~82% volumetric over-recovery intervention hook, FLIC/E3 sensitivities, and the Stage-B allocator MD-enablement dependency. | -| ct_eversource_dist_mc_methodology.md | eversource (CT) sub-TX + dist MC: how to derive CL&P/Eversource CT's `sub_tx_and_dist` MC from Exhibits CLP-MCOS-1/2. Stage A: system-wide diluted $20.17/kW-yr (Table 3) vs locational $86.58/kW-yr, why diluted mirrors NY's FLIC convention. Stage B: CL&P's own hourly probability-of-peak method (std-dev of 2022-2025 substation load, BTM solar/HP-adoption forecast adjustment) vs. what data we actually have (Table 2A TOU/season buckets only — Back-Up 36's raw PoP curve is listed in the TOC but not in the PDF); Tier 1 TOU-bucket mapping (implementable now) vs Tier 2 load-based PoP proxy (stretch); task list and open questions (UI gap, locational sensitivity, TOU-window confirmation). | -| md_bulk_transmission.md | How to construct bulk TX marginal cost for MD BAT: PJM NITS rates as OATT ATRR proxy (E3 ICC-VDER precedent), zone-to-utility mapping (BGE/DPL/PEPCO/APS covering all 13 MD utilities), calendar-year day-weighted blending, full-year top-150 PCAF allocation (following E3), reference data at `data/pjm/bulk_tx/nits/`, implementation in `bulk_tx_pjm.py`. | +| File | Purpose | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ny_dist_mc_bat_methodology.md | Shared methodology for deriving sub-TX and distribution BAT marginal cost inputs from the seven NY MCOS workbooks: pipeline overview, key conceptual choices (incremental not cumulative, levelized 7-year window, no discounting, 2026 dollar year), per-utility deviations and harmonization, final values and sanity check, research appendices. | +| ny_mcos_studies_comparison.md | Cross-utility comparison of the seven NY 2025 MCOS studies: what MCOS numbers mean (undiluted/diluted, grid topology), methodology comparison, annualization mechanics (workbook-verified step-by-step), system-wide diluted/undiluted tables, component taxonomy mapping (physical hierarchy → cost-center labels), bulk vs local TX scrub, reassessment of bulk TX MC gap. | +| ny_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for NY BAT: the gap, data sources (OATT ATRR, CLCPA project costs, FERC Order 1920), five options with tradeoffs, recommended approach (OATT as upper-bound proxy), hourly PoP allocation. | +| ri_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for RI BAT: data sources (RNS rate, AESC 2024 avoided PTF cost, ISO-NE 2050 study), four options with tradeoffs, recommended approach (AESC primary, RNS upper bound), hourly PoP allocation, comparison with NY. | +| ct_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for CT BAT: same AESC 2024 $69/kW-year and top-100 exceedance as RI, plus `--allocation-load` flag to toggle between NE system aggregate and CT zone load for peak-hour identification; utilities `ct_eversource` and `ct_ui`; implementation files and Just commands. | +| capacity_market_comparison_nyiso_isone.md | Comparison of NYISO ICAP (Strip/Monthly/Spot) vs ISO-NE FCM (FCA/ARA/MRA) capacity market structures: structural analogy table, forward horizons, price volatility differences, implications for BAT marginal cost signal (Spot vs FCA vs MRA), current approach in NY vs RI, trade-offs. | +| dist_mc_definition_choice.md | Cross-cutting: which LRMC definition the `sub_tx_and_dist` BAT input uses, and why it differs by state. Taxonomy (Turvey / AIC-avoided / FLIC) and why AIC and FLIC differ ~10-50x, source-number table (NY FLIC $1-8, RI AESC $80, BGE Brattle $32, BGE E3 $203-258, OPC $25-34, BAT paper $54.46), what is implemented per state (NY=FLIC, RI/MD=avoided), the platform tension (NY-vs-RI/MD; methodology-doc §3-vs-§7), rationale, and BGE sensitivities. | +| md_bge_dist_mc_methodology.md | BGE (MD) sub-TX + distribution MC implementation: adopts Brattle avoided distribution cost $32/kW-yr (2022$), RI-style single-scalar config + CPI handling, why avoided cost is consistent with BGE's own peak-driven COS, the ~82% volumetric over-recovery intervention hook, FLIC/E3 sensitivities, and the Stage-B allocator MD-enablement dependency. | +| ct_eversource_dist_mc_methodology.md | eversource (CT) sub-TX + dist MC: how to derive CL&P/Eversource CT's `sub_tx_and_dist` MC from Exhibits CLP-MCOS-1/2. Stage A: system-wide diluted $20.17/kW-yr (Table 3) vs locational $86.58/kW-yr, why diluted mirrors NY's FLIC convention, and the secondary loss gross-up to the implemented $21.22/kW-yr (OCC-863 Attachment 2). Stage B: CL&P's own hourly probability-of-peak method (std-dev of 2022-2025 substation load, BTM solar/HP-adoption forecast adjustment) vs. what data we actually have (Table 2A TOU/season buckets only — Back-Up 36's raw PoP curve is listed in the TOC but not in the PDF); Tier 1 TOU-bucket mapping (implementable now) vs Tier 2 load-based PoP proxy (stretch); task list and open questions (UI gap, locational sensitivity, TOU-window confirmation). | +| md_bulk_transmission.md | How to construct bulk TX marginal cost for MD BAT: PJM NITS rates as OATT ATRR proxy (E3 ICC-VDER precedent), zone-to-utility mapping (BGE/DPL/PEPCO/APS covering all 13 MD utilities), calendar-year day-weighted blending, full-year top-150 PCAF allocation (following E3), reference data at `data/pjm/bulk_tx/nits/`, implementation in `bulk_tx_pjm.py`. | ### methods/ (root) diff --git a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md index 6501afec..087bb467 100644 --- a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md +++ b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md @@ -68,18 +68,37 @@ structural consequence of the platform's architecture. closer to NY's FLIC convention than to RI/BGE's published avoided-cost-scalar convention**, even though the dollar magnitude (`$20.17`) is roughly comparable to RI/BGE. -### 2.2 Recommendation: use the system-wide diluted figure (`$20.17/kW-yr`) +### 2.2 Recommendation: system-wide diluted figure, grossed up to secondary (`$21.22/kW-yr`) + +Table 3's `$20.17/kW-yr` is the cost per kW of upstream peak capability, before line losses. +Residential customers take service at secondary voltage, so a kW measured at the meter requires +more than a kW of that upstream capacity. Eversource's response to OCC-863, Attachment 2, page 1 +("Average hourly marginal cost by month, secondary service") applies the gross-up in cell J3: + +$$20.17 \times 1.051845136935937 = 21.2157\ldots$$ + +where `1.051845136935937` is the secondary loss factor in cell N3. Rounded to the cent, that is +**`$21.22/kW-yr`**, and that is the scalar in the config CSV. + +The same workbook's page 2 uses the primary loss factor 1.0460400426767955, which gives +$21.10/kW-yr. We use secondary because the BAT population is residential. The choice also +matches MCOS-2 Table 2A: the secondary "Annual Average" row is $1.768/kW-mo, and +$1.768 × 12 = $21.22/kW-yr. + +Three reasons for starting from the system-wide diluted base rather than the locational `$86.58`: 1. **It's CL&P's own primary framing.** Table 1 (comparing current rates to marginal cost) and the rate-design testimony (MCOS-1 §IV) both use the system-wide figure as the reference marginal cost when arguing current volumetric rates recover far more than marginal cost. Using the same figure CL&P uses against itself keeps the intervention grounded in the utility's own numbers (same logic - as the BGE precedent in `dist_mc_definition_choice.md` §5). + as the BGE precedent in `dist_mc_definition_choice.md` §5). The loss gross-up is likewise + CL&P's own, from the OCC-863 workbook rather than a factor we estimated. 2. **Consistent with the platform's dilution convention.** Diluting by expansion-area load share is exactly the mechanic NY's FLIC values use; adopting it keeps CT internally consistent with NY rather than introducing a third convention. 3. Carry `$86.58/kW-yr` as a documented **sensitivity/upper bound** (parallel to how BGE's `$203–258` - E3 figure and NY's undiluted numbers are retained as sensitivities elsewhere). + E3 figure and NY's undiluted numbers are retained as sensitivities elsewhere). That sensitivity + stays at the Table 3 locational figure; we do not also loss-adjust it. ### 2.3 Config file @@ -88,9 +107,11 @@ Following the RI/MD pattern (`ri_marginal_costs_2025.csv`, `md_marginal_costs_20 ```csv utility,sub_tx_and_dist_mc_kw_yr,dollar_year -ct_eversource,20.17,2026 +ct_eversource,21.22,2026 ``` +`$21.22` is Table 3's `$20.17` times the OCC-863 secondary loss factor, rounded to the cent (§2.2). + `ct_ui` (United Illuminating) is **not** covered by this MCOS — it needs its own source (see [Open questions](#4-open-questions--decisions-needed) below). The optional `dollar_year=2026` lets the existing CPI-inflation logic in `generate_utility_tx_dx_mc.py` handle any run year other than @@ -171,11 +192,14 @@ bulk-TX MC. 2. **Created the config CSV** (§2.3). 3. **Filled the utility-level load gap** (§3.3) — `ct_eversource`/`ct_ui` zone-mapping rows, ran the ISO-NE utility aggregation for 2025, uploaded to S3. -4. **Refreshed CPI data through 2026**: the MCOS-2 Table 3 figure is filed in 2026$ +4. **Refreshed CPI data through 2026**: the config scalar is filed in 2026$ (`dollar_year=2026` in the config CSV), but `data/fred/cpi/parquet/` only had annual averages through 2025. Ran `just -f data/fred/cpi/Justfile fetch-cpi CPIAUCSL 2019 2026` (2026 is a partial-year average — 6 months as of this run — since FRED lags by ~1 month) and uploaded. - `$20.17 → $19.65/kW-yr` in 2025$ (CPI factor 0.9740). + The run below used the pre-loss Table 3 scalar: `$20.17 → $19.65/kW-yr` in 2025$ (CPI factor + 0.9740). The config is now the secondary loss-adjusted `$21.22/kW-yr` (§2.2); the same factor + applied to that scalar is about `$20.67/kW-yr` in 2025$. Re-run `create-dist-mc-data` to + replace the S3 parquet, which still reflects the pre-loss scalar. 5. **Ran the standard PoP allocation** via the new `just -f ct/Justfile create-dist-mc-data 2025 --upload` recipe (§3.5): ```bash @@ -186,7 +210,8 @@ bulk-TX MC. --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ \ --upload ``` -6. **Validated**: the built-in `validate_allocation`-equivalent 1-kW-constant-load check passed +6. **Validated** (against the pre-loss `$20.17` scalar; re-run after the `$21.22` change): the + built-in `validate_allocation`-equivalent 1-kW-constant-load check passed exactly ($19.6460/kW-yr, 0.0000% error). The top-100 PoP hours for 2025 fall entirely in June–August (30 in June, 62 in July, 8 in August, 0 elsewhere) — consistent with CL&P's own finding of concentrated summer peak-probability, though the specific split differs somewhat from @@ -211,14 +236,19 @@ over `UTILITIES=ct_eversource,ct_ui` the way the generic `create-dist-and-sub-tx recipe does would raise `ValueError: No marginal cost data found for ct_ui`. `create-dist-mc-data` therefore takes an explicit year argument and always targets `ct_eversource`. -### 3.6 Primary vs. secondary voltage column (Table 2A reference values) +### 3.6 Secondary loss adjustment, and Table 2A as a cross-check + +The annual scalar is grossed up by the secondary loss factor before PoP allocation (§2.2), so every +allocated hour carries the same gross-up. We still do not use Table 2A's `$/kWh` rates as the hourly +signal. PoP spreads the annual scalar across the top load hours. -Table 2A gives near-identical Primary and Secondary `$/kWh` columns (e.g. system-wide annual average -is `$0.00241` primary vs `$0.00242` secondary [DocumentCloud p. 5](https://www.documentcloud.org/documents/28540599-exhibit-clp-mcos-2/#document/p5/a2826760)). The difference is negligible (loss-adjustment). Since -we're using the PoP method with the `$20.17/kW-yr` Table 3 figure directly (not the Table 2A `$/kWh` -rates), the primary/secondary distinction doesn't affect the implementation. Table 2A's values remain -useful as a cross-check: our PoP-allocated 8760 should produce similar seasonal concentration to -what Table 2A shows (nearly all cost in summer, near-zero winter). +Table 2A remains the seasonal cross-check, and it is also where the primary/secondary split shows up +as a rate. The system-wide annual average is `$0.00241/kWh` primary vs `$0.00242/kWh` secondary +[DocumentCloud p. 5](https://www.documentcloud.org/documents/28540599-exhibit-clp-mcos-2/#document/p5/a2826760). +The secondary "Annual Average" `$1.768/kW-mo` row is the same `$21.22/kW-yr` we store +(`$1.768 × 12`). Our PoP-allocated 8760 should still produce the seasonal concentration Table 2A +shows (nearly all cost in summer, near-zero winter). The hours that concentration lands on come from +CT zone load, not from Table 2A's own probability-of-peak weights. ### 3.7 Back-Up 36 (if obtainable) @@ -253,7 +283,7 @@ allocation or as a direct replacement. This is a nice-to-have, not a blocker for - **2026 CPI is a partial-year average.** The CPI inflation factor (§3.4 point 4) uses a 2026 annual average computed from only the months FRED had published as of this implementation. As more 2026 months are published, re-running `just fetch-cpi` will shift the 2026 average slightly, which - would change the `$20.17 → $19.65` inflated value by a small amount. Not expected to matter + would change the `$21.22 → ~$20.67` inflated value by a small amount. Not expected to matter materially, but worth knowing if the output value changes on a future re-run. --- @@ -269,7 +299,10 @@ allocation or as a direct replacement. This is a nice-to-have, not a blocker for `tests/test_ct_dist_mc.py`. 6. ~~Update this doc and [dist_mc_definition_choice.md](dist_mc_definition_choice.md) §2–3, adding CT to the source-number and per-state tables.~~ Done. -7. **Remaining**: wire `path_dist_and_sub_tx_mc` into a CT scenario config (`scenarios_ct_eversource.yaml` +7. **Remaining**: re-run `create-dist-mc-data 2025 --upload`. The parquet at + `s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet` + was built from the pre-loss `$20.17/kW-yr` scalar. The config CSV is now `$21.22/kW-yr`. +8. **Remaining**: wire `path_dist_and_sub_tx_mc` into a CT scenario config (`scenarios_ct_eversource.yaml` or equivalent) once CT scenario YAMLs exist, so a CAIRO run actually consumes this MC output. No CT scenario configs exist yet in `rate_design/hp_rates/ct/config/scenarios/` — that's a separate, larger piece of CT onboarding beyond this MC-generation task. diff --git a/context/methods/marginal_costs/dist_mc_definition_choice.md b/context/methods/marginal_costs/dist_mc_definition_choice.md index 0ee632de..2f3beb06 100644 --- a/context/methods/marginal_costs/dist_mc_definition_choice.md +++ b/context/methods/marginal_costs/dist_mc_definition_choice.md @@ -59,21 +59,23 @@ everyone?"; AIC/avoided asks "what does a marginal kW of peak cost?". ## 2. Source numbers (all retained, for every alternative) -| Source | Figure ($/kW-yr) | Dollar yr | Definition | Scope | Citation | -| --------------------------------------------- | ---------------- | --------- | ------------------------------------------------ | ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| NY MCOS (7 utilities) | 1.16 – 8.52 | 2026 | FLIC (incremental diluted) | sub-TX + dist | `ny_sub_tx_and_dist_mc_levelized.csv`; [ny_dist_mc_bat_methodology.md](ny_dist_mc_bat_methodology.md) | -| RI AESC 2024 (RIE) | 80.24 | 2019 | AIC / avoided | dist (non-PTF) | `ri_marginal_costs_2025.csv` | -| **BGE Brattle (MD)** | **32** | **2022** | **AIC / avoided** | **dist incl. 34 kV sub-TX** | [BGE MYP testimony p.138](https://www.documentcloud.org/documents/28269392-001-2-bge-myp-2-case-direct-testimony-final-w-att/#document/p138/a2821808); [bge_case9692_dist_mc.md](../../sources/mcos/bge_case9692_dist_mc.md) §1 | -| BGE OPC/Takahashi (MD) | 25.1 – 34.09 | 2022 | AIC / avoided (corroborates Brattle) | dist | [bge_case9692_dist_mc.md](../../sources/mcos/bge_case9692_dist_mc.md) §1 | -| BGE E3 (MD) | 203 – 258 | 2022 | AIC / avoided | **combined T&D incl. FERC TX** | [bge_case9692_dist_mc.md](../../sources/mcos/bge_case9692_dist_mc.md) §2 | -| BAT paper (PG&E) | 54.46 | — | AIC / avoided (CPUC ACC) | dist CapEx | Simeone et al. 2023, App. A | -| CL&P MCOS-2 Table 3 (CT, system-wide diluted) | 20.17 | 2026 | FLIC-like (diluted by expansion-area load share) | upstream distribution (station + trunkline) | [exhibit_clp_mcos_2.md](../../sources/mcos/exhibit_clp_mcos_2.md) Table 3; [ct_eversource_dist_mc_methodology.md](ct_eversource_dist_mc_methodology.md) §2 | -| CL&P MCOS-2 Table 3 (CT, locational) | 86.58 | 2026 | AIC-like (undiluted, expansion areas only) | upstream distribution (station + trunkline) | [exhibit_clp_mcos_2.md](../../sources/mcos/exhibit_clp_mcos_2.md) Table 3; [ct_eversource_dist_mc_methodology.md](ct_eversource_dist_mc_methodology.md) §2 | +| Source | Figure ($/kW-yr) | Dollar yr | Definition | Scope | Citation | +| ------------------------------------------------------ | ---------------- | --------- | ------------------------------------------------ | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| NY MCOS (7 utilities) | 1.16 – 8.52 | 2026 | FLIC (incremental diluted) | sub-TX + dist | `ny_sub_tx_and_dist_mc_levelized.csv`; [ny_dist_mc_bat_methodology.md](ny_dist_mc_bat_methodology.md) | +| RI AESC 2024 (RIE) | 80.24 | 2019 | AIC / avoided | dist (non-PTF) | `ri_marginal_costs_2025.csv` | +| **BGE Brattle (MD)** | **32** | **2022** | **AIC / avoided** | **dist incl. 34 kV sub-TX** | [BGE MYP testimony p.138](https://www.documentcloud.org/documents/28269392-001-2-bge-myp-2-case-direct-testimony-final-w-att/#document/p138/a2821808); [bge_case9692_dist_mc.md](../../sources/mcos/bge_case9692_dist_mc.md) §1 | +| BGE OPC/Takahashi (MD) | 25.1 – 34.09 | 2022 | AIC / avoided (corroborates Brattle) | dist | [bge_case9692_dist_mc.md](../../sources/mcos/bge_case9692_dist_mc.md) §1 | +| BGE E3 (MD) | 203 – 258 | 2022 | AIC / avoided | **combined T&D incl. FERC TX** | [bge_case9692_dist_mc.md](../../sources/mcos/bge_case9692_dist_mc.md) §2 | +| BAT paper (PG&E) | 54.46 | — | AIC / avoided (CPUC ACC) | dist CapEx | Simeone et al. 2023, App. A | +| CL&P MCOS-2 Table 3 (CT, system-wide diluted) | 20.17 | 2026 | FLIC-like (diluted by expansion-area load share) | upstream distribution (station + trunkline), before losses | [exhibit_clp_mcos_2.md](../../sources/mcos/exhibit_clp_mcos_2.md) Table 3; [ct_eversource_dist_mc_methodology.md](ct_eversource_dist_mc_methodology.md) §2 | +| CL&P Table 3 × secondary loss factor (CT, implemented) | 21.22 | 2026 | Same diluted figure, grossed up to secondary | upstream distribution at the residential meter | OCC-863 Attachment 2 p. 1: `$20.17 × 1.051845137`; [ct_eversource_dist_mc_methodology.md](ct_eversource_dist_mc_methodology.md) §2.2 | +| CL&P MCOS-2 Table 3 (CT, locational) | 86.58 | 2026 | AIC-like (undiluted, expansion areas only) | upstream distribution (station + trunkline) | [exhibit_clp_mcos_2.md](../../sources/mcos/exhibit_clp_mcos_2.md) Table 3; [ct_eversource_dist_mc_methodology.md](ct_eversource_dist_mc_methodology.md) §2 | Note the pattern: **every figure except NY's and CT's is an avoided/AIC number**, and they cluster at `$25–80/kW-yr` for distribution alone (E3 is higher because it bundles FERC transmission). NY's FLIC values are the outlier in magnitude — by construction (§1). CT's system-wide diluted figure -(`$20.17`) is a coincidental exception: it's built the same way as FLIC (locational cost diluted by +(`$20.17` before losses, `$21.22` after the secondary loss gross-up we implement) is a coincidental +exception: it's built the same way as FLIC (locational cost diluted by load share) but happens to land in the AIC/avoided-cost dollar range because CL&P's undiluted locational figure (`$86.58`) is itself much larger than NY's undiluted MCOS figures. @@ -81,12 +83,12 @@ locational figure (`$86.58`) is itself much larger than NY's undiluted MCOS figu ## 3. What we implement, by state (current production) -| State / utility | Definition implemented | Value | Why | -| ------------------------- | ------------------------------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| NY (7 utilities) | **FLIC** (incremental diluted from MCOS) | 1.16 – 8.52 | MCOS workbooks give project-level capital; PSC-blessed NERA method | -| RI (RIE) | **AIC / avoided** (AESC 2024 avoided non-PTF T&D) | 80.24 (2019$) | AESC is the New England institutional standard; no MCOS workbook | -| **MD (BGE)** | **AIC / avoided** (Brattle replacement-value ÷ peak) | 32 (2022$) | BGE's own COS classifies the plant 100% peak-driven; BGE's own marginal numbers use a peak denominator (see §5) | -| CT (ct_eversource / CL&P) | **FLIC-like** (system-wide diluted from MCOS-2 Table 3) | 20.17 (2026$) | CL&P's own MCOS dilutes locational cost by expansion-area load share, structurally the same move as NY's diluted FLIC (see [ct_eversource_dist_mc_methodology.md](ct_eversource_dist_mc_methodology.md) §2) | +| State / utility | Definition implemented | Value | Why | +| ------------------------- | ------------------------------------------------------------------------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| NY (7 utilities) | **FLIC** (incremental diluted from MCOS) | 1.16 – 8.52 | MCOS workbooks give project-level capital; PSC-blessed NERA method | +| RI (RIE) | **AIC / avoided** (AESC 2024 avoided non-PTF T&D) | 80.24 (2019$) | AESC is the New England institutional standard; no MCOS workbook | +| **MD (BGE)** | **AIC / avoided** (Brattle replacement-value ÷ peak) | 32 (2022$) | BGE's own COS classifies the plant 100% peak-driven; BGE's own marginal numbers use a peak denominator (see §5) | +| CT (ct_eversource / CL&P) | **FLIC-like** (system-wide diluted from MCOS-2 Table 3, then × secondary loss factor) | 21.22 (2026$) | CL&P's own MCOS dilutes locational cost by expansion-area load share, structurally the same move as NY's diluted FLIC. The config scalar grosses Table 3's `$20.17` up by the OCC-863 secondary loss factor 1.051845137 (see [ct_eversource_dist_mc_methodology.md](ct_eversource_dist_mc_methodology.md) §2) | All four feed the _same_ PoP allocator ([generate_utility_tx_dx_mc.py](../../../utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py)) diff --git a/rate_design/hp_rates/ct/Justfile b/rate_design/hp_rates/ct/Justfile index e9ad9ada..8edbad9d 100644 --- a/rate_design/hp_rates/ct/Justfile +++ b/rate_design/hp_rates/ct/Justfile @@ -114,7 +114,8 @@ create-bulk-tx-mc-data-all *extra_args: # ============================================================================= # CT-only: distribution + sub-transmission marginal costs (CL&P MCOS-2 Table 3) # -# Uses CL&P's own system-wide diluted marginal cost ($20.17/kW-yr, 2026$) +# Uses CL&P's system-wide diluted marginal cost grossed up to secondary +# voltage ($20.17 × 1.051845137 = $21.22/kW-yr, 2026$; OCC-863 Attachment 2). # allocated via the standard Probability-of-Peak (PoP) method — same # methodology as NY/RI/MD — on CT zone hourly load (top-100 hours by default). # See context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md. diff --git a/rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv b/rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv index 2b7e60af..6f251b14 100644 --- a/rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv +++ b/rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv @@ -1,2 +1,2 @@ utility,sub_tx_and_dist_mc_kw_yr,dollar_year -ct_eversource,20.17,2026 +ct_eversource,21.22,2026 diff --git a/tests/test_ct_dist_mc.py b/tests/test_ct_dist_mc.py index 15819a19..96353c80 100644 --- a/tests/test_ct_dist_mc.py +++ b/tests/test_ct_dist_mc.py @@ -4,8 +4,9 @@ tested for NY/RI/MD in test_marginal_cost_allocation.py and test_tx_dx_load_layouts.py. These tests cover the CT-specific wiring: the ISO-NE zone mapping (both CT utilities map to the single CT load zone), and an end-to-end PoP allocation on a synthetic -CT-shaped (summer-peaking) load profile using the actual $20.17/kW-yr MCOS-2 Table 3 -figure. See context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md. +CT-shaped (summer-peaking) load profile using the secondary loss-adjusted +$21.22/kW-yr scalar (MCOS-2 Table 3 × the OCC-863 secondary loss factor). +See context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md. """ from __future__ import annotations @@ -57,7 +58,10 @@ def test_get_utility_zone_mapping_includes_ct(self) -> None: assert utility_zone_map["ct_ui"] == ["CT"] -# ── CT marginal cost config CSV (MCOS-2 Table 3) ───────────────────────────── +# ── CT marginal cost config CSV (Table 3 × secondary loss factor) ──────────── + +# OCC-863 Attachment 2, page 1, cell N3 ("Secondary" loss factor). J3 = 20.17 * N3. +_SECONDARY_LOSS_FACTOR = 1.051845136935937 class TestCtMarginalCostTable: @@ -66,10 +70,16 @@ def test_config_csv_exists(self) -> None: f"Expected CT marginal cost config at {CT_MC_TABLE_PATH}" ) - def test_ct_eversource_value_matches_mcos2_table3(self) -> None: + def test_ct_eversource_value_is_table3_times_secondary_loss_factor(self) -> None: + """MCOS-2 Table 3 ($20.17/kW-yr) grossed up to secondary service. + + OCC-863 Attachment 2 page 1 sets the annual secondary cost to + 20.17 times the secondary loss factor, which rounds to $21.22/kW-yr. + """ mc_df = pl.read_csv(CT_MC_TABLE_PATH) mc = get_marginal_cost_for_utility(mc_df, "ct_eversource") - assert mc == pytest.approx(20.17) + assert mc == pytest.approx(21.22) + assert mc == pytest.approx(round(20.17 * _SECONDARY_LOSS_FACTOR, 2)) def test_dollar_year_is_2026(self) -> None: """MCOS-2 Table 3 is filed in 2026$; CPI inflation converts to run year.""" From 9cddada1f0eed197613d011724b74ea15cc20db0 Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Wed, 7 Oct 2026 18:36:24 +0000 Subject: [PATCH 02/12] upload and transcribe OCC discovery files --- context/README.md | 1 + context/code/data/ct_occ863_system_load.md | 70 +++++++ .../ct_eversource_dist_mc_methodology.md | 30 +++ rate_design/hp_rates/ct/Justfile | 16 ++ tests/test_ct_occ863_system_load.py | 141 +++++++++++++ .../convert_ct_occ863_system_load.py | 190 ++++++++++++++++++ 6 files changed, 448 insertions(+) create mode 100644 context/code/data/ct_occ863_system_load.md create mode 100644 tests/test_ct_occ863_system_load.py create mode 100644 utils/data_prep/marginal_costs/convert_ct_occ863_system_load.py diff --git a/context/README.md b/context/README.md index f3f1cb84..f2cb06cd 100644 --- a/context/README.md +++ b/context/README.md @@ -170,6 +170,7 @@ ResStock, EIA, NYISO, ISO-NE: where data comes from, how to read and prepare it. | ct_tariff_fetch.md | CT gas tariff **fetch** (ticket scope: RateAcuity URDB JSONs + `null_gas_tariff` + envelope only): schedules for CNG / Yankee / SCG (three residential classes each) plus Norwich muni (GRES + GSHRES/`_mf`), evidence. Omits (reference only, with Zotero links): CNG Rate SE Seasonal Gas Service; System Expansion `*-SE` (RSH-SE / 02-SE etc.); third-party supply. **Known limitation:** IOU `_mf` demand / MDQ charges dropped by `tariff_fetch`; CAIRO gas path has no MDQ billing. Mapper / tariff maps are follow-up. | | ct_eversource_custom_rates.md | CT Eversource hand-encoded proposed rates (Docket 26-05-10): `ct_eversource_rate6` (HP space-heating) + `ct_eversource_rate1_proposed` (matched comparison baseline), each as delivery + `_supply` JSON pairs with top-up YAMLs. Documents the filed-2026 delivery vintage + Genability 2025 supply, the ESI/RDM roll-in and SBC/CTA/FMCC decisions, full per-number provenance to the CLP-Rates exhibits, the transmission vintage note (why filed 0.05050 differs from RateAcuity 0.03401/0.04433), and the **pre-calc caveat** (fixed rates, no calibrated RR — bill-calc runs only). | | ct_eversource_class_rev_requirement.md | CT Eversource **whole residential rate class** revenue-requirement files (Docket 26-05-10): the test-year (`ct_eversource_rate_case_test_year.yaml`, current rates) and rate-year-1 (`ct_eversource_rate_case_rate_year_1.yaml`, Proposed+GET) Track 2 YAMLs that model every residential ResStock building on Rate 1 but size base DRR / kWh / customer count to the class (Rate 1 + 5 + 7). Documents the bug it fixes (whole-class ResStock pinned to a Rate-1 target), how top-ups scale off the two class-total scalars, full per-number provenance to the CLP-RATES-2 exhibit with derivations, and the Proposed+GET vs pre-GET caveat. | +| ct_occ863_system_load.md | OCC-863 (Docket 26-05-10) source files: where the four workbooks and the discovery PDF live on S3 and the EBS mirror. Attachment 3 page 2 becomes `system_load.parquet` (hourly distribution-substation load, MW, 2022–2025); page 1 becomes `system_monthly_peak_mw.parquet`. Notes how to feed the hourly file into the CT distribution MC run without changing `--utility-load-s3-base`. | ### code/marginal_costs/ diff --git a/context/code/data/ct_occ863_system_load.md b/context/code/data/ct_occ863_system_load.md new file mode 100644 index 00000000..7ab2b5e0 --- /dev/null +++ b/context/code/data/ct_occ863_system_load.md @@ -0,0 +1,70 @@ +# OCC-863 system load (Eversource CT, Docket 26-05-10) + +OCC-863 asked for hourly system load, hourly marginal cost, monthly system peak, and a residential +class 8760, for three historical years and three forecast years. Eversource's response (filed +October 6, 2026) points each subpart at an attachment. Subparts (a) and (c) use hourly +distribution-substation loads. The response says no forecasted hourly system loads are available. +Subpart (b) assigns the system-wide marginal substation and trunkline cost per kW to each hour type +within the month. + +## Where the files are + +Raw workbooks, the discovery-request PDF, and the parquets built from Attachment 3: + +```text +s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/ +/ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/ +├── pdf/OCC-863_Final.pdf +├── xlsx/attachment-1.xlsx # (a) hourly system load, 2023–2025 +├── xlsx/attachment-2.xlsx # (b) hourly marginal cost by month, primary and secondary +├── xlsx/attachment-3.xlsx # (c) monthly maximum MW, plus the hourly loads for 2022–2025 +├── xlsx/attachment-4.xlsx # (d) 2025 hourly kW for Rates 1, 5, and 7 +└── parquet/ + ├── system_load.parquet + └── system_monthly_peak_mw.parquet +``` + +The local path is the EBS mirror of the S3 prefix (`/ebs/data/` mirrors `s3://data.sb/`). The +workbooks are not in git. + +## Why page 2, not Attachment 1 + +Attachment 1 (subpart a) and Attachment 3 page 2 are the same substation-sum series where they +overlap. They differ on three hours: 1:00 a.m. on the fall-back Sunday in 2023, 2024, and 2025. +Attachment 1 stores exactly twice the page 2 value there. Page 1's November "Max MW" matches page +2, so the doubled hour is the one to drop. Page 2 also includes 2022, which Attachment 1 does not. +Both sheets fill the missing spring-forward hour (2:00 a.m.) with the average of the hours on +either side. + +`system_load.parquet` is built from page 2. `system_monthly_peak_mw.parquet` is page 1. The +converter checks that each page 1 peak equals the maximum of that month on page 2. + +## Columns + +`system_load.parquet` has `timestamp` (hour-beginning, no timezone) and `load_mw`. One file holds +2022–2025. 2024 has 8,784 hours because February 29 is kept. A few Excel timestamps sit a fraction +of a second before the hour; the converter rounds those to the nearest hour. The load is the sum +across distribution substations, in megawatts. It is not a per-substation average and not ISO-NE +zone load. + +`system_monthly_peak_mw.parquet` has `year`, `month` (1–12), and `peak_mw`. Four years, 48 rows. + +Rebuild both with: + +```bash +just -f rate_design/hp_rates/ct/Justfile convert-occ863-system-load \ + /ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/xlsx/attachment-3.xlsx \ + /ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet +``` + +Then sync the `occ-863/` directory to the S3 prefix above. Attachments 2 and 4 are archived only. + +## Using this load for CT distribution marginal cost + +The distribution marginal-cost recipe still reads ISO-NE Connecticut zone load through +`--utility-load-s3-base` (`s3://data.sb/isone/hourly_demand/utilities/`). That flag scans a +hive-partitioned folder and filters on `utility` and `year`. Do not point it at this prefix. + +When the CT run should use this substation series instead, read `system_load.parquet`, keep the +load year, and pass that table to `normalize_load_to_cairo_8760`. That function already requires +`timestamp` and `load_mw`. The ISO-NE reader stays as it is for the other states. diff --git a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md index 087bb467..2dd1ef1e 100644 --- a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md +++ b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md @@ -306,3 +306,33 @@ allocation or as a direct replacement. This is a nice-to-have, not a blocker for or equivalent) once CT scenario YAMLs exist, so a CAIRO run actually consumes this MC output. No CT scenario configs exist yet in `rate_design/hp_rates/ct/config/scenarios/` — that's a separate, larger piece of CT onboarding beyond this MC-generation task. + +--- + +## 6. OCC-863 source files + +OCC-863 (Docket 26-05-10, filed October 6, 2026) asked for three historical years and three +forecast years of: (a) average hourly system load by month, (b) average hourly marginal cost by +month, (c) maximum monthly system peak, and (d) an 8760 for the residential class. Eversource +answered with four workbooks. Subparts (a) and (c) are historical only: both are built from hourly +distribution-substation loads for the last three years, and the response says no forecasted hourly +system loads are available. Subpart (b) is the system-wide marginal distribution substation and +trunkline cost per kW, assigned to each hour type within the month. That workbook is also where the +secondary loss factor behind the `$21.22/kW-yr` scalar comes from (§2.2). + +The raw workbooks and the discovery-request PDF live at +`s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/`, mirrored locally at +`/ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/`. + +The hourly series we keep is Attachment 3 page 2, not Attachment 1. Page 2 is the substation sum +for 2022–2025. Attachment 1 covers 2023–2025 and, on the three fall-back Sundays, stores twice the +page 2 value at 1:00 a.m. Page 1's monthly "Max MW" matches page 2, including those Novembers. +`parquet/system_load.parquet` (`timestamp`, `load_mw`) is page 2. `parquet/system_monthly_peak_mw.parquet` +(`year`, `month`, `peak_mw`) is page 1. 2024 includes February 29 (8784 hours). + +Neither file is an input to the PoP allocator yet. That run still reads ISO-NE CT zone load via +`--utility-load-s3-base`. When CT switches to this substation series, read `system_load.parquet`, +filter to the load year, and pass the table to `normalize_load_to_cairo_8760`. Do not point +`--utility-load-s3-base` at the OCC-863 prefix: that flag expects the hive `utility`/`year` layout +the other states use. How to rebuild the parquets: +[ct_occ863_system_load.md](../../code/data/ct_occ863_system_load.md). diff --git a/rate_design/hp_rates/ct/Justfile b/rate_design/hp_rates/ct/Justfile index 8edbad9d..ac9cbb20 100644 --- a/rate_design/hp_rates/ct/Justfile +++ b/rate_design/hp_rates/ct/Justfile @@ -141,6 +141,22 @@ create-dist-mc-data year_arg *extra_args: --n-hours {{ upstream_hours }} \ {{ extra_args }} +# ============================================================================= +# CT-only: OCC-863 Attachment 3 → hourly system load and monthly peaks (MW) +# +# Page 2 is the hourly distribution-substation load for 2022–2025 (2024 keeps +# the leap day). Page 1 is each month's maximum MW. Writes +# system_load.parquet and system_monthly_peak_mw.parquet. See +# context/code/data/ct_occ863_system_load.md. +# +# just -f ct/Justfile convert-occ863-system-load +# ============================================================================= + +convert-occ863-system-load path_xlsx path_output_dir: + cd {{ path_repo }} && uv run python {{ path_repo }}/utils/data_prep/marginal_costs/convert_ct_occ863_system_load.py \ + --path-xlsx {{ path_xlsx }} \ + --path-output-dir {{ path_output_dir }} + # ============================================================================= # CT-only: gas tariff fetch # diff --git a/tests/test_ct_occ863_system_load.py b/tests/test_ct_occ863_system_load.py new file mode 100644 index 00000000..799c934b --- /dev/null +++ b/tests/test_ct_occ863_system_load.py @@ -0,0 +1,141 @@ +"""OCC-863 Attachment 3 → hourly system load and monthly peaks. No S3. + +The real workbook is not in git. These tests build a small stand-in with the +timestamp drift the workbook has, and a page-1 table with a title row and a +source note. +""" + +from __future__ import annotations + +import polars as pl +import pytest + +from utils.data_prep.marginal_costs.convert_ct_occ863_system_load import ( + assert_monthly_peaks_match_hourly, + hourly_system_load, + monthly_peak_mw, +) + + +def _year_hours(year: int) -> pl.Series: + n_hours = 8784 if year % 4 == 0 else 8760 + end = pl.select( + pl.lit(f"{year}-01-01").str.to_datetime() + pl.duration(hours=n_hours - 1) + ).item() + return pl.datetime_range( + pl.select(pl.lit(f"{year}-01-01").str.to_datetime()).item(), + end, + interval="1h", + eager=True, + ) + + +def _page2( + years: list[int], *, drift_first_of_second_year: bool = False +) -> pl.DataFrame: + frames = [pl.DataFrame({"DateTime": _year_hours(year)}) for year in years] + raw = pl.concat(frames).with_columns( + (1000.0 + (pl.int_range(pl.len()) % 24)).alias("Actual Load"), + ) + if drift_first_of_second_year: + first_of_second = _year_hours(years[1])[0] + raw = raw.with_columns( + pl.when(pl.col("DateTime") == first_of_second) + .then(pl.col("DateTime") - pl.duration(microseconds=2000)) + .otherwise(pl.col("DateTime")) + .alias("DateTime") + ) + return raw + + +def _page1_sheet() -> pl.DataFrame: + """Title, year header, unit row, twelve months, then a source note.""" + rows: list[tuple[str | None, ...]] = [ + ("Monthly Max MW", None, None, None, None), + ("Month", "2025", "2024", "2023", "2022"), + (None, "Max MW", "Max MW", "Max MW", "Max MW"), + ] + for month in range(1, 13): + rows.append( + ( + str(month), + str(3000 + month), + str(2000 + month), + str(1000 + month), + str(500 + month), + ) + ) + rows.append(("Source:", "from substations", None, None, None)) + return pl.DataFrame( + rows, + schema=["column_1", "column_2", "column_3", "column_4", "column_5"], + orient="row", + ) + + +def test_rounds_a_timestamp_that_excel_stored_just_before_the_hour() -> None: + load = hourly_system_load( + _page2([2022, 2023, 2024, 2025], drift_first_of_second_year=True) + ) + new_year = load.filter( + pl.col("timestamp").dt.strftime("%Y-%m-%d %H") == "2023-01-01 00" + ) + assert new_year.height == 1 + assert load["timestamp"].n_unique() == load.height + + +def test_keeps_the_leap_day_and_names_the_load_column_in_mw() -> None: + load = hourly_system_load(_page2([2022, 2023, 2024, 2025])) + assert load.columns == ["timestamp", "load_mw"] + counts = dict( + load.with_columns(pl.col("timestamp").dt.year().alias("year")) + .group_by("year") + .len() + .iter_rows() + ) + assert counts == {2022: 8760, 2023: 8760, 2024: 8784, 2025: 8760} + leap = load.filter( + pl.col("timestamp").dt.strftime("%Y-%m-%d %H") == "2024-02-29 00" + ) + assert leap.height == 1 + + +def test_rejects_a_non_positive_load() -> None: + raw = _page2([2022, 2023, 2024, 2025]).with_columns( + pl.when(pl.col("DateTime").dt.strftime("%Y-%m-%d %H") == "2023-06-01 12") + .then(0.0) + .otherwise(pl.col("Actual Load")) + .alias("Actual Load") + ) + with pytest.raises(ValueError, match="non-positive"): + hourly_system_load(raw) + + +def test_monthly_peaks_skip_the_title_and_the_source_note() -> None: + peaks = monthly_peak_mw(_page1_sheet()) + assert peaks.columns == ["year", "month", "peak_mw"] + assert peaks.height == 48 + jan_2025 = peaks.filter((pl.col("year") == 2025) & (pl.col("month") == 1)) + assert jan_2025["peak_mw"][0] == 3001.0 + assert peaks["year"].unique().sort().to_list() == [2022, 2023, 2024, 2025] + + +def test_rejects_a_monthly_peak_that_is_not_the_hourly_maximum() -> None: + load = hourly_system_load(_page2([2022, 2023, 2024, 2025])) + peaks = ( + load.with_columns( + pl.col("timestamp").dt.year().alias("year"), + pl.col("timestamp").dt.month().alias("month"), + ) + .group_by("year", "month") + .agg(pl.col("load_mw").max().alias("peak_mw")) + .sort("year", "month") + ) + broken = peaks.with_columns( + pl.when((pl.col("year") == 2022) & (pl.col("month") == 11)) + .then(pl.col("peak_mw") + 100) + .otherwise(pl.col("peak_mw")) + .alias("peak_mw") + ) + with pytest.raises(ValueError, match="do not match"): + assert_monthly_peaks_match_hourly(load, broken) diff --git a/utils/data_prep/marginal_costs/convert_ct_occ863_system_load.py b/utils/data_prep/marginal_costs/convert_ct_occ863_system_load.py new file mode 100644 index 00000000..a76cabe7 --- /dev/null +++ b/utils/data_prep/marginal_costs/convert_ct_occ863_system_load.py @@ -0,0 +1,190 @@ +"""Convert OCC-863 Attachment 3 to hourly system load and monthly peaks. + +Attachment 3 answers OCC-863 subpart (c) for PURA Docket 26-05-10. Page 2 is the +hourly sum of Eversource CT distribution-substation loads for 2022–2025, in +megawatts. Page 1 is the monthly maximum of that series. + +Page 2 is the source for ``system_load.parquet``, not Attachment 1. The two +sheets agree on every hour except 1:00 a.m. on the three fall-back Sundays, +where Attachment 1 stores twice the page 2 value. Page 1's November maximum +matches page 2. + +Excel stores a handful of timestamps a fraction of a second before the hour. +Those are rounded to the nearest hour. 2024 keeps its leap day (8784 hours). +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import polars as pl + +_HOURLY_SHEET = "OCC-863 Attachment 3 Page 2" +_MONTHLY_SHEET = "OCC-863 Attachment 3 Page 1" +_HOURLY_FILENAME = "system_load.parquet" +_MONTHLY_FILENAME = "system_monthly_peak_mw.parquet" +_EXPECTED_HOURS = {2022: 8760, 2023: 8760, 2024: 8784, 2025: 8760} + + +def hourly_system_load(raw: pl.DataFrame) -> pl.DataFrame: + """Return ``timestamp`` and ``load_mw`` from Attachment 3 page 2. + + ``raw`` must have ``DateTime`` and ``Actual Load``. Blank rows are dropped. + Timestamps are rounded to the nearest hour and must form one contiguous + hour-beginning series for each of 2022, 2023, 2024, and 2025. + """ + required = {"DateTime", "Actual Load"} + missing = required - set(raw.columns) + if missing: + raise ValueError(f"Attachment 3 page 2 is missing columns: {sorted(missing)}") + + load = ( + raw.select( + pl.col("DateTime").alias("timestamp"), + pl.col("Actual Load").cast(pl.Float64).alias("load_mw"), + ) + .drop_nulls(subset=["timestamp", "load_mw"]) + .with_columns(pl.col("timestamp").dt.round("1h")) + .sort("timestamp") + ) + _validate_system_load(load) + return load + + +def monthly_peak_mw(raw: pl.DataFrame) -> pl.DataFrame: + """Return ``year``, ``month``, and ``peak_mw`` from Attachment 3 page 1. + + ``raw`` is the sheet read with no header. The year labels sit on the row + whose first cell is ``Month``; the twelve data rows follow, and a source + note follows those. Years come out oldest-first. + """ + label_col = raw.columns[0] + labels = raw.get_column(label_col).to_list() + try: + header_idx = labels.index("Month") + except ValueError as exc: + raise ValueError("Attachment 3 page 1 has no Month header row") from exc + + year_cols: list[tuple[str, int]] = [] + for col in raw.columns[1:]: + year_label = raw[col][header_idx] + if year_label is not None and str(year_label).isdigit(): + year_cols.append((col, int(str(year_label)))) + if not year_cols: + raise ValueError("Attachment 3 page 1 has no year columns") + + rows: list[dict[str, int | float]] = [] + for record in raw.slice(header_idx + 1).iter_rows(named=True): + month_label = record[label_col] + if month_label is None or not str(month_label).isdigit(): + continue + month = int(str(month_label)) + for col, year in year_cols: + value = record[col] + if value is None: + raise ValueError(f"missing peak for {year}-{month:02d}") + rows.append({"year": year, "month": month, "peak_mw": float(value)}) + + peaks = pl.DataFrame(rows).sort("year", "month") + _validate_monthly_peaks(peaks) + return peaks + + +def assert_monthly_peaks_match_hourly(load: pl.DataFrame, peaks: pl.DataFrame) -> None: + """Raise if a page 1 peak is not the maximum of that month on page 2.""" + from_hourly = ( + load.with_columns( + pl.col("timestamp").dt.year().alias("year"), + pl.col("timestamp").dt.month().alias("month"), + ) + .group_by("year", "month") + .agg(pl.col("load_mw").max().alias("hourly_max")) + ) + compared = peaks.join(from_hourly, on=["year", "month"], how="left") + mismatched = compared.filter( + pl.col("hourly_max").is_null() + | ((pl.col("peak_mw") - pl.col("hourly_max")).abs() > 1e-6) + ) + if mismatched.height: + sample = mismatched.select("year", "month", "peak_mw", "hourly_max").head(4) + raise ValueError(f"monthly peaks do not match the hourly series:\n{sample}") + + +def _validate_system_load(load: pl.DataFrame) -> None: + if load["timestamp"].n_unique() != load.height: + raise ValueError("rounded timestamps are not unique") + if load.filter(pl.col("load_mw") <= 0).height: + raise ValueError("system load has a non-positive MW value") + + years = load.with_columns(pl.col("timestamp").dt.year().alias("year")) + counts = dict(years.group_by("year").len().iter_rows()) + if counts != _EXPECTED_HOURS: + raise ValueError(f"expected hours {_EXPECTED_HOURS}, got {counts}") + + for year, hours in _EXPECTED_HOURS.items(): + year_ts = years.filter(pl.col("year") == year)["timestamp"] + start = year_ts[0] + if (start.year, start.month, start.day, start.hour) != (year, 1, 1, 0): + raise ValueError(f"{year} does not start at Jan 1 hour 0: {start}") + if year_ts.len() != hours: + raise ValueError(f"{year} has {year_ts.len()} hours, expected {hours}") + + +def _validate_monthly_peaks(peaks: pl.DataFrame) -> None: + expected = {(year, month) for year in _EXPECTED_HOURS for month in range(1, 13)} + found = set(peaks.select("year", "month").iter_rows()) + if found != expected: + raise ValueError( + f"expected monthly peaks {sorted(expected)}, got {sorted(found)}" + ) + if peaks.filter(pl.col("peak_mw") <= 0).height: + raise ValueError("monthly peak has a non-positive MW value") + + +def write_occ863_load_parquets( + path_xlsx: Path, path_output_dir: Path +) -> tuple[pl.DataFrame, pl.DataFrame]: + """Read Attachment 3 and write both parquets under ``path_output_dir``.""" + if "{{" in str(path_xlsx) or "{{" in str(path_output_dir): + raise ValueError( + f"path looks like an uninterpolated Just variable: {path_xlsx} {path_output_dir}" + ) + raw_hourly = pl.read_excel( + path_xlsx, + sheet_name=_HOURLY_SHEET, + columns=["DateTime", "Actual Load"], + ) + raw_monthly = pl.read_excel(path_xlsx, sheet_name=_MONTHLY_SHEET, has_header=False) + load = hourly_system_load(raw_hourly) + peaks = monthly_peak_mw(raw_monthly) + assert_monthly_peaks_match_hourly(load, peaks) + + path_output_dir.mkdir(parents=True, exist_ok=True) + load.write_parquet(path_output_dir / _HOURLY_FILENAME) + peaks.write_parquet(path_output_dir / _MONTHLY_FILENAME) + return load, peaks + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--path-xlsx", type=Path, required=True, help="OCC-863 Attachment 3 workbook" + ) + parser.add_argument( + "--path-output-dir", + type=Path, + required=True, + help="Directory for system_load.parquet and system_monthly_peak_mw.parquet", + ) + args = parser.parse_args() + load, peaks = write_occ863_load_parquets(args.path_xlsx, args.path_output_dir) + print(f"Wrote {load.height:,} hours to {args.path_output_dir / _HOURLY_FILENAME}") + print(f" load_mw min={load['load_mw'].min():.1f} max={load['load_mw'].max():.1f}") + print( + f"Wrote {peaks.height} monthly peaks to {args.path_output_dir / _MONTHLY_FILENAME}" + ) + + +if __name__ == "__main__": + main() From badc46dd60c29c449f793408ee656c970e4b4852 Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Wed, 7 Oct 2026 18:40:55 +0000 Subject: [PATCH 03/12] Use rate year 1 (2026) dollar values for MC --- .../ct_eversource_dist_mc_methodology.md | 31 +++++++++---------- .../dist_mc_definition_choice.md | 5 +-- rate_design/hp_rates/ct/Justfile | 7 +++-- tests/test_ct_dist_mc.py | 2 +- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md index 2dd1ef1e..f75d55c2 100644 --- a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md +++ b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md @@ -113,9 +113,11 @@ ct_eversource,21.22,2026 `$21.22` is Table 3's `$20.17` times the OCC-863 secondary loss factor, rounded to the cent (§2.2). `ct_ui` (United Illuminating) is **not** covered by this MCOS — it needs its own source (see -[Open questions](#4-open-questions--decisions-needed) below). The optional `dollar_year=2026` lets -the existing CPI-inflation logic in `generate_utility_tx_dx_mc.py` handle any run year other than -2026 with no code change. +[Open questions](#4-open-questions--decisions-needed) below). `dollar_year=2026` records that +`$21.22` is the rate year 1 figure, in the filing's own dollars. The CT recipe passes +`--target-dollar-year 2026`, which matches that column, so `generate_utility_tx_dx_mc.py` allocates +`$21.22` as provided. It does not CPI-adjust the scalar onto the load year. RI and MD still inflate +from their `dollar_year` to the run year; this skip is CT-only. --- @@ -192,14 +194,11 @@ bulk-TX MC. 2. **Created the config CSV** (§2.3). 3. **Filled the utility-level load gap** (§3.3) — `ct_eversource`/`ct_ui` zone-mapping rows, ran the ISO-NE utility aggregation for 2025, uploaded to S3. -4. **Refreshed CPI data through 2026**: the config scalar is filed in 2026$ - (`dollar_year=2026` in the config CSV), but `data/fred/cpi/parquet/` only had annual averages - through 2025. Ran `just -f data/fred/cpi/Justfile fetch-cpi CPIAUCSL 2019 2026` (2026 is a - partial-year average — 6 months as of this run — since FRED lags by ~1 month) and uploaded. - The run below used the pre-loss Table 3 scalar: `$20.17 → $19.65/kW-yr` in 2025$ (CPI factor - 0.9740). The config is now the secondary loss-adjusted `$21.22/kW-yr` (§2.2); the same factor - applied to that scalar is about `$20.67/kW-yr` in 2025$. Re-run `create-dist-mc-data` to - replace the S3 parquet, which still reflects the pre-loss scalar. +4. **Rate year 1 uses `$21.22` with no CPI adjustment.** An earlier run deflated the pre-loss + Table 3 scalar from 2026$ to the 2025 load year (`$20.17 → $19.65`, CPI factor 0.9740). The + same factor on `$21.22` would be about `$20.67`. The CT analysis is rate year 1, so the recipe + now passes `--target-dollar-year 2026` and the allocator keeps `$21.22`. The S3 parquet from + that earlier run still has the deflated pre-loss scalar until `create-dist-mc-data` is re-run. 5. **Ran the standard PoP allocation** via the new `just -f ct/Justfile create-dist-mc-data 2025 --upload` recipe (§3.5): ```bash @@ -280,11 +279,8 @@ allocation or as a direct replacement. This is a nice-to-have, not a blocker for CT-specific question, but CT's MCOS happens to compute Bonbright-style customer/facilities marginal costs explicitly (Tables 4–7), which makes the gap visible. Worth raising with the team as a cross-cutting design question (see §1). -- **2026 CPI is a partial-year average.** The CPI inflation factor (§3.4 point 4) uses a 2026 annual - average computed from only the months FRED had published as of this implementation. As more 2026 - months are published, re-running `just fetch-cpi` will shift the 2026 average slightly, which - would change the `$21.22 → ~$20.67` inflated value by a small amount. Not expected to matter - materially, but worth knowing if the output value changes on a future re-run. +- **No CPI adjustment on the CT scalar.** Rate year 1 uses `$21.22/kW-yr` as filed (§2.3, §3.4 + point 4). A future FRED update to the partial-year 2026 CPI average does not change this figure. --- @@ -301,7 +297,8 @@ allocation or as a direct replacement. This is a nice-to-have, not a blocker for to the source-number and per-state tables.~~ Done. 7. **Remaining**: re-run `create-dist-mc-data 2025 --upload`. The parquet at `s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet` - was built from the pre-loss `$20.17/kW-yr` scalar. The config CSV is now `$21.22/kW-yr`. + was built from the pre-loss `$20.17/kW-yr` scalar after a CPI deflation to 2025$. The recipe now + allocates the unadjusted rate year 1 scalar, `$21.22/kW-yr`. 8. **Remaining**: wire `path_dist_and_sub_tx_mc` into a CT scenario config (`scenarios_ct_eversource.yaml` or equivalent) once CT scenario YAMLs exist, so a CAIRO run actually consumes this MC output. No CT scenario configs exist yet in `rate_design/hp_rates/ct/config/scenarios/` — that's a separate, diff --git a/context/methods/marginal_costs/dist_mc_definition_choice.md b/context/methods/marginal_costs/dist_mc_definition_choice.md index 2f3beb06..f6c64846 100644 --- a/context/methods/marginal_costs/dist_mc_definition_choice.md +++ b/context/methods/marginal_costs/dist_mc_definition_choice.md @@ -92,8 +92,9 @@ locational figure (`$86.58`) is itself much larger than NY's undiluted MCOS figu All four feed the _same_ PoP allocator ([generate_utility_tx_dx_mc.py](../../../utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py)) -in the identical `sub_tx_and_dist_mc_kw_yr` slot, with CPI inflation via the optional `dollar_year` -column. +in the identical `sub_tx_and_dist_mc_kw_yr` slot. RI and MD CPI-inflate from the optional +`dollar_year` column to the run year. CT's rate year 1 run does not: the recipe sets +`--target-dollar-year 2026`, so `$21.22/kW-yr` is allocated as filed. --- diff --git a/rate_design/hp_rates/ct/Justfile b/rate_design/hp_rates/ct/Justfile index ac9cbb20..d5367960 100644 --- a/rate_design/hp_rates/ct/Justfile +++ b/rate_design/hp_rates/ct/Justfile @@ -115,8 +115,10 @@ create-bulk-tx-mc-data-all *extra_args: # CT-only: distribution + sub-transmission marginal costs (CL&P MCOS-2 Table 3) # # Uses CL&P's system-wide diluted marginal cost grossed up to secondary -# voltage ($20.17 × 1.051845137 = $21.22/kW-yr, 2026$; OCC-863 Attachment 2). -# allocated via the standard Probability-of-Peak (PoP) method — same +# voltage ($20.17 × 1.051845137 = $21.22/kW-yr; OCC-863 Attachment 2). +# Rate year 1 is modeled in those filing dollars: --target-dollar-year 2026 +# matches the CSV, so the allocator does not CPI-adjust $21.22 to the load +# year. Allocated via the standard Probability-of-Peak (PoP) method — same # methodology as NY/RI/MD — on CT zone hourly load (top-100 hours by default). # See context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md. # @@ -135,6 +137,7 @@ create-dist-mc-data year_arg *extra_args: --utility ct_eversource \ --year {{ year_arg }} \ --load-year {{ year_arg }} \ + --target-dollar-year 2026 \ --mc-table-path {{ path_config }}/marginal_costs/ct_marginal_costs_{{ year_arg }}.csv \ --utility-load-s3-base {{ path_s3_utility_loads }} \ --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ \ diff --git a/tests/test_ct_dist_mc.py b/tests/test_ct_dist_mc.py index 96353c80..b355bea9 100644 --- a/tests/test_ct_dist_mc.py +++ b/tests/test_ct_dist_mc.py @@ -82,7 +82,7 @@ def test_ct_eversource_value_is_table3_times_secondary_loss_factor(self) -> None assert mc == pytest.approx(round(20.17 * _SECONDARY_LOSS_FACTOR, 2)) def test_dollar_year_is_2026(self) -> None: - """MCOS-2 Table 3 is filed in 2026$; CPI inflation converts to run year.""" + """Rate year 1 dollars. The CT recipe sets --target-dollar-year 2026 so this is not CPI-adjusted.""" mc_df = pl.read_csv(CT_MC_TABLE_PATH) row = mc_df.filter(pl.col("utility") == "ct_eversource") assert int(row["dollar_year"][0]) == 2026 From d94dd8f95591943bcfcfe3035fccb9d5ee01ef19 Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Wed, 7 Oct 2026 18:43:15 +0000 Subject: [PATCH 04/12] Use rate year 1 (2026) dollar values for MC --- context/README.md | 24 +++++++++---------- .../ct_eversource_dist_mc_methodology.md | 6 +++-- .../generate_utility_tx_dx_mc.py | 12 +++++----- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/context/README.md b/context/README.md index f2cb06cd..5c971f8b 100644 --- a/context/README.md +++ b/context/README.md @@ -77,18 +77,18 @@ Cost-reflective TOU design: theory and window optimization. How we derive BAT marginal-cost inputs from regulatory studies. -| File | Purpose | -| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| ny_dist_mc_bat_methodology.md | Shared methodology for deriving sub-TX and distribution BAT marginal cost inputs from the seven NY MCOS workbooks: pipeline overview, key conceptual choices (incremental not cumulative, levelized 7-year window, no discounting, 2026 dollar year), per-utility deviations and harmonization, final values and sanity check, research appendices. | -| ny_mcos_studies_comparison.md | Cross-utility comparison of the seven NY 2025 MCOS studies: what MCOS numbers mean (undiluted/diluted, grid topology), methodology comparison, annualization mechanics (workbook-verified step-by-step), system-wide diluted/undiluted tables, component taxonomy mapping (physical hierarchy → cost-center labels), bulk vs local TX scrub, reassessment of bulk TX MC gap. | -| ny_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for NY BAT: the gap, data sources (OATT ATRR, CLCPA project costs, FERC Order 1920), five options with tradeoffs, recommended approach (OATT as upper-bound proxy), hourly PoP allocation. | -| ri_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for RI BAT: data sources (RNS rate, AESC 2024 avoided PTF cost, ISO-NE 2050 study), four options with tradeoffs, recommended approach (AESC primary, RNS upper bound), hourly PoP allocation, comparison with NY. | -| ct_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for CT BAT: same AESC 2024 $69/kW-year and top-100 exceedance as RI, plus `--allocation-load` flag to toggle between NE system aggregate and CT zone load for peak-hour identification; utilities `ct_eversource` and `ct_ui`; implementation files and Just commands. | -| capacity_market_comparison_nyiso_isone.md | Comparison of NYISO ICAP (Strip/Monthly/Spot) vs ISO-NE FCM (FCA/ARA/MRA) capacity market structures: structural analogy table, forward horizons, price volatility differences, implications for BAT marginal cost signal (Spot vs FCA vs MRA), current approach in NY vs RI, trade-offs. | -| dist_mc_definition_choice.md | Cross-cutting: which LRMC definition the `sub_tx_and_dist` BAT input uses, and why it differs by state. Taxonomy (Turvey / AIC-avoided / FLIC) and why AIC and FLIC differ ~10-50x, source-number table (NY FLIC $1-8, RI AESC $80, BGE Brattle $32, BGE E3 $203-258, OPC $25-34, BAT paper $54.46), what is implemented per state (NY=FLIC, RI/MD=avoided), the platform tension (NY-vs-RI/MD; methodology-doc §3-vs-§7), rationale, and BGE sensitivities. | -| md_bge_dist_mc_methodology.md | BGE (MD) sub-TX + distribution MC implementation: adopts Brattle avoided distribution cost $32/kW-yr (2022$), RI-style single-scalar config + CPI handling, why avoided cost is consistent with BGE's own peak-driven COS, the ~82% volumetric over-recovery intervention hook, FLIC/E3 sensitivities, and the Stage-B allocator MD-enablement dependency. | -| ct_eversource_dist_mc_methodology.md | eversource (CT) sub-TX + dist MC: how to derive CL&P/Eversource CT's `sub_tx_and_dist` MC from Exhibits CLP-MCOS-1/2. Stage A: system-wide diluted $20.17/kW-yr (Table 3) vs locational $86.58/kW-yr, why diluted mirrors NY's FLIC convention, and the secondary loss gross-up to the implemented $21.22/kW-yr (OCC-863 Attachment 2). Stage B: CL&P's own hourly probability-of-peak method (std-dev of 2022-2025 substation load, BTM solar/HP-adoption forecast adjustment) vs. what data we actually have (Table 2A TOU/season buckets only — Back-Up 36's raw PoP curve is listed in the TOC but not in the PDF); Tier 1 TOU-bucket mapping (implementable now) vs Tier 2 load-based PoP proxy (stretch); task list and open questions (UI gap, locational sensitivity, TOU-window confirmation). | -| md_bulk_transmission.md | How to construct bulk TX marginal cost for MD BAT: PJM NITS rates as OATT ATRR proxy (E3 ICC-VDER precedent), zone-to-utility mapping (BGE/DPL/PEPCO/APS covering all 13 MD utilities), calendar-year day-weighted blending, full-year top-150 PCAF allocation (following E3), reference data at `data/pjm/bulk_tx/nits/`, implementation in `bulk_tx_pjm.py`. | +| File | Purpose | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| ny_dist_mc_bat_methodology.md | Shared methodology for deriving sub-TX and distribution BAT marginal cost inputs from the seven NY MCOS workbooks: pipeline overview, key conceptual choices (incremental not cumulative, levelized 7-year window, no discounting, 2026 dollar year), per-utility deviations and harmonization, final values and sanity check, research appendices. | +| ny_mcos_studies_comparison.md | Cross-utility comparison of the seven NY 2025 MCOS studies: what MCOS numbers mean (undiluted/diluted, grid topology), methodology comparison, annualization mechanics (workbook-verified step-by-step), system-wide diluted/undiluted tables, component taxonomy mapping (physical hierarchy → cost-center labels), bulk vs local TX scrub, reassessment of bulk TX MC gap. | +| ny_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for NY BAT: the gap, data sources (OATT ATRR, CLCPA project costs, FERC Order 1920), five options with tradeoffs, recommended approach (OATT as upper-bound proxy), hourly PoP allocation. | +| ri_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for RI BAT: data sources (RNS rate, AESC 2024 avoided PTF cost, ISO-NE 2050 study), four options with tradeoffs, recommended approach (AESC primary, RNS upper bound), hourly PoP allocation, comparison with NY. | +| ct_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for CT BAT: same AESC 2024 $69/kW-year and top-100 exceedance as RI, plus `--allocation-load` flag to toggle between NE system aggregate and CT zone load for peak-hour identification; utilities `ct_eversource` and `ct_ui`; implementation files and Just commands. | +| capacity_market_comparison_nyiso_isone.md | Comparison of NYISO ICAP (Strip/Monthly/Spot) vs ISO-NE FCM (FCA/ARA/MRA) capacity market structures: structural analogy table, forward horizons, price volatility differences, implications for BAT marginal cost signal (Spot vs FCA vs MRA), current approach in NY vs RI, trade-offs. | +| dist_mc_definition_choice.md | Cross-cutting: which LRMC definition the `sub_tx_and_dist` BAT input uses, and why it differs by state. Taxonomy (Turvey / AIC-avoided / FLIC) and why AIC and FLIC differ ~10-50x, source-number table (NY FLIC $1-8, RI AESC $80, BGE Brattle $32, BGE E3 $203-258, OPC $25-34, BAT paper $54.46), what is implemented per state (NY=FLIC, RI/MD=avoided), the platform tension (NY-vs-RI/MD; methodology-doc §3-vs-§7), rationale, and BGE sensitivities. | +| md_bge_dist_mc_methodology.md | BGE (MD) sub-TX + distribution MC implementation: adopts Brattle avoided distribution cost $32/kW-yr (2022$), RI-style single-scalar config + CPI handling, why avoided cost is consistent with BGE's own peak-driven COS, the ~82% volumetric over-recovery intervention hook, FLIC/E3 sensitivities, and the Stage-B allocator MD-enablement dependency. | +| ct_eversource_dist_mc_methodology.md | eversource (CT) sub-TX + dist MC: how to derive CL&P/Eversource CT's `sub_tx_and_dist` MC from Exhibits CLP-MCOS-1/2. Stage A: system-wide diluted $20.17/kW-yr (Table 3) vs locational $86.58/kW-yr, why diluted mirrors NY's FLIC convention, and the secondary loss gross-up to the implemented $21.22/kW-yr (OCC-863 Attachment 2), used as filed for rate year 1 with no CPI adjustment to the load year. Stage B: CL&P's own hourly probability-of-peak method (std-dev of 2022-2025 substation load, BTM solar/HP-adoption forecast adjustment) vs. what data we actually have (Table 2A TOU/season buckets only — Back-Up 36's raw PoP curve is listed in the TOC but not in the PDF); Tier 1 TOU-bucket mapping (implementable now) vs Tier 2 load-based PoP proxy (stretch); task list and open questions (UI gap, locational sensitivity, TOU-window confirmation). | +| md_bulk_transmission.md | How to construct bulk TX marginal cost for MD BAT: PJM NITS rates as OATT ATRR proxy (E3 ICC-VDER precedent), zone-to-utility mapping (BGE/DPL/PEPCO/APS covering all 13 MD utilities), calendar-year day-weighted blending, full-year top-150 PCAF allocation (following E3), reference data at `data/pjm/bulk_tx/nits/`, implementation in `bulk_tx_pjm.py`. | ### methods/ (root) diff --git a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md index f75d55c2..c103248c 100644 --- a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md +++ b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md @@ -3,8 +3,9 @@ **Status: IMPLEMENTED.** This document lays out how the `sub_tx_and_dist` BAT marginal-cost input for Eversource CT (CL&P) is derived from the two MCOS exhibits, and how it is allocated to an 8760 hourly signal. The config CSV, allocator wiring, and S3 output described below exist and have been -run; see §3.4 for the implementation notes (including two upstream data gaps — ISO-NE utility-level -load and 2026 CPI data — that had to be filled first). Only CT **bulk transmission** MC was +run; see §3.4 for the implementation notes (including the ISO-NE utility-level load gap that +had to be filled first). Rate year 1 uses the filed `$21.22/kW-yr` with no CPI adjustment (§2.3). +Only CT **bulk transmission** MC was implemented before this ([ct_bulk_transmission_marginal_cost.md](ct_bulk_transmission_marginal_cost.md)). @@ -204,6 +205,7 @@ bulk-TX MC. ```bash uv run python utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py \ --state CT --utility ct_eversource --year 2025 \ + --target-dollar-year 2026 \ --mc-table-path rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv \ --utility-load-s3-base s3://data.sb/isone/hourly_demand/utilities/ \ --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ \ diff --git a/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py b/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py index 147e8a02..3c3b8d13 100644 --- a/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py +++ b/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py @@ -12,11 +12,10 @@ - Load year (determines which load profile year to use) The optional ``dollar_year`` column in the MC table specifies the dollar year of -the cost estimate. When present, the script inflates the value to -``--target-dollar-year`` (defaults to ``--year``) using the annual-average -CPIAUCSL index stored at ``--cpi-s3-base`` (default: -``s3://data.sb/fred/cpi/``). If the column is absent (e.g. the existing NY -table), the raw value is used as-is with no inflation applied. +the cost estimate. When it differs from ``--target-dollar-year`` (default: +``--year``), the script inflates the value using the annual-average CPIAUCSL +index stored at ``--cpi-s3-base`` (default: ``s3://data.sb/fred/cpi/``). When +the years match, or the column is absent (the NY table), the raw value is used. Output partitions written as: - NY default base: s3://data.sb/switchbox/marginal_costs/ny/dist_and_sub_tx/ @@ -49,8 +48,9 @@ --output-s3-base s3://data.sb/switchbox/marginal_costs/ny/dist_and_sub_tx/ \ --upload - # CT/Eversource (CL&P) on ISO-NE native loads, inflates 2026$ → 2025$ + # CT/Eversource (CL&P): rate year 1 keeps the filed 2026$ scalar (no CPI adjustment) python generate_utility_tx_dx_mc.py --state CT --utility ct_eversource --year 2025 \ + --target-dollar-year 2026 \ --mc-table-path rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv \ --utility-load-s3-base s3://data.sb/isone/hourly_demand/utilities/ \ --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ \ From 7c6618f523e192ec482614f344520e165e6e0365 Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Wed, 7 Oct 2026 18:55:16 +0000 Subject: [PATCH 05/12] Add rate level load parquet files --- context/README.md | 2 +- context/code/data/ct_occ863_system_load.md | 32 ++- .../ct_eversource_dist_mc_methodology.md | 5 +- rate_design/hp_rates/ct/Justfile | 16 ++ tests/test_ct_occ863_rate_class_load.py | 79 ++++++++ .../convert_ct_occ863_rate_class_load.py | 186 ++++++++++++++++++ 6 files changed, 315 insertions(+), 5 deletions(-) create mode 100644 tests/test_ct_occ863_rate_class_load.py create mode 100644 utils/data_prep/marginal_costs/convert_ct_occ863_rate_class_load.py diff --git a/context/README.md b/context/README.md index 5c971f8b..3b70e7e0 100644 --- a/context/README.md +++ b/context/README.md @@ -170,7 +170,7 @@ ResStock, EIA, NYISO, ISO-NE: where data comes from, how to read and prepare it. | ct_tariff_fetch.md | CT gas tariff **fetch** (ticket scope: RateAcuity URDB JSONs + `null_gas_tariff` + envelope only): schedules for CNG / Yankee / SCG (three residential classes each) plus Norwich muni (GRES + GSHRES/`_mf`), evidence. Omits (reference only, with Zotero links): CNG Rate SE Seasonal Gas Service; System Expansion `*-SE` (RSH-SE / 02-SE etc.); third-party supply. **Known limitation:** IOU `_mf` demand / MDQ charges dropped by `tariff_fetch`; CAIRO gas path has no MDQ billing. Mapper / tariff maps are follow-up. | | ct_eversource_custom_rates.md | CT Eversource hand-encoded proposed rates (Docket 26-05-10): `ct_eversource_rate6` (HP space-heating) + `ct_eversource_rate1_proposed` (matched comparison baseline), each as delivery + `_supply` JSON pairs with top-up YAMLs. Documents the filed-2026 delivery vintage + Genability 2025 supply, the ESI/RDM roll-in and SBC/CTA/FMCC decisions, full per-number provenance to the CLP-Rates exhibits, the transmission vintage note (why filed 0.05050 differs from RateAcuity 0.03401/0.04433), and the **pre-calc caveat** (fixed rates, no calibrated RR — bill-calc runs only). | | ct_eversource_class_rev_requirement.md | CT Eversource **whole residential rate class** revenue-requirement files (Docket 26-05-10): the test-year (`ct_eversource_rate_case_test_year.yaml`, current rates) and rate-year-1 (`ct_eversource_rate_case_rate_year_1.yaml`, Proposed+GET) Track 2 YAMLs that model every residential ResStock building on Rate 1 but size base DRR / kWh / customer count to the class (Rate 1 + 5 + 7). Documents the bug it fixes (whole-class ResStock pinned to a Rate-1 target), how top-ups scale off the two class-total scalars, full per-number provenance to the CLP-RATES-2 exhibit with derivations, and the Proposed+GET vs pre-GET caveat. | -| ct_occ863_system_load.md | OCC-863 (Docket 26-05-10) source files: where the four workbooks and the discovery PDF live on S3 and the EBS mirror. Attachment 3 page 2 becomes `system_load.parquet` (hourly distribution-substation load, MW, 2022–2025); page 1 becomes `system_monthly_peak_mw.parquet`. Notes how to feed the hourly file into the CT distribution MC run without changing `--utility-load-s3-base`. | +| ct_occ863_system_load.md | OCC-863 (Docket 26-05-10) source files: where the four workbooks and the discovery PDF live on S3 and the EBS mirror. Attachment 3 page 2 becomes `system_load.parquet` (hourly distribution-substation load, MW, 2022–2025); page 1 becomes `system_monthly_peak_mw.parquet`. Attachment 4 becomes `rate_1_load.parquet`, `rate_5_load.parquet`, and `rate_7_load.parquet` (2025 hourly class kW, shifted from interval-ending to hour-beginning). Notes how to feed the system-load file into the CT distribution MC run without changing `--utility-load-s3-base`. | ### code/marginal_costs/ diff --git a/context/code/data/ct_occ863_system_load.md b/context/code/data/ct_occ863_system_load.md index 7ab2b5e0..59d760b9 100644 --- a/context/code/data/ct_occ863_system_load.md +++ b/context/code/data/ct_occ863_system_load.md @@ -9,7 +9,7 @@ within the month. ## Where the files are -Raw workbooks, the discovery-request PDF, and the parquets built from Attachment 3: +Raw workbooks, the discovery-request PDF, and the parquets built from Attachments 3 and 4: ```text s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/ @@ -21,7 +21,10 @@ s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/ ├── xlsx/attachment-4.xlsx # (d) 2025 hourly kW for Rates 1, 5, and 7 └── parquet/ ├── system_load.parquet - └── system_monthly_peak_mw.parquet + ├── system_monthly_peak_mw.parquet + ├── rate_1_load.parquet + ├── rate_5_load.parquet + └── rate_7_load.parquet ``` The local path is the EBS mirror of the S3 prefix (`/ebs/data/` mirrors `s3://data.sb/`). The @@ -57,7 +60,30 @@ just -f rate_design/hp_rates/ct/Justfile convert-occ863-system-load \ /ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet ``` -Then sync the `occ-863/` directory to the S3 prefix above. Attachments 2 and 4 are archived only. +Then sync the `occ-863/` directory to the S3 prefix above. Attachment 2 is archived only. + +## Rate-class loads (Attachment 4) + +Attachment 4 is the 2025 hourly class load for residential Rates 1, 5, and 7, in kW. Excel row 8 +is the header `Interval Ending EST`. The hourly rows begin on the next row. Each stamp is the end +of the hour, in Eastern Standard Time, so the parquet timestamp is one hour earlier: +`2025-01-01 01:00:00` becomes `2025-01-01 00:00:00`, and `2026-01-01 00:00:00` becomes +`2025-12-31 23:00:00`. The sheet already has 8,760 stamps, including a normal 24-hour day on both +the spring-forward and fall-back Sundays, so this is a one-hour shift rather than a timezone +conversion. + +The title block, Peak Demand, Total Usage, and the footer check are not stored. The converter +checks that the hourly maximum equals Peak Demand and the hourly sum equals Total Usage, then +drops those rows. + +Each parquet has `timestamp` (hour-beginning, no timezone), `load_kw`, and `load_mw` (`load_kw / +1000`): + +```bash +just -f rate_design/hp_rates/ct/Justfile convert-occ863-rate-class-load \ + /ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/xlsx/attachment-4.xlsx \ + /ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet +``` ## Using this load for CT distribution marginal cost diff --git a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md index c103248c..bc84c945 100644 --- a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md +++ b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md @@ -327,7 +327,10 @@ The hourly series we keep is Attachment 3 page 2, not Attachment 1. Page 2 is th for 2022–2025. Attachment 1 covers 2023–2025 and, on the three fall-back Sundays, stores twice the page 2 value at 1:00 a.m. Page 1's monthly "Max MW" matches page 2, including those Novembers. `parquet/system_load.parquet` (`timestamp`, `load_mw`) is page 2. `parquet/system_monthly_peak_mw.parquet` -(`year`, `month`, `peak_mw`) is page 1. 2024 includes February 29 (8784 hours). +(`year`, `month`, `peak_mw`) is page 1. 2024 includes February 29 (8784 hours). Attachment 4, the +2025 residential class 8760, is `parquet/rate_1_load.parquet`, `rate_5_load.parquet`, and +`rate_7_load.parquet` (`timestamp`, `load_kw`, `load_mw`). The sheet's interval-ending stamps are +shifted back one hour. Neither file is an input to the PoP allocator yet. That run still reads ISO-NE CT zone load via `--utility-load-s3-base`. When CT switches to this substation series, read `system_load.parquet`, diff --git a/rate_design/hp_rates/ct/Justfile b/rate_design/hp_rates/ct/Justfile index d5367960..144cabc5 100644 --- a/rate_design/hp_rates/ct/Justfile +++ b/rate_design/hp_rates/ct/Justfile @@ -160,6 +160,22 @@ convert-occ863-system-load path_xlsx path_output_dir: --path-xlsx {{ path_xlsx }} \ --path-output-dir {{ path_output_dir }} +# ============================================================================= +# CT-only: OCC-863 Attachment 4 → 2025 hourly load for Rates 1, 5, and 7 +# +# Interval Ending EST is shifted back one hour to a hour-beginning timestamp. +# Writes rate_1_load.parquet, rate_5_load.parquet, and rate_7_load.parquet +# with columns timestamp, load_kw, load_mw. See +# context/code/data/ct_occ863_system_load.md. +# +# just -f ct/Justfile convert-occ863-rate-class-load +# ============================================================================= + +convert-occ863-rate-class-load path_xlsx path_output_dir: + cd {{ path_repo }} && uv run python {{ path_repo }}/utils/data_prep/marginal_costs/convert_ct_occ863_rate_class_load.py \ + --path-xlsx {{ path_xlsx }} \ + --path-output-dir {{ path_output_dir }} + # ============================================================================= # CT-only: gas tariff fetch # diff --git a/tests/test_ct_occ863_rate_class_load.py b/tests/test_ct_occ863_rate_class_load.py new file mode 100644 index 00000000..65c4f52d --- /dev/null +++ b/tests/test_ct_occ863_rate_class_load.py @@ -0,0 +1,79 @@ +"""OCC-863 Attachment 4 → hourly rate-class loads. No S3. + +The real workbook is not in git. These tests build a one-year stand-in with the +title block, a footer, and interval-ending stamps. +""" + +from __future__ import annotations + +import polars as pl +import pytest + +from utils.data_prep.marginal_costs.convert_ct_occ863_rate_class_load import ( + rate_class_loads, +) + + +def _interval_ending_hours() -> pl.Series: + start = pl.select(pl.lit("2025-01-01 01:00:00").str.to_datetime()).item() + end = pl.select(pl.lit("2026-01-01 00:00:00").str.to_datetime()).item() + return pl.datetime_range(start, end, interval="1h", eager=True) + + +def _sheet() -> pl.DataFrame: + stamps = _interval_ending_hours() + n = stamps.len() + rate1 = pl.Series("r1", [1000.0 + (i % 24) for i in range(n)]) + rate5 = pl.Series("r5", [500.0 + (i % 24) for i in range(n)]) + rate7 = pl.Series("r7", [10.0 + (i % 10) for i in range(n)]) + rows: list[tuple[str | None, str | None, str | None, str | None]] = [ + ("Connecticut Light and Power Company", None, None, None), + ("Year 2025 Hourly Rate Class Loads", None, None, None), + (None, None, None, None), + ("Rate", "1", "5", "7"), + ("Peak Demand", str(rate1.max()), str(rate5.max()), str(rate7.max())), + ("Total Usage", str(rate1.sum()), str(rate5.sum()), str(rate7.sum())), + (None, None, None, None), + ("Interval Ending EST", "kW", "kW", "kW"), + ] + for stamp, a, b, c in zip( + stamps.to_list(), rate1.to_list(), rate5.to_list(), rate7.to_list(), strict=True + ): + rows.append((str(stamp), str(a), str(b), str(c))) + rows.append((None, str(rate1.max()), str(rate5.max()), str(rate7.max()))) + rows.append((None, "true", "true", "true")) + return pl.DataFrame( + rows, + schema=["column_1", "column_2", "column_3", "column_4"], + orient="row", + ) + + +def test_shifts_interval_ending_back_one_hour_and_drops_the_summary() -> None: + loads = rate_class_loads(_sheet()) + assert set(loads) == {1, 5, 7} + rate1 = loads[1] + assert rate1.columns == ["timestamp", "load_kw", "load_mw"] + assert rate1.height == 8760 + assert ( + rate1["timestamp"][0] + == pl.select(pl.lit("2025-01-01 00:00:00").str.to_datetime()).item() + ) + assert ( + rate1["timestamp"][-1] + == pl.select(pl.lit("2025-12-31 23:00:00").str.to_datetime()).item() + ) + first = rate1.row(0, named=True) + assert first["load_kw"] == 1000.0 + assert first["load_mw"] == pytest.approx(1.0) + + +def test_rejects_a_peak_that_does_not_match_the_hourly_column() -> None: + raw = _sheet().with_columns( + pl.when(pl.col("column_1") == "Peak Demand") + .then(pl.lit("1")) + .otherwise(pl.col("column_2")) + .alias("column_2") + ) + with pytest.raises(ValueError, match="Peak Demand"): + rate_class_loads(raw) diff --git a/utils/data_prep/marginal_costs/convert_ct_occ863_rate_class_load.py b/utils/data_prep/marginal_costs/convert_ct_occ863_rate_class_load.py new file mode 100644 index 00000000..760b0d06 --- /dev/null +++ b/utils/data_prep/marginal_costs/convert_ct_occ863_rate_class_load.py @@ -0,0 +1,186 @@ +"""Convert OCC-863 Attachment 4 to hourly rate-class load parquets. + +Attachment 4 answers OCC-863 subpart (d) for PURA Docket 26-05-10: 2025 hourly +load for Eversource CT residential Rates 1, 5, and 7. The sheet labels each +stamp "Interval Ending EST". The hour that stamp closes is the previous hour, +so 2025-01-01 01:00:00 becomes a hour-beginning timestamp of 2025-01-01 00:00:00. +The last stamp, 2026-01-01 00:00:00, becomes 2025-12-31 23:00:00. + +The series is already 8760 hours in standard time. Spring-forward and fall-back +days each have 24 stamps, so the conversion is a one-hour shift and not a +timezone conversion. + +Title rows, the Peak Demand and Total Usage summaries, and the footer check +are dropped. Those summaries are checked against the hourly column before +they are dropped. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import polars as pl + +_SHEET = "OCC-863 Attachment 4" +_RATES = (1, 5, 7) + + +def rate_class_filename(rate: int) -> str: + """Parquet name for one residential rate class.""" + return f"rate_{rate}_load.parquet" + + +def rate_class_loads(raw: pl.DataFrame) -> dict[int, pl.DataFrame]: + """Return one ``timestamp`` / ``load_kw`` / ``load_mw`` frame per rate. + + ``raw`` is Attachment 4 read with no header and with empty rows kept, so + the title block stays aligned with the sheet. Blank rows, the summary + block, and the footer are not part of the returned frames. + """ + label_col = raw.columns[0] + labels = [ + None if value is None else str(value) + for value in raw.get_column(label_col).to_list() + ] + rate_idx = _label_index(labels, "Rate") + peak_idx = _label_index(labels, "Peak Demand") + usage_idx = _label_index(labels, "Total Usage") + header_idx = _label_index(labels, "Interval Ending EST") + + frames: dict[int, pl.DataFrame] = {} + for col in raw.columns[1:]: + rate_label = raw[col][rate_idx] + if rate_label is None or not str(rate_label).isdigit(): + continue + rate = int(str(rate_label)) + if rate not in _RATES: + raise ValueError(f"unexpected rate column {rate}") + peak_kw = float(str(raw[col][peak_idx])) + usage_kwh = float(str(raw[col][usage_idx])) + frames[rate] = _one_rate(raw, label_col, col, header_idx, peak_kw, usage_kwh) + + missing = [rate for rate in _RATES if rate not in frames] + if missing: + raise ValueError(f"Attachment 4 is missing rates {missing}") + return frames + + +def _one_rate( + raw: pl.DataFrame, + label_col: str, + value_col: str, + header_idx: int, + peak_kw: float, + usage_kwh: float, +) -> pl.DataFrame: + load = ( + raw.slice(header_idx + 1) + .select( + pl.col(label_col).str.to_datetime(strict=False).alias("interval_ending"), + pl.col(value_col).cast(pl.Float64, strict=False).alias("load_kw"), + ) + .drop_nulls(subset=["interval_ending", "load_kw"]) + .with_columns( + (pl.col("interval_ending") - pl.duration(hours=1)).alias("timestamp"), + (pl.col("load_kw") / 1000.0).alias("load_mw"), + ) + .select("timestamp", "load_kw", "load_mw") + .sort("timestamp") + ) + _validate_rate_load(load, peak_kw, usage_kwh) + return load + + +def _validate_rate_load(load: pl.DataFrame, peak_kw: float, usage_kwh: float) -> None: + if load["timestamp"].n_unique() != load.height: + raise ValueError("rate-class timestamps are not unique") + if load.filter(pl.col("load_kw") <= 0).height: + raise ValueError("rate-class load has a non-positive kW value") + + start = load["timestamp"][0] + end = load["timestamp"][-1] + year = start.year + expected = 8784 if year % 4 == 0 else 8760 + if load.height != expected: + raise ValueError(f"expected {expected} hours in {year}, got {load.height}") + if (start.month, start.day, start.hour) != (1, 1, 0): + raise ValueError(f"series does not start at Jan 1 hour 0: {start}") + if (end.year, end.month, end.day, end.hour) != (year, 12, 31, 23): + raise ValueError(f"series does not end at Dec 31 hour 23: {end}") + steps = load.select( + pl.col("timestamp").diff().dt.total_hours().alias("dh") + ).drop_nulls() + if steps.filter(pl.col("dh") != 1).height: + raise ValueError("rate-class timestamps are not one hour apart") + + hourly_peak = load.select(pl.col("load_kw").max()).item() + hourly_sum = load.select(pl.col("load_kw").sum()).item() + if not isinstance(hourly_peak, (int, float)) or not isinstance( + hourly_sum, (int, float) + ): + raise TypeError("rate-class load did not aggregate to a number") + if abs(hourly_peak - peak_kw) > 0.05: + raise ValueError( + f"hourly peak {hourly_peak} does not match Peak Demand {peak_kw}" + ) + if abs(hourly_sum - usage_kwh) > 1.0: + raise ValueError( + f"hourly sum {hourly_sum} does not match Total Usage {usage_kwh}" + ) + + +def _label_index(labels: list[str | None], label: str) -> int: + try: + return labels.index(label) + except ValueError as exc: + raise ValueError(f"Attachment 4 has no {label!r} row") from exc + + +def write_rate_class_loads( + path_xlsx: Path, path_output_dir: Path +) -> dict[int, pl.DataFrame]: + """Read Attachment 4 and write one parquet per rate under ``path_output_dir``.""" + if "{{" in str(path_xlsx) or "{{" in str(path_output_dir): + raise ValueError( + f"path looks like an uninterpolated Just variable: {path_xlsx} {path_output_dir}" + ) + raw = pl.read_excel( + path_xlsx, + sheet_name=_SHEET, + has_header=False, + drop_empty_rows=False, + drop_empty_cols=False, + ) + if not isinstance(raw, pl.DataFrame): + raise TypeError("expected one sheet from Attachment 4") + loads = rate_class_loads(raw) + path_output_dir.mkdir(parents=True, exist_ok=True) + for rate, load in loads.items(): + load.write_parquet(path_output_dir / rate_class_filename(rate)) + return loads + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--path-xlsx", type=Path, required=True, help="OCC-863 Attachment 4 workbook" + ) + parser.add_argument( + "--path-output-dir", + type=Path, + required=True, + help="Directory for rate_1_load.parquet, rate_5_load.parquet, and rate_7_load.parquet", + ) + args = parser.parse_args() + loads = write_rate_class_loads(args.path_xlsx, args.path_output_dir) + for rate, load in loads.items(): + path = args.path_output_dir / rate_class_filename(rate) + print( + f"Wrote {load.height:,} hours to {path} " + f"(load_kw min={load['load_kw'].min():.1f} max={load['load_kw'].max():.1f})" + ) + + +if __name__ == "__main__": + main() From 1ca383e70e075eb18a3929dfaee979f1866392d5 Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Wed, 7 Oct 2026 20:00:43 +0000 Subject: [PATCH 06/12] Update dist mc hourly allocation --- context/README.md | 2 +- context/code/data/ct_occ863_system_load.md | 34 +++--- .../ct_eversource_dist_mc_methodology.md | 97 ++++++++--------- rate_design/hp_rates/ct/Justfile | 11 +- tests/test_ct_dist_mc.py | 64 +++++++++++ tests/test_ct_occ863_rate_class_load.py | 23 ++++ tests/test_ct_occ863_system_load.py | 2 + .../convert_ct_occ863_rate_class_load.py | 93 +++++++++++++++- .../convert_ct_occ863_system_load.py | 2 +- .../generate_utility_tx_dx_mc.py | 101 ++++++++++++++++-- 10 files changed, 349 insertions(+), 80 deletions(-) diff --git a/context/README.md b/context/README.md index 3b70e7e0..30d9d7e8 100644 --- a/context/README.md +++ b/context/README.md @@ -170,7 +170,7 @@ ResStock, EIA, NYISO, ISO-NE: where data comes from, how to read and prepare it. | ct_tariff_fetch.md | CT gas tariff **fetch** (ticket scope: RateAcuity URDB JSONs + `null_gas_tariff` + envelope only): schedules for CNG / Yankee / SCG (three residential classes each) plus Norwich muni (GRES + GSHRES/`_mf`), evidence. Omits (reference only, with Zotero links): CNG Rate SE Seasonal Gas Service; System Expansion `*-SE` (RSH-SE / 02-SE etc.); third-party supply. **Known limitation:** IOU `_mf` demand / MDQ charges dropped by `tariff_fetch`; CAIRO gas path has no MDQ billing. Mapper / tariff maps are follow-up. | | ct_eversource_custom_rates.md | CT Eversource hand-encoded proposed rates (Docket 26-05-10): `ct_eversource_rate6` (HP space-heating) + `ct_eversource_rate1_proposed` (matched comparison baseline), each as delivery + `_supply` JSON pairs with top-up YAMLs. Documents the filed-2026 delivery vintage + Genability 2025 supply, the ESI/RDM roll-in and SBC/CTA/FMCC decisions, full per-number provenance to the CLP-Rates exhibits, the transmission vintage note (why filed 0.05050 differs from RateAcuity 0.03401/0.04433), and the **pre-calc caveat** (fixed rates, no calibrated RR — bill-calc runs only). | | ct_eversource_class_rev_requirement.md | CT Eversource **whole residential rate class** revenue-requirement files (Docket 26-05-10): the test-year (`ct_eversource_rate_case_test_year.yaml`, current rates) and rate-year-1 (`ct_eversource_rate_case_rate_year_1.yaml`, Proposed+GET) Track 2 YAMLs that model every residential ResStock building on Rate 1 but size base DRR / kWh / customer count to the class (Rate 1 + 5 + 7). Documents the bug it fixes (whole-class ResStock pinned to a Rate-1 target), how top-ups scale off the two class-total scalars, full per-number provenance to the CLP-RATES-2 exhibit with derivations, and the Proposed+GET vs pre-GET caveat. | -| ct_occ863_system_load.md | OCC-863 (Docket 26-05-10) source files: where the four workbooks and the discovery PDF live on S3 and the EBS mirror. Attachment 3 page 2 becomes `system_load.parquet` (hourly distribution-substation load, MW, 2022–2025); page 1 becomes `system_monthly_peak_mw.parquet`. Attachment 4 becomes `rate_1_load.parquet`, `rate_5_load.parquet`, and `rate_7_load.parquet` (2025 hourly class kW, shifted from interval-ending to hour-beginning). Notes how to feed the system-load file into the CT distribution MC run without changing `--utility-load-s3-base`. | +| ct_occ863_system_load.md | OCC-863 (Docket 26-05-10) source files: where the four workbooks and the discovery PDF live on S3 and the EBS mirror. Attachment 3 page 2 becomes `system_load.parquet` (hourly distribution-substation load, MW, 2022–2025); page 1 becomes `system_monthly_peak_mw.parquet`. Attachment 4 becomes `rate_1_load.parquet`, `rate_5_load.parquet`, and `rate_7_load.parquet` (2025 hourly class kW; interval-ending EST shifted to hour-beginning local time). `create-dist-mc-data` reads `system_load.parquet` through `--path-utility-load`. | ### code/marginal_costs/ diff --git a/context/code/data/ct_occ863_system_load.md b/context/code/data/ct_occ863_system_load.md index 59d760b9..2abed78d 100644 --- a/context/code/data/ct_occ863_system_load.md +++ b/context/code/data/ct_occ863_system_load.md @@ -66,18 +66,22 @@ Then sync the `occ-863/` directory to the S3 prefix above. Attachment 2 is archi Attachment 4 is the 2025 hourly class load for residential Rates 1, 5, and 7, in kW. Excel row 8 is the header `Interval Ending EST`. The hourly rows begin on the next row. Each stamp is the end -of the hour, in Eastern Standard Time, so the parquet timestamp is one hour earlier: +of the hour, in Eastern Standard Time, so the first step is one hour earlier: `2025-01-01 01:00:00` becomes `2025-01-01 00:00:00`, and `2026-01-01 00:00:00` becomes `2025-12-31 23:00:00`. The sheet already has 8,760 stamps, including a normal 24-hour day on both -the spring-forward and fall-back Sundays, so this is a one-hour shift rather than a timezone -conversion. +the spring-forward and fall-back Sundays. The title block, Peak Demand, Total Usage, and the footer check are not stored. The converter -checks that the hourly maximum equals Peak Demand and the hourly sum equals Total Usage, then -drops those rows. +checks that the hourly maximum equals Peak Demand and the hourly sum equals Total Usage while the +stamps are still Eastern Standard Time, then drops those rows. -Each parquet has `timestamp` (hour-beginning, no timezone), `load_kw`, and `load_mw` (`load_kw / -1000`): +The stored `timestamp` is then local wall-clock time, with no timezone, matching +`system_load.parquet`. Winter hours are unchanged. Summer hours move forward one hour. On the +fall-back morning the two standard-time hours that share local 01:00 are averaged. The +spring-forward 02:00 hour, which the standard-time series does not have, is the average of local +01:00 and 03:00. + +Each parquet has `timestamp`, `load_kw`, and `load_mw` (`load_kw / 1000`). ```bash just -f rate_design/hp_rates/ct/Justfile convert-occ863-rate-class-load \ @@ -87,10 +91,14 @@ just -f rate_design/hp_rates/ct/Justfile convert-occ863-rate-class-load \ ## Using this load for CT distribution marginal cost -The distribution marginal-cost recipe still reads ISO-NE Connecticut zone load through -`--utility-load-s3-base` (`s3://data.sb/isone/hourly_demand/utilities/`). That flag scans a -hive-partitioned folder and filters on `utility` and `year`. Do not point it at this prefix. +`create-dist-mc-data` passes `system_load.parquet` as `--path-utility-load`. The allocator keeps +the requested year and ranks the top 100 hours of that substation series. The residential +rate-class files are a class shape; they are not the PoP load. See +`context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md` §3.3. + +`--utility-load-s3-base` still scans a hive `utility`/`year` folder for the other states. This +prefix is one multi-year file, so it is not a value for that flag. -When the CT run should use this substation series instead, read `system_load.parquet`, keep the -load year, and pass that table to `normalize_load_to_cairo_8760`. That function already requires -`timestamp` and `load_mw`. The ISO-NE reader stays as it is for the other states. +```bash +just -f rate_design/hp_rates/ct/Justfile create-dist-mc-data 2025 --upload +``` diff --git a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md index bc84c945..077e329c 100644 --- a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md +++ b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md @@ -155,38 +155,37 @@ CL&P's MCOS used an hourly probability-of-peak analysis (MCOS-1, p. 14)(https:// We don't have CL&P's underlying hourly PoP curve (the MCOS-2 back-up tables, including Back-Up 36 "Probabilities of peak by month and time of day period," were listed in the table of contents but not -included in the 17-page exhibit we have). However, the standard PoP method on CT zone load is a -reasonable proxy — CT is summer-peaking (CL&P's testimony confirms ~80% of annual peak probability -falls in July–August, ~20% in June/September, <1% winter) [DocumentCloud p. 20](https://www.documentcloud.org/documents/28540606-exhibit-clp-mcos-1/#document/p20/a2826759), which our PoP allocator will naturally -reproduce from the load data. +included in the 17-page exhibit we have). The PoP run uses the substation series from OCC-863 +Attachment 3 page 2 (§3.3). CT is summer-peaking (CL&P's testimony confirms ~80% of annual peak +probability falls in July–August, ~20% in June/September, <1% winter) [DocumentCloud p. 20](https://www.documentcloud.org/documents/28540606-exhibit-clp-mcos-1/#document/p20/a2826759). ### 3.3 Load data source -Use the CT zone hourly load already on S3 for the CT bulk-TX pipeline: -`s3://data.sb/isone/hourly_demand/zones/`, zone label `CT`. Both `ct_eversource` and `ct_ui` map to -the single `CT` ISO-NE zone. `generate_utility_tx_dx_mc.py` loads **utility-level** (not zone-level) -data from `s3://data.sb/isone/hourly_demand/utilities/utility=ct_eversource/...` via the -`--utility-load-s3-base` argument — the same layout NY/RI/MD use. - -That utility-level partition **did not exist** before this implementation (only `utility=rie` was -present under `s3://data.sb/isone/hourly_demand/utilities/`); CT bulk-TX reads zone-level data -directly via `load_isone_zone_loads()` and never needed the utility-level aggregation. To fill the -gap: - -1. Added `ct_eversource` and `ct_ui` rows (zone `CT`, location 4004) to - `data/isone/zone_mapping/generate_zone_mapping_csv.py` and regenerated - `data/isone/zone_mapping/csv/isone_utility_zone_mapping.csv`. -2. Ran `data/isone/hourly_demand/aggregate_isone_utility_loads.py` (via the - `aggregate-utility-loads` Justfile recipe) for `ct_eversource` and `ct_ui`, year 2025 — a 1:1 - zone→utility relabel (both utilities share the single CT zone, same as RI's `rie`→`RI`). -3. Uploaded the resulting `utility=ct_eversource/year=2025/` and `utility=ct_ui/year=2025/` - partitions to `s3://data.sb/isone/hourly_demand/utilities/`. - -**Caveat**: this is CT-zone transmission-level load (both utilities combined), not CL&P's -substation-level distribution load. It doesn't reflect CL&P's BTM-solar/HP-adoption forward -adjustments. This is the same trade-off as in other states where we use zone-level rather than -utility-specific substation data — sufficient for the BAT, and matches the approach used for CT -bulk-TX MC. +PoP ranks the top 100 hours of Eversource's distribution-substation load: OCC-863 Attachment 3 +page 2, stored as `system_load.parquet` (`timestamp`, `load_mw`, 2022–2025). `create-dist-mc-data` +passes that file as `--path-utility-load`. `normalize_load_to_cairo_8760` keeps `--load-year`. +The hive reader `--utility-load-s3-base` stays what NY, RI, and MD use. The OCC-863 prefix is one +multi-year file, so it is not a valid value for that flag. + +This is the substation-sum series behind CL&P's own probability-of-peak (MCOS-1, p. 14). The +`$21.22/kW-yr` scalar prices upstream station and trunkline capacity, which is added when that +system load peaks. On the 2025 series the top 100 hours are 28 in June, 66 in July, and 6 in August. + +Attachment 4 (Rates 1, 5, and 7 summed) is the residential class shape. We do not rank it. The +upstream investment is sized to the substation sum, which includes non-residential load, so the +system series is the cost driver. The rate-class parquets are stored on the same local clock as +`system_load.parquet` (see +[ct_occ863_system_load.md](../../code/data/ct_occ863_system_load.md)). On that clock the two 2025 +top-100 lists share 84 hours. The residential class peaks at 17:00 on June 24. The system peaks +at 18:00 on July 29. The rate-class parquets stay available for class-shape work. + +The first run ranked ISO-NE Connecticut zone load +(`s3://data.sb/isone/hourly_demand/utilities/utility=ct_eversource/`). That series is +transmission-level load for both CT utilities combined. It is still the bulk-transmission load. +Building it required a `ct_eversource` / `ct_ui` zone-mapping row (zone `CT`, location 4004) and +`aggregate_isone_utility_loads.py` for 2025, because that hive partition did not exist before. It +does not include CL&P's BTM-solar or heat-pump forward adjustments, and neither does the +substation series: OCC-863 says no forecasted hourly system loads are available. ### 3.4 Implementation notes @@ -207,7 +206,7 @@ bulk-TX MC. --state CT --utility ct_eversource --year 2025 \ --target-dollar-year 2026 \ --mc-table-path rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv \ - --utility-load-s3-base s3://data.sb/isone/hourly_demand/utilities/ \ + --path-utility-load s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet/system_load.parquet \ --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ \ --upload ``` @@ -216,8 +215,10 @@ bulk-TX MC. exactly ($19.6460/kW-yr, 0.0000% error). The top-100 PoP hours for 2025 fall entirely in June–August (30 in June, 62 in July, 8 in August, 0 elsewhere) — consistent with CL&P's own finding of concentrated summer peak-probability, though the specific split differs somewhat from - CL&P's multi-year-normalized ~80% Jul–Aug / ~20% Jun–Sep because this uses a single year (2025) - of CT zone load rather than CL&P's normalized 2022–2025 substation analysis. + CL&P's multi-year-normalized ~80% Jul–Aug / ~20% Jun–Sep because that run used a single year + (2025) of ISO-NE CT zone load. The recipe now ranks the OCC-863 substation series instead; + its 2025 top 100 hours are 28 in June, 66 in July, and 6 in August (§3.3). The S3 parquet + still reflects the zone-load run until task 7 is re-run. 7. **Output**: same schema as all other states (`timestamp, utility, year, mc_total_per_kwh`) at `s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet`. Downstream CAIRO wiring is unchanged. @@ -249,7 +250,7 @@ as a rate. The system-wide annual average is `$0.00241/kWh` primary vs `$0.00242 The secondary "Annual Average" `$1.768/kW-mo` row is the same `$21.22/kW-yr` we store (`$1.768 × 12`). Our PoP-allocated 8760 should still produce the seasonal concentration Table 2A shows (nearly all cost in summer, near-zero winter). The hours that concentration lands on come from -CT zone load, not from Table 2A's own probability-of-peak weights. +the OCC-863 substation series (§3.3), not from Table 2A's own probability-of-peak weights. ### 3.7 Back-Up 36 (if obtainable) @@ -272,11 +273,11 @@ allocation or as a direct replacement. This is a nice-to-have, not a blocker for only the config-CSV row is missing. - **Locational sensitivity.** Confirm whether we want `$86.58/kW-yr` carried as a formal sensitivity run (§2.2 point 3) or just documented here. Not yet run. -- **`--n-hours` parameter.** Ran with the platform default of `100` for 2025: the resulting top-100 - hours fall entirely in June–August (30/62/8 split), directionally consistent with but not an exact - match to CL&P's normalized ~80% Jul–Aug / ~20% Jun–Sep testimony finding (see §3.4 point 6 for why - the exact split differs). Not tuned further; revisit if a closer match to CL&P's own split matters - for a given analysis. +- **`--n-hours` parameter.** Platform default of `100`. On the 2025 OCC-863 substation series the + top 100 hours are 28 in June, 66 in July, and 6 in August (§3.3). That is close to, but not + the same as, CL&P's normalized ~80% Jul–Aug / ~20% Jun–Sep testimony finding, which pools + 2022–2025. Not tuned further; revisit if a closer match to CL&P's own split matters for a given + analysis. - **Whether "marginal customer/facilities cost" should ever get its own BAT MC term.** Not a CT-specific question, but CT's MCOS happens to compute Bonbright-style customer/facilities marginal costs explicitly (Tables 4–7), which makes the gap visible. Worth raising with the team as a @@ -299,8 +300,9 @@ allocation or as a direct replacement. This is a nice-to-have, not a blocker for to the source-number and per-state tables.~~ Done. 7. **Remaining**: re-run `create-dist-mc-data 2025 --upload`. The parquet at `s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet` - was built from the pre-loss `$20.17/kW-yr` scalar after a CPI deflation to 2025$. The recipe now - allocates the unadjusted rate year 1 scalar, `$21.22/kW-yr`. + was built from the pre-loss `$20.17/kW-yr` scalar after a CPI deflation to 2025$, on ISO-NE zone + load. The recipe now allocates the unadjusted rate year 1 scalar, `$21.22/kW-yr`, on the OCC-863 + substation series (§3.3). 8. **Remaining**: wire `path_dist_and_sub_tx_mc` into a CT scenario config (`scenarios_ct_eversource.yaml` or equivalent) once CT scenario YAMLs exist, so a CAIRO run actually consumes this MC output. No CT scenario configs exist yet in `rate_design/hp_rates/ct/config/scenarios/` — that's a separate, @@ -329,12 +331,11 @@ page 2 value at 1:00 a.m. Page 1's monthly "Max MW" matches page 2, including th `parquet/system_load.parquet` (`timestamp`, `load_mw`) is page 2. `parquet/system_monthly_peak_mw.parquet` (`year`, `month`, `peak_mw`) is page 1. 2024 includes February 29 (8784 hours). Attachment 4, the 2025 residential class 8760, is `parquet/rate_1_load.parquet`, `rate_5_load.parquet`, and -`rate_7_load.parquet` (`timestamp`, `load_kw`, `load_mw`). The sheet's interval-ending stamps are -shifted back one hour. - -Neither file is an input to the PoP allocator yet. That run still reads ISO-NE CT zone load via -`--utility-load-s3-base`. When CT switches to this substation series, read `system_load.parquet`, -filter to the load year, and pass the table to `normalize_load_to_cairo_8760`. Do not point -`--utility-load-s3-base` at the OCC-863 prefix: that flag expects the hive `utility`/`year` layout -the other states use. How to rebuild the parquets: +`rate_7_load.parquet` (`timestamp`, `load_kw`, `load_mw`). Interval-ending stamps are shifted +back one hour, then mapped from Eastern Standard Time onto local wall-clock time. + +`create-dist-mc-data` reads `system_load.parquet` through `--path-utility-load` and keeps the +load year inside `normalize_load_to_cairo_8760` (§3.3). The rate-class files are not the PoP +load. `--utility-load-s3-base` still expects the hive `utility`/`year` layout the other states +use. How to rebuild the parquets: [ct_occ863_system_load.md](../../code/data/ct_occ863_system_load.md). diff --git a/rate_design/hp_rates/ct/Justfile b/rate_design/hp_rates/ct/Justfile index 144cabc5..dc2f46f8 100644 --- a/rate_design/hp_rates/ct/Justfile +++ b/rate_design/hp_rates/ct/Justfile @@ -14,6 +14,7 @@ set dotenv-path := "state.env" state := env_var_or_default('STATE', 'ct') path_s3_utility_loads := "s3://data.sb/isone/hourly_demand/utilities/" +path_s3_occ863_system_load := "s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet/system_load.parquet" import '../Justfile' @@ -119,7 +120,10 @@ create-bulk-tx-mc-data-all *extra_args: # Rate year 1 is modeled in those filing dollars: --target-dollar-year 2026 # matches the CSV, so the allocator does not CPI-adjust $21.22 to the load # year. Allocated via the standard Probability-of-Peak (PoP) method — same -# methodology as NY/RI/MD — on CT zone hourly load (top-100 hours by default). +# methodology as NY/RI/MD — on OCC-863 Attachment 3 page 2, the +# distribution-substation load (top-100 hours by default). That file is one +# multi-year parquet, so this recipe passes --path-utility-load rather than +# the hive --utility-load-s3-base the other states use. # See context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md. # # CL&P's MCOS-2 covers ct_eversource only; ct_ui has no MCOS-derived value yet @@ -139,7 +143,7 @@ create-dist-mc-data year_arg *extra_args: --load-year {{ year_arg }} \ --target-dollar-year 2026 \ --mc-table-path {{ path_config }}/marginal_costs/ct_marginal_costs_{{ year_arg }}.csv \ - --utility-load-s3-base {{ path_s3_utility_loads }} \ + --path-utility-load {{ path_s3_occ863_system_load }} \ --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ \ --n-hours {{ upstream_hours }} \ {{ extra_args }} @@ -163,7 +167,8 @@ convert-occ863-system-load path_xlsx path_output_dir: # ============================================================================= # CT-only: OCC-863 Attachment 4 → 2025 hourly load for Rates 1, 5, and 7 # -# Interval Ending EST is shifted back one hour to a hour-beginning timestamp. +# Interval Ending EST is shifted back one hour, then mapped from Eastern +# Standard Time onto local wall-clock time so the stamps match system_load. # Writes rate_1_load.parquet, rate_5_load.parquet, and rate_7_load.parquet # with columns timestamp, load_kw, load_mw. See # context/code/data/ct_occ863_system_load.md. diff --git a/tests/test_ct_dist_mc.py b/tests/test_ct_dist_mc.py index b355bea9..3e2b9525 100644 --- a/tests/test_ct_dist_mc.py +++ b/tests/test_ct_dist_mc.py @@ -12,6 +12,7 @@ from __future__ import annotations import argparse +import tempfile from datetime import datetime from pathlib import Path @@ -26,7 +27,9 @@ allocate_costs_to_hours, calculate_pop_weights, get_marginal_cost_for_utility, + load_utility_load_file, normalize_load_to_cairo_8760, + resolve_utility_load, ) CT_MC_TABLE_PATH = ( @@ -94,6 +97,67 @@ def test_ct_ui_not_in_table(self) -> None: get_marginal_cost_for_utility(mc_df, "ct_ui") +# ── OCC-863 system load as the PoP profile ────────────────────────────────── + + +class TestCtSystemLoadSource: + def test_resolve_rejects_both_load_sources(self) -> None: + with pytest.raises(ValueError, match="not both"): + resolve_utility_load( + "system_load.parquet", + "s3://data.sb/isone/hourly_demand/utilities/", + 2025, + "ct_eversource", + {}, + ) + + def test_resolve_rejects_a_missing_load_source(self) -> None: + with pytest.raises(ValueError, match="path-utility-load"): + resolve_utility_load(None, None, 2025, "ct_eversource", {}) + + def test_load_file_reads_a_single_parquet(self) -> None: + df = pl.DataFrame( + { + "timestamp": ["2025-01-01 00:00:00", "2025-01-01 01:00:00"], + "load_mw": [10.0, 11.0], + } + ).with_columns(pl.col("timestamp").str.to_datetime()) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "system_load.parquet" + df.write_parquet(path) + loaded = load_utility_load_file(str(path), {}) + assert loaded.height == 2 + assert set(loaded.columns) >= {"timestamp", "load_mw"} + + def test_millisecond_timestamps_normalize_to_8760(self) -> None: + """A datetime[ms] load still normalizes; the 8760 index is datetime[us]. + + A few hours are enough: the join fails as soon as an ms stamp meets + the us index. Missing 2025 hours are interpolated. + """ + df = pl.DataFrame( + { + "timestamp": pl.select( + pl.datetime_range( + pl.lit("2024-12-31 22:00:00").str.to_datetime(time_unit="ms"), + pl.lit("2025-01-02 02:00:00").str.to_datetime(time_unit="ms"), + interval="1h", + time_unit="ms", + ) + ).to_series(), + "load_mw": 1000.0, + } + ) + assert df.schema["timestamp"] == pl.Datetime("ms") + normalized = normalize_load_to_cairo_8760(df, "ct_eversource", 2025) + assert normalized.height == 8760 + assert normalized.schema["timestamp"] == pl.Datetime("us") + kept = normalized.filter( + pl.col("timestamp") == pl.lit("2025-01-01 00:00:00").str.to_datetime() + ) + assert float(kept["load_mw"][0]) == pytest.approx(1000.0) + + # ── End-to-end PoP allocation on synthetic CT-shaped load ─────────────────── diff --git a/tests/test_ct_occ863_rate_class_load.py b/tests/test_ct_occ863_rate_class_load.py index 65c4f52d..1805445a 100644 --- a/tests/test_ct_occ863_rate_class_load.py +++ b/tests/test_ct_occ863_rate_class_load.py @@ -54,6 +54,7 @@ def test_shifts_interval_ending_back_one_hour_and_drops_the_summary() -> None: assert set(loads) == {1, 5, 7} rate1 = loads[1] assert rate1.columns == ["timestamp", "load_kw", "load_mw"] + assert rate1.schema["timestamp"] == pl.Datetime("us") assert rate1.height == 8760 assert ( rate1["timestamp"][0] @@ -68,6 +69,28 @@ def test_shifts_interval_ending_back_one_hour_and_drops_the_summary() -> None: assert first["load_mw"] == pytest.approx(1.0) +def _kw_at(load: pl.DataFrame, stamp: str) -> float: + row = load.filter(pl.col("timestamp") == pl.lit(stamp).str.to_datetime()) + assert row.height == 1 + return float(row["load_kw"][0]) + + +def test_maps_standard_time_onto_local_clock() -> None: + """Winter hours stay put. Summer hours move forward one hour. + + The synthetic load is 1000 plus the standard-time hour of day. Local + 16:00 in July therefore carries the 15:00 standard-time value. + """ + rate1 = rate_class_loads(_sheet())[1] + assert _kw_at(rate1, "2025-01-15 15:00:00") == 1015.0 + assert _kw_at(rate1, "2025-07-15 16:00:00") == 1015.0 + # Spring-forward 02:00 is the average of local 01:00 (EST 01:00) and + # local 03:00 (EST 02:00): (1001 + 1002) / 2. + assert _kw_at(rate1, "2025-03-09 02:00:00") == pytest.approx(1001.5) + # Fall-back local 01:00 is the average of EST 00:00 and EST 01:00. + assert _kw_at(rate1, "2025-11-02 01:00:00") == pytest.approx(1000.5) + + def test_rejects_a_peak_that_does_not_match_the_hourly_column() -> None: raw = _sheet().with_columns( pl.when(pl.col("column_1") == "Peak Demand") diff --git a/tests/test_ct_occ863_system_load.py b/tests/test_ct_occ863_system_load.py index 799c934b..e3f00822 100644 --- a/tests/test_ct_occ863_system_load.py +++ b/tests/test_ct_occ863_system_load.py @@ -87,6 +87,8 @@ def test_rounds_a_timestamp_that_excel_stored_just_before_the_hour() -> None: def test_keeps_the_leap_day_and_names_the_load_column_in_mw() -> None: load = hourly_system_load(_page2([2022, 2023, 2024, 2025])) assert load.columns == ["timestamp", "load_mw"] + # Same unit as the rate-class files and the ISO-NE loads, so joins need no cast. + assert load.schema["timestamp"] == pl.Datetime("us") counts = dict( load.with_columns(pl.col("timestamp").dt.year().alias("year")) .group_by("year") diff --git a/utils/data_prep/marginal_costs/convert_ct_occ863_rate_class_load.py b/utils/data_prep/marginal_costs/convert_ct_occ863_rate_class_load.py index 760b0d06..3f984e28 100644 --- a/utils/data_prep/marginal_costs/convert_ct_occ863_rate_class_load.py +++ b/utils/data_prep/marginal_costs/convert_ct_occ863_rate_class_load.py @@ -6,9 +6,12 @@ so 2025-01-01 01:00:00 becomes a hour-beginning timestamp of 2025-01-01 00:00:00. The last stamp, 2026-01-01 00:00:00, becomes 2025-12-31 23:00:00. -The series is already 8760 hours in standard time. Spring-forward and fall-back -days each have 24 stamps, so the conversion is a one-hour shift and not a -timezone conversion. +Those hour-beginning stamps are Eastern Standard Time all year. The other +hourly files (``system_load.parquet``, the CAIRO 8760) are local wall-clock +time with no timezone stored. This converter maps standard time onto that +clock: winter hours stay put, summer hours move forward one hour, the +fall-back hour is the average of the two standard-time hours that land on it, +and the missing spring-forward hour is the average of the hours on either side. Title rows, the Peak Demand and Total Usage summaries, and the footer check are dropped. Those summaries are checked against the hourly column before @@ -89,7 +92,89 @@ def _one_rate( .sort("timestamp") ) _validate_rate_load(load, peak_kw, usage_kwh) - return load + local = est_hour_beginning_to_local(load) + _validate_local_clock(local) + return local + + +def est_hour_beginning_to_local(load: pl.DataFrame) -> pl.DataFrame: + """Map hour-beginning Eastern Standard Time onto local wall-clock time. + + ``load`` has ``timestamp``, ``load_kw``, and ``load_mw``. The timestamps + are naive and mean Eastern Standard Time (UTC-5), which is how Attachment + 4 labels the year. The returned timestamps are naive ``America/New_York`` + wall time, matching ``system_load.parquet``. + + Two clock-change hours are not a one-hour shift. On the fall-back morning + two standard-time hours share local 01:00; those loads are averaged. On + the spring-forward morning local 02:00 does not exist in the standard-time + series; it is filled with the average of local 01:00 and 03:00. + """ + local = load.with_columns( + pl.col("timestamp") + .dt.replace_time_zone("Etc/GMT+5") + .dt.convert_time_zone("America/New_York") + .dt.replace_time_zone(None) + .cast(pl.Datetime("us")) + .alias("timestamp") + ) + local = ( + local.group_by("timestamp") + .agg(pl.col("load_kw").mean(), pl.col("load_mw").mean()) + .sort("timestamp") + ) + year = local["timestamp"][0].year + expected_n = 8784 if year % 4 == 0 else 8760 + if local.height != expected_n - 1: + raise ValueError( + f"expected {expected_n - 1} local hours before the spring-forward " + f"fill, got {local.height}" + ) + + start = pl.select(pl.lit(f"{year}-01-01 00:00:00").str.to_datetime()).item() + end = pl.select(pl.lit(f"{year}-12-31 23:00:00").str.to_datetime()).item() + expected = pl.DataFrame( + {"timestamp": pl.datetime_range(start, end, interval="1h", eager=True)} + ).with_columns(pl.col("timestamp").cast(pl.Datetime("us"))) + joined = expected.join(local, on="timestamp", how="left").sort("timestamp") + missing = joined.filter(pl.col("load_kw").is_null()) + if missing.height != 1: + raise ValueError( + f"expected one missing local hour (spring-forward 02:00), got {missing.height}" + ) + gap = missing["timestamp"][0] + if (gap.month, gap.hour) != (3, 2): + raise ValueError(f"unexpected gap in local rate-class load: {gap}") + # One missing hour between its neighbors is their average. + filled = joined.with_columns( + pl.col("load_kw").interpolate(), + pl.col("load_mw").interpolate(), + ) + if filled.filter(pl.col("load_kw").is_null()).height: + raise ValueError("spring-forward hour was not filled") + return filled + + +def _validate_local_clock(load: pl.DataFrame) -> None: + if load["timestamp"].n_unique() != load.height: + raise ValueError("local rate-class timestamps are not unique") + start = load["timestamp"][0] + end = load["timestamp"][-1] + year = start.year + expected = 8784 if year % 4 == 0 else 8760 + if load.height != expected: + raise ValueError( + f"expected {expected} local hours in {year}, got {load.height}" + ) + if (start.month, start.day, start.hour) != (1, 1, 0): + raise ValueError(f"local series does not start at Jan 1 hour 0: {start}") + if (end.year, end.month, end.day, end.hour) != (year, 12, 31, 23): + raise ValueError(f"local series does not end at Dec 31 hour 23: {end}") + steps = load.select( + pl.col("timestamp").diff().dt.total_hours().alias("dh") + ).drop_nulls() + if steps.filter(pl.col("dh") != 1).height: + raise ValueError("local rate-class timestamps are not one hour apart") def _validate_rate_load(load: pl.DataFrame, peak_kw: float, usage_kwh: float) -> None: diff --git a/utils/data_prep/marginal_costs/convert_ct_occ863_system_load.py b/utils/data_prep/marginal_costs/convert_ct_occ863_system_load.py index a76cabe7..16d353f8 100644 --- a/utils/data_prep/marginal_costs/convert_ct_occ863_system_load.py +++ b/utils/data_prep/marginal_costs/convert_ct_occ863_system_load.py @@ -45,7 +45,7 @@ def hourly_system_load(raw: pl.DataFrame) -> pl.DataFrame: pl.col("Actual Load").cast(pl.Float64).alias("load_mw"), ) .drop_nulls(subset=["timestamp", "load_mw"]) - .with_columns(pl.col("timestamp").dt.round("1h")) + .with_columns(pl.col("timestamp").dt.round("1h").cast(pl.Datetime("us"))) .sort("timestamp") ) _validate_system_load(load) diff --git a/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py b/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py index 3c3b8d13..af6ddb4b 100644 --- a/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py +++ b/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py @@ -48,11 +48,12 @@ --output-s3-base s3://data.sb/switchbox/marginal_costs/ny/dist_and_sub_tx/ \ --upload - # CT/Eversource (CL&P): rate year 1 keeps the filed 2026$ scalar (no CPI adjustment) + # CT/Eversource (CL&P): rate year 1 keeps the filed 2026$ scalar (no CPI adjustment). + # PoP hours come from OCC-863 Attachment 3 page 2, not the ISO-NE hive layout. python generate_utility_tx_dx_mc.py --state CT --utility ct_eversource --year 2025 \ --target-dollar-year 2026 \ --mc-table-path rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv \ - --utility-load-s3-base s3://data.sb/isone/hourly_demand/utilities/ \ + --path-utility-load s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet/system_load.parquet \ --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ \ --upload """ @@ -117,6 +118,67 @@ def load_utility_load_profile( return df +def load_utility_load_file( + path: str, + storage_options: dict[str, str], +) -> pl.DataFrame: + """Load one load parquet that is not in the hive utility/year layout. + + The file must have ``timestamp`` and ``load_mw``. Year filtering happens + in ``normalize_load_to_cairo_8760``. CT uses this for the OCC-863 + Eversource system load, which is one multi-year file. + + Args: + path: Local path or ``s3://`` URI of the parquet. + storage_options: Polars S3 storage options. Ignored for a local path. + + Returns: + DataFrame with at least timestamp and load_mw. + """ + if path.startswith("s3://"): + lf = pl.scan_parquet(path, storage_options=storage_options) + else: + lf = pl.scan_parquet(path) + collected = lf.collect() + if not isinstance(collected, pl.DataFrame): + raise TypeError("Expected DataFrame from utility load file collect()") + df = collected + missing = {"timestamp", "load_mw"} - set(df.columns) + if missing: + raise ValueError( + f"Utility load file {path} is missing columns: {sorted(missing)}" + ) + if df.is_empty(): + raise FileNotFoundError(f"Utility load file is empty: {path}") + print(f"Loaded {len(df):,} hourly load records from {path}") + return df + + +def resolve_utility_load( + path_utility_load: str | None, + utility_load_s3_base: str | None, + year_load: int, + utility: str, + storage_options: dict[str, str], +) -> pl.DataFrame: + """Load the PoP profile from one parquet or from the hive utility/year scan. + + Exactly one of the two sources must be set. The hive scan is what NY, RI, + and MD use. The single file is the CT OCC-863 system load. + """ + if path_utility_load and utility_load_s3_base: + raise ValueError( + "Pass either --path-utility-load or --utility-load-s3-base, not both." + ) + if path_utility_load: + return load_utility_load_file(path_utility_load, storage_options) + if utility_load_s3_base: + return load_utility_load_profile( + utility_load_s3_base, year_load, utility, storage_options + ) + raise ValueError("Pass --path-utility-load or --utility-load-s3-base.") + + def normalize_load_to_cairo_8760( load_df: pl.DataFrame, utility: str, year_load: int ) -> pl.DataFrame: @@ -154,6 +216,10 @@ def normalize_load_to_cairo_8760( .alias("timestamp") ) + # The expected 8760 index is datetime[us]. A load file in another unit + # (e.g. ms from an Excel-sourced parquet) would fail the join without this. + df = df.with_columns(pl.col("timestamp").cast(pl.Datetime("us"))) + # Keep only requested load year. df = df.filter(pl.col("timestamp").dt.year() == year_load) if df.is_empty(): @@ -197,7 +263,9 @@ def normalize_load_to_cairo_8760( cur += timedelta(hours=1) if has_feb29: expected = [t for t in expected if not (t.month == 12 and t.day == 31)] - expected_df = pl.DataFrame({"timestamp": expected}) + expected_df = pl.DataFrame({"timestamp": expected}).with_columns( + pl.col("timestamp").cast(pl.Datetime("us")) + ) # Reindex and fill any missing hours. df = expected_df.join(df, on="timestamp", how="left").sort("timestamp") @@ -563,10 +631,20 @@ def main(): "--nyiso-s3-base", dest="utility_load_s3_base", type=str, - required=True, + default=None, help=( - "Base S3 path for utility loads " - "(e.g. s3://data.sb/eia/hourly_demand/utilities/)" + "Base S3 path for hive-partitioned utility loads " + "(utility=X/year=YYYY). Required unless --path-utility-load is set." + ), + ) + parser.add_argument( + "--path-utility-load", + dest="path_utility_load", + type=str, + default=None, + help=( + "Single parquet with timestamp and load_mw, used instead of the " + "hive scan. CT passes the OCC-863 system_load.parquet." ), ) parser.add_argument( @@ -622,8 +700,6 @@ def main(): load_year, ) - s3_base = args.utility_load_s3_base - print("=" * 60) print("MARGINAL COST ALLOCATION") print(f"State: {args.state}") @@ -634,12 +710,17 @@ def main(): print(f"Load year: {load_year}") print(f"Target dollar year: {target_dollar_year}") print(f"Output S3 base: {output_s3_base}") + if args.path_utility_load: + print(f"Load file: {args.path_utility_load}") + else: + print(f"Load hive base: {args.utility_load_s3_base}") print(f"Allocation window: Top {args.n_hours} hours") print(f"Upload to S3: {'Yes' if args.upload else 'No (inspection only)'}") print("=" * 60) - load_df = load_utility_load_profile( - s3_base, + load_df = resolve_utility_load( + args.path_utility_load, + args.utility_load_s3_base, load_year, args.utility, storage_options, From 8da40bca1c3a8803811a03b65430ffb9b4da150f Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Wed, 7 Oct 2026 20:07:31 +0000 Subject: [PATCH 07/12] Change output location for new OCC 863 based allocation --- .../ct_eversource_dist_mc_methodology.md | 36 +++++++++++-------- rate_design/hp_rates/ct/Justfile | 10 +++++- .../generate_utility_tx_dx_mc.py | 4 ++- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md index 077e329c..ba89f9ab 100644 --- a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md +++ b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md @@ -197,8 +197,8 @@ substation series: OCC-863 says no forecasted hourly system loads are available. 4. **Rate year 1 uses `$21.22` with no CPI adjustment.** An earlier run deflated the pre-loss Table 3 scalar from 2026$ to the 2025 load year (`$20.17 → $19.65`, CPI factor 0.9740). The same factor on `$21.22` would be about `$20.67`. The CT analysis is rate year 1, so the recipe - now passes `--target-dollar-year 2026` and the allocator keeps `$21.22`. The S3 parquet from - that earlier run still has the deflated pre-loss scalar until `create-dist-mc-data` is re-run. + now passes `--target-dollar-year 2026` and the allocator keeps `$21.22`. The earlier run's + parquet keeps the deflated pre-loss scalar (point 7). 5. **Ran the standard PoP allocation** via the new `just -f ct/Justfile create-dist-mc-data 2025 --upload` recipe (§3.5): ```bash @@ -207,7 +207,7 @@ substation series: OCC-863 says no forecasted hourly system loads are available. --target-dollar-year 2026 \ --mc-table-path rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv \ --path-utility-load s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet/system_load.parquet \ - --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ \ + --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/ \ --upload ``` 6. **Validated** (against the pre-loss `$20.17` scalar; re-run after the `$21.22` change): the @@ -217,11 +217,22 @@ substation series: OCC-863 says no forecasted hourly system loads are available. finding of concentrated summer peak-probability, though the specific split differs somewhat from CL&P's multi-year-normalized ~80% Jul–Aug / ~20% Jun–Sep because that run used a single year (2025) of ISO-NE CT zone load. The recipe now ranks the OCC-863 substation series instead; - its 2025 top 100 hours are 28 in June, 66 in July, and 6 in August (§3.3). The S3 parquet - still reflects the zone-load run until task 7 is re-run. -7. **Output**: same schema as all other states (`timestamp, utility, year, mc_total_per_kwh`) at - `s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet`. - Downstream CAIRO wiring is unchanged. + its 2025 top 100 hours are 28 in June, 66 in July, and 6 in August (§3.3). +7. **Output**: same schema as all other states (`timestamp, utility, year, mc_total_per_kwh`). + There are two CT datasets, so the OCC-863 run does not overwrite the earlier one: + + | Dataset | Scalar | Allocation load | + | ------------------------------------------------------------------ | ------------------------------------- | -------------------------------------- | + | `s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/` | `$20.17` deflated to `$19.65` (2025$) | ISO-NE CT zone load, 2025 | + | `s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/` | `$21.22`, rate year 1, no CPI | OCC-863 Attachment 3 system load, 2025 | + + Both use `utility=ct_eversource/year=2025/data.parquet`. The `_occ863` suffix names the + source of both changes: the secondary loss factor (Attachment 2) and the system load + (Attachment 3). It follows the sibling-prefix pattern of `dist_and_sub_tx_load{year}/`, so a + recursive scan of one dataset never picks up the other's 8760. `create-dist-mc-data` writes + only the `_occ863` dataset. The earlier one cannot be rebuilt from the current inputs, since + the config CSV and the load both changed. The CT scenario YAMLs still read `dist_and_sub_tx/`; + switching a scenario to the new 8760 means changing its `path_dist_and_sub_tx_mc`. 8. **Wired a `just` recipe**, `create-dist-mc-data`, in `rate_design/hp_rates/ct/Justfile` (mirrors `create-bulk-tx-mc-data`'s style, but scoped to `ct_eversource` only — see §3.5). 9. **Added tests** in `tests/test_ct_dist_mc.py`: ISO-NE zone-mapping coverage for both CT @@ -292,17 +303,14 @@ allocation or as a direct replacement. This is a nice-to-have, not a blocker for 1. ~~Add `"CT"` to `--state` choices in `generate_utility_tx_dx_mc.py`.~~ Done. 2. ~~Create `rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv` (§2.3).~~ Done. 3. ~~Wire a `just` recipe in `rate_design/hp_rates/ct/Justfile` (`create-dist-mc-data`).~~ Done. -4. ~~Run PoP allocation and validate (§3.4).~~ Done — output at +4. ~~Run PoP allocation and validate (§3.4).~~ Done for the earlier zone-load run, at `s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet`. 5. ~~Add tests (8760-hour coverage, annual reconciliation, seasonal concentration check).~~ Done — `tests/test_ct_dist_mc.py`. 6. ~~Update this doc and [dist_mc_definition_choice.md](dist_mc_definition_choice.md) §2–3, adding CT to the source-number and per-state tables.~~ Done. -7. **Remaining**: re-run `create-dist-mc-data 2025 --upload`. The parquet at - `s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet` - was built from the pre-loss `$20.17/kW-yr` scalar after a CPI deflation to 2025$, on ISO-NE zone - load. The recipe now allocates the unadjusted rate year 1 scalar, `$21.22/kW-yr`, on the OCC-863 - substation series (§3.3). +7. **Remaining**: run `create-dist-mc-data 2025 --upload`. It writes the OCC-863 8760 (§3.4 point 7) + to `dist_and_sub_tx_occ863/` and leaves the earlier `dist_and_sub_tx/` parquet in place. 8. **Remaining**: wire `path_dist_and_sub_tx_mc` into a CT scenario config (`scenarios_ct_eversource.yaml` or equivalent) once CT scenario YAMLs exist, so a CAIRO run actually consumes this MC output. No CT scenario configs exist yet in `rate_design/hp_rates/ct/config/scenarios/` — that's a separate, diff --git a/rate_design/hp_rates/ct/Justfile b/rate_design/hp_rates/ct/Justfile index dc2f46f8..de780ee3 100644 --- a/rate_design/hp_rates/ct/Justfile +++ b/rate_design/hp_rates/ct/Justfile @@ -15,6 +15,8 @@ state := env_var_or_default('STATE', 'ct') path_s3_utility_loads := "s3://data.sb/isone/hourly_demand/utilities/" path_s3_occ863_system_load := "s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet/system_load.parquet" +# Sibling of dist_and_sub_tx/, which keeps the earlier ISO-NE-zone-load 8760. +path_s3_dist_mc_occ863 := "s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/" import '../Justfile' @@ -124,6 +126,12 @@ create-bulk-tx-mc-data-all *extra_args: # distribution-substation load (top-100 hours by default). That file is one # multi-year parquet, so this recipe passes --path-utility-load rather than # the hive --utility-load-s3-base the other states use. +# +# Output goes to dist_and_sub_tx_occ863/, not dist_and_sub_tx/. The suffix +# names the source of both inputs: the secondary loss factor (Attachment 2) +# and the system load (Attachment 3). dist_and_sub_tx/ keeps the earlier +# 8760 ($20.17 deflated to 2025$, on ISO-NE CT zone load), which this recipe +# can no longer reproduce. Scenario YAMLs still point at dist_and_sub_tx/. # See context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md. # # CL&P's MCOS-2 covers ct_eversource only; ct_ui has no MCOS-derived value yet @@ -144,7 +152,7 @@ create-dist-mc-data year_arg *extra_args: --target-dollar-year 2026 \ --mc-table-path {{ path_config }}/marginal_costs/ct_marginal_costs_{{ year_arg }}.csv \ --path-utility-load {{ path_s3_occ863_system_load }} \ - --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ \ + --output-s3-base {{ path_s3_dist_mc_occ863 }} \ --n-hours {{ upstream_hours }} \ {{ extra_args }} diff --git a/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py b/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py index af6ddb4b..a5087faa 100644 --- a/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py +++ b/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py @@ -22,6 +22,8 @@ - RI default base: s3://data.sb/switchbox/marginal_costs/ri/dist_and_sub_tx/ - MD default base: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/ - CT default base: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ + (earlier ISO-NE zone-load run; the OCC-863 run writes to + dist_and_sub_tx_occ863/) - Default partition: utility=X/year=YYYY/data.parquet - Alternate-load-year base: _loadYYYY/ Example: dist_and_sub_tx_load2018/utility=X/year=2025/data.parquet @@ -54,7 +56,7 @@ --target-dollar-year 2026 \ --mc-table-path rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv \ --path-utility-load s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet/system_load.parquet \ - --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ \ + --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/ \ --upload """ From f8714a1d8b065c72a29940b7df19bc427356ba39 Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Thu, 8 Oct 2026 02:22:30 +0000 Subject: [PATCH 08/12] Set up new marginal cost path and start cairo run --- .../hp_rates/ct/config/scenarios/pipeline_ct_eversource.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rate_design/hp_rates/ct/config/scenarios/pipeline_ct_eversource.yaml b/rate_design/hp_rates/ct/config/scenarios/pipeline_ct_eversource.yaml index e7804ffb..b98c6d8e 100644 --- a/rate_design/hp_rates/ct/config/scenarios/pipeline_ct_eversource.yaml +++ b/rate_design/hp_rates/ct/config/scenarios/pipeline_ct_eversource.yaml @@ -15,7 +15,10 @@ resstock: upgrade_precalc: "00" upgrade_calibrated: "02" marginal_costs: - dist_and_sub_tx: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + # OCC-863 run: $21.22/kW-yr (secondary, rate year 1, no CPI) allocated on + # Eversource's system load. dist_and_sub_tx/ (no suffix) is the earlier + # ISO-NE zone-load 8760. See ct_eversource_dist_mc_methodology.md §3.4. + dist_and_sub_tx: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet bulk_tx: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet # No supply MC 8760s exist for CT (no ISO-NE LMP/FCM pipeline wired up yet); # point directly at zero.parquet so both delivery and supply runs get zero supply MCs. From cee772b660d0ee335c29f995e454e55ba52fe5ed Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Thu, 8 Oct 2026 03:05:50 +0000 Subject: [PATCH 09/12] Transcribe marginal cost excel file from OCC 863 --- context/README.md | 54 ++-- context/code/data/ct_occ863_system_load.md | 44 ++- rate_design/hp_rates/ct/Justfile | 17 + tests/test_ct_occ863_month_hour_mc.py | 106 +++++++ .../convert_ct_occ863_month_hour_mc.py | 292 ++++++++++++++++++ 5 files changed, 485 insertions(+), 28 deletions(-) create mode 100644 tests/test_ct_occ863_month_hour_mc.py create mode 100644 utils/data_prep/marginal_costs/convert_ct_occ863_month_hour_mc.py diff --git a/context/README.md b/context/README.md index 30d9d7e8..3ff21482 100644 --- a/context/README.md +++ b/context/README.md @@ -144,33 +144,33 @@ How CAIRO works: LMI, demand flex, tax rate, tiered rates, performance, parallel ResStock, EIA, NYISO, ISO-NE: where data comes from, how to read and prepare it. -| File | Purpose | -| -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| resstock_data_preparation_run_order.md | Run order for ResStock `_sb` release prep: recommended `main.py` invocation (single command covers fetch → metadata → approx → MF adj → utility assignment → monthly loads → upload), plus legacy individual Justfile step reference for debugging | -| resstock_sb_release_pipeline_main_py.md | Unified `data/resstock/main.py` pipeline: step-by-step reference for all implemented steps (2b through 2d), constants (`SB_CLONE_EXCLUDED_FILE_TYPES`, `sb_file_types`, column sets), config.yaml, state_configs.yaml (`SUPPORTED_UTILITY_STATES`, polygon filenames, per-state settings), sample mode, `path_raw` vs `path_sb` data flow, file types included/excluded in `_sb`, how to expand `_sb` scope | -| resstock_sb_annual_load_curves.md | `_sb` `load_curve_annual` from modified hourly (combined with monthly in `aggregate_loads`): one hourly read → monthly write + annual row; keep/drop/aggregate decisions; upgrade 01–05 savings; `upgrade_name`; energy_delivered 4× bug (bsf before 1.6.6); pipeline step 2d | -| resstock_lmi_metadata_guide.md | ResStock 2024.2 parquet metadata: columns for LMI tier assignment, FPL/SMI, income | -| parquet_reads_local_vs_s3.md | Reading Parquet from local disk vs S3: per-GET overhead, file discovery, Hive filters vs path construction, best practices for whole-state and per-utility reads | -| polars_laziness_and_validation.md | Polars LazyFrame best practices: when to collect, runtime data-quality asserts vs laziness/streaming, strategies for small and large data | -| nyiso_lbmp_zonal_data_sources.md | NYISO Day-Ahead/Real-Time zonal LBMP: MIS ZIP vs gridstatus vs NYISOToolkit; data samples | -| pjm_hourly_loads.md | PJM-native hourly zonal loads (Data Miner 2 `hrl_load_metered`): why PJM over EIA-930, feed quirks (legacy `BC`/`PEP`/`DPL`/`AP` zone codes, `load_area`-grained → sum to zone), archive/standard 731-day boundary handling in `data/pjm/dataminer.py`, EPT-year vs UTC-built timestamp / 8760-8784 completeness, zone→utility mapping, output layout, and how to run the pipeline | -| eia_861_supply_vs_delivery_stats.md | EIA-861 supply vs delivery stats: definitions and use | -| compare_resstock_eia861_loads.md | ResStock hourly load vs EIA-861 residential sales comparison: script usage, defaults, interpretation | -| investigate_resstock_eia_load_discrepancy.md | ResStock vs EIA load discrepancy by utility: MF non-HVAC adjustment, column-by-column SF/MF ratios, workflow, design decisions, and relation to compare_resstock_eia861_loads | -| isone_fca_yaml_data_guide.md | ISO-NE FCA assumptions YAML data dictionary: what each field means (clearing price, CSO, months), where to find source data online (FCA results report, CELT Report Table 4.1), and how to update for a new year | -| utility_assignment_resstock.md | ResStock utility assignment: generic architecture (`assign_utility.py`, `utils.py`), NY GIS-based implementation (excluded gas utilities, PUMA overlap, nearest-neighbor donor, name crosswalk), MD GIS-based implementation (HIFLD electric/gas sources, no crosswalk, nearest-neighbor PUMA fill for ~26% coverage gap, data quality analysis), RI rule-based pattern, `SUPPORTED_UTILITY_STATES` derivation from `state_configs.yaml`, invocation via main.py (step 2b), and a full checklist for adding a new state | -| ct_utility_gis_data_sources.md | CT electric utility service territory GIS sources: current NASA NCCS HIFLD mirror (actual polygon boundaries, 2022 vintage), PUMA TIGER/Line download, HIFLD Open shutdown background, mirror comparison table, and PUDL/EIA-861 county-level alternative for future use | -| approximate_non_hp_load.md | MF highrise non-HP load approximation: nearest-neighbor RMSE, replace HVAC columns, update/sink, validation; invocation via main.py (step 2c-i) vs legacy Justfile recipe | -| ny_lmi_discounts_genability_encoding.md | EAP/LMI encoding in NY Genability electric tariffs: which utilities encode EAP, tier/heating structure, amount comparison to lmi_discounts_in_ny.md | -| tariff_rates_and_genability.md | Tariff/rate/rider/charge terminology, Genability/Arcadia data model (rateUnit, calculationFactor, applicabilityValue, masterTariffId vs tariffId, rider pattern), two tariff_fetch library bugs and their patches (BGE empty-band BOOLEAN applicability bug; Pepco/DPL calculation_factor fold for GRT/DSIC charges), why NY utilities did not trigger the same bugs, tariff versioning/date-boundary behavior, and how to reconstruct historical monthly charges via the Calculate API | -| ny_genability_charge_fetch_map.md | Exhaustive charge-level table for NY Genability tariffs: tariffRateId, fetch_type, variableRateKey, master_charge, decision; used to fetch 2025 monthly rates from Arcadia API for top-up implementation. | -| psegli_revenue_requirement_estimation.md | PSEG-LI revenue requirement estimation: why no rate case, TOU overcount problem, bill-proportional method from LIPA budget, charge classification, implementation via `estimate_psegli_rr.py`. | -| tariff_generation_pipeline.md | Tariff generation pipeline: Genability → monthly_rates YAML → flat and default-structure URDB v7 JSONs. Covers fetch_monthly_rates.py, charge classification, monthly_rates YAML structure, create_flat_tariffs.py (top-down from RR), create_default_structure_tariffs.py (bottom-up from filed rates), BASE_TARIFF_PATTERN, Justfile wiring (all-pre), CAIRO calibration, and why flat vs default rates differ. | -| md_tariff_fetch.md | MD electric and gas tariff pipeline: Arcadia electric URDBs, RateAcuity gas tariffs, Chesapeake county and RES-1/RES-2 mapping from `_sb` annual therms, UGI/Easton commodity rates, `tariff_fetch` bug patches, verified rate values, and BGE Schedule RD fetch (`masterTariffId` 3350180, effective date 2026-01-01). | -| ct_tariff_fetch.md | CT gas tariff **fetch** (ticket scope: RateAcuity URDB JSONs + `null_gas_tariff` + envelope only): schedules for CNG / Yankee / SCG (three residential classes each) plus Norwich muni (GRES + GSHRES/`_mf`), evidence. Omits (reference only, with Zotero links): CNG Rate SE Seasonal Gas Service; System Expansion `*-SE` (RSH-SE / 02-SE etc.); third-party supply. **Known limitation:** IOU `_mf` demand / MDQ charges dropped by `tariff_fetch`; CAIRO gas path has no MDQ billing. Mapper / tariff maps are follow-up. | -| ct_eversource_custom_rates.md | CT Eversource hand-encoded proposed rates (Docket 26-05-10): `ct_eversource_rate6` (HP space-heating) + `ct_eversource_rate1_proposed` (matched comparison baseline), each as delivery + `_supply` JSON pairs with top-up YAMLs. Documents the filed-2026 delivery vintage + Genability 2025 supply, the ESI/RDM roll-in and SBC/CTA/FMCC decisions, full per-number provenance to the CLP-Rates exhibits, the transmission vintage note (why filed 0.05050 differs from RateAcuity 0.03401/0.04433), and the **pre-calc caveat** (fixed rates, no calibrated RR — bill-calc runs only). | -| ct_eversource_class_rev_requirement.md | CT Eversource **whole residential rate class** revenue-requirement files (Docket 26-05-10): the test-year (`ct_eversource_rate_case_test_year.yaml`, current rates) and rate-year-1 (`ct_eversource_rate_case_rate_year_1.yaml`, Proposed+GET) Track 2 YAMLs that model every residential ResStock building on Rate 1 but size base DRR / kWh / customer count to the class (Rate 1 + 5 + 7). Documents the bug it fixes (whole-class ResStock pinned to a Rate-1 target), how top-ups scale off the two class-total scalars, full per-number provenance to the CLP-RATES-2 exhibit with derivations, and the Proposed+GET vs pre-GET caveat. | -| ct_occ863_system_load.md | OCC-863 (Docket 26-05-10) source files: where the four workbooks and the discovery PDF live on S3 and the EBS mirror. Attachment 3 page 2 becomes `system_load.parquet` (hourly distribution-substation load, MW, 2022–2025); page 1 becomes `system_monthly_peak_mw.parquet`. Attachment 4 becomes `rate_1_load.parquet`, `rate_5_load.parquet`, and `rate_7_load.parquet` (2025 hourly class kW; interval-ending EST shifted to hour-beginning local time). `create-dist-mc-data` reads `system_load.parquet` through `--path-utility-load`. | +| File | Purpose | +| -------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| resstock_data_preparation_run_order.md | Run order for ResStock `_sb` release prep: recommended `main.py` invocation (single command covers fetch → metadata → approx → MF adj → utility assignment → monthly loads → upload), plus legacy individual Justfile step reference for debugging | +| resstock_sb_release_pipeline_main_py.md | Unified `data/resstock/main.py` pipeline: step-by-step reference for all implemented steps (2b through 2d), constants (`SB_CLONE_EXCLUDED_FILE_TYPES`, `sb_file_types`, column sets), config.yaml, state_configs.yaml (`SUPPORTED_UTILITY_STATES`, polygon filenames, per-state settings), sample mode, `path_raw` vs `path_sb` data flow, file types included/excluded in `_sb`, how to expand `_sb` scope | +| resstock_sb_annual_load_curves.md | `_sb` `load_curve_annual` from modified hourly (combined with monthly in `aggregate_loads`): one hourly read → monthly write + annual row; keep/drop/aggregate decisions; upgrade 01–05 savings; `upgrade_name`; energy_delivered 4× bug (bsf before 1.6.6); pipeline step 2d | +| resstock_lmi_metadata_guide.md | ResStock 2024.2 parquet metadata: columns for LMI tier assignment, FPL/SMI, income | +| parquet_reads_local_vs_s3.md | Reading Parquet from local disk vs S3: per-GET overhead, file discovery, Hive filters vs path construction, best practices for whole-state and per-utility reads | +| polars_laziness_and_validation.md | Polars LazyFrame best practices: when to collect, runtime data-quality asserts vs laziness/streaming, strategies for small and large data | +| nyiso_lbmp_zonal_data_sources.md | NYISO Day-Ahead/Real-Time zonal LBMP: MIS ZIP vs gridstatus vs NYISOToolkit; data samples | +| pjm_hourly_loads.md | PJM-native hourly zonal loads (Data Miner 2 `hrl_load_metered`): why PJM over EIA-930, feed quirks (legacy `BC`/`PEP`/`DPL`/`AP` zone codes, `load_area`-grained → sum to zone), archive/standard 731-day boundary handling in `data/pjm/dataminer.py`, EPT-year vs UTC-built timestamp / 8760-8784 completeness, zone→utility mapping, output layout, and how to run the pipeline | +| eia_861_supply_vs_delivery_stats.md | EIA-861 supply vs delivery stats: definitions and use | +| compare_resstock_eia861_loads.md | ResStock hourly load vs EIA-861 residential sales comparison: script usage, defaults, interpretation | +| investigate_resstock_eia_load_discrepancy.md | ResStock vs EIA load discrepancy by utility: MF non-HVAC adjustment, column-by-column SF/MF ratios, workflow, design decisions, and relation to compare_resstock_eia861_loads | +| isone_fca_yaml_data_guide.md | ISO-NE FCA assumptions YAML data dictionary: what each field means (clearing price, CSO, months), where to find source data online (FCA results report, CELT Report Table 4.1), and how to update for a new year | +| utility_assignment_resstock.md | ResStock utility assignment: generic architecture (`assign_utility.py`, `utils.py`), NY GIS-based implementation (excluded gas utilities, PUMA overlap, nearest-neighbor donor, name crosswalk), MD GIS-based implementation (HIFLD electric/gas sources, no crosswalk, nearest-neighbor PUMA fill for ~26% coverage gap, data quality analysis), RI rule-based pattern, `SUPPORTED_UTILITY_STATES` derivation from `state_configs.yaml`, invocation via main.py (step 2b), and a full checklist for adding a new state | +| ct_utility_gis_data_sources.md | CT electric utility service territory GIS sources: current NASA NCCS HIFLD mirror (actual polygon boundaries, 2022 vintage), PUMA TIGER/Line download, HIFLD Open shutdown background, mirror comparison table, and PUDL/EIA-861 county-level alternative for future use | +| approximate_non_hp_load.md | MF highrise non-HP load approximation: nearest-neighbor RMSE, replace HVAC columns, update/sink, validation; invocation via main.py (step 2c-i) vs legacy Justfile recipe | +| ny_lmi_discounts_genability_encoding.md | EAP/LMI encoding in NY Genability electric tariffs: which utilities encode EAP, tier/heating structure, amount comparison to lmi_discounts_in_ny.md | +| tariff_rates_and_genability.md | Tariff/rate/rider/charge terminology, Genability/Arcadia data model (rateUnit, calculationFactor, applicabilityValue, masterTariffId vs tariffId, rider pattern), two tariff_fetch library bugs and their patches (BGE empty-band BOOLEAN applicability bug; Pepco/DPL calculation_factor fold for GRT/DSIC charges), why NY utilities did not trigger the same bugs, tariff versioning/date-boundary behavior, and how to reconstruct historical monthly charges via the Calculate API | +| ny_genability_charge_fetch_map.md | Exhaustive charge-level table for NY Genability tariffs: tariffRateId, fetch_type, variableRateKey, master_charge, decision; used to fetch 2025 monthly rates from Arcadia API for top-up implementation. | +| psegli_revenue_requirement_estimation.md | PSEG-LI revenue requirement estimation: why no rate case, TOU overcount problem, bill-proportional method from LIPA budget, charge classification, implementation via `estimate_psegli_rr.py`. | +| tariff_generation_pipeline.md | Tariff generation pipeline: Genability → monthly_rates YAML → flat and default-structure URDB v7 JSONs. Covers fetch_monthly_rates.py, charge classification, monthly_rates YAML structure, create_flat_tariffs.py (top-down from RR), create_default_structure_tariffs.py (bottom-up from filed rates), BASE_TARIFF_PATTERN, Justfile wiring (all-pre), CAIRO calibration, and why flat vs default rates differ. | +| md_tariff_fetch.md | MD electric and gas tariff pipeline: Arcadia electric URDBs, RateAcuity gas tariffs, Chesapeake county and RES-1/RES-2 mapping from `_sb` annual therms, UGI/Easton commodity rates, `tariff_fetch` bug patches, verified rate values, and BGE Schedule RD fetch (`masterTariffId` 3350180, effective date 2026-01-01). | +| ct_tariff_fetch.md | CT gas tariff **fetch** (ticket scope: RateAcuity URDB JSONs + `null_gas_tariff` + envelope only): schedules for CNG / Yankee / SCG (three residential classes each) plus Norwich muni (GRES + GSHRES/`_mf`), evidence. Omits (reference only, with Zotero links): CNG Rate SE Seasonal Gas Service; System Expansion `*-SE` (RSH-SE / 02-SE etc.); third-party supply. **Known limitation:** IOU `_mf` demand / MDQ charges dropped by `tariff_fetch`; CAIRO gas path has no MDQ billing. Mapper / tariff maps are follow-up. | +| ct_eversource_custom_rates.md | CT Eversource hand-encoded proposed rates (Docket 26-05-10): `ct_eversource_rate6` (HP space-heating) + `ct_eversource_rate1_proposed` (matched comparison baseline), each as delivery + `_supply` JSON pairs with top-up YAMLs. Documents the filed-2026 delivery vintage + Genability 2025 supply, the ESI/RDM roll-in and SBC/CTA/FMCC decisions, full per-number provenance to the CLP-Rates exhibits, the transmission vintage note (why filed 0.05050 differs from RateAcuity 0.03401/0.04433), and the **pre-calc caveat** (fixed rates, no calibrated RR — bill-calc runs only). | +| ct_eversource_class_rev_requirement.md | CT Eversource **whole residential rate class** revenue-requirement files (Docket 26-05-10): the test-year (`ct_eversource_rate_case_test_year.yaml`, current rates) and rate-year-1 (`ct_eversource_rate_case_rate_year_1.yaml`, Proposed+GET) Track 2 YAMLs that model every residential ResStock building on Rate 1 but size base DRR / kWh / customer count to the class (Rate 1 + 5 + 7). Documents the bug it fixes (whole-class ResStock pinned to a Rate-1 target), how top-ups scale off the two class-total scalars, full per-number provenance to the CLP-RATES-2 exhibit with derivations, and the Proposed+GET vs pre-GET caveat. | +| ct_occ863_system_load.md | OCC-863 (Docket 26-05-10) source files: where the four workbooks and the discovery PDF live on S3 and the EBS mirror. Attachment 3 page 2 becomes `system_load.parquet` (hourly distribution-substation load, MW, 2022–2025); page 1 becomes `system_monthly_peak_mw.parquet`. Attachment 4 becomes `rate_1_load.parquet`, `rate_5_load.parquet`, and `rate_7_load.parquet` (2025 hourly class kW; interval-ending EST shifted to hour-beginning local time). Attachment 2 becomes six month-hour marginal-cost parquets: `all_days`, `weekday`, and `weekend_holidays`, each for secondary (page 1) and primary (page 2). `create-dist-mc-data` reads `system_load.parquet` through `--path-utility-load`. | ### code/marginal_costs/ diff --git a/context/code/data/ct_occ863_system_load.md b/context/code/data/ct_occ863_system_load.md index 2abed78d..80dab14b 100644 --- a/context/code/data/ct_occ863_system_load.md +++ b/context/code/data/ct_occ863_system_load.md @@ -24,7 +24,13 @@ s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/ ├── system_monthly_peak_mw.parquet ├── rate_1_load.parquet ├── rate_5_load.parquet - └── rate_7_load.parquet + ├── rate_7_load.parquet + ├── month_hour_marginal_costs_all_days_secondary.parquet + ├── month_hour_marginal_costs_weekday_secondary.parquet + ├── month_hour_marginal_costs_weekend_holidays_secondary.parquet + ├── month_hour_marginal_costs_all_days_primary.parquet + ├── month_hour_marginal_costs_weekday_primary.parquet + └── month_hour_marginal_costs_weekend_holidays_primary.parquet ``` The local path is the EBS mirror of the S3 prefix (`/ebs/data/` mirrors `s3://data.sb/`). The @@ -89,6 +95,42 @@ just -f rate_design/hp_rates/ct/Justfile convert-occ863-rate-class-load \ /ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet ``` +## Attachment 2: marginal cost by month and hour + +Page 1 is secondary service and page 2 is primary. Each has three month-by-hour matrices in +`$/kWh`: all day types, weekdays, and weekends plus public holidays. Above each matrix a "No Days" +row gives the day count for that month. `--voltage secondary` reads page 1 and `--voltage primary` +reads page 2. Each run writes three parquets, melted to one row per month and hour. The names end +in `_secondary` or `_primary`: + +| File stem | Matrix | +| -------------------------------------------- | ---------------------------- | +| `month_hour_marginal_costs_all_days` | all day types | +| `month_hour_marginal_costs_weekday` | weekdays only | +| `month_hour_marginal_costs_weekend_holidays` | weekends and public holidays | + +| Column | Meaning | +| ------------ | ------------------------------------------------------------------- | +| `month` | 1–12 | +| `hour` | 0–23, the hour label on the sheet | +| `num_days` | That matrix's "No Days" count for the month, repeated on every hour | +| `mc_per_kwh` | The matrix value, in `$/kWh` | + +For each month the all-day-types day count must equal the weekday count plus the +weekend-and-holiday count. The fourth sheet is a copy of page 3 (the probability-of-peak table) +with an extra `dist_mc` column, not a copy of page 2. + +```bash +just -f rate_design/hp_rates/ct/Justfile convert-occ863-month-hour-mc \ + /ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/xlsx/attachment-2.xlsx \ + /ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet \ + secondary +just -f rate_design/hp_rates/ct/Justfile convert-occ863-month-hour-mc \ + /ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/xlsx/attachment-2.xlsx \ + /ebs/data/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet \ + primary +``` + ## Using this load for CT distribution marginal cost `create-dist-mc-data` passes `system_load.parquet` as `--path-utility-load`. The allocator keeps diff --git a/rate_design/hp_rates/ct/Justfile b/rate_design/hp_rates/ct/Justfile index de780ee3..b6031493 100644 --- a/rate_design/hp_rates/ct/Justfile +++ b/rate_design/hp_rates/ct/Justfile @@ -189,6 +189,23 @@ convert-occ863-rate-class-load path_xlsx path_output_dir: --path-xlsx {{ path_xlsx }} \ --path-output-dir {{ path_output_dir }} +# ============================================================================= +# CT-only: OCC-863 Attachment 2 → month-hour marginal costs +# +# voltage is secondary (page 1) or primary (page 2). Writes the all-day-types, +# weekday, and weekend-and-holiday matrices. Each row is one month (1–12) and +# hour (0–23), with that month's "No Days" count and the $/kWh value. See +# context/code/data/ct_occ863_system_load.md. +# +# just -f ct/Justfile convert-occ863-month-hour-mc secondary +# ============================================================================= + +convert-occ863-month-hour-mc path_xlsx path_output_dir voltage: + cd {{ path_repo }} && uv run python {{ path_repo }}/utils/data_prep/marginal_costs/convert_ct_occ863_month_hour_mc.py \ + --path-xlsx {{ path_xlsx }} \ + --path-output-dir {{ path_output_dir }} \ + --voltage {{ voltage }} + # ============================================================================= # CT-only: gas tariff fetch # diff --git a/tests/test_ct_occ863_month_hour_mc.py b/tests/test_ct_occ863_month_hour_mc.py new file mode 100644 index 00000000..80a56904 --- /dev/null +++ b/tests/test_ct_occ863_month_hour_mc.py @@ -0,0 +1,106 @@ +"""OCC-863 Attachment 2 page 1 → month-hour marginal costs. No S3. + +The real workbook is not in git. These tests build a small stand-in with the +three matrices page 1 has: all day types, weekdays, and weekends plus holidays. +""" + +from __future__ import annotations + +import polars as pl +import pytest + +from utils.data_prep.marginal_costs.convert_ct_occ863_month_hour_mc import ( + Voltage, + _MATRIX_TITLES, + month_hour_marginal_costs, +) + + +def _matrix_block( + title: str, day_counts: list[int], value: float +) -> list[tuple[object, ...]]: + width = 16 + rows: list[tuple[object, ...]] = [] + title_row: list[object] = [None] * width + title_row[2] = title + rows.append(tuple(title_row)) + + days_row: list[object] = [None] * width + days_row[3] = "No Days" + month_row: list[object] = [None] * width + month_row[3] = "Month >" + for month, days in enumerate(day_counts, start=1): + days_row[3 + month] = days + month_row[3 + month] = month + rows.append(tuple(days_row)) + rows.append(tuple(month_row)) + + header: list[object] = [None] * width + header[2] = "Hour" + rows.append(tuple(header)) + for hour in range(24): + hour_row: list[object] = [None] * width + hour_row[2] = hour + for month in range(1, 13): + hour_row[3 + month] = value if month == 8 and hour == 16 else 0 + rows.append(tuple(hour_row)) + average: list[object] = [None] * width + average[3] = "Daily Average" + rows.append(tuple(average)) + return rows + + +def _page(voltage: Voltage, *, weekday_days: list[int] | None = None) -> pl.DataFrame: + default_weekday = [19, 19, 23, 22, 20, 21, 21, 22, 21, 20, 20, 22] + weekend_days = [10, 9, 10, 8, 10, 10, 9, 10, 9, 9, 12, 9] + if weekday_days is None: + weekday_days = default_weekday + all_days = [a + b for a, b in zip(default_weekday, weekend_days, strict=True)] + all_days_title, weekday_title, weekend_title = _MATRIX_TITLES[voltage] + rows = ( + [("title",)] + + _matrix_block(all_days_title, all_days, 0.001) + + _matrix_block(weekday_title, weekday_days, 0.02557) + + _matrix_block(weekend_title, weekend_days, 0.00011) + ) + width = max(len(row) for row in rows) + padded = [row + (None,) * (width - len(row)) for row in rows] + return pl.DataFrame(padded, orient="row") + + +def test_melts_weekdays_with_the_day_count_repeated_on_every_hour() -> None: + all_days, weekday, weekend = month_hour_marginal_costs( + _page("secondary"), "secondary" + ) + assert weekday.columns == ["month", "hour", "num_days", "mc_per_kwh"] + assert all_days.height == 288 + assert weekday.height == 288 + assert weekend.height == 288 + january = weekday.filter(pl.col("month") == 1) + assert january["hour"].to_list() == list(range(24)) + assert january["num_days"].unique().to_list() == [19] + assert january["mc_per_kwh"].sum() == 0 + august_hour_16 = weekday.filter((pl.col("month") == 8) & (pl.col("hour") == 16)) + assert august_hour_16["mc_per_kwh"].item() == pytest.approx(0.02557) + assert august_hour_16["num_days"].item() == 22 + weekend_august = weekend.filter((pl.col("month") == 8) & (pl.col("hour") == 16)) + assert weekend_august["mc_per_kwh"].item() == pytest.approx(0.00011) + assert weekend_august["num_days"].item() == 10 + all_days_august = all_days.filter((pl.col("month") == 8) & (pl.col("hour") == 16)) + assert all_days_august["mc_per_kwh"].item() == pytest.approx(0.001) + assert all_days_august["num_days"].item() == 32 + + +def test_reads_primary_titles() -> None: + all_days, _, _ = month_hour_marginal_costs(_page("primary"), "primary") + assert all_days.filter((pl.col("month") == 8) & (pl.col("hour") == 16))[ + "mc_per_kwh" + ].item() == pytest.approx(0.001) + + +def test_rejects_day_counts_that_do_not_add_up() -> None: + weekday_days = [18, 19, 23, 22, 20, 21, 21, 22, 21, 20, 20, 22] + with pytest.raises(ValueError, match="day counts"): + month_hour_marginal_costs( + _page("secondary", weekday_days=weekday_days), "secondary" + ) diff --git a/utils/data_prep/marginal_costs/convert_ct_occ863_month_hour_mc.py b/utils/data_prep/marginal_costs/convert_ct_occ863_month_hour_mc.py new file mode 100644 index 00000000..2ae18f63 --- /dev/null +++ b/utils/data_prep/marginal_costs/convert_ct_occ863_month_hour_mc.py @@ -0,0 +1,292 @@ +"""Convert OCC-863 Attachment 2 into month-hour marginal costs. + +Attachment 2 answers OCC-863 subpart (b) for PURA Docket 26-05-10. Page 1 is +secondary service and page 2 is primary. Each holds three month-by-hour +matrices of average marginal cost in $/kWh: all day types, weekdays, and +weekends plus public holidays. Each matrix has a "No Days" row giving the day +count behind that month. + +``--voltage`` selects the sheet. For each voltage this module writes all three +matrices. It checks that the all-day-types day count equals the weekday count +plus the weekend-and-holiday count. Page 3 is the probability-of-peak table +those matrices are built from. A fourth sheet is a copy of page 3 with an +extra ``dist_mc`` column; it is not page 2. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Literal + +import polars as pl + +Voltage = Literal["primary", "secondary"] + +_MARGINAL_COST_SHEETS: dict[Voltage, str] = { + "secondary": "OCC-863 Attachment 2 Page 1", + "primary": "OCC-863 Attachment 2 Page 2", +} +_N_MONTHS = 12 +_N_HOURS = 24 +# Titles as stored on the sheet, with line breaks collapsed to single spaces. +# The weekend secondary title has no space before "("; the primary one does. +_MATRIX_TITLES: dict[Voltage, tuple[str, str, str]] = { + "secondary": ( + "Average of Hourly Marginal Cost, across all day types, Secondary $/kWh", + "Average Hourly Marginal Cost by Month, Weekdays Only. Secondary ($/kWh)", + "Average Hourly Marginal Cost by Month, Weekend & Public Holiday, Secondary( $/kWh)", + ), + "primary": ( + "Average of Hourly Marginal Cost Across day types, Primary $/kWh", + "Average Hourly Marginal Cost by Month, Weekdays Only. Primary ($/kWh)", + "Average Hourly Marginal Cost by Month, Weekend & Public Holiday, Primary ($/kWh)", + ), +} + + +def month_hour_marginal_costs( + raw: pl.DataFrame, voltage: Voltage +) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]: + """Return the all-days, weekday, and weekend-and-holiday matrices. + + ``voltage`` is ``primary`` or ``secondary``. It selects the three matrix + titles on that sheet. The caller reads the matching sheet. + + Each frame has ``month`` (1–12), ``hour`` (0–23), ``num_days`` (the "No + Days" count for that month, repeated on every hour), and ``mc_per_kwh``. + """ + all_days_title, weekday_title, weekend_title = _MATRIX_TITLES[voltage] + rows = raw.rows() + all_days = _melt_matrix(rows, all_days_title) + weekday = _melt_matrix(rows, weekday_title) + weekend = _melt_matrix(rows, weekend_title) + _check_day_counts_add(all_days, weekday, weekend) + return all_days, weekday, weekend + + +def _melt_matrix(rows: list[tuple[object, ...]], title: str) -> pl.DataFrame: + title_idx = _find_title(rows, title) + days_idx = _find_label(rows, title_idx, "No Days") + month_idx = _find_label(rows, days_idx, "Month") + hour_header_idx, hour_col = _find_hour_header(rows, month_idx) + month_cols = _month_columns(rows[month_idx]) + day_counts = _day_counts(rows[days_idx], month_cols) + + records: list[dict[str, int | float]] = [] + for row in rows[hour_header_idx + 1 :]: + hour = _as_int(row[hour_col] if hour_col < len(row) else None) + if hour is None: + break + if not 0 <= hour < _N_HOURS: + raise ValueError(f"hour {hour} is outside 0–23 in {title!r}") + for col, month in month_cols: + mc = _as_number(row[col] if col < len(row) else None) + if mc is None: + raise ValueError( + f"missing {title!r} value at month {month} hour {hour}" + ) + if mc < 0: + raise ValueError( + f"negative {title!r} value at month {month} hour {hour}" + ) + records.append( + { + "month": month, + "hour": hour, + "num_days": day_counts[month], + "mc_per_kwh": mc, + } + ) + + frame = pl.DataFrame(records).cast( + { + "month": pl.Int64, + "hour": pl.Int64, + "num_days": pl.Int64, + "mc_per_kwh": pl.Float64, + } + ) + _validate_matrix(frame, title) + return frame.sort("month", "hour") + + +def _find_title(rows: list[tuple[object, ...]], title: str) -> int: + for idx, row in enumerate(rows): + for cell in row: + if _cell_text(cell) == title: + return idx + raise ValueError(f"Attachment 2 has no matrix titled {title!r}") + + +def _find_label(rows: list[tuple[object, ...]], start: int, label: str) -> int: + for idx in range(start, len(rows)): + if any(_cell_text(cell).startswith(label) for cell in rows[idx]): + return idx + raise ValueError(f"Attachment 2 has no {label!r} row after the matrix title") + + +def _find_hour_header(rows: list[tuple[object, ...]], start: int) -> tuple[int, int]: + for idx in range(start, len(rows)): + for col, cell in enumerate(rows[idx]): + if _cell_text(cell) == "Hour": + return idx, col + raise ValueError("Attachment 2 has no Hour header") + + +def _month_columns(row: tuple[object, ...]) -> list[tuple[int, int]]: + found: list[tuple[int, int]] = [] + for col, cell in enumerate(row): + month = _as_int(cell) + if month is not None and 1 <= month <= 12: + found.append((col, month)) + months = [month for _, month in found] + if months != list(range(1, 13)): + raise ValueError(f"month header is {months}, expected 1–12 in order") + return found + + +def _day_counts( + row: tuple[object, ...], month_cols: list[tuple[int, int]] +) -> dict[int, int]: + counts: dict[int, int] = {} + for col, month in month_cols: + days = _as_int(row[col] if col < len(row) else None) + if days is None or days <= 0: + raise ValueError(f"month {month} has no positive day count") + counts[month] = days + return counts + + +def _validate_matrix(frame: pl.DataFrame, title: str) -> None: + if frame.height != _N_MONTHS * _N_HOURS: + raise ValueError(f"{title!r} has {frame.height} rows, expected 288") + if frame.select("month", "hour").n_unique() != frame.height: + raise ValueError(f"{title!r} repeats a month-hour") + hours = set(frame["hour"].unique().to_list()) + months = set(frame["month"].unique().to_list()) + if hours != set(range(_N_HOURS)) or months != set(range(1, 13)): + raise ValueError(f"{title!r} does not cover months 1–12 and hours 0–23") + varying = ( + frame.group_by("month") + .agg(pl.col("num_days").n_unique()) + .filter(pl.col("num_days") != 1) + ) + if varying.height: + raise ValueError(f"{title!r} day count changes within a month") + + +def _check_day_counts_add( + all_days: pl.DataFrame, weekday: pl.DataFrame, weekend: pl.DataFrame +) -> None: + """Weekday days plus weekend-and-holiday days equal the all-days count.""" + combined = ( + weekday.select("month", pl.col("num_days").alias("weekday")) + .unique() + .join( + weekend.select("month", pl.col("num_days").alias("weekend")).unique(), + on="month", + ) + .join( + all_days.select("month", pl.col("num_days").alias("all_days")).unique(), + on="month", + ) + .filter(pl.col("weekday") + pl.col("weekend") != pl.col("all_days")) + ) + if combined.height: + raise ValueError( + "weekday plus weekend day counts do not match the all-days row: " + f"{combined.rows()}" + ) + + +def _cell_text(value: object) -> str: + if value is None: + return "" + return " ".join(str(value).split()) + + +def _as_number(value: object) -> float | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, (int, float)): + return float(value) + text = str(value).strip().replace(",", "") + if not text: + return None + try: + return float(text) + except ValueError: + return None + + +def _as_int(value: object) -> int | None: + number = _as_number(value) + if number is None or not number.is_integer(): + return None + return int(number) + + +def _filename(kind: str, voltage: Voltage) -> str: + return f"month_hour_marginal_costs_{kind}_{voltage}.parquet" + + +def write_month_hour_parquets( + path_xlsx: Path, path_output_dir: Path, voltage: Voltage +) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]: + """Read one Attachment 2 sheet and write its three parquets.""" + if "{{" in str(path_xlsx) or "{{" in str(path_output_dir): + raise ValueError( + f"path looks like an uninterpolated Just variable: {path_xlsx} {path_output_dir}" + ) + raw = pl.read_excel( + path_xlsx, sheet_name=_MARGINAL_COST_SHEETS[voltage], has_header=False + ) + if not isinstance(raw, pl.DataFrame): + raise TypeError(f"expected one sheet from Attachment 2 {voltage}") + all_days, weekday, weekend = month_hour_marginal_costs(raw, voltage) + path_output_dir.mkdir(parents=True, exist_ok=True) + for kind, frame in ( + ("all_days", all_days), + ("weekday", weekday), + ("weekend_holidays", weekend), + ): + frame.write_parquet(path_output_dir / _filename(kind, voltage)) + return all_days, weekday, weekend + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--path-xlsx", type=Path, required=True, help="OCC-863 Attachment 2 workbook" + ) + parser.add_argument( + "--path-output-dir", + type=Path, + required=True, + help="Directory for the three month_hour_marginal_costs_* parquets", + ) + parser.add_argument( + "--voltage", + choices=tuple(_MARGINAL_COST_SHEETS), + required=True, + help="secondary is Attachment 2 page 1; primary is page 2", + ) + args = parser.parse_args() + voltage: Voltage = args.voltage + all_days, weekday, weekend = write_month_hour_parquets( + args.path_xlsx, args.path_output_dir, voltage + ) + for kind, frame in ( + ("all_days", all_days), + ("weekday", weekday), + ("weekend_holidays", weekend), + ): + print( + f"Wrote {frame.height} rows to {args.path_output_dir / _filename(kind, voltage)} " + f"(mc_per_kwh max={frame['mc_per_kwh'].max():.6f})" + ) + + +if __name__ == "__main__": + main() From 6d6926d96aaf323b229f2996b421a9627a08e50d Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Thu, 8 Oct 2026 03:07:43 +0000 Subject: [PATCH 10/12] Add new updated files from the CAIRO run --- .../ct_eversource_hp_vs_non-hp.yaml | 30 ++--- .../scenarios/scenarios_ct_eversource.yaml | 112 +++++++++--------- ...ource_hp_flat_percustomer_passthrough.json | 2 +- ...at_percustomer_passthrough_calibrated.json | 2 +- ...p_flat_percustomer_passthrough_supply.json | 2 +- ...ustomer_passthrough_supply_calibrated.json | 2 +- ...e_hp_seasonal_percustomer_passthrough.json | 2 +- ...al_percustomer_passthrough_calibrated.json | 4 +- ...asonal_percustomer_passthrough_supply.json | 2 +- ...ustomer_passthrough_supply_calibrated.json | 4 +- ...se_percustomer_passthrough_calibrated.json | 6 +- ...ustomer_passthrough_supply_calibrated.json | 8 +- 12 files changed, 88 insertions(+), 88 deletions(-) diff --git a/rate_design/hp_rates/ct/config/rev_requirement/ct_eversource_hp_vs_non-hp.yaml b/rate_design/hp_rates/ct/config/rev_requirement/ct_eversource_hp_vs_non-hp.yaml index 2a8de7f0..db0edf68 100644 --- a/rate_design/hp_rates/ct/config/rev_requirement/ct_eversource_hp_vs_non-hp.yaml +++ b/rate_design/hp_rates/ct/config/rev_requirement/ct_eversource_hp_vs_non-hp.yaml @@ -1,6 +1,6 @@ utility: ct_eversource group_col: has_hp -source_run_dir: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/20261001_142749_ct_ct_eversource_default_precalc_delivery +source_run_dir: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/20261008_015841_ct_ct_eversource_default_precalc_delivery test_year_customer_count: 1182119.1 resstock_kwh_scale_factor: 0.9935444283506427 total_delivery_revenue_requirement: 1689584509.3 @@ -8,24 +8,24 @@ total_delivery_and_supply_revenue_requirement: 2742274995.0 subclass_revenue_requirements: delivery: percustomer: - non-hp: 1633529947.9599276 - hp: 56054561.340073064 + non-hp: 1633512165.6752164 + hp: 56072343.62478405 epmc: - non-hp: 1619587472.4360082 - hp: 69997036.86399275 + non-hp: 1619624264.624463 + hp: 69960244.67553735 volumetric: - non-hp: 1595672197.662744 - hp: 93912311.63725626 + non-hp: 1595710456.7359679 + hp: 93874052.56403273 passthrough: - non-hp: 1596231837.1069303 - hp: 93352672.19307567 + non-hp: 1596231837.10693 + hp: 93352672.19307569 supply: passthrough: - non-hp: 992561380.9702992 - hp: 60129104.72970189 + non-hp: 992561380.970299 + hp: 60129104.72970188 percustomer: - non-hp: 1018483250.363605 - hp: 34207235.33639449 + non-hp: 1018483250.3636043 + hp: 34207235.33639446 volumetric: - non-hp: 992948182.3618731 - hp: 59742303.33812502 + non-hp: 992948182.3618729 + hp: 59742303.33812499 diff --git a/rate_design/hp_rates/ct/config/scenarios/scenarios_ct_eversource.yaml b/rate_design/hp_rates/ct/config/scenarios/scenarios_ct_eversource.yaml index d62acc1e..7efe0ae0 100644 --- a/rate_design/hp_rates/ct/config/scenarios/scenarios_ct_eversource.yaml +++ b/rate_design/hp_rates/ct/config/scenarios/scenarios_ct_eversource.yaml @@ -12,11 +12,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_default.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_precalc_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_precalc_delivery utility_revenue_requirement: rev_requirement/ct_eversource_rate_case_test_year.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -39,11 +39,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_default_supply.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_precalc_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_precalc_supply utility_revenue_requirement: rev_requirement/ct_eversource_rate_case_test_year.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -66,11 +66,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_default_calibrated.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_calibrated_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_calibrated_delivery utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_test_year.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -93,11 +93,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_default_supply_calibrated.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_calibrated_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_calibrated_supply utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_test_year.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -120,11 +120,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate1_proposed.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r1_uncalibrated_precalc_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r1_uncalibrated_precalc_delivery utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -147,11 +147,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate1_proposed_supply.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r1_uncalibrated_precalc_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r1_uncalibrated_precalc_supply utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -174,11 +174,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate1_proposed.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r1_uncalibrated_calibrated_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r1_uncalibrated_calibrated_delivery utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -201,11 +201,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate1_proposed_supply.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r1_uncalibrated_calibrated_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r1_uncalibrated_calibrated_supply utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -228,11 +228,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_uncalibrated_precalc_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_uncalibrated_precalc_delivery utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -255,11 +255,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6_supply.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_uncalibrated_precalc_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_uncalibrated_precalc_supply utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -282,11 +282,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_uncalibrated_calibrated_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_uncalibrated_calibrated_delivery utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -309,11 +309,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6_supply.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_uncalibrated_calibrated_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_uncalibrated_calibrated_supply utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -336,11 +336,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6_ratemc.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_ratemc_uncalibrated_precalc_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_ratemc_uncalibrated_precalc_delivery utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -363,11 +363,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6_ratemc_supply.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_ratemc_uncalibrated_precalc_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_ratemc_uncalibrated_precalc_supply utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -390,11 +390,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6_ratemc.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_ratemc_uncalibrated_calibrated_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_ratemc_uncalibrated_calibrated_delivery utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -417,11 +417,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6_ratemc_supply.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_ratemc_uncalibrated_calibrated_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_ratemc_uncalibrated_calibrated_supply utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -444,11 +444,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6_fixed_ratemc.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_fixed_ratemc_uncalibrated_precalc_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_fixed_ratemc_uncalibrated_precalc_delivery utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -471,11 +471,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6_fixed_ratemc_supply.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_fixed_ratemc_uncalibrated_precalc_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_fixed_ratemc_uncalibrated_precalc_supply utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -498,11 +498,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6_fixed_ratemc.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_fixed_ratemc_uncalibrated_calibrated_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_fixed_ratemc_uncalibrated_calibrated_delivery utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -525,11 +525,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_rate6_fixed_ratemc_supply.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_default_ry1_r6_fixed_ratemc_uncalibrated_calibrated_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_default_ry1_r6_fixed_ratemc_uncalibrated_calibrated_supply utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_rate_year_1.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -553,11 +553,11 @@ runs: hp: tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough.json non-hp: tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_hp_seasonal_percustomer_passthrough_precalc_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_hp_seasonal_percustomer_passthrough_precalc_delivery utility_revenue_requirement: rev_requirement/ct_eversource_hp_vs_non-hp.yaml run_includes_supply: false run_includes_subclasses: true - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -588,11 +588,11 @@ runs: hp: tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_supply.json non-hp: tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough_supply.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_hp_seasonal_percustomer_passthrough_precalc_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_hp_seasonal_percustomer_passthrough_precalc_supply utility_revenue_requirement: rev_requirement/ct_eversource_hp_vs_non-hp.yaml run_includes_supply: true run_includes_subclasses: true - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -622,11 +622,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_calibrated.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_hp_seasonal_percustomer_passthrough_calibrated_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_hp_seasonal_percustomer_passthrough_calibrated_delivery utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_test_year.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -651,11 +651,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_supply_calibrated.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_hp_seasonal_percustomer_passthrough_calibrated_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_hp_seasonal_percustomer_passthrough_calibrated_supply utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_test_year.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -681,11 +681,11 @@ runs: hp: tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough.json non-hp: tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_hp_flat_percustomer_passthrough_precalc_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_hp_flat_percustomer_passthrough_precalc_delivery utility_revenue_requirement: rev_requirement/ct_eversource_hp_vs_non-hp.yaml run_includes_supply: false run_includes_subclasses: true - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -716,11 +716,11 @@ runs: hp: tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_supply.json non-hp: tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough_supply.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_hp_flat_percustomer_passthrough_precalc_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_hp_flat_percustomer_passthrough_precalc_supply utility_revenue_requirement: rev_requirement/ct_eversource_hp_vs_non-hp.yaml run_includes_supply: true run_includes_subclasses: true - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -750,11 +750,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_calibrated.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_hp_flat_percustomer_passthrough_calibrated_delivery + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_hp_flat_percustomer_passthrough_calibrated_delivery utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_test_year.yaml run_includes_supply: false run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet @@ -779,11 +779,11 @@ runs: path_tariffs_electric: all: tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_supply_calibrated.json path_tariffs_gas: tariffs/gas - path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261001_a/ct_ct_eversource_hp_flat_percustomer_passthrough_calibrated_supply + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ct/ct_eversource/ct_20261007_a/ct_ct_eversource_hp_flat_percustomer_passthrough_calibrated_supply utility_revenue_requirement: rev_requirement/ct_eversource_large_number_rate_case_test_year.yaml run_includes_supply: true run_includes_subclasses: false - path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=2025/data.parquet + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/utility=ct_eversource/year=2025/data.parquet path_bulk_tx_mc: s3://data.sb/switchbox/marginal_costs/ct/bulk_tx/utility=ct_eversource/year=2025/data.parquet path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/energy/utility=ct_eversource/year=2025/zero.parquet path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ct/supply/capacity/utility=ct_eversource/year=2025/zero.parquet diff --git a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough.json b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough.json index e58d7558..28a661d7 100644 --- a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough.json +++ b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough.json @@ -640,7 +640,7 @@ "energyratestructure": [ [ { - "rate": 0.08936387859680141, + "rate": 0.08939466296243935, "adj": 0.0, "unit": "kWh" } diff --git a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_calibrated.json b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_calibrated.json index 92aba370..9676047e 100644 --- a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_calibrated.json +++ b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_calibrated.json @@ -640,7 +640,7 @@ "energyratestructure": [ [ { - "rate": 0.0899445219024096, + "rate": 0.08997550628998163, "adj": 0.0, "unit": "kWh" } diff --git a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_supply.json b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_supply.json index 0980e6ad..e6f8556b 100644 --- a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_supply.json +++ b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_supply.json @@ -640,7 +640,7 @@ "energyratestructure": [ [ { - "rate": 0.14858281796126255, + "rate": 0.14861360232690063, "adj": 0.0, "unit": "kWh" } diff --git a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_supply_calibrated.json b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_supply_calibrated.json index f29d742a..51bffc7c 100644 --- a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_supply_calibrated.json +++ b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_flat_percustomer_passthrough_supply_calibrated.json @@ -640,7 +640,7 @@ "energyratestructure": [ [ { - "rate": 0.19471528199218457, + "rate": 0.19474626637975656, "adj": 0.0, "unit": "kWh" } diff --git a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough.json b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough.json index e9495a31..e17e0e5a 100644 --- a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough.json +++ b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough.json @@ -647,7 +647,7 @@ ], [ { - "rate": 0.04985960419221816, + "rate": 0.04990922265531289, "adj": 0.0, "unit": "kWh" } diff --git a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_calibrated.json b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_calibrated.json index f4fc2a5f..5e720f8f 100644 --- a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_calibrated.json +++ b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_calibrated.json @@ -640,14 +640,14 @@ "energyratestructure": [ [ { - "rate": 0.15546663131980376, + "rate": 0.155467411489478, "adj": 0.0, "unit": "kWh" } ], [ { - "rate": 0.050356826780618584, + "rate": 0.0504060602047395, "adj": 0.0, "unit": "kWh" } diff --git a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_supply.json b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_supply.json index bd6d0406..66e109b0 100644 --- a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_supply.json +++ b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_supply.json @@ -647,7 +647,7 @@ ], [ { - "rate": 0.08162340928382522, + "rate": 0.08167302774692001, "adj": 0.0, "unit": "kWh" } diff --git a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_supply_calibrated.json b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_supply_calibrated.json index c3bc8ed5..18bbb618 100644 --- a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_supply_calibrated.json +++ b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_hp_seasonal_percustomer_passthrough_supply_calibrated.json @@ -640,14 +640,14 @@ "energyratestructure": [ [ { - "rate": 0.3393284853244324, + "rate": 0.3393123561233851, "adj": 0.0, "unit": "kWh" } ], [ { - "rate": 0.1073416694054259, + "rate": 0.10740111927432822, "adj": 0.0, "unit": "kWh" } diff --git a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough_calibrated.json b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough_calibrated.json index 6b2b60e7..f057858f 100644 --- a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough_calibrated.json +++ b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough_calibrated.json @@ -640,21 +640,21 @@ "energyratestructure": [ [ { - "rate": 0.16874088346405935, + "rate": 0.16873888506419557, "adj": 0.0, "unit": "kWh" } ], [ { - "rate": 0.15335896043117192, + "rate": 0.15335714419956673, "adj": 0.0, "unit": "kWh" } ], [ { - "rate": 0.14847416054910634, + "rate": 0.1484724021682319, "adj": 0.0, "unit": "kWh" } diff --git a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough_supply_calibrated.json b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough_supply_calibrated.json index 21ab4f8e..d795bdfd 100644 --- a/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough_supply_calibrated.json +++ b/rate_design/hp_rates/ct/config/tariffs/electric/ct_eversource_non-hp_base_percustomer_passthrough_supply_calibrated.json @@ -640,28 +640,28 @@ "energyratestructure": [ [ { - "rate": 0.27992769016412195, + "rate": 0.2799256943219635, "adj": 0.0, "unit": "kWh" } ], [ { - "rate": 0.26462010286568954, + "rate": 0.2646182161643169, "adj": 0.0, "unit": "kWh" } ], [ { - "rate": 0.25017111421380384, + "rate": 0.25016933053153834, "adj": 0.0, "unit": "kWh" } ], [ { - "rate": 0.24531389901333972, + "rate": 0.2453121499622851, "adj": 0.0, "unit": "kWh" } From 6fbeb8d87f32f94b7ce96a21ea11a4761e7b989b Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Thu, 8 Oct 2026 03:11:34 +0000 Subject: [PATCH 11/12] Update schema names --- context/code/data/ct_occ863_system_load.md | 12 ++++++------ rate_design/hp_rates/ct/Justfile | 2 +- tests/test_ct_occ863_month_hour_mc.py | 12 ++++++------ .../convert_ct_occ863_month_hour_mc.py | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/context/code/data/ct_occ863_system_load.md b/context/code/data/ct_occ863_system_load.md index 80dab14b..cabfaf78 100644 --- a/context/code/data/ct_occ863_system_load.md +++ b/context/code/data/ct_occ863_system_load.md @@ -109,12 +109,12 @@ in `_secondary` or `_primary`: | `month_hour_marginal_costs_weekday` | weekdays only | | `month_hour_marginal_costs_weekend_holidays` | weekends and public holidays | -| Column | Meaning | -| ------------ | ------------------------------------------------------------------- | -| `month` | 1–12 | -| `hour` | 0–23, the hour label on the sheet | -| `num_days` | That matrix's "No Days" count for the month, repeated on every hour | -| `mc_per_kwh` | The matrix value, in `$/kWh` | +| Column | Meaning | +| ------------------ | ------------------------------------------------------------------- | +| `month` | 1–12 | +| `hour` | 0–23, the hour label on the sheet | +| `num_days` | That matrix's "No Days" count for the month, repeated on every hour | +| `mc_total_per_kwh` | The matrix value, in `$/kWh`, same name as the dist-and-sub-tx 8760 | For each month the all-day-types day count must equal the weekday count plus the weekend-and-holiday count. The fourth sheet is a copy of page 3 (the probability-of-peak table) diff --git a/rate_design/hp_rates/ct/Justfile b/rate_design/hp_rates/ct/Justfile index b6031493..77983530 100644 --- a/rate_design/hp_rates/ct/Justfile +++ b/rate_design/hp_rates/ct/Justfile @@ -194,7 +194,7 @@ convert-occ863-rate-class-load path_xlsx path_output_dir: # # voltage is secondary (page 1) or primary (page 2). Writes the all-day-types, # weekday, and weekend-and-holiday matrices. Each row is one month (1–12) and -# hour (0–23), with that month's "No Days" count and the $/kWh value. See +# hour (0–23), with that month's "No Days" count and mc_total_per_kwh. See # context/code/data/ct_occ863_system_load.md. # # just -f ct/Justfile convert-occ863-month-hour-mc secondary diff --git a/tests/test_ct_occ863_month_hour_mc.py b/tests/test_ct_occ863_month_hour_mc.py index 80a56904..651fe13b 100644 --- a/tests/test_ct_occ863_month_hour_mc.py +++ b/tests/test_ct_occ863_month_hour_mc.py @@ -72,29 +72,29 @@ def test_melts_weekdays_with_the_day_count_repeated_on_every_hour() -> None: all_days, weekday, weekend = month_hour_marginal_costs( _page("secondary"), "secondary" ) - assert weekday.columns == ["month", "hour", "num_days", "mc_per_kwh"] + assert weekday.columns == ["month", "hour", "num_days", "mc_total_per_kwh"] assert all_days.height == 288 assert weekday.height == 288 assert weekend.height == 288 january = weekday.filter(pl.col("month") == 1) assert january["hour"].to_list() == list(range(24)) assert january["num_days"].unique().to_list() == [19] - assert january["mc_per_kwh"].sum() == 0 + assert january["mc_total_per_kwh"].sum() == 0 august_hour_16 = weekday.filter((pl.col("month") == 8) & (pl.col("hour") == 16)) - assert august_hour_16["mc_per_kwh"].item() == pytest.approx(0.02557) + assert august_hour_16["mc_total_per_kwh"].item() == pytest.approx(0.02557) assert august_hour_16["num_days"].item() == 22 weekend_august = weekend.filter((pl.col("month") == 8) & (pl.col("hour") == 16)) - assert weekend_august["mc_per_kwh"].item() == pytest.approx(0.00011) + assert weekend_august["mc_total_per_kwh"].item() == pytest.approx(0.00011) assert weekend_august["num_days"].item() == 10 all_days_august = all_days.filter((pl.col("month") == 8) & (pl.col("hour") == 16)) - assert all_days_august["mc_per_kwh"].item() == pytest.approx(0.001) + assert all_days_august["mc_total_per_kwh"].item() == pytest.approx(0.001) assert all_days_august["num_days"].item() == 32 def test_reads_primary_titles() -> None: all_days, _, _ = month_hour_marginal_costs(_page("primary"), "primary") assert all_days.filter((pl.col("month") == 8) & (pl.col("hour") == 16))[ - "mc_per_kwh" + "mc_total_per_kwh" ].item() == pytest.approx(0.001) diff --git a/utils/data_prep/marginal_costs/convert_ct_occ863_month_hour_mc.py b/utils/data_prep/marginal_costs/convert_ct_occ863_month_hour_mc.py index 2ae18f63..f9cb525c 100644 --- a/utils/data_prep/marginal_costs/convert_ct_occ863_month_hour_mc.py +++ b/utils/data_prep/marginal_costs/convert_ct_occ863_month_hour_mc.py @@ -54,7 +54,7 @@ def month_hour_marginal_costs( titles on that sheet. The caller reads the matching sheet. Each frame has ``month`` (1–12), ``hour`` (0–23), ``num_days`` (the "No - Days" count for that month, repeated on every hour), and ``mc_per_kwh``. + Days" count for that month, repeated on every hour), and ``mc_total_per_kwh``. """ all_days_title, weekday_title, weekend_title = _MATRIX_TITLES[voltage] rows = raw.rows() @@ -95,7 +95,7 @@ def _melt_matrix(rows: list[tuple[object, ...]], title: str) -> pl.DataFrame: "month": month, "hour": hour, "num_days": day_counts[month], - "mc_per_kwh": mc, + "mc_total_per_kwh": mc, } ) @@ -104,7 +104,7 @@ def _melt_matrix(rows: list[tuple[object, ...]], title: str) -> pl.DataFrame: "month": pl.Int64, "hour": pl.Int64, "num_days": pl.Int64, - "mc_per_kwh": pl.Float64, + "mc_total_per_kwh": pl.Float64, } ) _validate_matrix(frame, title) @@ -284,7 +284,7 @@ def main() -> None: ): print( f"Wrote {frame.height} rows to {args.path_output_dir / _filename(kind, voltage)} " - f"(mc_per_kwh max={frame['mc_per_kwh'].max():.6f})" + f"(mc_total_per_kwh max={frame['mc_total_per_kwh'].max():.6f})" ) From 7d70e8f33b0ebe2bad38555f28f1e4b1dee3384a Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Thu, 8 Oct 2026 20:56:02 +0000 Subject: [PATCH 12/12] Refactor scripts and files --- context/README.md | 78 ++++----- context/code/data/ct_occ863_system_load.md | 16 +- .../ct_eversource_dist_mc_methodology.md | 73 ++++---- .../dist_mc_definition_choice.md | 2 +- rate_design/hp_rates/ct/Justfile | 103 +++++++---- .../generate_utility_tx_dx_mc.py | 161 +++++++++++------- 6 files changed, 250 insertions(+), 183 deletions(-) diff --git a/context/README.md b/context/README.md index 2cc6e64b..e8feeb39 100644 --- a/context/README.md +++ b/context/README.md @@ -77,18 +77,18 @@ Cost-reflective TOU design: theory and window optimization. How we derive BAT marginal-cost inputs from regulatory studies. -| File | Purpose | -| ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| ny_dist_mc_bat_methodology.md | Shared methodology for deriving sub-TX and distribution BAT marginal cost inputs from the seven NY MCOS workbooks: pipeline overview, key conceptual choices (incremental not cumulative, levelized 7-year window, no discounting, 2026 dollar year), per-utility deviations and harmonization, final values and sanity check, research appendices. | -| ny_mcos_studies_comparison.md | Cross-utility comparison of the seven NY 2025 MCOS studies: what MCOS numbers mean (undiluted/diluted, grid topology), methodology comparison, annualization mechanics (workbook-verified step-by-step), system-wide diluted/undiluted tables, component taxonomy mapping (physical hierarchy → cost-center labels), bulk vs local TX scrub, reassessment of bulk TX MC gap. | -| ny_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for NY BAT: the gap, data sources (OATT ATRR, CLCPA project costs, FERC Order 1920), five options with tradeoffs, recommended approach (OATT as upper-bound proxy), hourly PoP allocation. | -| ri_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for RI BAT: data sources (RNS rate, AESC 2024 avoided PTF cost, ISO-NE 2050 study), four options with tradeoffs, recommended approach (AESC primary, RNS upper bound), hourly PoP allocation, comparison with NY. | -| ct_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for CT BAT: same AESC 2024 $69/kW-year and top-100 exceedance as RI, plus `--allocation-load` flag to toggle between NE system aggregate and CT zone load for peak-hour identification; utilities `ct_eversource` and `ct_ui`; implementation files and Just commands. | -| capacity_market_comparison_nyiso_isone.md | Comparison of NYISO ICAP (Strip/Monthly/Spot) vs ISO-NE FCM (FCA/ARA/MRA) capacity market structures: structural analogy table, forward horizons, price volatility differences, implications for BAT marginal cost signal (Spot vs FCA vs MRA), current approach in NY vs RI, trade-offs. | -| dist_mc_definition_choice.md | Cross-cutting: which LRMC definition the `sub_tx_and_dist` BAT input uses, and why it differs by state. Taxonomy (Turvey / AIC-avoided / FLIC) and why AIC and FLIC differ ~10-50x, source-number table (NY FLIC $1-8, RI AESC $80, BGE Brattle $32, BGE E3 $203-258, OPC $25-34, BAT paper $54.46), what is implemented per state (NY=FLIC, RI/MD=avoided), the platform tension (NY-vs-RI/MD; methodology-doc §3-vs-§7), rationale, and BGE sensitivities. | -| md_bge_dist_mc_methodology.md | BGE (MD) sub-TX + distribution MC implementation: adopts Brattle avoided distribution cost $32/kW-yr (2022$), RI-style single-scalar config + CPI handling, why avoided cost is consistent with BGE's own peak-driven COS, the ~82% volumetric over-recovery intervention hook, FLIC/E3 sensitivities, and the Stage-B allocator MD-enablement dependency. | -| ct_eversource_dist_mc_methodology.md | Eversource (CT) sub-TX + dist MC. Three 8760s. Default (§7): OCC-863 Attachment 2 secondary month-hour $/kWh tables expanded by weekday/weekend, with `include_holidays` (default true; federal holidays, Saturday observed Friday and Sunday observed Monday). One `--year` for the calendar, timestamps, and Hive partition, written to `dist_and_sub_tx_eversource_derived/`. Second: `$21.22/kW-yr`(Table 3` $20.17` × the secondary loss factor), allocated as filed with no CPI because the analysis is rate year 1, on the top 100 hours of OCC-863 substation load, written to `dist_and_sub_tx_occ863/`. Oldest, not preferred: the stored `dist_and_sub_tx/` parquet allocated `$20.17` after CPI deflation onto ISO-NE CT zone load. | -| md_bulk_transmission.md | How to construct bulk TX marginal cost for MD BAT: PJM NITS rates as OATT ATRR proxy (E3 ICC-VDER precedent), zone-to-utility mapping (BGE/DPL/PEPCO/APS covering all 13 MD utilities), calendar-year day-weighted blending, full-year top-150 PCAF allocation (following E3), reference data at `data/pjm/bulk_tx/nits/`, implementation in `bulk_tx_pjm.py`. | +| File | Purpose | +| ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ny_dist_mc_bat_methodology.md | Shared methodology for deriving sub-TX and distribution BAT marginal cost inputs from the seven NY MCOS workbooks: pipeline overview, key conceptual choices (incremental not cumulative, levelized 7-year window, no discounting, 2026 dollar year), per-utility deviations and harmonization, final values and sanity check, research appendices. | +| ny_mcos_studies_comparison.md | Cross-utility comparison of the seven NY 2025 MCOS studies: what MCOS numbers mean (undiluted/diluted, grid topology), methodology comparison, annualization mechanics (workbook-verified step-by-step), system-wide diluted/undiluted tables, component taxonomy mapping (physical hierarchy → cost-center labels), bulk vs local TX scrub, reassessment of bulk TX MC gap. | +| ny_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for NY BAT: the gap, data sources (OATT ATRR, CLCPA project costs, FERC Order 1920), five options with tradeoffs, recommended approach (OATT as upper-bound proxy), hourly PoP allocation. | +| ri_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for RI BAT: data sources (RNS rate, AESC 2024 avoided PTF cost, ISO-NE 2050 study), four options with tradeoffs, recommended approach (AESC primary, RNS upper bound), hourly PoP allocation, comparison with NY. | +| ct_bulk_transmission_marginal_cost.md | How to construct bulk TX marginal cost for CT BAT: same AESC 2024 $69/kW-year and top-100 exceedance as RI, plus `--allocation-load` flag to toggle between NE system aggregate and CT zone load for peak-hour identification; utilities `ct_eversource` and `ct_ui`; implementation files and Just commands. | +| capacity_market_comparison_nyiso_isone.md | Comparison of NYISO ICAP (Strip/Monthly/Spot) vs ISO-NE FCM (FCA/ARA/MRA) capacity market structures: structural analogy table, forward horizons, price volatility differences, implications for BAT marginal cost signal (Spot vs FCA vs MRA), current approach in NY vs RI, trade-offs. | +| dist_mc_definition_choice.md | Cross-cutting: which LRMC definition the `sub_tx_and_dist` BAT input uses, and why it differs by state. Taxonomy (Turvey / AIC-avoided / FLIC) and why AIC and FLIC differ ~10-50x, source-number table (NY FLIC $1-8, RI AESC $80, BGE Brattle $32, BGE E3 $203-258, OPC $25-34, BAT paper $54.46), what is implemented per state (NY=FLIC, RI/MD=avoided), the platform tension (NY-vs-RI/MD; methodology-doc §3-vs-§7), rationale, and BGE sensitivities. | +| md_bge_dist_mc_methodology.md | BGE (MD) sub-TX + distribution MC implementation: adopts Brattle avoided distribution cost $32/kW-yr (2022$), RI-style single-scalar config + CPI handling, why avoided cost is consistent with BGE's own peak-driven COS, the ~82% volumetric over-recovery intervention hook, FLIC/E3 sensitivities, and the Stage-B allocator MD-enablement dependency. | +| ct_eversource_dist_mc_methodology.md | Eversource (CT) sub-TX + dist MC. Three 8760s, picked by `create-dist-mc-data`'s `dist_mc_method` (`month_hour`, `pop_occ863`, `pop_isone_zone`). Default (§7): OCC-863 Attachment 2 secondary month-hour $/kWh tables expanded by weekday/weekend, with `include_holidays` (default true; federal holidays, Saturday observed Friday and Sunday observed Monday). One `--year` for the calendar, timestamps, and Hive partition, written to `dist_and_sub_tx_eversource_derived/`. Second: $21.22/kW-yr (Table 3 $20.17 × the secondary loss factor), allocated as filed with no CPI because the analysis is rate year 1, on the top 100 hours of OCC-863 substation load, written to `dist_and_sub_tx_occ863/`. Oldest, not preferred: the same $21.22 on ISO-NE CT zone load, written to `dist_and_sub_tx/`; the parquet stored there now allocated $20.17 after CPI deflation. | +| md_bulk_transmission.md | How to construct bulk TX marginal cost for MD BAT: PJM NITS rates as OATT ATRR proxy (E3 ICC-VDER precedent), zone-to-utility mapping (BGE/DPL/PEPCO/APS covering all 13 MD utilities), calendar-year day-weighted blending, full-year top-150 PCAF allocation (following E3), reference data at `data/pjm/bulk_tx/nits/`, implementation in `bulk_tx_pjm.py`. | ### methods/ (root) @@ -144,33 +144,33 @@ How CAIRO works: LMI, demand flex, tax rate, tiered rates, performance, parallel ResStock, EIA, NYISO, ISO-NE: where data comes from, how to read and prepare it. -| File | Purpose | -| -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| resstock_data_preparation_run_order.md | Run order for ResStock `_sb` release prep: recommended `main.py` invocation (single command covers fetch → metadata → approx → MF adj → utility assignment → monthly loads → upload), plus legacy individual Justfile step reference for debugging | -| resstock_sb_release_pipeline_main_py.md | Unified `data/resstock/main.py` pipeline: step-by-step reference for all implemented steps (2b through 2d), constants (`SB_CLONE_EXCLUDED_FILE_TYPES`, `sb_file_types`, column sets), config.yaml, state_configs.yaml (`SUPPORTED_UTILITY_STATES`, polygon filenames, per-state settings), sample mode, `path_raw` vs `path_sb` data flow, file types included/excluded in `_sb`, how to expand `_sb` scope | -| resstock_sb_annual_load_curves.md | `_sb` `load_curve_annual` from modified hourly (combined with monthly in `aggregate_loads`): one hourly read → monthly write + annual row; keep/drop/aggregate decisions; upgrade 01–05 savings; `upgrade_name`; energy_delivered 4× bug (bsf before 1.6.6); pipeline step 2d | -| resstock_lmi_metadata_guide.md | ResStock 2024.2 parquet metadata: columns for LMI tier assignment, FPL/SMI, income | -| parquet_reads_local_vs_s3.md | Reading Parquet from local disk vs S3: per-GET overhead, file discovery, Hive filters vs path construction, best practices for whole-state and per-utility reads | -| polars_laziness_and_validation.md | Polars LazyFrame best practices: when to collect, runtime data-quality asserts vs laziness/streaming, strategies for small and large data | -| nyiso_lbmp_zonal_data_sources.md | NYISO Day-Ahead/Real-Time zonal LBMP: MIS ZIP vs gridstatus vs NYISOToolkit; data samples | -| pjm_hourly_loads.md | PJM-native hourly zonal loads (Data Miner 2 `hrl_load_metered`): why PJM over EIA-930, feed quirks (legacy `BC`/`PEP`/`DPL`/`AP` zone codes, `load_area`-grained → sum to zone), archive/standard 731-day boundary handling in `data/pjm/dataminer.py`, EPT-year vs UTC-built timestamp / 8760-8784 completeness, zone→utility mapping, output layout, and how to run the pipeline | -| eia_861_supply_vs_delivery_stats.md | EIA-861 supply vs delivery stats: definitions and use | -| compare_resstock_eia861_loads.md | ResStock hourly load vs EIA-861 residential sales comparison: script usage, defaults, interpretation | -| investigate_resstock_eia_load_discrepancy.md | ResStock vs EIA load discrepancy by utility: MF non-HVAC adjustment, column-by-column SF/MF ratios, workflow, design decisions, and relation to compare_resstock_eia861_loads | -| isone_fca_yaml_data_guide.md | ISO-NE FCA assumptions YAML data dictionary: what each field means (clearing price, CSO, months), where to find source data online (FCA results report, CELT Report Table 4.1), and how to update for a new year | -| utility_assignment_resstock.md | ResStock utility assignment: generic architecture (`assign_utility.py`, `utils.py`), NY GIS-based implementation (excluded gas utilities, PUMA overlap, nearest-neighbor donor, name crosswalk), MD GIS-based implementation (HIFLD electric/gas sources, no crosswalk, nearest-neighbor PUMA fill for ~26% coverage gap, data quality analysis), RI rule-based pattern, `SUPPORTED_UTILITY_STATES` derivation from `state_configs.yaml`, invocation via main.py (step 2b), and a full checklist for adding a new state | -| ct_utility_gis_data_sources.md | CT electric utility service territory GIS sources: current NASA NCCS HIFLD mirror (actual polygon boundaries, 2022 vintage), PUMA TIGER/Line download, HIFLD Open shutdown background, mirror comparison table, and PUDL/EIA-861 county-level alternative for future use | -| approximate_non_hp_load.md | MF highrise non-HP load approximation: nearest-neighbor RMSE, replace HVAC columns, update/sink, validation; invocation via main.py (step 2c-i) vs legacy Justfile recipe | -| ny_lmi_discounts_genability_encoding.md | EAP/LMI encoding in NY Genability electric tariffs: which utilities encode EAP, tier/heating structure, amount comparison to lmi_discounts_in_ny.md | -| tariff_rates_and_genability.md | Tariff/rate/rider/charge terminology, Genability/Arcadia data model (rateUnit, calculationFactor, applicabilityValue, masterTariffId vs tariffId, rider pattern), two tariff_fetch library bugs and their patches (BGE empty-band BOOLEAN applicability bug; Pepco/DPL calculation_factor fold for GRT/DSIC charges), why NY utilities did not trigger the same bugs, tariff versioning/date-boundary behavior, and how to reconstruct historical monthly charges via the Calculate API | -| ny_genability_charge_fetch_map.md | Exhaustive charge-level table for NY Genability tariffs: tariffRateId, fetch_type, variableRateKey, master_charge, decision; used to fetch 2025 monthly rates from Arcadia API for top-up implementation. | -| psegli_revenue_requirement_estimation.md | PSEG-LI revenue requirement estimation: why no rate case, TOU overcount problem, bill-proportional method from LIPA budget, charge classification, implementation via `estimate_psegli_rr.py`. | -| tariff_generation_pipeline.md | Tariff generation pipeline: Genability → monthly_rates YAML → flat and default-structure URDB v7 JSONs. Covers fetch_monthly_rates.py, charge classification, monthly_rates YAML structure, create_flat_tariffs.py (top-down from RR), create_default_structure_tariffs.py (bottom-up from filed rates), BASE_TARIFF_PATTERN, Justfile wiring (all-pre), CAIRO calibration, and why flat vs default rates differ. | -| md_tariff_fetch.md | MD electric and gas tariff pipeline: Arcadia electric URDBs, RateAcuity gas tariffs, Chesapeake county and RES-1/RES-2 mapping from `_sb` annual therms, UGI/Easton commodity rates, `tariff_fetch` bug patches, verified rate values, and BGE Schedule RD fetch (`masterTariffId` 3350180, effective date 2026-01-01). | -| ct_tariff_fetch.md | CT gas tariff **fetch** (ticket scope: RateAcuity URDB JSONs + `null_gas_tariff` + envelope only): schedules for CNG / Yankee / SCG (three residential classes each) plus Norwich muni (GRES + GSHRES/`_mf`), evidence. Omits (reference only, with Zotero links): CNG Rate SE Seasonal Gas Service; System Expansion `*-SE` (RSH-SE / 02-SE etc.); third-party supply. **Known limitation:** IOU `_mf` demand / MDQ charges dropped by `tariff_fetch`; CAIRO gas path has no MDQ billing. Mapper / tariff maps are follow-up. | -| ct_eversource_custom_rates.md | CT Eversource hand-encoded proposed rates (Docket 26-05-10): `ct_eversource_rate6` (HP space-heating) + `ct_eversource_rate1_proposed` (matched comparison baseline), each as delivery + `_supply` JSON pairs with top-up YAMLs. Documents the filed-2026 delivery vintage + Genability 2025 supply, the ESI/RDM roll-in and SBC/CTA/FMCC decisions, full per-number provenance to the CLP-Rates exhibits, the transmission vintage note (why filed 0.05050 differs from RateAcuity 0.03401/0.04433), and the **pre-calc caveat** (fixed rates, no calibrated RR — bill-calc runs only). | -| ct_eversource_class_rev_requirement.md | CT Eversource **whole residential rate class** revenue-requirement files (Docket 26-05-10): the test-year (`ct_eversource_rate_case_test_year.yaml`, current rates) and rate-year-1 (`ct_eversource_rate_case_rate_year_1.yaml`, Proposed+GET) Track 2 YAMLs that model every residential ResStock building on Rate 1 but size base DRR / kWh / customer count to the class (Rate 1 + 5 + 7). Documents the bug it fixes (whole-class ResStock pinned to a Rate-1 target), how top-ups scale off the two class-total scalars, full per-number provenance to the CLP-RATES-2 exhibit with derivations, and the Proposed+GET vs pre-GET caveat. | -| ct_occ863_system_load.md | OCC-863 (Docket 26-05-10) source files: where the four workbooks and the discovery PDF live on S3 and the EBS mirror. Attachment 3 page 2 becomes `system_load.parquet` (hourly distribution-substation load, MW, 2022–2025); page 1 becomes `system_monthly_peak_mw.parquet`. Attachment 4 becomes `rate_1_load.parquet`, `rate_5_load.parquet`, and `rate_7_load.parquet` (2025 hourly class kW; interval-ending EST shifted to hour-beginning local time). Attachment 2 becomes six month-hour marginal-cost parquets: `all_days`, `weekday`, and `weekend_holidays`, each for secondary (page 1) and primary (page 2). The default `create-dist-mc-data` recipe expands the secondary weekday and weekend-holiday month-hour parquets. The substation-load probability-of-peak path reads `system_load.parquet` through `--path-utility-load`. | +| File | Purpose | +| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| resstock_data_preparation_run_order.md | Run order for ResStock `_sb` release prep: recommended `main.py` invocation (single command covers fetch → metadata → approx → MF adj → utility assignment → monthly loads → upload), plus legacy individual Justfile step reference for debugging | +| resstock_sb_release_pipeline_main_py.md | Unified `data/resstock/main.py` pipeline: step-by-step reference for all implemented steps (2b through 2d), constants (`SB_CLONE_EXCLUDED_FILE_TYPES`, `sb_file_types`, column sets), config.yaml, state_configs.yaml (`SUPPORTED_UTILITY_STATES`, polygon filenames, per-state settings), sample mode, `path_raw` vs `path_sb` data flow, file types included/excluded in `_sb`, how to expand `_sb` scope | +| resstock_sb_annual_load_curves.md | `_sb` `load_curve_annual` from modified hourly (combined with monthly in `aggregate_loads`): one hourly read → monthly write + annual row; keep/drop/aggregate decisions; upgrade 01–05 savings; `upgrade_name`; energy_delivered 4× bug (bsf before 1.6.6); pipeline step 2d | +| resstock_lmi_metadata_guide.md | ResStock 2024.2 parquet metadata: columns for LMI tier assignment, FPL/SMI, income | +| parquet_reads_local_vs_s3.md | Reading Parquet from local disk vs S3: per-GET overhead, file discovery, Hive filters vs path construction, best practices for whole-state and per-utility reads | +| polars_laziness_and_validation.md | Polars LazyFrame best practices: when to collect, runtime data-quality asserts vs laziness/streaming, strategies for small and large data | +| nyiso_lbmp_zonal_data_sources.md | NYISO Day-Ahead/Real-Time zonal LBMP: MIS ZIP vs gridstatus vs NYISOToolkit; data samples | +| pjm_hourly_loads.md | PJM-native hourly zonal loads (Data Miner 2 `hrl_load_metered`): why PJM over EIA-930, feed quirks (legacy `BC`/`PEP`/`DPL`/`AP` zone codes, `load_area`-grained → sum to zone), archive/standard 731-day boundary handling in `data/pjm/dataminer.py`, EPT-year vs UTC-built timestamp / 8760-8784 completeness, zone→utility mapping, output layout, and how to run the pipeline | +| eia_861_supply_vs_delivery_stats.md | EIA-861 supply vs delivery stats: definitions and use | +| compare_resstock_eia861_loads.md | ResStock hourly load vs EIA-861 residential sales comparison: script usage, defaults, interpretation | +| investigate_resstock_eia_load_discrepancy.md | ResStock vs EIA load discrepancy by utility: MF non-HVAC adjustment, column-by-column SF/MF ratios, workflow, design decisions, and relation to compare_resstock_eia861_loads | +| isone_fca_yaml_data_guide.md | ISO-NE FCA assumptions YAML data dictionary: what each field means (clearing price, CSO, months), where to find source data online (FCA results report, CELT Report Table 4.1), and how to update for a new year | +| utility_assignment_resstock.md | ResStock utility assignment: generic architecture (`assign_utility.py`, `utils.py`), NY GIS-based implementation (excluded gas utilities, PUMA overlap, nearest-neighbor donor, name crosswalk), MD GIS-based implementation (HIFLD electric/gas sources, no crosswalk, nearest-neighbor PUMA fill for ~26% coverage gap, data quality analysis), RI rule-based pattern, `SUPPORTED_UTILITY_STATES` derivation from `state_configs.yaml`, invocation via main.py (step 2b), and a full checklist for adding a new state | +| ct_utility_gis_data_sources.md | CT electric utility service territory GIS sources: current NASA NCCS HIFLD mirror (actual polygon boundaries, 2022 vintage), PUMA TIGER/Line download, HIFLD Open shutdown background, mirror comparison table, and PUDL/EIA-861 county-level alternative for future use | +| approximate_non_hp_load.md | MF highrise non-HP load approximation: nearest-neighbor RMSE, replace HVAC columns, update/sink, validation; invocation via main.py (step 2c-i) vs legacy Justfile recipe | +| ny_lmi_discounts_genability_encoding.md | EAP/LMI encoding in NY Genability electric tariffs: which utilities encode EAP, tier/heating structure, amount comparison to lmi_discounts_in_ny.md | +| tariff_rates_and_genability.md | Tariff/rate/rider/charge terminology, Genability/Arcadia data model (rateUnit, calculationFactor, applicabilityValue, masterTariffId vs tariffId, rider pattern), two tariff_fetch library bugs and their patches (BGE empty-band BOOLEAN applicability bug; Pepco/DPL calculation_factor fold for GRT/DSIC charges), why NY utilities did not trigger the same bugs, tariff versioning/date-boundary behavior, and how to reconstruct historical monthly charges via the Calculate API | +| ny_genability_charge_fetch_map.md | Exhaustive charge-level table for NY Genability tariffs: tariffRateId, fetch_type, variableRateKey, master_charge, decision; used to fetch 2025 monthly rates from Arcadia API for top-up implementation. | +| psegli_revenue_requirement_estimation.md | PSEG-LI revenue requirement estimation: why no rate case, TOU overcount problem, bill-proportional method from LIPA budget, charge classification, implementation via `estimate_psegli_rr.py`. | +| tariff_generation_pipeline.md | Tariff generation pipeline: Genability → monthly_rates YAML → flat and default-structure URDB v7 JSONs. Covers fetch_monthly_rates.py, charge classification, monthly_rates YAML structure, create_flat_tariffs.py (top-down from RR), create_default_structure_tariffs.py (bottom-up from filed rates), BASE_TARIFF_PATTERN, Justfile wiring (all-pre), CAIRO calibration, and why flat vs default rates differ. | +| md_tariff_fetch.md | MD electric and gas tariff pipeline: Arcadia electric URDBs, RateAcuity gas tariffs, Chesapeake county and RES-1/RES-2 mapping from `_sb` annual therms, UGI/Easton commodity rates, `tariff_fetch` bug patches, verified rate values, and BGE Schedule RD fetch (`masterTariffId` 3350180, effective date 2026-01-01). | +| ct_tariff_fetch.md | CT gas tariff **fetch** (ticket scope: RateAcuity URDB JSONs + `null_gas_tariff` + envelope only): schedules for CNG / Yankee / SCG (three residential classes each) plus Norwich muni (GRES + GSHRES/`_mf`), evidence. Omits (reference only, with Zotero links): CNG Rate SE Seasonal Gas Service; System Expansion `*-SE` (RSH-SE / 02-SE etc.); third-party supply. **Known limitation:** IOU `_mf` demand / MDQ charges dropped by `tariff_fetch`; CAIRO gas path has no MDQ billing. Mapper / tariff maps are follow-up. | +| ct_eversource_custom_rates.md | CT Eversource hand-encoded proposed rates (Docket 26-05-10): `ct_eversource_rate6` (HP space-heating) + `ct_eversource_rate1_proposed` (matched comparison baseline), each as delivery + `_supply` JSON pairs with top-up YAMLs. Documents the filed-2026 delivery vintage + Genability 2025 supply, the ESI/RDM roll-in and SBC/CTA/FMCC decisions, full per-number provenance to the CLP-Rates exhibits, the transmission vintage note (why filed 0.05050 differs from RateAcuity 0.03401/0.04433), and the **pre-calc caveat** (fixed rates, no calibrated RR — bill-calc runs only). | +| ct_eversource_class_rev_requirement.md | CT Eversource **whole residential rate class** revenue-requirement files (Docket 26-05-10): the test-year (`ct_eversource_rate_case_test_year.yaml`, current rates) and rate-year-1 (`ct_eversource_rate_case_rate_year_1.yaml`, Proposed+GET) Track 2 YAMLs that model every residential ResStock building on Rate 1 but size base DRR / kWh / customer count to the class (Rate 1 + 5 + 7). Documents the bug it fixes (whole-class ResStock pinned to a Rate-1 target), how top-ups scale off the two class-total scalars, full per-number provenance to the CLP-RATES-2 exhibit with derivations, and the Proposed+GET vs pre-GET caveat. | +| ct_occ863_system_load.md | OCC-863 (Docket 26-05-10) source files: where the four workbooks and the discovery PDF live on S3 and the EBS mirror. Attachment 3 page 2 becomes `system_load.parquet` (hourly distribution-substation load, MW, 2022–2025); page 1 becomes `system_monthly_peak_mw.parquet`. Attachment 4 becomes `rate_1_load.parquet`, `rate_5_load.parquet`, and `rate_7_load.parquet` (2025 hourly class kW; interval-ending EST shifted to hour-beginning local time). Attachment 2 becomes six month-hour marginal-cost parquets: `all_days`, `weekday`, and `weekend_holidays`, each for secondary (page 1) and primary (page 2). `create-dist-mc-data`'s default (`dist_mc_method=month_hour`) expands the secondary weekday and weekend-holiday month-hour parquets; `dist_mc_method=pop_occ863` reads `system_load.parquet` through `--path-utility-load`. | ### code/marginal_costs/ diff --git a/context/code/data/ct_occ863_system_load.md b/context/code/data/ct_occ863_system_load.md index cabfaf78..ad949c9f 100644 --- a/context/code/data/ct_occ863_system_load.md +++ b/context/code/data/ct_occ863_system_load.md @@ -133,14 +133,18 @@ just -f rate_design/hp_rates/ct/Justfile convert-occ863-month-hour-mc \ ## Using this load for CT distribution marginal cost -`create-dist-mc-data` passes `system_load.parquet` as `--path-utility-load`. The allocator keeps -the requested year and ranks the top 100 hours of that substation series. The residential -rate-class files are a class shape; they are not the PoP load. See -`context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md` §3.3. +`create-dist-mc-data` uses two of these outputs. Its default, `dist_mc_method=month_hour`, +expands the secondary weekday and weekend-and-holiday month-hour parquets. Its first fallback, +`dist_mc_method=pop_occ863`, passes `system_load.parquet` as `--path-utility-load`. The +allocator keeps the requested year and ranks the top 100 hours of that substation series. The +residential rate-class files are a class shape; they are not the PoP load. See +`context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md` §3.3 and §7. -`--utility-load-s3-base` still scans a hive `utility`/`year` folder for the other states. This -prefix is one multi-year file, so it is not a value for that flag. +`--utility-load-s3-base` still scans a hive `utility`/`year` folder (the other states, and CT's +`pop_isone_zone` fallback). This prefix is one multi-year file, so it is not a value for that +flag. ```bash just -f rate_design/hp_rates/ct/Justfile create-dist-mc-data 2025 --upload +just -f rate_design/hp_rates/ct/Justfile dist_mc_method=pop_occ863 create-dist-mc-data 2025 --upload ``` diff --git a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md index 8fbf9293..8e6c1d2b 100644 --- a/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md +++ b/context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md @@ -1,8 +1,10 @@ # CT Eversource (CL&P) sub-TX + distribution marginal cost: plan of action **Status: IMPLEMENTED.** Three 8760s can be built for Eversource CT (CL&P) distribution -marginal cost. `create-dist-mc-data` runs the first. The other two stay available in -`generate_utility_tx_dx_mc.py`. +marginal cost, listed in order of preference. `create-dist-mc-data` builds any of them; its +`dist_mc_method` variable picks `month_hour` (the default), `pop_occ863`, or `pop_isone_zone`. +Each method passes the matching `--method` (`month_hour` or `pop`) and load source to +`generate_utility_tx_dx_mc.py` and writes its own dataset. 1. **Month-hour expansion (default).** Eversource's OCC-863 Attachment 2 secondary month-hour `$/kWh` tables are looked up onto one calendar year. Saturday and Sunday take the @@ -10,24 +12,22 @@ marginal cost. `create-dist-mc-data` runs the first. The other two stay availabl eleven federal holidays in 5 U.S.C. § 6103 when `include_holidays` is true (the default): a Saturday holiday is observed Friday, and a Sunday holiday is observed Monday. `--year` is that calendar, and the same year is stamped on `timestamp`, the `year` column, and the Hive - partition. The recipe passes a `dist_and_sub_tx/` output base; the script rewrites it to - `dist_and_sub_tx_eversource_derived/`. Detail is in §7. This path does not use the annual - `$/kW-yr` scalar or a load curve. + partition. Written to `dist_and_sub_tx_eversource_derived/`. Detail is in §7. This path + does not use the annual `$/kW-yr` scalar or a load curve. 2. **`$21.22/kW-yr` on Eversource's substation load.** The scalar is Table 3's `$20.17/kW-yr` times the OCC-863 secondary loss factor (§2.2). Probability-of-peak (§3.1) spreads it over the top 100 hours of OCC-863 Attachment 3 page 2, the distribution-substation load (`system_load.parquet`). It is written to `dist_and_sub_tx_occ863/`. There is no CPI adjustment: the CT analysis is rate year 1, in the filing's own 2026 dollars, not the test year, so `--target-dollar-year 2026` matches the CSV and the allocator keeps `$21.22` - (§2.3). The `create-dist-mc-data` recipe does not pass this load or this output base; the - Python invocation is in §3.4. + (§2.3). `dist_mc_method=pop_occ863` (§3.4). 3. **ISO-NE Connecticut zone load (oldest, not preferred).** The file that exists at `dist_and_sub_tx/utility=ct_eversource/year=2025/` allocated Table 3's `$20.17/kW-yr` after CPI deflation onto the 2025 load year (about `$19.65`), ranked on ISO-NE CT zone aggregate load. That was the best allocation before the OCC-863 substation load and the loss-adjusted scalar were in hand. It is not the same file as a fresh zone-load run: the CSV is now - `$21.22` with no CPI, so `use_eversource_marginal_cost=false` would allocate `$21.22` on - that zone load into `dist_and_sub_tx/` and would not reproduce the stored parquet. + `$21.22` with no CPI, so `dist_mc_method=pop_isone_zone` allocates `$21.22` on that zone + load into `dist_and_sub_tx/`, and uploading year 2025 replaces the stored parquet. Only CT **bulk transmission** MC was implemented before this ([ct_bulk_transmission_marginal_cost.md](ct_bulk_transmission_marginal_cost.md)). @@ -151,7 +151,7 @@ The default recipe does not run this allocator. It expands the month-hour tables two probability-of-peak 8760s differ in which load picks the top 100 hours. The substation-load run allocates `$21.22/kW-yr` with no CPI adjustment and writes `dist_and_sub_tx_occ863/` (§3.4). The stored zone-load file at `dist_and_sub_tx/` allocated the pre-loss `$20.17` -after CPI deflation. `use_eversource_marginal_cost=false` still runs zone-load +after CPI deflation. `dist_mc_method=pop_isone_zone` still runs zone-load probability-of-peak into `dist_and_sub_tx/`, using the current CSV scalar. ### 3.1 Method: standard PoP allocation (same as NY, RI, MD) @@ -192,9 +192,9 @@ probability falls in July–August, ~20% in June/September, <1% winter) [Documen ### 3.3 Load data source PoP ranks the top 100 hours of Eversource's distribution-substation load: OCC-863 Attachment 3 -page 2, stored as `system_load.parquet` (`timestamp`, `load_mw`, 2022–2025). The Python -invocation in §3.4 passes that file as `--path-utility-load`. The `create-dist-mc-data` recipe -does not. `normalize_load_to_cairo_8760` keeps `--load-year`. +page 2, stored as `system_load.parquet` (`timestamp`, `load_mw`, 2022–2025). +`dist_mc_method=pop_occ863` passes that file as `--path-utility-load`. +`normalize_load_to_cairo_8760` keeps `--load-year`. The hive reader `--utility-load-s3-base` stays what NY, RI, and MD use. The OCC-863 prefix is one multi-year file, so it is not a valid value for that flag. @@ -230,17 +230,16 @@ substation series: OCC-863 says no forecasted hourly system loads are available. same factor on `$21.22` would be about `$20.67`. The CT analysis is rate year 1, so the recipe now passes `--target-dollar-year 2026` and the allocator keeps `$21.22`. The earlier run's parquet keeps the deflated pre-loss scalar (point 7). -5. **Ran the standard PoP allocation** via the new `just -f ct/Justfile create-dist-mc-data 2025 - --upload` recipe (§3.5): +5. **Ran the standard PoP allocation** (§3.5): ```bash - uv run python utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py \ - --state CT --utility ct_eversource --year 2025 \ - --target-dollar-year 2026 \ - --mc-table-path rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv \ - --path-utility-load s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet/system_load.parquet \ - --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/ \ - --upload + just -f ct/Justfile dist_mc_method=pop_occ863 create-dist-mc-data 2025 --upload + just -f ct/Justfile dist_mc_method=pop_isone_zone create-dist-mc-data 2025 --upload ``` + Both call `generate_utility_tx_dx_mc.py --method pop --target-dollar-year 2026` with the + CT config CSV. `pop_occ863` passes `--path-utility-load .../occ-863/parquet/system_load.parquet` + and writes `dist_and_sub_tx_occ863/`. `pop_isone_zone` passes + `--utility-load-s3-base s3://data.sb/isone/hourly_demand/utilities/` and writes + `dist_and_sub_tx/`. 6. **Validated** (against the pre-loss `$20.17` scalar; re-run after the `$21.22` change): the built-in `validate_allocation`-equivalent 1-kW-constant-load check passed exactly ($19.6460/kW-yr, 0.0000% error). The top-100 PoP hours for 2025 fall entirely in @@ -260,10 +259,10 @@ substation series: OCC-863 says no forecasted hourly system loads are available. Both use `utility=ct_eversource/year=2025/data.parquet`. The `_occ863` suffix names the source of both changes: the secondary loss factor (Attachment 2) and the system load (Attachment 3). It follows the sibling-prefix pattern of `dist_and_sub_tx_load{year}/`, so a - recursive scan of one dataset never picks up the other's 8760. `create-dist-mc-data` now - writes the month-hour file under `dist_and_sub_tx_eversource_derived/` (§7), not either of - these. The zone-load parquet cannot be rebuilt from the current inputs, since the config - CSV and the preferred load both changed. Point a scenario's `path_dist_and_sub_tx_mc` at + recursive scan of one dataset never picks up the other's 8760. The default + `dist_mc_method=month_hour` writes `dist_and_sub_tx_eversource_derived/` (§7), not either of + these. The stored zone-load parquet cannot be rebuilt from the current inputs, since the + config CSV changed. Point a scenario's `path_dist_and_sub_tx_mc` at whichever of the three 8760s that run should read. 8. **Wired a `just` recipe**, `create-dist-mc-data`, in `rate_design/hp_rates/ct/Justfile` (mirrors `create-bulk-tx-mc-data`'s style, but scoped to `ct_eversource` only — see §3.5). @@ -372,7 +371,7 @@ page 2 value at 1:00 a.m. Page 1's monthly "Max MW" matches page 2, including th `rate_7_load.parquet` (`timestamp`, `load_kw`, `load_mw`). Interval-ending stamps are shifted back one hour, then mapped from Eastern Standard Time onto local wall-clock time. -The §3.4 invocation reads `system_load.parquet` through `--path-utility-load` and keeps the +`dist_mc_method=pop_occ863` reads `system_load.parquet` through `--path-utility-load` and keeps the load year inside `normalize_load_to_cairo_8760` (§3.3). The rate-class files are not the PoP load. `--utility-load-s3-base` still expects the hive `utility`/`year` layout the other states use. How to rebuild the parquets: @@ -382,7 +381,8 @@ use. How to rebuild the parquets: ## 7. OCC-863 Attachment 2 month-hour 8760 (current default) -`create-dist-mc-data` defaults to `--use-eversource-marginal-cost true`. It does not +`create-dist-mc-data` defaults to `dist_mc_method=month_hour`, which passes +`--method month_hour`. It does not probability-of-peak allocate the `$21.22/kW-yr` scalar. It looks up each hour of one calendar year in Eversource's OCC-863 Attachment 2 secondary month-hour `$/kWh` tables (Docket 26-05-10) and writes an 8760 in the same schema as the PoP file (`timestamp`, `utility`, `year`, `mc_total_per_kwh`). @@ -428,24 +428,21 @@ from a non-leap run after February 28. ### 7.3 Where it is written -With the flag true, the script rewrites an output base that ends in `dist_and_sub_tx/` to -`dist_and_sub_tx_eversource_derived/`. The recipe still passes the `dist_and_sub_tx/` base; the -redirect is what keeps this file off the PoP path. +The recipe passes the `dist_and_sub_tx_eversource_derived/` base directly: ```text s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_eversource_derived/utility=ct_eversource/year=YYYY/data.parquet /ebs/data/switchbox/marginal_costs/ct/dist_and_sub_tx_eversource_derived/utility=ct_eversource/year=YYYY/data.parquet ``` -`use_eversource_marginal_cost=false` leaves the base alone and writes the §3 PoP file: - -```text -s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/utility=ct_eversource/year=YYYY/data.parquet -``` +As a guard, `--method month_hour` also rewrites a base ending in `dist_and_sub_tx/` to that +dataset, so a month-hour 8760 never lands on a PoP path. The two PoP fallbacks (§3) write their +own datasets: ```bash -just -f ct/Justfile use_eversource_marginal_cost=false create-dist-mc-data 2025 --upload -USE_EVERSOURCE_MARGINAL_COST=false just s ct create-dist-mc-data 2025 --upload +just -f ct/Justfile dist_mc_method=pop_occ863 create-dist-mc-data 2025 --upload # dist_and_sub_tx_occ863/ +just -f ct/Justfile dist_mc_method=pop_isone_zone create-dist-mc-data 2025 --upload # dist_and_sub_tx/ +DIST_MC_METHOD=pop_occ863 just s ct create-dist-mc-data 2025 --upload ``` Tests for the lookup, the observed-holiday rule, the holiday toggle, leap-year length, and the diff --git a/context/methods/marginal_costs/dist_mc_definition_choice.md b/context/methods/marginal_costs/dist_mc_definition_choice.md index 5ab5f184..328f2e09 100644 --- a/context/methods/marginal_costs/dist_mc_definition_choice.md +++ b/context/methods/marginal_costs/dist_mc_definition_choice.md @@ -98,7 +98,7 @@ in the identical `sub_tx_and_dist_mc_kw_yr` slot. RI and MD CPI-inflate from the CT's default 8760 does not use this scalar. It is the OCC-863 Attachment 2 month-hour expansion ([ct_eversource_dist_mc_methodology.md](ct_eversource_dist_mc_methodology.md) §7). The `$21.22/kW-yr` figure is the allocator's input only when -`use_eversource_marginal_cost=false`. That path does not CPI-adjust: the analysis is rate +`dist_mc_method` is `pop_occ863` or `pop_isone_zone`. That path does not CPI-adjust: the analysis is rate year 1, in the filing's 2026 dollars, so `--target-dollar-year 2026` matches the CSV and `$21.22` is allocated as filed. Spread over the top 100 hours of OCC-863 substation load, it is written to `dist_and_sub_tx_occ863/`. The same flag with ISO-NE CT zone load writes diff --git a/rate_design/hp_rates/ct/Justfile b/rate_design/hp_rates/ct/Justfile index ba67a1f2..9c66d9fb 100644 --- a/rate_design/hp_rates/ct/Justfile +++ b/rate_design/hp_rates/ct/Justfile @@ -14,12 +14,11 @@ set dotenv-path := "state.env" state := env_var_or_default('STATE', 'ct') path_s3_utility_loads := "s3://data.sb/isone/hourly_demand/utilities/" -path_s3_occ863_system_load := "s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet/system_load.parquet" -# Sibling of dist_and_sub_tx/, which keeps the earlier ISO-NE-zone-load 8760. -path_s3_dist_mc_occ863 := "s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/" path_s3_occ863_parquet := "s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet" -path_local_dist_mc := "/ebs/data/switchbox/marginal_costs/ct/dist_and_sub_tx/" -use_eversource_marginal_cost := env_var_or_default('USE_EVERSOURCE_MARGINAL_COST', 'true') +path_s3_dist_mc_root := "s3://data.sb/switchbox/marginal_costs/ct/" +path_local_dist_mc_root := "/ebs/data/switchbox/marginal_costs/ct/" +# month_hour (default), pop_occ863, or pop_isone_zone. See create-dist-mc-data. +dist_mc_method := env_var_or_default('DIST_MC_METHOD', 'month_hour') include_holidays := env_var_or_default('INCLUDE_HOLIDAYS', 'true') import '../Justfile' @@ -119,21 +118,29 @@ create-bulk-tx-mc-data-all *extra_args: echo ">> Done: all CT bulk TX MC (${#utils[@]} utilities × $(echo {{ ct_bulk_tx_years }} | wc -w) years)" >&2 # ============================================================================= -# CT-only: distribution + sub-transmission marginal costs (CL&P MCOS-2 Table 3) -# -# This recipe's default is the OCC-863 Attachment 2 secondary month-hour -# $/kWh tables, expanded to an 8760 on year_arg's calendar: weekends take the -# weekend table, and federal holidays do too unless include_holidays=false. -# That same year is stamped on the timestamps and the Hive partition. The -# script rewrites the dist_and_sub_tx/ output base to -# dist_and_sub_tx_eversource_derived/. -# -# use_eversource_marginal_cost=false still runs the older zone-load -# probability-of-peak allocation into dist_and_sub_tx/. The substation-load -# probability-of-peak path ($21.22/kW-yr, no CPI, written to -# dist_and_sub_tx_occ863/) stays in generate_utility_tx_dx_mc.py: -# --use-eversource-marginal-cost false, --path-utility-load -# path_s3_occ863_system_load, --output-s3-base path_s3_dist_mc_occ863. +# CT-only: distribution + sub-transmission marginal costs (Eversource CL&P) +# +# dist_mc_method picks one of three 8760s, in order of preference: +# +# month_hour (default) +# OCC-863 Attachment 2 secondary month-hour $/kWh tables, expanded onto +# year_arg's calendar. Saturday and Sunday take the weekend table, and +# federal holidays do too unless include_holidays=false. year_arg is +# stamped on the timestamps and the Hive partition. +# -> dist_and_sub_tx_eversource_derived/ +# +# pop_occ863 +# $21.22/kW-yr (ct_marginal_costs_.csv) allocated over the top +# upstream_hours of Eversource's OCC-863 Attachment 3 substation load. +# -> dist_and_sub_tx_occ863/ +# +# pop_isone_zone (oldest, not preferred) +# The same $21.22/kW-yr over the top hours of ISO-NE CT zone load. +# -> dist_and_sub_tx/ (also holds the older CPI-deflated $20.17 parquet +# at year=2025; uploading 2025 replaces it) +# +# Both PoP methods pass --target-dollar-year 2026: the analysis is rate year 1, +# in the filing's 2026 dollars, so $21.22 is allocated with no CPI adjustment. # See context/methods/marginal_costs/ct_eversource_dist_mc_methodology.md. # # CL&P's MCOS-2 covers ct_eversource only; ct_ui has no MCOS-derived value yet @@ -144,25 +151,53 @@ create-bulk-tx-mc-data-all *extra_args: # Generate dist+sub-tx MC for ct_eversource and one year. # # just -f ct/Justfile create-dist-mc-data 2025 --upload -# just -f ct/Justfile use_eversource_marginal_cost=false create-dist-mc-data 2025 --upload # just -f ct/Justfile include_holidays=false create-dist-mc-data 2025 --upload -# USE_EVERSOURCE_MARGINAL_COST=false just s ct create-dist-mc-data 2025 --upload +# just -f ct/Justfile dist_mc_method=pop_occ863 create-dist-mc-data 2025 --upload +# just -f ct/Justfile dist_mc_method=pop_isone_zone create-dist-mc-data 2025 --upload +# DIST_MC_METHOD=pop_occ863 just s ct create-dist-mc-data 2025 --upload create-dist-mc-data year_arg *extra_args: - cd {{ path_repo }} && uv run python {{ path_repo }}/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py \ + #!/usr/bin/env bash + set -euo pipefail + case "{{ dist_mc_method }}" in + month_hour) + dataset=dist_and_sub_tx_eversource_derived + method_args=( + --method month_hour + --path-weekday-mc "{{ path_s3_occ863_parquet }}/month_hour_marginal_costs_weekday_secondary.parquet" + --path-weekend-mc "{{ path_s3_occ863_parquet }}/month_hour_marginal_costs_weekend_holidays_secondary.parquet" + --include-holidays "{{ include_holidays }}" + ) + ;; + pop_occ863) + dataset=dist_and_sub_tx_occ863 + method_args=(--method pop --path-utility-load "{{ path_s3_occ863_parquet }}/system_load.parquet") + ;; + pop_isone_zone) + dataset=dist_and_sub_tx + method_args=(--method pop --utility-load-s3-base "{{ path_s3_utility_loads }}") + ;; + *) + echo "dist_mc_method must be month_hour, pop_occ863, or pop_isone_zone (got '{{ dist_mc_method }}')" >&2 + exit 1 + ;; + esac + if [[ "${method_args[1]}" == "pop" ]]; then + method_args+=( + --load-year "{{ year_arg }}" + --target-dollar-year 2026 + --mc-table-path "{{ path_config }}/marginal_costs/ct_marginal_costs_{{ year_arg }}.csv" + --n-hours "{{ upstream_hours }}" + ) + fi + echo ">> CT dist MC: method={{ dist_mc_method }} year={{ year_arg }} -> ${dataset}/" >&2 + cd {{ path_repo }} + uv run python utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py \ --state CT \ --utility ct_eversource \ --year {{ year_arg }} \ - --load-year {{ year_arg }} \ - --target-dollar-year 2026 \ - --mc-table-path {{ path_config }}/marginal_costs/ct_marginal_costs_{{ year_arg }}.csv \ - --utility-load-s3-base {{ path_s3_utility_loads }} \ - --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ \ - --path-local-output-base {{ path_local_dist_mc }} \ - --n-hours {{ upstream_hours }} \ - --use-eversource-marginal-cost {{ use_eversource_marginal_cost }} \ - --path-weekday-mc {{ path_s3_occ863_parquet }}/month_hour_marginal_costs_weekday_secondary.parquet \ - --path-weekend-mc {{ path_s3_occ863_parquet }}/month_hour_marginal_costs_weekend_holidays_secondary.parquet \ - --include-holidays {{ include_holidays }} \ + --output-s3-base "{{ path_s3_dist_mc_root }}${dataset}/" \ + --path-local-output-base "{{ path_local_dist_mc_root }}${dataset}/" \ + "${method_args[@]}" \ {{ extra_args }} # ============================================================================= diff --git a/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py b/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py index 224dc830..a70ac2f8 100644 --- a/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py +++ b/utils/data_prep/marginal_costs/generate_utility_tx_dx_mc.py @@ -1,16 +1,42 @@ -"""Allocate marginal sub-transmission and distribution costs to hourly price signals. - -This script implements the diluted marginal cost allocation methodology using -the Probability of Peak (PoP) method to allocate $/kW-yr costs to $/kWh hourly -price signals. - -Input: - - Utility hourly load profile (ISO-native layout): utility=X/year=YYYY[/month=MM]/data.parquet +"""Build an hourly sub-transmission and distribution marginal cost 8760. + +``--method`` picks how the 8760 is built: + +- ``pop`` (default; NY, RI, MD, and CT's fallback): Probability of Peak (PoP) + allocation of one $/kW-yr value from the MC table over the top + ``--n-hours`` hours of a load profile. The load comes from exactly one of + ``--utility-load-s3-base`` (hive utility=X/year=YYYY scan) or + ``--path-utility-load`` (one parquet with timestamp and load_mw). +- ``month_hour`` (CT only): look up Eversource's OCC-863 Attachment 2 + month-hour $/kWh tables onto one calendar year. Uses no $/kW-yr value and no + load profile. + +CT Eversource, in order of preference (the ct Justfile's ``dist_mc_method``): + +1. ``month_hour``: OCC-863 Attachment 2 secondary tables, weekday vs. + weekend-and-holiday. Written to ``dist_and_sub_tx_eversource_derived/``. +2. ``pop`` on ``--path-utility-load`` = OCC-863 Attachment 3 system load: the + $21.22/kW-yr secondary value on Eversource's own substation peaks. Written to + ``dist_and_sub_tx_occ863/``. +3. ``pop`` on ``--utility-load-s3-base`` = ISO-NE CT zone load: the same + $21.22/kW-yr on zone peaks. The oldest approach. Written to + ``dist_and_sub_tx/``. + +Both CT PoP paths pass ``--target-dollar-year 2026``: the analysis is rate year +1, already in the filing's 2026 dollars, so no CPI adjustment applies. + +PoP input: + - Utility hourly load profile, either ISO-native hive layout + utility=X/year=YYYY[/month=MM]/data.parquet (e.g. s3://data.sb/nyiso/hourly_demand/utilities/, s3://data.sb/pjm/hourly_demand/utilities/, - s3://data.sb/isone/hourly_demand/utilities/) + s3://data.sb/isone/hourly_demand/utilities/), or one parquet file - Marginal cost table CSV with columns: utility, sub_tx_and_dist_mc_kw_yr[, dollar_year] - Load year (determines which load profile year to use) +Month-hour input: + - Weekday and weekend-and-holiday parquets, each 288 rows of + month (1-12), hour (0-23), mc_total_per_kwh + The optional ``dollar_year`` column in the MC table specifies the dollar year of the cost estimate. When it differs from ``--target-dollar-year`` (default: ``--year``), the script inflates the value using the annual-average CPIAUCSL @@ -21,12 +47,10 @@ - NY default base: s3://data.sb/switchbox/marginal_costs/ny/dist_and_sub_tx/ - RI default base: s3://data.sb/switchbox/marginal_costs/ri/dist_and_sub_tx/ - MD default base: s3://data.sb/switchbox/marginal_costs/md/dist_and_sub_tx/ - - CT default base: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ - (earlier ISO-NE zone-load run; the OCC-863 run writes to - dist_and_sub_tx_occ863/) - - CT Eversource month-hour (--use-eversource-marginal-cost true): - s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_eversource_derived/ + - CT month_hour: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_eversource_derived/ The partition year is --year, the same calendar stamped on the timestamps. + - CT pop on OCC-863 system load: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/ + - CT pop on ISO-NE CT zone load: s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx/ - Default partition: utility=X/year=YYYY/data.parquet - Alternate-load-year base: _loadYYYY/ Example: dist_and_sub_tx_load2018/utility=X/year=2025/data.parquet @@ -53,14 +77,10 @@ --output-s3-base s3://data.sb/switchbox/marginal_costs/ny/dist_and_sub_tx/ \ --upload - # CT/Eversource (CL&P): rate year 1 keeps the filed 2026$ scalar (no CPI adjustment). - # PoP hours come from OCC-863 Attachment 3 page 2, not the ISO-NE hive layout. - python generate_utility_tx_dx_mc.py --state CT --utility ct_eversource --year 2025 \ - --target-dollar-year 2026 \ - --mc-table-path rate_design/hp_rates/ct/config/marginal_costs/ct_marginal_costs_2025.csv \ - --path-utility-load s3://data.sb/switchbox/sources/ct/eversource/docket-26-05-10/occ-863/parquet/system_load.parquet \ - --output-s3-base s3://data.sb/switchbox/marginal_costs/ct/dist_and_sub_tx_occ863/ \ - --upload + # CT/Eversource: use the ct Justfile, which wires all three methods. + just -f rate_design/hp_rates/ct/Justfile create-dist-mc-data 2025 --upload + just -f rate_design/hp_rates/ct/Justfile dist_mc_method=pop_occ863 create-dist-mc-data 2025 --upload + just -f rate_design/hp_rates/ct/Justfile dist_mc_method=pop_isone_zone create-dist-mc-data 2025 --upload """ import argparse @@ -339,7 +359,7 @@ def eversource_derived_output_base(base: str) -> str: return f"{trimmed}/" if leaf != "dist_and_sub_tx": raise ValueError( - "--use-eversource-marginal-cost writes under " + "--method month_hour writes under " "dist_and_sub_tx_eversource_derived/, so the output base must end " f"in dist_and_sub_tx/ (got {base!r})" ) @@ -764,7 +784,20 @@ def save_allocated_costs( def main(): """Main entry point for the script.""" parser = argparse.ArgumentParser( - description="Allocate marginal costs to hourly price signals using PoP method" + description=( + "Build an hourly dist + sub-tx marginal cost 8760 by PoP allocation " + "or (CT) Eversource month-hour lookup" + ) + ) + parser.add_argument( + "--method", + choices=["pop", "month_hour"], + default="pop", + help=( + "pop: allocate the MC table's $/kW-yr over the top --n-hours of a " + "load profile. month_hour (CT only): look up OCC-863 Attachment 2 " + "month-hour $/kWh tables onto --year. Default: pop." + ), ) parser.add_argument( "--state", @@ -783,21 +816,26 @@ def main(): "--year", type=int, required=True, - help="Output year for partition key and timestamps (e.g. 2025).", + help=( + "Output year for partition key and timestamps (e.g. 2025). For " + "month_hour, also the calendar that sets weekdays and holidays." + ), ) - parser.add_argument( + + pop = parser.add_argument_group("--method pop") + pop.add_argument( "--load-year", type=int, default=None, help=("Year of load profile to use for PoP allocation (defaults to --year)."), ) - parser.add_argument( + pop.add_argument( "--mc-table-path", type=str, - required=True, - help="Path to marginal cost table CSV (local or s3://)", + default=None, + help="Path to marginal cost table CSV (local or s3://). Required for pop.", ) - parser.add_argument( + pop.add_argument( "--utility-load-s3-base", "--nyiso-s3-base", dest="utility_load_s3_base", @@ -805,17 +843,18 @@ def main(): default=None, help=( "Base S3 path for hive-partitioned utility loads " - "(utility=X/year=YYYY). Required unless --path-utility-load is set." + "(utility=X/year=YYYY). CT: ISO-NE CT zone load. " + "Pass this or --path-utility-load, not both." ), ) - parser.add_argument( + pop.add_argument( "--path-utility-load", dest="path_utility_load", type=str, default=None, help=( "Single parquet with timestamp and load_mw, used instead of the " - "hive scan. CT passes the OCC-863 system_load.parquet." + "hive scan. CT: the OCC-863 system_load.parquet." ), ) parser.add_argument( @@ -824,14 +863,14 @@ def main(): required=True, help="Base S3 path for output (e.g., s3://data.sb/switchbox/marginal_costs/ny/dist_and_sub_tx/)", ) - parser.add_argument( + pop.add_argument( "--n-hours", type=int, choices=range(8761), default=100, help="Number of top load hours for PoP allocation (0-8760, default: 100)", ) - parser.add_argument( + pop.add_argument( "--target-dollar-year", type=int, default=None, @@ -840,7 +879,7 @@ def main(): "Only applied when the MC table has a dollar_year column." ), ) - parser.add_argument( + pop.add_argument( "--cpi-s3-base", type=str, default="s3://data.sb/fred/cpi/", @@ -854,33 +893,20 @@ def main(): action="store_true", help="Upload results to S3 (default: False, for data inspection only)", ) - parser.add_argument( - "--use-eversource-marginal-cost", - type=_parse_bool, - default=False, - metavar="{true,false}", - help=( - "true: build the 8760 from Eversource's OCC-863 Attachment 2 " - "month-hour $/kWh tables (weekday vs. weekend-and-holiday). " - "false: PoP allocation of the $/kW-yr value. CT only; default false." - ), - ) - parser.add_argument( + month_hour = parser.add_argument_group("--method month_hour") + month_hour.add_argument( "--path-weekday-mc", type=str, default=None, - help="Month-hour weekday $/kWh parquet (with --use-eversource-marginal-cost)", + help="Month-hour weekday $/kWh parquet. Required for month_hour.", ) - parser.add_argument( + month_hour.add_argument( "--path-weekend-mc", type=str, default=None, - help=( - "Month-hour weekend-and-holiday $/kWh parquet " - "(with --use-eversource-marginal-cost)" - ), + help="Month-hour weekend-and-holiday $/kWh parquet. Required for month_hour.", ) - parser.add_argument( + month_hour.add_argument( "--include-holidays", type=_parse_bool, default=True, @@ -888,7 +914,7 @@ def main(): help=( "true: federal holidays take the weekend-and-holiday table. " "false: only Saturday and Sunday do; a weekday holiday stays on " - "the weekday table. Used with --use-eversource-marginal-cost." + "the weekday table." ), ) parser.add_argument( @@ -902,10 +928,18 @@ def main(): load_dotenv() storage_options = get_aws_storage_options() - if args.use_eversource_marginal_cost: - _run_eversource_month_hour(args, storage_options) - return + if args.method == "month_hour": + _run_month_hour(args, storage_options) + else: + _run_pop_allocation(args, storage_options) + +def _run_pop_allocation( + args: argparse.Namespace, storage_options: dict[str, str] +) -> None: + """PoP-allocate the MC table's $/kW-yr over the top load hours, then save.""" + if not args.mc_table_path: + raise ValueError("--method pop needs --mc-table-path") validate_mc_table_path(args.mc_table_path) output_year = args.year @@ -1017,16 +1051,13 @@ def main(): print("=" * 60) -def _run_eversource_month_hour( - args: argparse.Namespace, storage_options: dict[str, str] -) -> None: +def _run_month_hour(args: argparse.Namespace, storage_options: dict[str, str]) -> None: """Build and optionally save the 8760 from OCC-863 month-hour tables.""" if args.state != "CT": - raise ValueError("--use-eversource-marginal-cost is only defined for CT") + raise ValueError("--method month_hour is only defined for CT") if not (args.path_weekday_mc and args.path_weekend_mc): raise ValueError( - "--use-eversource-marginal-cost needs --path-weekday-mc and " - "--path-weekend-mc" + "--method month_hour needs --path-weekday-mc and --path-weekend-mc" ) year = args.year output_s3_base = eversource_derived_output_base(args.output_s3_base)