Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions src/core/outputs_by_time.jl
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,49 @@ function make_dataframes(
)
end

"""
Restrict a wide-format outputs table to `realized` timestamps, stitching multiple windows
into one continuous `DataFrame` when necessary.

`table` is either the flat `DataFrame` a single-window outputs type (e.g.
`OptimizationProblemOutputs`) returns from `read_variable`/`read_key_wide`, or the
`SortedDict{Dates.DateTime, DataFrame}` that [`make_dataframes`](@ref) returns for a
multi-window [`OutputsByTime`](@ref) under `TableFormat.WIDE`, one entry per window's own
start time — what a rolling-horizon `Simulation`'s per-problem results produce, since
consecutive windows overlap and only each window's non-overlapping prefix is realized.
`realized` is the target set of timestamps to keep, typically from
[`get_realized_timestamps`](@ref).

This is the shared implementation behind "give me the realized time series, not the raw
windows," so every `IS.Outputs`-consuming caller reads through the same stitching logic
instead of each reimplementing it. Add a method here — not a special case in the caller —
to support a new windowed shape.
"""
function make_realized_dataframe end

function make_realized_dataframe(table::DataFrame, realized)
return filter("DateTime" => in(realized), table)
end

function make_realized_dataframe(
table::AbstractDict{Dates.DateTime, <:DataFrame},
realized,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's probably a more efficient way: under typical usage, I suspect your data is already in contiguous chunks. (Assumption: the rows of each value in table are sorted by timestep and realized is a time interval.) But it's output code, so performance isn't terribly important

)
isempty(table) && return DataFrame()
realized_set = Set(realized)
window_starts = sort(collect(keys(table)))
parts = Vector{DataFrame}(undef, length(window_starts))
for (i, window_start) in enumerate(window_starts)
# Consecutive windows overlap (each one's look-ahead runs into the next window's
# own start), so truncate to [window_start, next_window_start) — the last window
# has no successor to truncate against and keeps everything through `realized`.
next_start = i < length(window_starts) ? window_starts[i + 1] : nothing
in_range(t) = (t in realized_set) && (isnothing(next_start) || t < next_start)
parts[i] = filter("DateTime" => in_range, table[window_start])
end
return sort!(reduce(vcat, parts), "DateTime")
end

struct OutputsByKeyAndTime
"Contains all keys stored in the model."
output_keys::Vector{OptimizationContainerKey}
Expand Down
52 changes: 52 additions & 0 deletions test/test_optimization_outputs.jl
Original file line number Diff line number Diff line change
Expand Up @@ -268,3 +268,55 @@ end
10,
)
end

@testset "Test make_realized_dataframe" begin
# Single-window (flat DataFrame) shape: OptimizationProblemOutputs' read_variable
# returns this directly. Filters straight through.
flat = DataFrame(
"DateTime" => DateTime.(["2024-01-01T00:00:00", "2024-01-01T01:00:00"]),
"c1" => [1.0, 2.0],
)
realized = [DateTime("2024-01-01T01:00:00")]
result = IOM.make_realized_dataframe(flat, realized)
@test result == DataFrame("DateTime" => realized, "c1" => [2.0])

# Multi-window shape: what `make_dataframes(::OutputsByTime; table_format=WIDE)`
# returns for a rolling-horizon Simulation, keyed by each window's own start time.
# Consecutive windows overlap; only each window's non-overlapping prefix is realized.
windows = SortedDict(
DateTime("2024-01-01T00:00:00") => DataFrame(
"DateTime" =>
DateTime.([
"2024-01-01T00:00:00",
"2024-01-01T01:00:00",
"2024-01-01T02:00:00",
]),
"c1" => [1.0, 2.0, 3.0],
),
DateTime("2024-01-01T02:00:00") => DataFrame(
"DateTime" =>
DateTime.([
"2024-01-01T02:00:00",
"2024-01-01T03:00:00",
"2024-01-01T04:00:00",
]),
"c1" => [30.0, 40.0, 50.0],
),
)
windows_realized = DateTime.([
"2024-01-01T00:00:00",
"2024-01-01T01:00:00",
"2024-01-01T02:00:00",
"2024-01-01T03:00:00",
"2024-01-01T04:00:00",
])
Comment on lines +306 to +312

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[JuliaFormatter] reported by reviewdog 🐶

Suggested change
windows_realized = DateTime.([
"2024-01-01T00:00:00",
"2024-01-01T01:00:00",
"2024-01-01T02:00:00",
"2024-01-01T03:00:00",
"2024-01-01T04:00:00",
])
windows_realized =
DateTime.([
"2024-01-01T00:00:00",
"2024-01-01T01:00:00",
"2024-01-01T02:00:00",
"2024-01-01T03:00:00",
"2024-01-01T04:00:00",
])

stitched = IOM.make_realized_dataframe(windows, windows_realized)
@test stitched.DateTime == windows_realized
# The second window's own value for 02:00 (30.0) wins over the first window's
# look-ahead value (3.0) because `realized` names exactly one row per timestamp and
# `filter` on the second window keeps its 02:00 row, not the first's.
@test stitched.c1 == [1.0, 2.0, 30.0, 40.0, 50.0]

@test IOM.make_realized_dataframe(SortedDict{DateTime, DataFrame}(), DateTime[]) ==
DataFrame()
end
Loading