diff --git a/Project.toml b/Project.toml index ec685aa..9e7f9c5 100644 --- a/Project.toml +++ b/Project.toml @@ -12,6 +12,12 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StringEncodings = "69024149-9ee7-55f6-a4c4-859efe599b68" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +[weakdeps] +TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" + +[extensions] +MATTimeZonesExt = "TimeZones" + [compat] CodecZlib = "0.5, 0.6, 0.7" Dates = "1" @@ -20,6 +26,7 @@ OrderedCollections = "1, 2" PooledArrays = "1.4.3" StringEncodings = "0.3.7" Tables = "1.12.1" +TimeZones = "1" julia = "1.6" [extras] @@ -27,6 +34,7 @@ DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" [targets] -test = ["DataStructures", "LinearAlgebra", "SparseArrays", "Test"] +test = ["DataStructures", "LinearAlgebra", "SparseArrays", "Test", "TimeZones"] diff --git a/docs/src/types.md b/docs/src/types.md index 4a14b6b..f4883d3 100644 --- a/docs/src/types.md +++ b/docs/src/types.md @@ -17,9 +17,35 @@ A few of the `MatlabOpaque` classes are automatically converted upon reading: | MATLAB | Julia | | -------- | ------- | | `string` | `String` | -| `datetime` | `Dates.DateTime` | +| `datetime` | `Dates.DateTime` (`TimeZones.ZonedDateTime` if zoned and TimeZones.jl is loaded) | | `duration` | `Dates.Millisecond` | | `category` | `PooledArrays.PooledArray` | | `table` | `MAT.MatlabTable` (or any other table) | +| `timetable` | `MAT.MatlabTable` (or any other table), with the row times as its first column | + +A timetable's row times come back as `Dates.DateTime` or `Dates.Millisecond`, in a +column named after the row dimension (`Time` unless renamed). A regular timetable, which +MATLAB stores as a start time and a sample rate or time step, has its row times +generated; one stepped in calendar units (months, for example) has no fixed rate and is +left as a `MatlabOpaque`, with a warning. Row times are rounded to whole milliseconds, +the resolution of `Dates.DateTime`. + +## Time zones + +MATLAB stores a `datetime` that has a time zone as its UTC instant together with the +zone's name. With [TimeZones.jl](https://github.com/JuliaTime/TimeZones.jl) loaded (Julia +1.9 and later, through a package extension), it is read as a `ZonedDateTime` in that zone: +an IANA zone such as `Europe/London`, `UTC`, or a fixed offset such as `+05:30`. + +```julia +using MAT, TimeZones +matread("file.mat")["t"] # 2022-07-20T12:00:00+01:00, in Europe/London +``` + +Without it, a zoned `datetime` is read as a `DateTime` holding the UTC instant, with a +warning. A `datetime` without a time zone is always a `DateTime` of the stored wall-clock +time. Zoned row times of a timetable follow the same rules. The `UTCLeapSeconds` zone counts +leap seconds, which neither type can represent, so such a `datetime` is left as a +`MatlabOpaque`, with a warning. Note that single element arrays are typically converted to scalars in Julia, because MATLAB cannot distinguish between scalars and `1x1` sized arrays. \ No newline at end of file diff --git a/ext/MATTimeZonesExt.jl b/ext/MATTimeZonesExt.jl new file mode 100644 index 0000000..8814144 --- /dev/null +++ b/ext/MATTimeZonesExt.jl @@ -0,0 +1,11 @@ +module MATTimeZonesExt + +# Zoned MATLAB datetimes as ZonedDateTime. MATLAB stores a zoned datetime as its UTC +# instant plus the zone's name: an IANA name ("Europe/London"), "UTC", or a fixed offset +# such as "+05:30", all of which TimeZone() parses. + +using MAT, Dates, TimeZones + +MAT.MAT_types.to_zoned(utc::DateTime, tz::String) = ZonedDateTime(utc, TimeZone(tz); from_utc = true) + +end diff --git a/src/MAT.jl b/src/MAT.jl index 17e6150..f6795e8 100644 --- a/src/MAT.jl +++ b/src/MAT.jl @@ -123,7 +123,8 @@ keyword argument only affects write operations. Use with `read`, `write`, `close`, `keys`, and `haskey`. -Optional keyword argument is the `table` type, for automatic conversion of Matlab tables. +Optional keyword argument is the `table` type, for automatic conversion of Matlab tables +and timetables (a timetable's row times become its first column). Note that Matlab tables may contain non-vector colums which cannot always be converted to a Julia table, like `DataFrame`. # Example @@ -164,7 +165,7 @@ matopen Return a dictionary of all the variables and values in a Matlab file, opening and closing it automatically. -Optionally provide the `table` type to convert Matlab tables into. Default uses a simple `MatlabTable` type. +Optionally provide the `table` type to convert Matlab tables and timetables into. Default uses a simple `MatlabTable` type. # Example diff --git a/src/MAT_types.jl b/src/MAT_types.jl index edaacec..8dc2e5f 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -473,6 +473,8 @@ function convert_opaque(obj::MatlabOpaque; table::Type=Nothing) return from_categorical(obj) elseif obj.class == "table" return from_table(obj, table) + elseif obj.class == "timetable" + return from_timetable(obj, table) else return obj end @@ -513,17 +515,43 @@ function from_string(obj::MatlabOpaque, encoding::Encoding=Encoding(Symbol("UTF- end end +""" + to_zoned(utc::DateTime, tz::String) + +A zoned datetime from its UTC instant and MATLAB's name for the zone (an IANA name such +as "Europe/London", "UTC", or a fixed offset such as "+05:30"). Defined by the TimeZones +extension (Julia 1.9 and later, when TimeZones.jl is loaded), which returns a +`ZonedDateTime`; without it, zoned datetimes are read as their UTC instant. +""" +function to_zoned end + function from_datetime(obj::MatlabOpaque) dat = obj["data"] if isnothing(dat) || isempty(dat) return DateTime[] end - if haskey(obj, "tz") && !isempty(obj["tz"]) - tz = obj["tz"] - @warn "no timezone conversion yet for datetime objects. timezone of \"$tz\" ignored" - end #isdate = obj["isDateOnly"] # optional: convert to Date instead of DateTime? - return map_or_not(ms_to_datetime, dat) + tz = haskey(obj, "tz") ? obj["tz"] : "" + if !(tz isa AbstractString) || isempty(tz) + return map_or_not(ms_to_datetime, dat) # no zone: the stored wall-clock time + end + # MATLAB stores a zoned datetime as its UTC instant, except in "UTCLeapSeconds", where + # the count includes leap seconds, which DateTime cannot represent + if tz == "UTCLeapSeconds" + @warn "datetime with timezone \"UTCLeapSeconds\" is not converted (leap seconds cannot be represented); returning the MatlabOpaque" + return obj + end + utc = map_or_not(ms_to_datetime, dat) + if !hasmethod(to_zoned, Tuple{DateTime,String}) + @warn "datetime with timezone \"$tz\" is returned as its UTC instant; load TimeZones.jl to read it as a ZonedDateTime" + return utc + end + try + return map_or_not(t -> ismissing(t) ? missing : to_zoned(t, String(tz)), utc) + catch err + @warn "timezone \"$tz\" was not recognised ($(sprint(showerror, err))); returning the UTC instant" + return utc + end end # is the complex part the submilliseconds? @@ -632,6 +660,39 @@ end # option to not convert and get the MatlabOpaque as table from_table(obj::MatlabOpaque, ::Type{Nothing}) = obj +# A timetable is a table whose first column holds the row times, named after the row +# dimension ("Time" unless renamed), then the variables. Row times are DateTime or +# Millisecond; they are stored one per row, or, for a regular timetable, as a start time +# and a sample rate (or time step), from which they are generated here. Row times that +# cannot be interpreted leave the timetable as the MatlabOpaque it was read as. +function from_timetable(obj::MatlabOpaque, ::Type{T}=MatlabTable) where {T} + tt = haskey(obj, "any") ? obj["any"] : obj + times = timetable_rowtimes(tt["rowTimes"], Int(tt["numRows"])) + if isnothing(times) + @warn "timetable row times of this kind are not converted; returning the MatlabOpaque" + return obj + end + names = Symbol[Symbol(tt["dimNames"][1]); Symbol.(vec(tt["varNames"]))] + cols = vcat(Any[times], Any[try_vec(c) for c in vec(tt["data"])]) + t = MatlabTable(names, cols) + return T(Tables.CopiedColumns(t)) +end +from_timetable(obj::MatlabOpaque, ::Type{Nothing}) = obj + +# row times stored one per row, already converted to DateTime or Millisecond +timetable_rowtimes(times::AbstractArray, n::Int) = vec(times) +# a regular timetable: a start time and a sample rate in Hz; MATLAB also stores the rate +# when the time step was given instead, so the rate serves for both +function timetable_rowtimes(regular::AbstractDict, n::Int) + origin, rate = get(regular, "origin", nothing), get(regular, "sampleRate", nothing) + # the start is a DateTime, a ZonedDateTime (TimeZones extension) or a duration + if !(origin isa Union{Dates.AbstractDateTime,Dates.Period} && rate isa Real && isfinite(rate) && rate > 0) + return nothing + end + return [origin + Millisecond(round(Int, 1000 * k / rate)) for k in 0:(n-1)] +end +timetable_rowtimes(times, n::Int) = nothing + try_vec(c::Vector) = c try_vec(c) = [c] function try_vec(c::AbstractArray) diff --git a/test/read.jl b/test/read.jl index 5f7b8ba..c95916d 100644 --- a/test/read.jl +++ b/test/read.jl @@ -274,6 +274,82 @@ for format in ["v7", "v7.3"] end end + # timetable.mat is written by timetable_gen.m: one variable per way MATLAB stores + # a timetable's row times + @testset "timetable $format" begin + filepath = joinpath(dirname(@__FILE__), format, "timetable.mat") + if isfile(filepath) + # tt_calendar's step is calendar months, which has no fixed rate: left as read + vars = @test_logs (:warn, r"timetable row times") match_mode=:any matread(filepath) + @test vars["tt_calendar"] isa MatlabOpaque + @test vars["tt_calendar"].class == "timetable" + + t = vars["tt_datetime"] # row times per row, as datetimes + @test t isa MatlabTable + @test t.names == [:Time, :Current, :Channel] + @test t[:Time] == DateTime(2022, 7, 20, 2, 27, 27) .+ Millisecond.([428, 496, 564]) + @test t[:Current] == [1.5e-11, 2.5e-11, -3e-12] + @test t[:Channel] == ["AB", "A", "B"] + + t = vars["tt_duration"] # row times per row, as durations + @test t.names == [:Time, :x] + @test t[:Time] == Millisecond.([0, 500, 1250]) + @test t[:x] == [1.0, 2.0, 3.0] + + t = vars["tt_rate"] # regular: start and sample rate + @test t.names == [:Time, :x] + @test t[:Time] == Millisecond.(0:3) + @test t[:x] == [10.0, 20.0, 30.0, 40.0] + + t = vars["tt_step"] # regular: datetime start and time step + @test t[:Time] == DateTime(2022, 7, 20, 2) .+ Millisecond.([0, 250, 500]) + @test t[:y] == [1.0, 2.0, 3.0] + + t = vars["tt_empty"] # no rows, no variables + @test t isa MatlabTable + @test t.names == [:Time] + @test isempty(t[:Time]) + + t = vars["tt_matrix"] # a two-column variable stays a matrix + @test t[:Time] == Millisecond.([1000, 2000]) + @test t[:m] == [1.0 2.0; 3.0 4.0] + + @test vars["tt_dimname"].names == [:Timestamp, :x] # the row-times dimension's own name + + # using Nothing keeps the MatlabOpaque + raw = matread(filepath; table=Nothing)["tt_rate"] + @test raw isa MatlabOpaque + @test raw.class == "timetable" + else + # generated in MATLAB by test/timetable_gen.m; skipped until it is committed + @test_skip isfile(filepath) + end + end + + # timezone.mat is written by timezone_gen.m. MATLAB stores a zoned datetime as its UTC + # instant; until TimeZones.jl is loaded (timezones.jl) that instant is what is returned. + @testset "time zones, without TimeZones.jl $format" begin + filepath = joinpath(dirname(@__FILE__), format, "timezone.mat") + ext_loaded = isdefined(Base, :get_extension) && Base.get_extension(MAT, :MATTimeZonesExt) !== nothing + if isfile(filepath) && !ext_loaded + vars = @test_logs (:warn, r"timezone") match_mode=:any matread(filepath) + @test vars["dt_unzoned"] == DateTime(2022, 7, 20, 12) # no zone: wall clock + @test vars["dt_utc"] == DateTime(2022, 7, 20, 12) + @test vars["dt_london_summer"] == DateTime(2022, 7, 20, 11) # 12:00 BST + @test vars["dt_london_winter"] == DateTime(2022, 1, 20, 12) # 12:00 GMT + @test vars["dt_offset"] == DateTime(2022, 7, 20, 6, 30) # 12:00 +05:30 + @test vars["dt_newyork"] == [DateTime(2022, 1, 20, 17) DateTime(2022, 7, 20, 16)] + # counted with leap seconds, which DateTime cannot represent: left as read + @test vars["dt_leap"] isa MatlabOpaque + @test vars["dt_leap"].class == "datetime" + @test vars["tt_zoned"][:Time] == DateTime(2022, 7, 20, 11) .+ Millisecond.([0, 1500]) + @test vars["tt_zoned_step"][:Time] == DateTime(2022, 7, 20, 11) .+ Millisecond.([0, 500, 1000]) + else + # generated in MATLAB by test/timezone_gen.m; skipped until it is committed + @test_skip isfile(filepath) && !ext_loaded + end + end + @testset "user defined classdef $format" begin let objtestfile = "user_defined_classdefs.mat" filepath = joinpath(dirname(@__FILE__), format, objtestfile) diff --git a/test/runtests.jl b/test/runtests.jl index 6789b43..88836ed 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,4 +6,5 @@ using Test, MAT include("read.jl") include("readwrite4.jl") include("write.jl") + include("timezones.jl") # last: loading TimeZones.jl changes how zoned datetimes read end diff --git a/test/timetable_gen.m b/test/timetable_gen.m new file mode 100644 index 0000000..6a235fe --- /dev/null +++ b/test/timetable_gen.m @@ -0,0 +1,38 @@ +% Generates test/v7/timetable.mat and test/v7.3/timetable.mat for the timetable tests +% in read.jl. Each variable is one way MATLAB stores a timetable's row times. +% Run from the test directory: matlab -batch "timetable_gen" + +% explicit datetime row times, with sub-millisecond parts, and a string variable +Time = datetime(2022, 7, 20, 2, 27, 27) + milliseconds([428.25; 495.75; 563.5]); +Current = [1.5e-11; 2.5e-11; -3e-12]; +Channel = ["AB"; "A"; "B"]; +tt_datetime = timetable(Time, Current, Channel); + +% explicit duration row times +tt_duration = timetable(seconds([0; 0.5; 1.25]), [1; 2; 3], 'VariableNames', {'x'}); + +% regular: a sample rate (row times start at 0 s) +x = [10; 20; 30; 40]; +tt_rate = timetable(x, 'SampleRate', 1000); + +% regular: a time step from a datetime start +y = [1; 2; 3]; +tt_step = timetable(y, 'TimeStep', seconds(0.25), 'StartTime', datetime(2022, 7, 20, 2, 0, 0)); + +% no rows and no variables, as an acquisition leaves an unused timetable +tt_empty = timetable(datetime.empty(0, 1)); + +% a variable with two columns +tt_matrix = timetable(seconds([1; 2]), [1 2; 3 4], 'VariableNames', {'m'}); + +% the row-times dimension renamed +tt_dimname = tt_duration; +tt_dimname.Properties.DimensionNames{1} = 'Timestamp'; + +% regular in calendar months: no fixed sample rate, so not converted +z = [1; 2; 3]; +tt_calendar = timetable(z, 'TimeStep', calmonths(1), 'StartTime', datetime(2022, 1, 1)); + +vars = {'tt_datetime', 'tt_duration', 'tt_rate', 'tt_step', 'tt_empty', 'tt_matrix', 'tt_dimname', 'tt_calendar'}; +save(fullfile('v7', 'timetable.mat'), vars{:}, '-v7'); +save(fullfile('v7.3', 'timetable.mat'), vars{:}, '-v7.3'); diff --git a/test/timezone_gen.m b/test/timezone_gen.m new file mode 100644 index 0000000..02691ce --- /dev/null +++ b/test/timezone_gen.m @@ -0,0 +1,21 @@ +% Generates test/v7/timezone.mat and test/v7.3/timezone.mat: datetimes with and without +% time zones, and timetables whose row times have one. +% Run from the test directory: matlab -batch "timezone_gen" + +dt_unzoned = datetime(2022, 7, 20, 12, 0, 0); +dt_utc = datetime(2022, 7, 20, 12, 0, 0, 'TimeZone', 'UTC'); +dt_london_summer = datetime(2022, 7, 20, 12, 0, 0, 'TimeZone', 'Europe/London'); % BST, UTC+1 +dt_london_winter = datetime(2022, 1, 20, 12, 0, 0, 'TimeZone', 'Europe/London'); % GMT +dt_offset = datetime(2022, 7, 20, 12, 0, 0, 'TimeZone', '+05:30'); +dt_newyork = datetime(2022, [1 7], 20, 12, 0, 0, 'TimeZone', 'America/New_York'); % EST, EDT +dt_leap = datetime(2016, 12, 31, 23, 59, 60, 'TimeZone', 'UTCLeapSeconds'); + +Time = datetime(2022, 7, 20, 12, 0, 0, 'TimeZone', 'Europe/London') + seconds([0; 1.5]); +tt_zoned = timetable(Time, [1; 2], 'VariableNames', {'x'}); +tt_zoned_step = timetable([1; 2; 3], 'TimeStep', seconds(0.5), ... + 'StartTime', datetime(2022, 7, 20, 12, 0, 0, 'TimeZone', 'Europe/London'), 'VariableNames', {'y'}); + +vars = {'dt_unzoned', 'dt_utc', 'dt_london_summer', 'dt_london_winter', 'dt_offset', ... + 'dt_newyork', 'dt_leap', 'tt_zoned', 'tt_zoned_step'}; +save(fullfile('v7', 'timezone.mat'), vars{:}, '-v7'); +save(fullfile('v7.3', 'timezone.mat'), vars{:}, '-v7.3'); diff --git a/test/timezones.jl b/test/timezones.jl new file mode 100644 index 0000000..356324c --- /dev/null +++ b/test/timezones.jl @@ -0,0 +1,37 @@ +# Zoned datetimes with TimeZones.jl loaded: ZonedDateTime, through the package extension +# (Julia 1.9 and later). read.jl reads the same file before TimeZones.jl is loaded. +using TimeZones, Dates + +@testset "time zones with TimeZones.jl" begin + for format in ("v7", "v7.3") + # timezone.mat is written in MATLAB by timezone_gen.m + filepath = joinpath(dirname(@__FILE__), format, "timezone.mat") + if !isfile(filepath) || VERSION < v"1.9" + @test_skip isfile(filepath) && VERSION >= v"1.9" + continue + end + vars = @test_logs (:warn, r"UTCLeapSeconds") match_mode=:any matread(filepath) + london, newyork = tz"Europe/London", tz"America/New_York" + + @test vars["dt_london_summer"] isa ZonedDateTime + @test vars["dt_london_summer"] == ZonedDateTime(2022, 7, 20, 12, london) # BST + @test timezone(vars["dt_london_summer"]) == london + @test vars["dt_london_winter"] == ZonedDateTime(2022, 1, 20, 12, london) # GMT + @test vars["dt_utc"] == ZonedDateTime(2022, 7, 20, 12, tz"UTC") + @test timezone(vars["dt_utc"]) == tz"UTC" + @test timezone(vars["dt_offset"]) == FixedTimeZone("+05:30") # a fixed offset + @test DateTime(vars["dt_offset"]) == DateTime(2022, 7, 20, 12) + @test vars["dt_newyork"] == [ZonedDateTime(2022, 1, 20, 12, newyork) ZonedDateTime(2022, 7, 20, 12, newyork)] + @test all(timezone.(vars["dt_newyork"]) .== newyork) + + @test vars["dt_unzoned"] == DateTime(2022, 7, 20, 12) # no zone: a DateTime, as before + @test vars["dt_leap"] isa MatlabOpaque # leap seconds counted: left as read + + t = vars["tt_zoned"] # zoned row times, one per row + @test t[:Time] == ZonedDateTime(2022, 7, 20, 12, london) .+ Millisecond.([0, 1500]) + @test all(timezone.(t[:Time]) .== london) + t = vars["tt_zoned_step"] # regular, from a zoned start + @test t[:Time] == ZonedDateTime(2022, 7, 20, 12, london) .+ Millisecond.([0, 500, 1000]) + @test all(timezone.(t[:Time]) .== london) + end +end