Read zoned datetimes as ZonedDateTime, through a TimeZones.jl extension

Project.toml

@@ -12,6 +12,12 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StringEncodings = "69024149-9ee7-55f6-a4c4-859efe599b68"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
+[weakdeps]
+TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
+
+[extensions]
+MATTimeZonesExt = "TimeZones"
+
 [compat]
 CodecZlib = "0.5, 0.6, 0.7"
 Dates = "1"
@@ -20,13 +26,15 @@ OrderedCollections = "1, 2"
 PooledArrays = "1.4.3"
 StringEncodings = "0.3.7"
 Tables = "1.12.1"
+TimeZones = "1"
 julia = "1.6"
 
 [extras]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 
 [targets]
-test = ["DataStructures", "LinearAlgebra", "SparseArrays", "Test"]
+test = ["DataStructures", "LinearAlgebra", "SparseArrays", "Test", "TimeZones"]

docs/src/types.md

@@ -17,9 +17,35 @@ A few of the `MatlabOpaque` classes are automatically converted upon reading:
 | MATLAB    | Julia |
 | -------- | ------- |
 | `string`    | `String`    |
-| `datetime`    | `Dates.DateTime`    |
+| `datetime`    | `Dates.DateTime` (`TimeZones.ZonedDateTime` if zoned and TimeZones.jl is loaded)    |
 | `duration`    | `Dates.Millisecond`    |
 | `category`    | `PooledArrays.PooledArray`    |
 | `table`    | `MAT.MatlabTable` (or any other table) |
+| `timetable`    | `MAT.MatlabTable` (or any other table), with the row times as its first column |
+
+A timetable's row times come back as `Dates.DateTime` or `Dates.Millisecond`, in a
+column named after the row dimension (`Time` unless renamed). A regular timetable, which
+MATLAB stores as a start time and a sample rate or time step, has its row times
+generated; one stepped in calendar units (months, for example) has no fixed rate and is
+left as a `MatlabOpaque`, with a warning. Row times are rounded to whole milliseconds,
+the resolution of `Dates.DateTime`.
+
+## Time zones
+
+MATLAB stores a `datetime` that has a time zone as its UTC instant together with the
+zone's name. With [TimeZones.jl](https://github.com/JuliaTime/TimeZones.jl) loaded (Julia
+1.9 and later, through a package extension), it is read as a `ZonedDateTime` in that zone:
+an IANA zone such as `Europe/London`, `UTC`, or a fixed offset such as `+05:30`.
+
+```julia
+using MAT, TimeZones
+matread("file.mat")["t"]   # 2022-07-20T12:00:00+01:00, in Europe/London
+```
+
+Without it, a zoned `datetime` is read as a `DateTime` holding the UTC instant, with a
+warning. A `datetime` without a time zone is always a `DateTime` of the stored wall-clock
+time. Zoned row times of a timetable follow the same rules. The `UTCLeapSeconds` zone counts
+leap seconds, which neither type can represent, so such a `datetime` is left as a
+`MatlabOpaque`, with a warning.
 
 Note that single element arrays are typically converted to scalars in Julia, because MATLAB cannot distinguish between scalars and `1x1` sized arrays.

ext/MATTimeZonesExt.jl

@@ -0,0 +1,11 @@
+module MATTimeZonesExt
+
+# Zoned MATLAB datetimes as ZonedDateTime. MATLAB stores a zoned datetime as its UTC
+# instant plus the zone's name: an IANA name ("Europe/London"), "UTC", or a fixed offset
+# such as "+05:30", all of which TimeZone() parses.
+
+using MAT, Dates, TimeZones
+
+MAT.MAT_types.to_zoned(utc::DateTime, tz::String) = ZonedDateTime(utc, TimeZone(tz); from_utc = true)
+
+end

src/MAT.jl

@@ -123,7 +123,8 @@ keyword argument only affects write operations.
 
 Use with `read`, `write`, `close`, `keys`, and `haskey`.
 
-Optional keyword argument is the `table` type, for automatic conversion of Matlab tables.
+Optional keyword argument is the `table` type, for automatic conversion of Matlab tables
+and timetables (a timetable's row times become its first column).
 Note that Matlab tables may contain non-vector colums which cannot always be converted to a Julia table, like `DataFrame`.
 
 # Example
@@ -164,7 +165,7 @@ matopen
 Return a dictionary of all the variables and values in a Matlab file,
 opening and closing it automatically.
 
-Optionally provide the `table` type to convert Matlab tables into. Default uses a simple `MatlabTable` type.
+Optionally provide the `table` type to convert Matlab tables and timetables into. Default uses a simple `MatlabTable` type.
 
 # Example
 

src/MAT_types.jl

@@ -473,6 +473,8 @@ function convert_opaque(obj::MatlabOpaque; table::Type=Nothing)
         return from_categorical(obj)
     elseif obj.class == "table"
         return from_table(obj, table)
+    elseif obj.class == "timetable"
+        return from_timetable(obj, table)
     else
         return obj
     end
@@ -513,17 +515,43 @@ function from_string(obj::MatlabOpaque, encoding::Encoding=Encoding(Symbol("UTF-
     end
 end
 
+"""
+    to_zoned(utc::DateTime, tz::String)
+
+A zoned datetime from its UTC instant and MATLAB's name for the zone (an IANA name such
+as "Europe/London", "UTC", or a fixed offset such as "+05:30"). Defined by the TimeZones
+extension (Julia 1.9 and later, when TimeZones.jl is loaded), which returns a
+`ZonedDateTime`; without it, zoned datetimes are read as their UTC instant.
+"""
+function to_zoned end
+
 function from_datetime(obj::MatlabOpaque)
     dat = obj["data"]
     if isnothing(dat) || isempty(dat)
         return DateTime[]
     end
-    if haskey(obj, "tz") && !isempty(obj["tz"])
-        tz = obj["tz"]
-        @warn "no timezone conversion yet for datetime objects. timezone of \"$tz\" ignored"
-    end
     #isdate = obj["isDateOnly"] # optional: convert to Date instead of DateTime?
-    return map_or_not(ms_to_datetime, dat)
+    tz = haskey(obj, "tz") ? obj["tz"] : ""
+    if !(tz isa AbstractString) || isempty(tz)
+        return map_or_not(ms_to_datetime, dat)  # no zone: the stored wall-clock time
+    end
+    # MATLAB stores a zoned datetime as its UTC instant, except in "UTCLeapSeconds", where
+    # the count includes leap seconds, which DateTime cannot represent
+    if tz == "UTCLeapSeconds"
+        @warn "datetime with timezone \"UTCLeapSeconds\" is not converted (leap seconds cannot be represented); returning the MatlabOpaque"
+        return obj
+    end
+    utc = map_or_not(ms_to_datetime, dat)
+    if !hasmethod(to_zoned, Tuple{DateTime,String})
+        @warn "datetime with timezone \"$tz\" is returned as its UTC instant; load TimeZones.jl to read it as a ZonedDateTime"
+        return utc
+    end
+    try
+        return map_or_not(t -> ismissing(t) ? missing : to_zoned(t, String(tz)), utc)
+    catch err
+        @warn "timezone \"$tz\" was not recognised ($(sprint(showerror, err))); returning the UTC instant"
+        return utc
+    end
 end
 
 # is the complex part the submilliseconds?
@@ -632,6 +660,39 @@ end
 # option to not convert and get the MatlabOpaque as table
 from_table(obj::MatlabOpaque, ::Type{Nothing}) = obj
 
+# A timetable is a table whose first column holds the row times, named after the row
+# dimension ("Time" unless renamed), then the variables. Row times are DateTime or
+# Millisecond; they are stored one per row, or, for a regular timetable, as a start time
+# and a sample rate (or time step), from which they are generated here. Row times that
+# cannot be interpreted leave the timetable as the MatlabOpaque it was read as.
+function from_timetable(obj::MatlabOpaque, ::Type{T}=MatlabTable) where {T}
+    tt = haskey(obj, "any") ? obj["any"] : obj
+    times = timetable_rowtimes(tt["rowTimes"], Int(tt["numRows"]))
+    if isnothing(times)
+        @warn "timetable row times of this kind are not converted; returning the MatlabOpaque"
+        return obj
+    end
+    names = Symbol[Symbol(tt["dimNames"][1]); Symbol.(vec(tt["varNames"]))]
+    cols = vcat(Any[times], Any[try_vec(c) for c in vec(tt["data"])])
+    t = MatlabTable(names, cols)
+    return T(Tables.CopiedColumns(t))
+end
+from_timetable(obj::MatlabOpaque, ::Type{Nothing}) = obj
+
+# row times stored one per row, already converted to DateTime or Millisecond
+timetable_rowtimes(times::AbstractArray, n::Int) = vec(times)
+# a regular timetable: a start time and a sample rate in Hz; MATLAB also stores the rate
+# when the time step was given instead, so the rate serves for both
+function timetable_rowtimes(regular::AbstractDict, n::Int)
+    origin, rate = get(regular, "origin", nothing), get(regular, "sampleRate", nothing)
+    # the start is a DateTime, a ZonedDateTime (TimeZones extension) or a duration
+    if !(origin isa Union{Dates.AbstractDateTime,Dates.Period} && rate isa Real && isfinite(rate) && rate > 0)
+        return nothing
+    end
+    return [origin + Millisecond(round(Int, 1000 * k / rate)) for k in 0:(n-1)]
+end
+timetable_rowtimes(times, n::Int) = nothing
+
 try_vec(c::Vector) = c
 try_vec(c) = [c]
 function try_vec(c::AbstractArray)

test/read.jl

@@ -274,6 +274,82 @@ for format in ["v7", "v7.3"]
     end
     end
 
+    # timetable.mat is written by timetable_gen.m: one variable per way MATLAB stores
+    # a timetable's row times
+    @testset "timetable $format" begin
+        filepath = joinpath(dirname(@__FILE__), format, "timetable.mat")
+        if isfile(filepath)
+            # tt_calendar's step is calendar months, which has no fixed rate: left as read
+            vars = @test_logs (:warn, r"timetable row times") match_mode=:any matread(filepath)
+            @test vars["tt_calendar"] isa MatlabOpaque
+            @test vars["tt_calendar"].class == "timetable"
+
+            t = vars["tt_datetime"]           # row times per row, as datetimes
+            @test t isa MatlabTable
+            @test t.names == [:Time, :Current, :Channel]
+            @test t[:Time] == DateTime(2022, 7, 20, 2, 27, 27) .+ Millisecond.([428, 496, 564])
+            @test t[:Current] == [1.5e-11, 2.5e-11, -3e-12]
+            @test t[:Channel] == ["AB", "A", "B"]
+
+            t = vars["tt_duration"]           # row times per row, as durations
+            @test t.names == [:Time, :x]
+            @test t[:Time] == Millisecond.([0, 500, 1250])
+            @test t[:x] == [1.0, 2.0, 3.0]
+
+            t = vars["tt_rate"]               # regular: start and sample rate
+            @test t.names == [:Time, :x]
+            @test t[:Time] == Millisecond.(0:3)
+            @test t[:x] == [10.0, 20.0, 30.0, 40.0]
+
+            t = vars["tt_step"]               # regular: datetime start and time step
+            @test t[:Time] == DateTime(2022, 7, 20, 2) .+ Millisecond.([0, 250, 500])
+            @test t[:y] == [1.0, 2.0, 3.0]
+
+            t = vars["tt_empty"]              # no rows, no variables
+            @test t isa MatlabTable
+            @test t.names == [:Time]
+            @test isempty(t[:Time])
+
+            t = vars["tt_matrix"]             # a two-column variable stays a matrix
+            @test t[:Time] == Millisecond.([1000, 2000])
+            @test t[:m] == [1.0 2.0; 3.0 4.0]
+
+            @test vars["tt_dimname"].names == [:Timestamp, :x]   # the row-times dimension's own name
+
+            # using Nothing keeps the MatlabOpaque
+            raw = matread(filepath; table=Nothing)["tt_rate"]
+            @test raw isa MatlabOpaque
+            @test raw.class == "timetable"
+        else
+            # generated in MATLAB by test/timetable_gen.m; skipped until it is committed
+            @test_skip isfile(filepath)
+        end
+    end
+
+    # timezone.mat is written by timezone_gen.m. MATLAB stores a zoned datetime as its UTC
+    # instant; until TimeZones.jl is loaded (timezones.jl) that instant is what is returned.
+    @testset "time zones, without TimeZones.jl $format" begin
+        filepath = joinpath(dirname(@__FILE__), format, "timezone.mat")
+        ext_loaded = isdefined(Base, :get_extension) && Base.get_extension(MAT, :MATTimeZonesExt) !== nothing
+        if isfile(filepath) && !ext_loaded
+            vars = @test_logs (:warn, r"timezone") match_mode=:any matread(filepath)
+            @test vars["dt_unzoned"] == DateTime(2022, 7, 20, 12)          # no zone: wall clock
+            @test vars["dt_utc"] == DateTime(2022, 7, 20, 12)
+            @test vars["dt_london_summer"] == DateTime(2022, 7, 20, 11)    # 12:00 BST
+            @test vars["dt_london_winter"] == DateTime(2022, 1, 20, 12)    # 12:00 GMT
+            @test vars["dt_offset"] == DateTime(2022, 7, 20, 6, 30)        # 12:00 +05:30
+            @test vars["dt_newyork"] == [DateTime(2022, 1, 20, 17) DateTime(2022, 7, 20, 16)]
+            # counted with leap seconds, which DateTime cannot represent: left as read
+            @test vars["dt_leap"] isa MatlabOpaque
+            @test vars["dt_leap"].class == "datetime"
+            @test vars["tt_zoned"][:Time] == DateTime(2022, 7, 20, 11) .+ Millisecond.([0, 1500])
+            @test vars["tt_zoned_step"][:Time] == DateTime(2022, 7, 20, 11) .+ Millisecond.([0, 500, 1000])
+        else
+            # generated in MATLAB by test/timezone_gen.m; skipped until it is committed
+            @test_skip isfile(filepath) && !ext_loaded
+        end
+    end
+
     @testset "user defined classdef $format" begin
     let objtestfile = "user_defined_classdefs.mat"
         filepath = joinpath(dirname(@__FILE__), format, objtestfile)

test/runtests.jl

@@ -6,4 +6,5 @@ using Test, MAT
     include("read.jl")
     include("readwrite4.jl")
     include("write.jl")
+    include("timezones.jl")   # last: loading TimeZones.jl changes how zoned datetimes read
 end

test/timetable_gen.m

@@ -0,0 +1,38 @@
+% Generates test/v7/timetable.mat and test/v7.3/timetable.mat for the timetable tests
+% in read.jl. Each variable is one way MATLAB stores a timetable's row times.
+% Run from the test directory: matlab -batch "timetable_gen"
+
+% explicit datetime row times, with sub-millisecond parts, and a string variable
+Time = datetime(2022, 7, 20, 2, 27, 27) + milliseconds([428.25; 495.75; 563.5]);
+Current = [1.5e-11; 2.5e-11; -3e-12];
+Channel = ["AB"; "A"; "B"];
+tt_datetime = timetable(Time, Current, Channel);
+
+% explicit duration row times
+tt_duration = timetable(seconds([0; 0.5; 1.25]), [1; 2; 3], 'VariableNames', {'x'});
+
+% regular: a sample rate (row times start at 0 s)
+x = [10; 20; 30; 40];
+tt_rate = timetable(x, 'SampleRate', 1000);
+
+% regular: a time step from a datetime start
+y = [1; 2; 3];
+tt_step = timetable(y, 'TimeStep', seconds(0.25), 'StartTime', datetime(2022, 7, 20, 2, 0, 0));
+
+% no rows and no variables, as an acquisition leaves an unused timetable
+tt_empty = timetable(datetime.empty(0, 1));
+
+% a variable with two columns
+tt_matrix = timetable(seconds([1; 2]), [1 2; 3 4], 'VariableNames', {'m'});
+
+% the row-times dimension renamed
+tt_dimname = tt_duration;
+tt_dimname.Properties.DimensionNames{1} = 'Timestamp';
+
+% regular in calendar months: no fixed sample rate, so not converted
+z = [1; 2; 3];
+tt_calendar = timetable(z, 'TimeStep', calmonths(1), 'StartTime', datetime(2022, 1, 1));
+
+vars = {'tt_datetime', 'tt_duration', 'tt_rate', 'tt_step', 'tt_empty', 'tt_matrix', 'tt_dimname', 'tt_calendar'};
+save(fullfile('v7', 'timetable.mat'), vars{:}, '-v7');
+save(fullfile('v7.3', 'timetable.mat'), vars{:}, '-v7.3');

test/timezone_gen.m

@@ -0,0 +1,21 @@
+% Generates test/v7/timezone.mat and test/v7.3/timezone.mat: datetimes with and without
+% time zones, and timetables whose row times have one.
+% Run from the test directory: matlab -batch "timezone_gen"
+
+dt_unzoned = datetime(2022, 7, 20, 12, 0, 0);
+dt_utc = datetime(2022, 7, 20, 12, 0, 0, 'TimeZone', 'UTC');
+dt_london_summer = datetime(2022, 7, 20, 12, 0, 0, 'TimeZone', 'Europe/London');   % BST, UTC+1
+dt_london_winter = datetime(2022, 1, 20, 12, 0, 0, 'TimeZone', 'Europe/London');   % GMT
+dt_offset = datetime(2022, 7, 20, 12, 0, 0, 'TimeZone', '+05:30');
+dt_newyork = datetime(2022, [1 7], 20, 12, 0, 0, 'TimeZone', 'America/New_York'); % EST, EDT
+dt_leap = datetime(2016, 12, 31, 23, 59, 60, 'TimeZone', 'UTCLeapSeconds');
+
+Time = datetime(2022, 7, 20, 12, 0, 0, 'TimeZone', 'Europe/London') + seconds([0; 1.5]);
+tt_zoned = timetable(Time, [1; 2], 'VariableNames', {'x'});
+tt_zoned_step = timetable([1; 2; 3], 'TimeStep', seconds(0.5), ...
+    'StartTime', datetime(2022, 7, 20, 12, 0, 0, 'TimeZone', 'Europe/London'), 'VariableNames', {'y'});
+
+vars = {'dt_unzoned', 'dt_utc', 'dt_london_summer', 'dt_london_winter', 'dt_offset', ...
+        'dt_newyork', 'dt_leap', 'tt_zoned', 'tt_zoned_step'};
+save(fullfile('v7', 'timezone.mat'), vars{:}, '-v7');
+save(fullfile('v7.3', 'timezone.mat'), vars{:}, '-v7.3');

test/timezones.jl

@@ -0,0 +1,37 @@
+# Zoned datetimes with TimeZones.jl loaded: ZonedDateTime, through the package extension
+# (Julia 1.9 and later). read.jl reads the same file before TimeZones.jl is loaded.
+using TimeZones, Dates
+
+@testset "time zones with TimeZones.jl" begin
+    for format in ("v7", "v7.3")
+        # timezone.mat is written in MATLAB by timezone_gen.m
+        filepath = joinpath(dirname(@__FILE__), format, "timezone.mat")
+        if !isfile(filepath) || VERSION < v"1.9"
+            @test_skip isfile(filepath) && VERSION >= v"1.9"
+            continue
+        end
+        vars = @test_logs (:warn, r"UTCLeapSeconds") match_mode=:any matread(filepath)
+        london, newyork = tz"Europe/London", tz"America/New_York"
+
+        @test vars["dt_london_summer"] isa ZonedDateTime
+        @test vars["dt_london_summer"] == ZonedDateTime(2022, 7, 20, 12, london)   # BST
+        @test timezone(vars["dt_london_summer"]) == london
+        @test vars["dt_london_winter"] == ZonedDateTime(2022, 1, 20, 12, london)   # GMT
+        @test vars["dt_utc"] == ZonedDateTime(2022, 7, 20, 12, tz"UTC")
+        @test timezone(vars["dt_utc"]) == tz"UTC"
+        @test timezone(vars["dt_offset"]) == FixedTimeZone("+05:30")               # a fixed offset
+        @test DateTime(vars["dt_offset"]) == DateTime(2022, 7, 20, 12)
+        @test vars["dt_newyork"] == [ZonedDateTime(2022, 1, 20, 12, newyork) ZonedDateTime(2022, 7, 20, 12, newyork)]
+        @test all(timezone.(vars["dt_newyork"]) .== newyork)
+
+        @test vars["dt_unzoned"] == DateTime(2022, 7, 20, 12)   # no zone: a DateTime, as before
+        @test vars["dt_leap"] isa MatlabOpaque                  # leap seconds counted: left as read
+
+        t = vars["tt_zoned"]                                    # zoned row times, one per row
+        @test t[:Time] == ZonedDateTime(2022, 7, 20, 12, london) .+ Millisecond.([0, 1500])
+        @test all(timezone.(t[:Time]) .== london)
+        t = vars["tt_zoned_step"]                               # regular, from a zoned start
+        @test t[:Time] == ZonedDateTime(2022, 7, 20, 12, london) .+ Millisecond.([0, 500, 1000])
+        @test all(timezone.(t[:Time]) .== london)
+    end
+end