From 522a7b3db717800709c1472ef8fed076877d982e Mon Sep 17 00:00:00 2001 From: Arkadii Kravchuk Date: Sat, 3 Oct 2026 06:22:24 +0300 Subject: [PATCH] DX-126900: [C++][Gandiva] Add TimeIR for unit-aware time64[us/ns] support Register time64 function signatures (extractHour/Minute/Second, comparisons, isnull/isnotnull, is_distinct_from/is_not_distinct_from) with time64[us]. Signature matching ignores the time64 unit, and LLVMGenerator::ResolveTimePcName() remaps time64[ns] calls to the _ns precompiled variants, mirroring TimestampIR. Unlike timestamp, a time64[ns] call without an _ns variant is rejected instead of falling through to the microsecond function, and mixed time64 units are rejected. Co-Authored-By: Claude Opus 5.5 --- cpp/src/gandiva/CMakeLists.txt | 1 + .../gandiva/function_registry_arithmetic.cc | 7 + cpp/src/gandiva/function_registry_datetime.cc | 8 + cpp/src/gandiva/function_registry_math_ops.cc | 4 + cpp/src/gandiva/function_signature.cc | 5 + cpp/src/gandiva/function_signature_test.cc | 16 ++ cpp/src/gandiva/llvm_generator.cc | 41 ++++ cpp/src/gandiva/llvm_generator.h | 8 + cpp/src/gandiva/llvm_generator_test.cc | 84 +++++++ cpp/src/gandiva/precompiled/CMakeLists.txt | 2 + cpp/src/gandiva/precompiled/arithmetic_ops.cc | 13 ++ cpp/src/gandiva/precompiled/time_test.cc | 22 ++ cpp/src/gandiva/precompiled/time_unit_ops.cc | 56 +++++ cpp/src/gandiva/precompiled/types.h | 7 + cpp/src/gandiva/tests/date_time_test.cc | 207 +++++++++++++++++- cpp/src/gandiva/time_ir.cc | 65 ++++++ cpp/src/gandiva/time_ir.h | 52 +++++ 17 files changed, 592 insertions(+), 6 deletions(-) create mode 100644 cpp/src/gandiva/precompiled/time_unit_ops.cc create mode 100644 cpp/src/gandiva/time_ir.cc create mode 100644 cpp/src/gandiva/time_ir.h diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index 84b447ccce6d..dc5fb02f5313 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -62,6 +62,7 @@ set(SRC_FILES decimal_ir.cc decimal_type_util.cc timestamp_ir.cc + time_ir.cc decimal_xlarge.cc engine.cc date_utils.cc diff --git a/cpp/src/gandiva/function_registry_arithmetic.cc b/cpp/src/gandiva/function_registry_arithmetic.cc index b2579eee0740..31d35779afe6 100644 --- a/cpp/src/gandiva/function_registry_arithmetic.cc +++ b/cpp/src/gandiva/function_registry_arithmetic.cc @@ -221,6 +221,13 @@ std::vector GetArithmeticFunctionRegistry() { BINARY_RELATIONAL_BOOL_DATE_FN(less_than_or_equal_to, {}), BINARY_RELATIONAL_BOOL_DATE_FN(greater_than, {}), BINARY_RELATIONAL_BOOL_DATE_FN(greater_than_or_equal_to, {}), + // time64 compare functions (unit-agnostic, see TimeIR) + BINARY_RELATIONAL_SAFE_NULL_IF_NULL(equal, ({"eq", "same"}), time64), + BINARY_RELATIONAL_SAFE_NULL_IF_NULL(not_equal, {}, time64), + BINARY_RELATIONAL_SAFE_NULL_IF_NULL(less_than, {}, time64), + BINARY_RELATIONAL_SAFE_NULL_IF_NULL(less_than_or_equal_to, {}, time64), + BINARY_RELATIONAL_SAFE_NULL_IF_NULL(greater_than, {}, time64), + BINARY_RELATIONAL_SAFE_NULL_IF_NULL(greater_than_or_equal_to, {}, time64), BASE_NUMERIC_TYPES(MULTIPLE_SAFE_NULL_IF_NULL, greatest, {}), BASE_NUMERIC_TYPES(MULTIPLE_SAFE_NULL_IF_NULL, least, {}), diff --git a/cpp/src/gandiva/function_registry_datetime.cc b/cpp/src/gandiva/function_registry_datetime.cc index a2fe75c80f3f..83453c67e8c9 100644 --- a/cpp/src/gandiva/function_registry_datetime.cc +++ b/cpp/src/gandiva/function_registry_datetime.cc @@ -45,6 +45,13 @@ namespace gandiva { TIME_TYPES(EXTRACT_SAFE_NULL_IF_NULL, name##Minute, {"minute"}), \ TIME_TYPES(EXTRACT_SAFE_NULL_IF_NULL, name##Second, {"second"}) +// time64 functions are registered for time64[us]; time64[ns] calls match the same +// signature and are remapped to the _ns variants by TimeIR at codegen time. +#define TIME64_EXTRACTION_FNS(name) \ + EXTRACT_SAFE_NULL_IF_NULL(name##Hour, {"hour"}, time64), \ + EXTRACT_SAFE_NULL_IF_NULL(name##Minute, {"minute"}, time64), \ + EXTRACT_SAFE_NULL_IF_NULL(name##Second, {"second"}, time64) + #define NEXT_DAY_FNS(name) DATE_TYPES(NEXT_DAY_SAFE_NULL_IF_NULL, name, {}) std::vector GetDateTimeFunctionRegistry() { @@ -59,6 +66,7 @@ std::vector GetDateTimeFunctionRegistry() { DATE_TYPES(EXTRACT_SAFE_NULL_IF_NULL, extractEpoch, {}), TIME_EXTRACTION_FNS(extract), + TIME64_EXTRACTION_FNS(extract), NEXT_DAY_FNS(next_day), diff --git a/cpp/src/gandiva/function_registry_math_ops.cc b/cpp/src/gandiva/function_registry_math_ops.cc index 3bfcfc180e7e..228c92dee82d 100644 --- a/cpp/src/gandiva/function_registry_math_ops.cc +++ b/cpp/src/gandiva/function_registry_math_ops.cc @@ -70,11 +70,15 @@ std::vector GetMathOpsFunctionRegistry() { UNARY_SAFE_NULL_NEVER_BOOL_FN(isnull, {}), UNARY_SAFE_NULL_NEVER_BOOL_FN(isnotnull, {}), + UNARY_SAFE_NULL_NEVER_BOOL(isnull, {}, time64), + UNARY_SAFE_NULL_NEVER_BOOL(isnotnull, {}, time64), NUMERIC_TYPES(UNARY_SAFE_NULL_NEVER_BOOL, isnumeric, {}), BINARY_SAFE_NULL_NEVER_BOOL_FN(is_distinct_from, {}), BINARY_SAFE_NULL_NEVER_BOOL_FN(is_not_distinct_from, {}), + BINARY_SAFE_NULL_NEVER_BOOL(is_distinct_from, {}, time64), + BINARY_SAFE_NULL_NEVER_BOOL(is_not_distinct_from, {}, time64), UNARY_UNSAFE_NULL_IF_NULL(factorial, {}, int32, int64), UNARY_UNSAFE_NULL_IF_NULL(factorial, {}, int64, int64), diff --git a/cpp/src/gandiva/function_signature.cc b/cpp/src/gandiva/function_signature.cc index 9a04fa42e1ea..ce9f325daa1a 100644 --- a/cpp/src/gandiva/function_signature.cc +++ b/cpp/src/gandiva/function_signature.cc @@ -56,6 +56,11 @@ bool DataTypeEquals(const DataTypePtr& left, const DataTypePtr& right) { auto tright = checked_cast(right.get()); return tleft->timezone() == tright->timezone(); } + case arrow::Type::TIME64: { + // For time64 types, the TimeUnit isn't part of the signature + // (conversion is handled at codegen time by TimeIR). + return true; + } default: return left->Equals(right); } diff --git a/cpp/src/gandiva/function_signature_test.cc b/cpp/src/gandiva/function_signature_test.cc index 0eb62d4e7bfb..7b5a4cf02cf2 100644 --- a/cpp/src/gandiva/function_signature_test.cc +++ b/cpp/src/gandiva/function_signature_test.cc @@ -110,4 +110,20 @@ TEST_F(TestFunctionSignature, TestHash) { EXPECT_EQ(f3.Hash(), f4.Hash()); } +TEST_F(TestFunctionSignature, TestTime64UnitNotPartOfSignature) { + // time64 units are resolved at codegen time by TimeIR, so time64[us] and time64[ns] + // match the same registered signature. + FunctionSignature micro("extractHour", {arrow::time64(arrow::TimeUnit::MICRO)}, + arrow::int64()); + FunctionSignature nano("extractHour", {arrow::time64(arrow::TimeUnit::NANO)}, + arrow::int64()); + EXPECT_EQ(micro, nano); + EXPECT_EQ(micro.Hash(), nano.Hash()); + + // time32 and time64 remain distinct types. + EXPECT_FALSE(micro == FunctionSignature("extractHour", + {arrow::time32(arrow::TimeUnit::MILLI)}, + arrow::int64())); +} + } // namespace gandiva diff --git a/cpp/src/gandiva/llvm_generator.cc b/cpp/src/gandiva/llvm_generator.cc index bf5207efe193..83d498ecfe14 100644 --- a/cpp/src/gandiva/llvm_generator.cc +++ b/cpp/src/gandiva/llvm_generator.cc @@ -30,6 +30,7 @@ #include "gandiva/expression.h" #include "gandiva/llvm_types.h" #include "gandiva/lvalue.h" +#include "gandiva/time_ir.h" #include "gandiva/timestamp_ir.h" namespace gandiva { @@ -1296,6 +1297,40 @@ Result LLVMGenerator::ResolveTimestampPcName(const std::string& pc_ return pc_name; } +Result LLVMGenerator::ResolveTimePcName(const std::string& pc_name, + const DataTypeVector& params) { + arrow::TimeUnit::type time_unit = arrow::TimeUnit::MICRO; + bool found_time = false; + for (const auto& param : params) { + if (param->id() == arrow::Type::TIME64) { + auto unit = arrow::internal::checked_cast(*param).unit(); + if (!found_time) { + time_unit = unit; + found_time = true; + } else if (unit != time_unit) { + return Status::Invalid( + "Gandiva cannot compile expression: mixed time64 units in function '", + pc_name, "'. All time64 arguments must have the same TimeUnit."); + } + } + } + // time64 signatures are registered with MICRO, so the unsuffixed function already + // handles MICRO arguments. + if (!found_time || time_unit == arrow::TimeUnit::MICRO || + TimeIR::IsUnitAgnosticFunction(pc_name)) { + return pc_name; + } + std::string remapped = pc_name + TimeIR::UnitSuffix(time_unit); + if (TimeIR::IsTimeIRFunction(remapped)) { + ARROW_LOG(DEBUG) << "TimeIR remap: " << pc_name << " -> " << remapped; + return remapped; + } + // Never fall through to the MICRO function: it would misinterpret the data. + return Status::Invalid("Gandiva cannot compile expression: function '", pc_name, + "' has no variant for time64 unit ", + arrow::internal::ToString(time_unit), "."); +} + LValuePtr LLVMGenerator::Visitor::BuildFunctionCall(const NativeFunction* func, DataTypePtr arrow_return_type, std::vector* params, @@ -1315,6 +1350,12 @@ LValuePtr LLVMGenerator::Visitor::BuildFunctionCall(const NativeFunction* func, return nullptr; } pc_name = resolve_result.MoveValueUnsafe(); + auto time_resolve_result = ResolveTimePcName(pc_name, descriptor->params()); + if (!time_resolve_result.ok()) { + status_ = time_resolve_result.status(); + return nullptr; + } + pc_name = time_resolve_result.MoveValueUnsafe(); } if (arrow_return_type_id == arrow::Type::DECIMAL) { diff --git a/cpp/src/gandiva/llvm_generator.h b/cpp/src/gandiva/llvm_generator.h index 1e7882d1bb2b..46a6fbdc5b03 100644 --- a/cpp/src/gandiva/llvm_generator.h +++ b/cpp/src/gandiva/llvm_generator.h @@ -67,6 +67,13 @@ class GANDIVA_EXPORT LLVMGenerator { static Result ResolveTimestampPcName(const std::string& pc_name, const DataTypeVector& params); + /// \brief Resolve the precompiled function name, remapping to a TimeIR variant + /// when the function's params include a time64[ns] argument. + /// Returns an error if params contain mixed time64 TimeUnits, or if the function + /// has no variant for the time64 unit and is not unit-agnostic. + static Result ResolveTimePcName(const std::string& pc_name, + const DataTypeVector& params); + /// \brief Build the code for the expression trees for default mode with a LLVM /// ObjectCache. Each element in the vector represents an expression tree Status Build(const ExpressionVector& exprs, SelectionVector::Mode mode); @@ -96,6 +103,7 @@ class GANDIVA_EXPORT LLVMGenerator { std::shared_ptr function_registry); FRIEND_TEST(TestLLVMGenerator, VerifyPCFunctions); + FRIEND_TEST(TestLLVMGenerator, VerifyTime64Functions); FRIEND_TEST(TestLLVMGenerator, TestAdd); FRIEND_TEST(TestLLVMGenerator, TestNullInternal); friend class TestLLVMGenerator; diff --git a/cpp/src/gandiva/llvm_generator_test.cc b/cpp/src/gandiva/llvm_generator_test.cc index 2d9a99e2fb79..93fc89131d47 100644 --- a/cpp/src/gandiva/llvm_generator_test.cc +++ b/cpp/src/gandiva/llvm_generator_test.cc @@ -28,6 +28,7 @@ #include "gandiva/func_descriptor.h" #include "gandiva/function_registry.h" #include "gandiva/tests/test_util.h" +#include "gandiva/time_ir.h" #include "gandiva/timestamp_ir.h" namespace gandiva { @@ -234,4 +235,87 @@ TEST_F(TestLLVMGenerator, ResolveTimestampPcName) { } } +TEST_F(TestLLVMGenerator, ResolveTimePcName) { + // Non-time64 params — name unchanged + { + DataTypeVector params{arrow::time32(arrow::TimeUnit::MILLI)}; + ASSERT_OK_AND_ASSIGN(auto name, + LLVMGenerator::ResolveTimePcName("extractHour_time32", params)); + EXPECT_EQ(name, "extractHour_time32"); + } + + // time64[us] — no remap (us is the registered time64 unit) + { + DataTypeVector params{arrow::time64(arrow::TimeUnit::MICRO)}; + ASSERT_OK_AND_ASSIGN(auto name, + LLVMGenerator::ResolveTimePcName("extractHour_time64", params)); + EXPECT_EQ(name, "extractHour_time64"); + } + + // time64[ns] with a unit-specific function — gets _ns suffix + { + DataTypeVector params{arrow::time64(arrow::TimeUnit::NANO)}; + ASSERT_OK_AND_ASSIGN(auto name, + LLVMGenerator::ResolveTimePcName("extractHour_time64", params)); + EXPECT_EQ(name, "extractHour_time64_ns"); + } + + // time64[ns] with a unit-agnostic function — name unchanged + { + DataTypeVector params{arrow::time64(arrow::TimeUnit::NANO), + arrow::time64(arrow::TimeUnit::NANO)}; + ASSERT_OK_AND_ASSIGN( + auto name, LLVMGenerator::ResolveTimePcName("less_than_time64_time64", params)); + EXPECT_EQ(name, "less_than_time64_time64"); + } + + // Mixed time64 units — returns Invalid status + { + DataTypeVector params{arrow::time64(arrow::TimeUnit::MICRO), + arrow::time64(arrow::TimeUnit::NANO)}; + auto result = LLVMGenerator::ResolveTimePcName("less_than_time64_time64", params); + EXPECT_FALSE(result.ok()); + EXPECT_THAT(result.status().message(), testing::HasSubstr("mixed time64 units")); + } + + // time64[ns] without an _ns variant must not fall back to the microsecond function + { + DataTypeVector params{arrow::time64(arrow::TimeUnit::NANO)}; + auto result = LLVMGenerator::ResolveTimePcName("no_such_function", params); + EXPECT_FALSE(result.ok()); + EXPECT_THAT(result.status().message(), testing::HasSubstr("no variant for time64")); + } +} + +// Every registered time64 function must resolve, for both time64 units, to a function +// that exists in the precompiled module. +TEST_F(TestLLVMGenerator, VerifyTime64Functions) { + ASSERT_OK_AND_ASSIGN(auto generator, LLVMGenerator::Make(TestConfiguration(), false)); + + llvm::Module* module = generator->module(); + ASSERT_OK(generator->engine_->LoadFunctionIRs()); + int num_time64_functions = 0; + for (auto& native_function : *registry_) { + for (auto& signature : native_function.signatures()) { + bool has_time64 = false; + for (auto& param : signature.param_types()) { + has_time64 |= param->id() == arrow::Type::TIME64; + } + if (!has_time64) continue; + ++num_time64_functions; + for (auto unit : {arrow::TimeUnit::MICRO, arrow::TimeUnit::NANO}) { + DataTypeVector params; + for (auto& param : signature.param_types()) { + params.push_back(param->id() == arrow::Type::TIME64 ? arrow::time64(unit) + : param); + } + ASSERT_OK_AND_ASSIGN(auto name, LLVMGenerator::ResolveTimePcName( + native_function.pc_name(), params)); + EXPECT_NE(module->getFunction(name), nullptr) << name; + } + } + } + EXPECT_GT(num_time64_functions, 0); +} + } // namespace gandiva diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index 77c85bde9d9d..580e27f4c03a 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -27,6 +27,7 @@ set(PRECOMPILED_SRCS time.cc timestamp_arithmetic.cc timestamp_unit_ops.cc + time_unit_ops.cc ../../arrow/util/basic_decimal.cc) set(GANDIVA_PRECOMPILED_BC_FILES) foreach(SOURCE ${PRECOMPILED_SRCS}) @@ -63,6 +64,7 @@ add_gandiva_test(precompiled-test time_test.cc time.cc timestamp_arithmetic.cc + time_unit_ops.cc ../cast_time.cc ../../arrow/vendored/datetime.cpp hash_test.cc diff --git a/cpp/src/gandiva/precompiled/arithmetic_ops.cc b/cpp/src/gandiva/precompiled/arithmetic_ops.cc index bebeb191a0f2..66bc5f881b1c 100644 --- a/cpp/src/gandiva/precompiled/arithmetic_ops.cc +++ b/cpp/src/gandiva/precompiled/arithmetic_ops.cc @@ -131,6 +131,15 @@ NUMERIC_DATE_TYPES(BINARY_RELATIONAL, less_than_or_equal_to, <=) NUMERIC_DATE_TYPES(BINARY_RELATIONAL, greater_than, >) NUMERIC_DATE_TYPES(BINARY_RELATIONAL, greater_than_or_equal_to, >=) +// time64 values of a single unit (micro or nano) compare directly, so these are +// unit-agnostic. Mixed units are rejected at codegen time (see TimeIR). +BINARY_RELATIONAL(equal, time64, ==) +BINARY_RELATIONAL(not_equal, time64, !=) +BINARY_RELATIONAL(less_than, time64, <) +BINARY_RELATIONAL(less_than_or_equal_to, time64, <=) +BINARY_RELATIONAL(greater_than, time64, >) +BINARY_RELATIONAL(greater_than_or_equal_to, time64, >=) + #undef BINARY_RELATIONAL // Returns the greatest or least value from a list of values @@ -236,6 +245,8 @@ CAST_INT_FLOAT(castINT, float64, int32) NUMERIC_BOOL_DATE_TYPES(VALIDITY_OP, isnull, !) NUMERIC_BOOL_DATE_TYPES(VALIDITY_OP, isnotnull, +) +VALIDITY_OP(isnull, time64, !) +VALIDITY_OP(isnotnull, time64, +) NUMERIC_TYPES(VALIDITY_OP, isnumeric, +) #undef VALIDITY_OP @@ -346,6 +357,8 @@ gdv_boolean not_boolean(gdv_boolean in) { return !in; } NUMERIC_BOOL_DATE_FUNCTION(IS_DISTINCT_FROM) NUMERIC_BOOL_DATE_FUNCTION(IS_NOT_DISTINCT_FROM) +IS_DISTINCT_FROM(time64) +IS_NOT_DISTINCT_FROM(time64) #undef IS_DISTINCT_FROM #undef IS_NOT_DISTINCT_FROM diff --git a/cpp/src/gandiva/precompiled/time_test.cc b/cpp/src/gandiva/precompiled/time_test.cc index 4310490a228b..a75c7cab6cc4 100644 --- a/cpp/src/gandiva/precompiled/time_test.cc +++ b/cpp/src/gandiva/precompiled/time_test.cc @@ -249,6 +249,28 @@ TEST(TestTime, TestExtractTime) { EXPECT_EQ(extractSecond_time32(time_as_millis_in_day), 33); } +TEST(TestTime, TestExtractTime64) { + // 10:20:33.123456 in micros + gdv_time64 time_as_micros_in_day = 37233123456LL; + EXPECT_EQ(extractHour_time64(time_as_micros_in_day), 10); + EXPECT_EQ(extractMinute_time64(time_as_micros_in_day), 20); + EXPECT_EQ(extractSecond_time64(time_as_micros_in_day), 33); + + // 10:20:33.123456789 in nanos + gdv_time64 time_as_nanos_in_day = 37233123456789LL; + EXPECT_EQ(extractHour_time64_ns(time_as_nanos_in_day), 10); + EXPECT_EQ(extractMinute_time64_ns(time_as_nanos_in_day), 20); + EXPECT_EQ(extractSecond_time64_ns(time_as_nanos_in_day), 33); + + // 23:59:59.999999 / 23:59:59.999999999 + EXPECT_EQ(extractHour_time64(86399999999LL), 23); + EXPECT_EQ(extractMinute_time64(86399999999LL), 59); + EXPECT_EQ(extractSecond_time64(86399999999LL), 59); + EXPECT_EQ(extractHour_time64_ns(86399999999999LL), 23); + EXPECT_EQ(extractMinute_time64_ns(86399999999999LL), 59); + EXPECT_EQ(extractSecond_time64_ns(86399999999999LL), 59); +} + TEST(TestTime, TestDateDiff) { gdv_timestamp ts1 = StringToTimestamp("2019-06-30 00:00:00"); gdv_timestamp ts2 = StringToTimestamp("2019-05-31 00:00:00"); diff --git a/cpp/src/gandiva/precompiled/time_unit_ops.cc b/cpp/src/gandiva/precompiled/time_unit_ops.cc new file mode 100644 index 000000000000..164a47145230 --- /dev/null +++ b/cpp/src/gandiva/precompiled/time_unit_ops.cc @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Precompiled time64 (time-of-day) functions. +// +// Gandiva registers time64 signatures with time64[us] (see function_registry_common.h). +// The unsuffixed functions below therefore take microseconds-of-day, and the _ns +// variants take nanoseconds-of-day. LLVMGenerator::ResolveTimePcName() remaps a call +// to the _ns variant when the expression's time64 argument has NANO unit, the same +// way timestamp[us/ns] calls are remapped to the variants in timestamp_unit_ops.cc. +// +// Values are offsets within a single day (0 .. 86'399'999'999 for micros, +// 0 .. 86'399'999'999'999 for nanos), not epoch offsets, so the semantics mirror the +// time32 (millis-of-day) functions in time.cc. + +#include "./types.h" + +extern "C" { + +#define EXTRACT_TIME64(SUFFIX, UNITS_PER_SECOND) \ + FORCE_INLINE \ + gdv_int64 extractHour_time64##SUFFIX(gdv_time64 in) { \ + return in / ((UNITS_PER_SECOND)*3600LL); \ + } \ + FORCE_INLINE \ + gdv_int64 extractMinute_time64##SUFFIX(gdv_time64 in) { \ + return (in / ((UNITS_PER_SECOND)*60LL)) % 60; \ + } \ + FORCE_INLINE \ + gdv_int64 extractSecond_time64##SUFFIX(gdv_time64 in) { \ + return (in / (UNITS_PER_SECOND)) % 60; \ + } + +// MICROSECOND (registered unit) +EXTRACT_TIME64(, 1000000LL) + +// NANOSECOND +EXTRACT_TIME64(_ns, 1000000000LL) + +#undef EXTRACT_TIME64 + +} // extern "C" diff --git a/cpp/src/gandiva/precompiled/types.h b/cpp/src/gandiva/precompiled/types.h index c93b694fc777..89e045683c1f 100644 --- a/cpp/src/gandiva/precompiled/types.h +++ b/cpp/src/gandiva/precompiled/types.h @@ -36,6 +36,7 @@ using gdv_float64 = double; using gdv_date64 = int64_t; using gdv_date32 = int32_t; using gdv_time32 = int32_t; +using gdv_time64 = int64_t; using gdv_timestamp = int64_t; using gdv_utf8 = char*; using gdv_binary = char*; @@ -70,6 +71,12 @@ gdv_int64 extractSecond_timestamp(gdv_timestamp millis); gdv_int64 extractHour_time32(gdv_int32 millis_in_day); gdv_int64 extractMinute_time32(gdv_int32 millis_in_day); gdv_int64 extractSecond_time32(gdv_int32 millis_in_day); +gdv_int64 extractHour_time64(gdv_time64 micros_in_day); +gdv_int64 extractMinute_time64(gdv_time64 micros_in_day); +gdv_int64 extractSecond_time64(gdv_time64 micros_in_day); +gdv_int64 extractHour_time64_ns(gdv_time64 nanos_in_day); +gdv_int64 extractMinute_time64_ns(gdv_time64 nanos_in_day); +gdv_int64 extractSecond_time64_ns(gdv_time64 nanos_in_day); gdv_int32 hash32(double val, gdv_int32 seed); gdv_int32 hash32_buf(const gdv_uint8* buf, int len, gdv_int32 seed); diff --git a/cpp/src/gandiva/tests/date_time_test.cc b/cpp/src/gandiva/tests/date_time_test.cc index 09f89b7ecbed..17ecf4d5a746 100644 --- a/cpp/src/gandiva/tests/date_time_test.cc +++ b/cpp/src/gandiva/tests/date_time_test.cc @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include @@ -1923,11 +1924,205 @@ TEST_F(DateTimeTestProjector, TestTimestampdiffSubSecondSensitivity) { kTestNanos + 1000000001LL, pool_)); } -// NOTE: TIME type handling -// time32 uses TimeUnit::SECOND or MILLISECOND. -// time64 uses TimeUnit::MICROSECOND or NANOSECOND. -// Gandiva's time functions (extractHour/Minute/Second on time32) operate on millis. -// time64[us]/time64[ns] will need the same TimestampIR treatment: convert to millis -// before calling precompiled functions. The pattern is identical to the extract wrappers. +// time64[us] / time64[ns] functions. Signatures are registered with time64[us]; +// time64[ns] calls are remapped to the _ns variants by TimeIR. + +int64_t UnitsInDay(arrow::TimeUnit::type unit, int64_t hh, int64_t mm, int64_t ss, + int64_t frac) { + int64_t units_per_second = unit == arrow::TimeUnit::NANO ? 1000000000LL : 1000000LL; + return ((hh * 60 + mm) * 60 + ss) * units_per_second + frac; +} + +// Helper: evaluate a unary time64 function returning int64 +static int64_t EvalTime64Extract(const std::string& func_name, arrow::TimeUnit::type unit, + int64_t value, arrow::MemoryPool* pool) { + auto time_type = arrow::time64(unit); + auto f0 = field("f0", time_type); + auto schema = arrow::schema({f0}); + auto result_field = field("result", int64()); + auto expr = TreeExprBuilder::MakeExpression(func_name, {f0}, result_field); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + auto in_array = + MakeArrowTypeArray(time_type, {value}, {true}); + auto in_batch = arrow::RecordBatch::Make(schema, 1, {in_array}); + + arrow::ArrayVector outputs; + status = projector->Evaluate(*in_batch, pool, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + + auto result_array = std::dynamic_pointer_cast(outputs.at(0)); + return result_array->Value(0); +} + +// Helper: evaluate a binary time64 function returning boolean +static bool EvalTime64Compare(const std::string& func_name, arrow::TimeUnit::type unit, + int64_t left, int64_t right, arrow::MemoryPool* pool) { + auto time_type = arrow::time64(unit); + auto f0 = field("f0", time_type); + auto f1 = field("f1", time_type); + auto schema = arrow::schema({f0, f1}); + auto result_field = field("result", boolean()); + auto expr = TreeExprBuilder::MakeExpression(func_name, {f0, f1}, result_field); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + auto left_array = + MakeArrowTypeArray(time_type, {left}, {true}); + auto right_array = + MakeArrowTypeArray(time_type, {right}, {true}); + auto in_batch = arrow::RecordBatch::Make(schema, 1, {left_array, right_array}); + + arrow::ArrayVector outputs; + status = projector->Evaluate(*in_batch, pool, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + + auto result_array = std::dynamic_pointer_cast(outputs.at(0)); + return result_array->Value(0); +} + +TEST_F(DateTimeTestProjector, TestTime64ExtractAcrossPrecisions) { + for (auto unit : {arrow::TimeUnit::MICRO, arrow::TimeUnit::NANO}) { + int64_t sub_second = unit == arrow::TimeUnit::NANO ? 999999999LL : 999999LL; + + // 00:00:00 + int64_t midnight = UnitsInDay(unit, 0, 0, 0, 0); + EXPECT_EQ(0, EvalTime64Extract("extractHour", unit, midnight, pool_)); + EXPECT_EQ(0, EvalTime64Extract("extractMinute", unit, midnight, pool_)); + EXPECT_EQ(0, EvalTime64Extract("extractSecond", unit, midnight, pool_)); + + // 05:35:25. + int64_t morning = UnitsInDay(unit, 5, 35, 25, sub_second); + EXPECT_EQ(5, EvalTime64Extract("extractHour", unit, morning, pool_)); + EXPECT_EQ(35, EvalTime64Extract("extractMinute", unit, morning, pool_)); + EXPECT_EQ(25, EvalTime64Extract("extractSecond", unit, morning, pool_)); + + // 23:00:00 is above INT32_MAX in both units (catches 32/64-bit mixups). + int64_t late = UnitsInDay(unit, 23, 0, 0, 0); + ASSERT_GT(late, static_cast(INT32_MAX)); + EXPECT_EQ(23, EvalTime64Extract("extractHour", unit, late, pool_)); + EXPECT_EQ(0, EvalTime64Extract("extractMinute", unit, late, pool_)); + EXPECT_EQ(0, EvalTime64Extract("extractSecond", unit, late, pool_)); + + // 23:59:59. + int64_t last = UnitsInDay(unit, 23, 59, 59, sub_second); + EXPECT_EQ(23, EvalTime64Extract("extractHour", unit, last, pool_)); + EXPECT_EQ(59, EvalTime64Extract("extractMinute", unit, last, pool_)); + EXPECT_EQ(59, EvalTime64Extract("extractSecond", unit, last, pool_)); + + // Aliases resolve to the same functions. + EXPECT_EQ(23, EvalTime64Extract("hour", unit, last, pool_)); + EXPECT_EQ(59, EvalTime64Extract("minute", unit, last, pool_)); + EXPECT_EQ(59, EvalTime64Extract("second", unit, last, pool_)); + } +} + +TEST_F(DateTimeTestProjector, TestTime64ExtractMatchesTime32) { + // The time64 extract functions agree with the time32 (millis) ones on values that + // are exactly representable in milliseconds. + auto field0 = field("f0", time32(arrow::TimeUnit::MILLI)); + auto schema = arrow::schema({field0}); + auto field_sec = field("ss", int64()); + auto expr = TreeExprBuilder::MakeExpression("extractSecond", {field0}, field_sec); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, TestConfiguration(), &projector)); + + std::vector millis = {MillisInDay(0, 0, 0, 0), MillisInDay(5, 35, 25, 123), + MillisInDay(23, 59, 59, 999)}; + auto array = MakeArrowTypeArray( + time32(arrow::TimeUnit::MILLI), millis, {true, true, true}); + auto in_batch = arrow::RecordBatch::Make(schema, 3, {array}); + arrow::ArrayVector outputs; + ASSERT_OK(projector->Evaluate(*in_batch, pool_, &outputs)); + auto seconds = std::dynamic_pointer_cast(outputs.at(0)); + + for (size_t i = 0; i < millis.size(); i++) { + EXPECT_EQ(seconds->Value(i), + EvalTime64Extract("extractSecond", arrow::TimeUnit::MICRO, + millis[i] * 1000LL, pool_)); + EXPECT_EQ(seconds->Value(i), EvalTime64Extract("extractSecond", arrow::TimeUnit::NANO, + millis[i] * 1000000LL, pool_)); + } +} + +TEST_F(DateTimeTestProjector, TestTime64CompareAcrossPrecisions) { + for (auto unit : {arrow::TimeUnit::MICRO, arrow::TimeUnit::NANO}) { + // Values one unit apart within the same millisecond: a millisecond-based + // comparison would consider them equal. + int64_t a = UnitsInDay(unit, 23, 59, 59, 1); + int64_t b = a + 1; + + EXPECT_FALSE(EvalTime64Compare("equal", unit, a, b, pool_)); + EXPECT_TRUE(EvalTime64Compare("equal", unit, a, a, pool_)); + EXPECT_TRUE(EvalTime64Compare("not_equal", unit, a, b, pool_)); + EXPECT_TRUE(EvalTime64Compare("less_than", unit, a, b, pool_)); + EXPECT_FALSE(EvalTime64Compare("less_than", unit, b, a, pool_)); + EXPECT_TRUE(EvalTime64Compare("less_than_or_equal_to", unit, a, a, pool_)); + EXPECT_TRUE(EvalTime64Compare("greater_than", unit, b, a, pool_)); + EXPECT_FALSE(EvalTime64Compare("greater_than", unit, a, b, pool_)); + EXPECT_TRUE(EvalTime64Compare("greater_than_or_equal_to", unit, b, b, pool_)); + } +} + +TEST_F(DateTimeTestProjector, TestTime64NullHandling) { + for (auto unit : {arrow::TimeUnit::MICRO, arrow::TimeUnit::NANO}) { + auto time_type = arrow::time64(unit); + auto f0 = field("f0", time_type); + auto f1 = field("f1", time_type); + auto schema = arrow::schema({f0, f1}); + auto out = field("out", boolean()); + + auto isnull_expr = TreeExprBuilder::MakeExpression("isnull", {f0}, out); + auto isnotnull_expr = TreeExprBuilder::MakeExpression("isnotnull", {f0}, out); + auto distinct_expr = + TreeExprBuilder::MakeExpression("is_distinct_from", {f0, f1}, out); + auto not_distinct_expr = + TreeExprBuilder::MakeExpression("is_not_distinct_from", {f0, f1}, out); + + std::shared_ptr projector; + ASSERT_OK(Projector::Make( + schema, {isnull_expr, isnotnull_expr, distinct_expr, not_distinct_expr}, + TestConfiguration(), &projector)); + + int64_t v = UnitsInDay(unit, 12, 0, 0, 1); + // rows: (v, v), (null, v), (null, null), (v, v + 1) + auto left = MakeArrowTypeArray( + time_type, {v, 0, 0, v}, {true, false, false, true}); + auto right = MakeArrowTypeArray( + time_type, {v, v, 0, v + 1}, {true, true, false, true}); + auto in_batch = arrow::RecordBatch::Make(schema, 4, {left, right}); + + arrow::ArrayVector outputs; + ASSERT_OK(projector->Evaluate(*in_batch, pool_, &outputs)); + + std::vector all_valid = {true, true, true, true}; + EXPECT_ARROW_ARRAY_EQUALS(MakeArrowArrayBool({false, true, true, false}, all_valid), + outputs.at(0)); + EXPECT_ARROW_ARRAY_EQUALS(MakeArrowArrayBool({true, false, false, true}, all_valid), + outputs.at(1)); + EXPECT_ARROW_ARRAY_EQUALS(MakeArrowArrayBool({false, true, false, true}, all_valid), + outputs.at(2)); + EXPECT_ARROW_ARRAY_EQUALS(MakeArrowArrayBool({true, false, true, false}, all_valid), + outputs.at(3)); + } +} + +TEST_F(DateTimeTestProjector, TestTime64MixedUnitsRejected) { + auto f0 = field("f0", arrow::time64(arrow::TimeUnit::MICRO)); + auto f1 = field("f1", arrow::time64(arrow::TimeUnit::NANO)); + auto schema = arrow::schema({f0, f1}); + auto expr = + TreeExprBuilder::MakeExpression("less_than", {f0, f1}, field("out", boolean())); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector); + EXPECT_FALSE(status.ok()); + EXPECT_THAT(status.message(), testing::HasSubstr("mixed time64 units")); +} } // namespace gandiva diff --git a/cpp/src/gandiva/time_ir.cc b/cpp/src/gandiva/time_ir.cc new file mode 100644 index 000000000000..6a396aab6421 --- /dev/null +++ b/cpp/src/gandiva/time_ir.cc @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "gandiva/time_ir.h" + +#include + +namespace gandiva { + +/*static*/ std::string TimeIR::UnitSuffix(arrow::TimeUnit::type unit) { + return unit == arrow::TimeUnit::NANO ? "_ns" : ""; +} + +// Unit-specific function names that exist in the precompiled bitcode +// (precompiled/time_unit_ops.cc). +static const std::unordered_set& UnitSpecificFunctionNames() { + static const std::unordered_set names = { + "extractHour_time64_ns", + "extractMinute_time64_ns", + "extractSecond_time64_ns", + }; + return names; +} + +// time64 functions whose result does not depend on the time unit, as long as all +// time64 arguments share a unit (precompiled/arithmetic_ops.cc). +static const std::unordered_set& UnitAgnosticFunctionNames() { + static const std::unordered_set names = { + "equal_time64_time64", + "not_equal_time64_time64", + "less_than_time64_time64", + "less_than_or_equal_to_time64_time64", + "greater_than_time64_time64", + "greater_than_or_equal_to_time64_time64", + "isnull_time64", + "isnotnull_time64", + "is_distinct_from_time64_time64", + "is_not_distinct_from_time64_time64", + }; + return names; +} + +/*static*/ bool TimeIR::IsTimeIRFunction(const std::string& name) { + return UnitSpecificFunctionNames().count(name) != 0; +} + +/*static*/ bool TimeIR::IsUnitAgnosticFunction(const std::string& pc_name) { + return UnitAgnosticFunctionNames().count(pc_name) != 0; +} + +} // namespace gandiva diff --git a/cpp/src/gandiva/time_ir.h b/cpp/src/gandiva/time_ir.h new file mode 100644 index 000000000000..e0c5f9cc57a4 --- /dev/null +++ b/cpp/src/gandiva/time_ir.h @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/type.h" + +namespace gandiva { + +/// @brief Registry of time64 functions and their unit-specific precompiled variants. +/// +/// Gandiva registers time64 signatures with time64[us]; the unsuffixed precompiled +/// function takes microseconds-of-day. Function signature matching ignores the time64 +/// unit, so time64[ns] arguments match the same signature, and +/// LLVMGenerator::ResolveTimePcName() remaps the call to the _ns variant in +/// precompiled/time_unit_ops.cc. +/// +/// Unlike timestamp, a time64[ns] call is never allowed to silently fall back to the +/// microsecond function: every time64 function must either have an _ns variant or be +/// unit-agnostic (e.g. comparisons, which only need both args to share a unit). +class TimeIR { + public: + /// Returns true if \p function_name is a unit-specific time64 variant + /// (e.g. "extractHour_time64_ns"). + static bool IsTimeIRFunction(const std::string& function_name); + + /// Returns true if the time64 precompiled function \p pc_name produces the same + /// result for any (single) time unit, so no unit-specific variant is needed. + static bool IsUnitAgnosticFunction(const std::string& pc_name); + + /// Suffix of the unit-specific variant for \p unit; empty for the registered unit + /// (MICRO). + static std::string UnitSuffix(arrow::TimeUnit::type unit); +}; + +} // namespace gandiva