From a0403ba3310a8743175e6bf7ea5e07f88bee5733 Mon Sep 17 00:00:00 2001 From: Davide Faconti Date: Sat, 14 Nov 2026 10:18:08 +0100 Subject: [PATCH] feat(sdk): MaterializedSeriesView reports its null samples (1.0.0) The view owns an Arrow struct whose value column carries a validity bitmap, but it exposed only the raw value pointers: a null sample (an empty CSV cell, an absent optional field) read as the zero the host puts in its slot, and a consumer had no way to tell. nullCount() and isNull(row) read the bitmap. Client-side helpers only: no host contract change, no feature floor. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 4 + pj_base/CMakeLists.txt | 1 + .../include/pj_base/sdk/plugin_data_api.hpp | 46 ++++++- .../tests/materialized_series_view_test.cpp | 118 ++++++++++++++++++ 4 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 pj_base/tests/materialized_series_view_test.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 586594e8..b822bdb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,10 @@ yet. Additive: no existing struct, slot, or wire format changes. DepthImage: +- `MaterializedSeriesView` (what `ToolboxHostView::readSeries` returns) gains `nullCount()` and `isNull(row)`. The + value column's validity bitmap was always there, but the view gave no way to read it, so a null sample (an empty + CSV cell, an absent optional field) read as the zero the host puts in its slot. Client-side helpers only: no host + contract change, no feature floor. - `sdk::DepthImage` gains `frame_id` (proto field 9 of `PJ.DepthImage`), the coordinate frame of the camera that produced the depth, so a consumer can pair it with a `CameraInfo` by id. Additive; readers built before this release skip the unknown field. diff --git a/pj_base/CMakeLists.txt b/pj_base/CMakeLists.txt index 03bfdeef..e8647097 100644 --- a/pj_base/CMakeLists.txt +++ b/pj_base/CMakeLists.txt @@ -163,6 +163,7 @@ if(PJ_BUILD_TESTS) tests/abi_layout_sentinels_test.cpp tests/platform_test.cpp tests/arrow_holders_test.cpp + tests/materialized_series_view_test.cpp tests/builtin_object_test.cpp tests/builtin_object_codec_test.cpp tests/frame_transforms_codec_test.cpp diff --git a/pj_base/include/pj_base/sdk/plugin_data_api.hpp b/pj_base/include/pj_base/sdk/plugin_data_api.hpp index 1e6f77b4..0814d6ce 100644 --- a/pj_base/include/pj_base/sdk/plugin_data_api.hpp +++ b/pj_base/include/pj_base/sdk/plugin_data_api.hpp @@ -1214,7 +1214,8 @@ template /// reimplementing the Arrow-format walk every time. /// /// Column layout: children[0] = int64 timestamp (ns epoch), -/// children[1] = typed field value. Validity bitmap is per Arrow spec. +/// children[1] = typed field value. Validity bitmap is per Arrow spec: +/// `nullCount()` / `isNull()` read it, the `valuesAs*()` pointers do not. class MaterializedSeriesView { public: MaterializedSeriesView() = default; @@ -1280,7 +1281,50 @@ class MaterializedSeriesView { #undef PJ_SDK_VALUES_AS + /// Number of null samples in the value column: samples the source did not provide (an empty CSV + /// cell, an absent optional field). The `valuesAs*()` slot of a null is unspecified (hosts zero + /// it), so a caller that computes over the values must leave those rows out; 0 means none to + /// leave out. + [[nodiscard]] size_t nullCount() const noexcept { + const uint8_t* bits = validityBits(); + if (bits == nullptr) { + return 0; + } + const auto* col = array_.get()->children[1]; + if (col->null_count >= 0) { + return static_cast(col->null_count); + } + size_t nulls = 0; // the producer did not count them + for (size_t row = 0; row < rowCount(); ++row) { + nulls += isNull(row) ? 1 : 0; + } + return nulls; + } + + /// Whether sample `row` is null (see nullCount()). False for a row past the end. + [[nodiscard]] bool isNull(size_t row) const noexcept { + const uint8_t* bits = validityBits(); + if (bits == nullptr || row >= rowCount()) { + return false; + } + const size_t bit = row + static_cast(array_.get()->children[1]->offset); + return ((bits[bit >> 3] >> (bit & 7)) & 1) == 0; + } + private: + /// The value column's validity bitmap (bit set: the sample holds a value), or nullptr when the + /// column has none, which per the Arrow spec means no sample is null. + [[nodiscard]] const uint8_t* validityBits() const noexcept { + if (!valid()) { + return nullptr; + } + const auto* col = array_.get()->children[1]; + if (col == nullptr || col->n_buffers < 1 || col->null_count == 0) { + return nullptr; + } + return static_cast(col->buffers[0]); + } + ArrowSchemaHolder schema_; ArrowArrayHolder array_; }; diff --git a/pj_base/tests/materialized_series_view_test.cpp b/pj_base/tests/materialized_series_view_test.cpp new file mode 100644 index 00000000..cc2d0830 --- /dev/null +++ b/pj_base/tests/materialized_series_view_test.cpp @@ -0,0 +1,118 @@ +/** + * @file materialized_series_view_test.cpp + * @brief MaterializedSeriesView over hand-built Arrow structs: the null samples it reports. + */ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include +#include + +#include "pj_base/sdk/plugin_data_api.hpp" + +namespace PJ::sdk { +namespace { + +void releaseSchema(::ArrowSchema* schema) { + std::memset(schema, 0, sizeof(*schema)); +} + +void releaseArray(::ArrowArray* array) { + std::memset(array, 0, sizeof(*array)); +} + +/// A {timestamp: int64, value: float64} struct of four samples, as a host's readSeriesArrow hands +/// it out. `validity` is the value column's bitmap (nullptr: none) and `null_count` what the +/// producer declares (-1: not counted). The storage outlives every view built from it. +struct Series { + std::array stamps{10, 20, 30, 40}; + std::array values{1.5, 0.0, 3.5, 0.0}; + std::array stamp_buffers{nullptr, stamps.data()}; + std::array value_buffers{nullptr, values.data()}; + ::ArrowSchema stamp_schema{}; + ::ArrowSchema value_schema{}; + std::array<::ArrowSchema*, 2> schema_children{&stamp_schema, &value_schema}; + ::ArrowArray stamp_array{}; + ::ArrowArray value_array{}; + std::array<::ArrowArray*, 2> array_children{&stamp_array, &value_array}; + + MaterializedSeriesView view(const uint8_t* validity, int64_t null_count) { + stamp_schema.format = "l"; + value_schema.format = "g"; + stamp_array.length = value_array.length = 4; + stamp_array.n_buffers = value_array.n_buffers = 2; + stamp_array.buffers = stamp_buffers.data(); + value_buffers[0] = validity; + value_array.buffers = value_buffers.data(); + value_array.null_count = null_count; + + ArrowSchemaHolder schema; + ::ArrowSchema* raw_schema = schema.out(); + raw_schema->format = "+s"; + raw_schema->n_children = 2; + raw_schema->children = schema_children.data(); + raw_schema->release = releaseSchema; + ArrowArrayHolder array; + ::ArrowArray* raw_array = array.out(); + raw_array->length = 4; + raw_array->n_children = 2; + raw_array->children = array_children.data(); + raw_array->release = releaseArray; + return MaterializedSeriesView(std::move(schema), std::move(array)); + } +}; + +TEST(MaterializedSeriesViewTest, AColumnWithoutABitmapHasNoNulls) { + Series series; + const MaterializedSeriesView view = series.view(nullptr, 0); + ASSERT_EQ(view.rowCount(), 4U); + EXPECT_EQ(view.nullCount(), 0U); + for (size_t row = 0; row < 4; ++row) { + EXPECT_FALSE(view.isNull(row)) << row; + } +} + +// Rows 1 and 3 are null: their zeroed value slots are not samples of 0. +TEST(MaterializedSeriesViewTest, ReportsTheNullSamplesOfItsValueColumn) { + Series series; + const uint8_t validity = 0b0101; + const MaterializedSeriesView view = series.view(&validity, 2); + EXPECT_EQ(view.nullCount(), 2U); + EXPECT_FALSE(view.isNull(0)); + EXPECT_TRUE(view.isNull(1)); + EXPECT_FALSE(view.isNull(2)); + EXPECT_TRUE(view.isNull(3)); + EXPECT_FALSE(view.isNull(4)) << "past the end"; + ASSERT_NE(view.valuesAsFloat64(), nullptr); + EXPECT_EQ(view.valuesAsFloat64()[2], 3.5); +} + +TEST(MaterializedSeriesViewTest, CountsNullsTheProducerDidNotCount) { + Series series; + const uint8_t validity = 0b0111; + const MaterializedSeriesView view = series.view(&validity, -1); + EXPECT_EQ(view.nullCount(), 1U); + EXPECT_TRUE(view.isNull(3)); +} + +// A bitmap may ride along with a declared count of zero: nothing is null then. +TEST(MaterializedSeriesViewTest, ADeclaredCountOfZeroWinsOverTheBitmap) { + Series series; + const uint8_t validity = 0b0000; + const MaterializedSeriesView view = series.view(&validity, 0); + EXPECT_EQ(view.nullCount(), 0U); + EXPECT_FALSE(view.isNull(1)); +} + +TEST(MaterializedSeriesViewTest, AnEmptyViewHasNoNulls) { + const MaterializedSeriesView view; + EXPECT_EQ(view.nullCount(), 0U); + EXPECT_FALSE(view.isNull(0)); +} + +} // namespace +} // namespace PJ::sdk