From be1d7c41a62035191fc74c6f7bc5455ff398cefb Mon Sep 17 00:00:00 2001 From: Davide Faconti Date: Fri, 13 Nov 2026 23:34:43 +0100 Subject: [PATCH] refactor(parser-module)!: move the functional parser-module kit to experimental/ (1.0.0) The kit (authoring headers, export ABI and block codecs, native loader and runtime, pj_add_parser_module) has no module and no host using it, so 1.0 would freeze an ABI nobody has exercised. It moves to experimental/parser_module/: still built and tested here, never installed or packaged, and outside the compatibility promise. It stays in this repository because the native loader depends on the plugin loaders' private detail/library_loader.hpp, the export ABI is a host-to-module contract that needs one home, and ObjectWriter is coupled to the builtin codecs and splice table it is tested against. Removed from the installed SDK: - component plotjuggler_sdk::parser_module and PjParserModule.cmake - pj_base/parser_module/*.hpp, parser_module_abi.h, parser_module_manifest.hpp - pj_plugins/host/native_parser_module.hpp, parser_module_runtime.hpp - the pj_parser_module_host archive linked by plugin_host - ParserClaimCatalog::ingestModuleManifest, decodeParserModuleManifest and ParserModuleManifest (now parser_module_claims.hpp in experimental, which validates through the public admitClaims path) - PJ_MODULE_PARSE_INPUT_FLAG_HAS_TIMESTAMP from the feature-floor baseline Claim admission, plugin-claim synthesis and ParserRouteResolver are unchanged. test_sdk_install.sh now fails if any parser-module file reaches the install tree. PJ_BUILD_EXPERIMENTAL (default: follows PJ_BUILD_TESTS) gates the build. Co-Authored-By: Claude Fable 5.2 --- .claude/skills/plotjuggler-plugin/SKILL.md | 10 +- .../references/message-parser.md | 4 - CHANGELOG.md | 17 ++ CLAUDE.md | 11 +- CMakeLists.txt | 8 +- README.md | 4 +- cmake/plotjuggler_sdkConfig.cmake.in | 5 - conanfile.py | 17 +- .../parser_module/AUTHORING.md | 16 +- experimental/parser_module/CMakeLists.txt | 242 +++++++++++++++++ experimental/parser_module/README.md | 144 ++++++++++ .../parser_module/cmake}/PjParserModule.cmake | 0 .../parser_module/cdr_field_locator.hpp | 0 .../pj_base/parser_module/cdr_reader.hpp | 0 .../include/pj_base/parser_module/core.hpp | 0 .../include/pj_base/parser_module/module.hpp | 0 .../pj_base/parser_module/object_writer.hpp | 0 .../parser_module/proto_field_locator.hpp | 0 .../pj_base/parser_module/proto_reader.hpp | 0 .../include/pj_base/parser_module/time.hpp | 0 .../include/pj_base/parser_module_abi.h | 0 .../pj_base/parser_module_manifest.hpp | 0 .../pj_plugins/host/native_parser_module.hpp | 0 .../pj_plugins/host/parser_module_claims.hpp | 41 +++ .../pj_plugins/host/parser_module_runtime.hpp | 0 .../detail/native_parser_module_loader.hpp | 0 .../src/detail/native_parser_module_state.hpp | 0 .../src/native_parser_module.cpp | 0 .../parser_module}/src/parser_module_abi.cpp | 0 .../src/parser_module_claims.cpp | 247 ++++++++++++++++++ .../src/parser_module_manifest.cpp | 0 .../src/parser_module_runtime.cpp | 0 .../tests/native_parser_module_fixture.cpp | 0 .../tests/native_parser_module_fixture.hpp | 0 .../tests/native_parser_module_test.cpp | 4 +- .../tests/parser_module_abi_test.cpp | 38 +++ .../parser_module_authoring_e2e_test.cpp | 4 +- .../tests/parser_module_claims_test.cpp | 122 +++++++++ .../tests/parser_module_kit_test.cpp | 0 .../tests/parser_module_manifest_test.cpp | 0 .../parser_module_object_writer_test.cpp | 0 .../tests/parser_module_runtime_test.cpp | 0 .../tests/parser_module_wasm_audit.cpp | 0 .../tests/toy_cdr_pointcloud.module.json | 0 .../tests/toy_cdr_pointcloud_module.cpp | 0 pj_base/CMakeLists.txt | 123 +-------- pj_base/feature_floors.json | 1 - .../include/pj_base/parser_module/README.md | 61 ----- pj_base/tests/abi_layout_sentinels_test.cpp | 38 --- pj_plugins/CLAUDE.md | 19 +- pj_plugins/CMakeLists.txt | 109 +------- pj_plugins/docs/ARCHITECTURE.md | 80 +----- pj_plugins/docs/message-parser-guide.md | 5 - .../pj_plugins/host/parser_claim_catalog.hpp | 27 +- pj_plugins/src/parser_claim_catalog.cpp | 218 ---------------- .../tests/parser_claim_catalog_test.cpp | 99 ------- recipe.yaml | 1 - test_sdk_install.sh | 8 +- tools/feature_floors/host_surfaces.snapshot | 1 - 59 files changed, 915 insertions(+), 809 deletions(-) rename .claude/skills/plotjuggler-plugin/references/parser-module.md => experimental/parser_module/AUTHORING.md (93%) create mode 100644 experimental/parser_module/CMakeLists.txt create mode 100644 experimental/parser_module/README.md rename {cmake => experimental/parser_module/cmake}/PjParserModule.cmake (100%) rename {pj_base => experimental/parser_module}/include/pj_base/parser_module/cdr_field_locator.hpp (100%) rename {pj_base => experimental/parser_module}/include/pj_base/parser_module/cdr_reader.hpp (100%) rename {pj_base => experimental/parser_module}/include/pj_base/parser_module/core.hpp (100%) rename {pj_base => experimental/parser_module}/include/pj_base/parser_module/module.hpp (100%) rename {pj_base => experimental/parser_module}/include/pj_base/parser_module/object_writer.hpp (100%) rename {pj_base => experimental/parser_module}/include/pj_base/parser_module/proto_field_locator.hpp (100%) rename {pj_base => experimental/parser_module}/include/pj_base/parser_module/proto_reader.hpp (100%) rename {pj_base => experimental/parser_module}/include/pj_base/parser_module/time.hpp (100%) rename {pj_base => experimental/parser_module}/include/pj_base/parser_module_abi.h (100%) rename {pj_base => experimental/parser_module}/include/pj_base/parser_module_manifest.hpp (100%) rename {pj_plugins => experimental/parser_module}/include/pj_plugins/host/native_parser_module.hpp (100%) create mode 100644 experimental/parser_module/include/pj_plugins/host/parser_module_claims.hpp rename {pj_plugins => experimental/parser_module}/include/pj_plugins/host/parser_module_runtime.hpp (100%) rename {pj_plugins => experimental/parser_module}/src/detail/native_parser_module_loader.hpp (100%) rename {pj_plugins => experimental/parser_module}/src/detail/native_parser_module_state.hpp (100%) rename {pj_plugins => experimental/parser_module}/src/native_parser_module.cpp (100%) rename {pj_base => experimental/parser_module}/src/parser_module_abi.cpp (100%) create mode 100644 experimental/parser_module/src/parser_module_claims.cpp rename {pj_base => experimental/parser_module}/src/parser_module_manifest.cpp (100%) rename {pj_plugins => experimental/parser_module}/src/parser_module_runtime.cpp (100%) rename {pj_plugins => experimental/parser_module}/tests/native_parser_module_fixture.cpp (100%) rename {pj_plugins => experimental/parser_module}/tests/native_parser_module_fixture.hpp (100%) rename {pj_plugins => experimental/parser_module}/tests/native_parser_module_test.cpp (96%) rename {pj_base => experimental/parser_module}/tests/parser_module_abi_test.cpp (83%) rename {pj_plugins => experimental/parser_module}/tests/parser_module_authoring_e2e_test.cpp (96%) create mode 100644 experimental/parser_module/tests/parser_module_claims_test.cpp rename {pj_base => experimental/parser_module}/tests/parser_module_kit_test.cpp (100%) rename {pj_base => experimental/parser_module}/tests/parser_module_manifest_test.cpp (100%) rename {pj_base => experimental/parser_module}/tests/parser_module_object_writer_test.cpp (100%) rename {pj_plugins => experimental/parser_module}/tests/parser_module_runtime_test.cpp (100%) rename {pj_base => experimental/parser_module}/tests/parser_module_wasm_audit.cpp (100%) rename {pj_base => experimental/parser_module}/tests/toy_cdr_pointcloud.module.json (100%) rename {pj_base => experimental/parser_module}/tests/toy_cdr_pointcloud_module.cpp (100%) delete mode 100644 pj_base/include/pj_base/parser_module/README.md diff --git a/.claude/skills/plotjuggler-plugin/SKILL.md b/.claude/skills/plotjuggler-plugin/SKILL.md index 798f525d..c2c47cfb 100644 --- a/.claude/skills/plotjuggler-plugin/SKILL.md +++ b/.claude/skills/plotjuggler-plugin/SKILL.md @@ -2,7 +2,7 @@ name: plotjuggler-plugin description: >- Write or modify a PlotJuggler plugin (DataSource, MessageParser, Toolbox, or - Dialog) or native functional parser module using the plotjuggler_sdk. Use this + Dialog) using the plotjuggler_sdk. Use this whenever the task is to implement, extend, debug, or build a PlotJuggler extension — importing a file format, streaming live data, parsing message payloads, mapping a custom type to a canonical object or scalars, adding a @@ -20,12 +20,6 @@ class and export it from a shared library (`.so`/`.dll`/`.dylib`). point, version symbol, exception trampolines and symbol folding. **You never touch the raw ABI.** Override a few virtual methods and ship the library. -There is also a lighter extension shape: a **functional parser module** is one -C++17 source compiled as a native shared library, with no plugin-family -boilerplate. Write a MessageParser plugin when you own a whole encoding. Write a -parser module when you only need one or a few custom message types rendered as -canonical objects or scalars. Start at `references/parser-module.md`. - This skill is the author-oriented fast path. The repo's `pj_plugins/docs/` guides are the full reference; this steers you to the right one and front-loads the things that silently break a plugin. @@ -69,7 +63,6 @@ plotjuggler_sdk::plugin_sdk)` — only the acquisition step differs: |---|---|---| | Turn a **file** or a **live source** (socket, serial, hardware) into topics | **DataSource** | `references/data-source.md` | | Decode a **byte payload** on a topic into named fields (JSON, protobuf, ROS, custom) | **MessageParser** | `references/message-parser.md` | -| Map **one or a few message types** to canonical objects or scalars without owning the encoding | **Functional parser module** | `references/parser-module.md` | | A **tool** that reads existing data, transforms it, and writes new topics | **Toolbox** | `references/toolbox.md` | | A **configuration UI** (for a source/parser/toolbox, or standalone) | **Dialog** | `references/dialog.md` | @@ -289,7 +282,6 @@ coalescing, etc.) — read them. | What each family may do; capabilities; permission matrix; config contract | `pj_plugins/docs/REQUIREMENTS.md` | | How the C ABI / loaders / host bridge work (mostly maintainer detail) | `pj_plugins/docs/ARCHITECTURE.md` | | Writing each family, in depth | `pj_plugins/docs/{data-source,message-parser,toolbox,dialog-plugin}-guide.md` | -| Authoring a native functional parser module | `references/parser-module.md`, `pj_base/include/pj_base/parser_module/README.md` | | Dialog `WidgetData` setters + event-handler signatures | `docs/dialog-sdk-reference.md` | | Builtin object types + their wire codecs | `docs/builtin_type.md`, `pj_base/include/pj_base/builtin/` | | Object store: publish/read objects, ownership, lazy fetch | `V4_STORE.md` | diff --git a/.claude/skills/plotjuggler-plugin/references/message-parser.md b/.claude/skills/plotjuggler-plugin/references/message-parser.md index e78dcaa8..c15f7cae 100644 --- a/.claude/skills/plotjuggler-plugin/references/message-parser.md +++ b/.claude/skills/plotjuggler-plugin/references/message-parser.md @@ -14,10 +14,6 @@ not a parser. A parser only decodes payloads the host hands it. A parser is righ when many payloads on a topic share an encoding (JSON, protobuf, ROS, a custom binary) and you decode each into fields. -If an existing encoding already carries one custom type that you want to render -as a canonical object or scalars, use a **functional parser module** instead of -forking or replacing the encoding's MessageParser. See `parser-module.md`. - ## Header (trap) ```cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index a5290b1b..5d66d543 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,23 @@ by hand, because an older host cannot decode the object. `Log` gains an optional one on the same wire, and the widget-protocol changes are documentation only; `pj.playback.v1` already has its floor in `feature_floors.json`. +Functional parser modules leave the installed SDK: + +- **Breaking (source, packaging):** the parser-module kit is no longer installed or packaged. It + moves to `experimental/parser_module/`, where it is still built and tested but carries no + compatibility promise; no host loads modules yet, so its export ABI and manifest schema are not + frozen by 1.0. Removed from the package: the `plotjuggler_sdk::parser_module` component and + `pj_add_parser_module()` (`PjParserModule.cmake`), the headers `pj_base/parser_module/*.hpp`, + `pj_base/parser_module_abi.h`, `pj_base/parser_module_manifest.hpp`, + `pj_plugins/host/native_parser_module.hpp` and `pj_plugins/host/parser_module_runtime.hpp`, and + the `pj_parser_module_host` archive that `plotjuggler_sdk::plugin_host` linked. The + `PJ::parser_module` block codecs leave the `pj_base` archive. +- **Breaking (source):** `ParserClaimCatalog::ingestModuleManifest`, `decodeParserModuleManifest` + and `ParserModuleManifest` leave `pj_plugins/host/parser_claim_catalog.hpp`. Claim admission, + `synthesizeParserPluginClaims` and `ParserRouteResolver` are unchanged. +- `PJ_MODULE_PARSE_INPUT_FLAG_HAS_TIMESTAMP` leaves the `feature_floors.json` baseline with the + header that declared it. + Log: - `PJ::sdk::Log` gains an optional `category` (proto field 5 of `PJ.Log`): a source-defined grouping diff --git a/CLAUDE.md b/CLAUDE.md index 3faa9997..bb5187eb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,8 +46,13 @@ protocol lives in `pj_plugins/dialog_protocol/` and the Widget protocol in plugin-resolution catalog belong to the application, not this SDK. The installed `cmake/` helpers (`pj_configure_plugin`, `pj_embed_file`, -`pj_harden_plugin_exports`, `pj_add_parser_module`, `pj_add_sdk_test_fixture`) -are public API and follow the same versioning contract as headers. +`pj_harden_plugin_exports`, `pj_add_sdk_test_fixture`) are public API and follow +the same versioning contract as headers. + +`experimental/` holds components that are built and tested here but never +installed or packaged, and that carry no compatibility promise. It currently +holds functional parser modules: see +[experimental/parser_module/README.md](experimental/parser_module/README.md). ## Build and test @@ -133,7 +138,7 @@ release. The real `X.Y.Z` is cut once, when the feature's contract is complete. - `CamelCase` types, `camelBack` functions, `lower_case` variables, `lower_case_` members, `kCamelCase` constants. - Match module ownership: vocabulary in `PJ`, SDK helpers in `PJ::sdk`, - provider helpers in `PJ::sdk::source`, standalone parser modules in `pj`. + provider helpers in `PJ::sdk::source`, experimental parser modules in `pj`. - Fallible operations use `PJ::Expected` / `PJ::Status`; invariants use `PJ_ASSERT`. Warnings: `-Wall -Wextra -Werror`. diff --git a/CMakeLists.txt b/CMakeLists.txt index 6daffce4..313c0429 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,6 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") include(GNUInstallDirs) # CMAKE_INSTALL_LIBDIR, etc. used by PjPlugin include(PjPlugin) -include(PjParserModule) include(PjSdkTestFixtures) # --------------------------------------------------------------------------- @@ -35,6 +34,8 @@ option(PJ_ENABLE_TSAN "Enable ThreadSanitizer for Debug builds" OFF) option(PJ_INSTALL_SDK "Install plotjuggler_sdk CMake package (base/plugin_sdk/plugin_host)" OFF) option(PJ_BUILD_PORTED_PLUGINS "Build pj_ported_plugins (ported plugins collection)" ON) option(PJ_BUILD_TESTS "Build tests, benchmarks, and examples" ON) +# experimental/ is never installed or packaged and carries no compatibility promise. +option(PJ_BUILD_EXPERIMENTAL "Build experimental components (functional parser modules)" ${PJ_BUILD_TESTS}) option(PJ_ENABLE_ABI_CHECK "Enable abidiff-based ABI drift gate (requires libabigail)" OFF) # -Werror is on for our own builds/CI, but must be off for third-party # packaging (Conan Center, distro builds) where an unpredicted compiler version @@ -133,6 +134,9 @@ endif() add_subdirectory(pj_base) add_subdirectory(pj_plugins) +if(PJ_BUILD_EXPERIMENTAL) + add_subdirectory(experimental/parser_module) +endif() if(PJ_BUILD_PORTED_PLUGINS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/pj_ported_plugins/CMakeLists.txt") set(PJ_HAS_PORTED_PLUGINS TRUE) @@ -145,7 +149,6 @@ endif() # Exported CMake namespace: plotjuggler_sdk:: # Components: # base — vocabulary types (always available) -# parser_module — standalone header-only functional module authoring kit # plugin_sdk — plugin-author surface: base + dialog SDK + parser SDK # source — compiled provider helpers (descriptor_import_support is an alias) # plugin_host — host-side loaders (data_source, message_parser, toolbox, @@ -196,7 +199,6 @@ if(PJ_INSTALL_SDK) "${CMAKE_CURRENT_BINARY_DIR}/plotjuggler_sdkConfigVersion.cmake" cmake/PjPlugin.cmake cmake/PjCheckElfPluginExports.cmake - cmake/PjParserModule.cmake cmake/PjSdkTestFixtures.cmake DESTINATION ${PJ_PACKAGE_CMAKE_DIR} ) diff --git a/README.md b/README.md index c8878e0d..f17531d5 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Both guides ship under `share/plotjuggler_sdk/docs/` in the installed package. | **pj_base** | Vocabulary types: `Timestamp`, `DatasetId`, `TopicId`, type trees, `Expected`, `Span` | None | Apache-2.0 | | **pj_source** | Provider descriptors, origins, artifact cache, jobs, limits, presentation, completion and Stop; link `plotjuggler_sdk::source` | pj_base, nlohmann/json | Apache-2.0 | | **pj_plugins** | C-ABI plugin protocol (DataSource, MessageParser, Dialog, Toolbox, Widget families), C++ SDK base classes, plugin discovery, host-side loaders, and config helpers | pj_base, nlohmann/json | Apache-2.0 | -| **cmake/** | Plugin-authoring CMake helpers shipped with `plugin_sdk`: `pj_configure_plugin()` (symbol isolation, export allowlist + gate, manifest validation/sidecar/embed), `pj_embed_file()`, `pj_harden_plugin_exports()`, `pj_add_parser_module()`, `pj_add_sdk_test_fixture()` — see `examples/sdk_consumer` | — | Apache-2.0 | +| **cmake/** | Plugin-authoring CMake helpers shipped with `plugin_sdk`: `pj_configure_plugin()` (symbol isolation, export allowlist + gate, manifest validation/sidecar/embed), `pj_embed_file()`, `pj_harden_plugin_exports()`, `pj_add_sdk_test_fixture()` — see `examples/sdk_consumer` | — | Apache-2.0 | ## Getting Started @@ -48,7 +48,7 @@ git ls-files -z '*.cpp' | xargs -0 clang-tidy-22 -p build # clang-tidy ``` pj_base/ Vocabulary, builtin codecs and sdk/source/ provider support pj_plugins/ C-ABI plugin protocol, SDK, host loaders -cmake/ Installed CMake helpers for plugin authors (PjPlugin.cmake, PjParserModule.cmake, PjSdkTestFixtures.cmake) +cmake/ Installed CMake helpers for plugin authors (PjPlugin.cmake, PjSdkTestFixtures.cmake) examples/sdk_consumer/ find_package() consumer exercising the helpers (built by the release + conda checks) docs/ Project-wide design guides ``` diff --git a/cmake/plotjuggler_sdkConfig.cmake.in b/cmake/plotjuggler_sdkConfig.cmake.in index b4af8c4f..705f170f 100644 --- a/cmake/plotjuggler_sdkConfig.cmake.in +++ b/cmake/plotjuggler_sdkConfig.cmake.in @@ -16,11 +16,6 @@ foreach(_comp ${plotjuggler_sdk_FIND_COMPONENTS}) if(_comp STREQUAL "base") set(plotjuggler_sdk_base_FOUND TRUE) - elseif(_comp STREQUAL "parser_module") - # Header-only authoring surface plus its native module target helper. - include("${CMAKE_CURRENT_LIST_DIR}/PjParserModule.cmake") - set(plotjuggler_sdk_parser_module_FOUND TRUE) - elseif(_comp STREQUAL "plugin_sdk") find_dependency(nlohmann_json) # Plugin-authoring CMake helpers (pj_configure_plugin, pj_embed_file, diff --git a/conanfile.py b/conanfile.py index 2eeeb716..ca8afc49 100644 --- a/conanfile.py +++ b/conanfile.py @@ -5,7 +5,6 @@ base — pj_base, vocabulary types (always available) plugin_sdk — umbrella for plugin authors (base + dialog SDK + parser SDK) plugin_host — umbrella for host loaders (data_source/parser/toolbox/dialog) - parser_module — standalone functional parser-module authoring headers source — compiled source-provider support descriptor_import_support — compatibility alias for source @@ -70,6 +69,7 @@ class PlotjugglerSdkConan(ConanFile): "pj_base/*", "tools/feature_floors/*", "pj_plugins/*", + "experimental/*", # built only with_tests=True; never packaged "examples/*", "!*/__pycache__/*", ) @@ -150,12 +150,11 @@ def package_info(self): # the package level (self.cpp_info), not at component level — declaring # it on the `sdk` component below silently produced an empty # plotjuggler_sdk_BUILD_MODULES_PATHS_RELEASE in the generated data - # file. Ship the plugin-authoring helpers (PjPlugin.cmake) and the - # parser-module helper from the package root so CMakeDeps actually - # include()s them after find_package() returns. + # file. Ship the plugin-authoring helpers (PjPlugin.cmake) from the + # package root so CMakeDeps actually include()s them after + # find_package() returns. self.cpp_info.set_property("cmake_build_modules", [ os.path.join("lib", "cmake", "plotjuggler_sdk", "PjPlugin.cmake"), - os.path.join("lib", "cmake", "plotjuggler_sdk", "PjParserModule.cmake"), os.path.join("lib", "cmake", "plotjuggler_sdk", "PjSdkTestFixtures.cmake"), ]) @@ -172,13 +171,6 @@ def package_info(self): sdk.includedirs = ["include"] sdk.requires = ["base", "nlohmann_json::nlohmann_json"] - # --- parser_module (header-only functional parser-module authoring kit; - # pj_add_parser_module() comes from the PjParserModule build module) --- - kit = self.cpp_info.components["parser_module"] - kit.set_property("cmake_target_name", "plotjuggler_sdk::parser_module") - kit.libs = [] # INTERFACE only: authored modules link no SDK library - kit.includedirs = ["include"] - # --- source (descriptor-import provider support) --- source = self.cpp_info.components["source"] source.set_property("cmake_target_name", "plotjuggler_sdk::source") @@ -204,7 +196,6 @@ def package_info(self): "pj_dialog_library", "pj_widget_host", "pj_plugin_catalog", - "pj_parser_module_host", "pj_parser_claim_catalog", "pj_plugin_loader_detail", ] diff --git a/.claude/skills/plotjuggler-plugin/references/parser-module.md b/experimental/parser_module/AUTHORING.md similarity index 93% rename from .claude/skills/plotjuggler-plugin/references/parser-module.md rename to experimental/parser_module/AUTHORING.md index 55ab6538..73f117ce 100644 --- a/.claude/skills/plotjuggler-plugin/references/parser-module.md +++ b/experimental/parser_module/AUTHORING.md @@ -1,5 +1,7 @@ # Functional parser module +> Experimental: see [README.md](README.md). Not part of the installed SDK. + A functional parser module maps one or a few message types from an existing encoding to canonical PlotJuggler objects or scalar fields. It is a single C++17 source compiled as a native shared library, with no plugin-family vtable or @@ -110,9 +112,11 @@ embedded bytes; catalog ingestion validates the complete JSON transactionally. ## Build -```cmake -find_package(plotjuggler_sdk 0.22 REQUIRED COMPONENTS parser_module) +`pj_add_parser_module()` exists only inside this repository's build (this +directory's `CMakeLists.txt` includes `cmake/PjParserModule.cmake`); there is no +installed component to `find_package`. +```cmake pj_add_parser_module(raw_mono_image_parser SOURCE raw_mono_image_parser.cpp MANIFEST raw_mono_image_parser.module.json @@ -121,9 +125,9 @@ pj_add_parser_module(raw_mono_image_parser ``` The helper embeds the manifest, hides every non-ABI symbol, and exports the -complete native `pj_module_*` set. SDK 0.22 accepts only `TARGETS native`; +complete native `pj_module_*` set. Only `TARGETS native` is accepted; requesting `TARGETS wasm` stops configuration with “wasm support arrives with -the SDK wasm loader milestone”. The shipped WASI check is structural +the SDK wasm loader milestone”. The WASI check is structural conformance testing, not a wasm authoring or execution target. ## Choose a schema-compatibility strategy @@ -171,8 +175,8 @@ selection before it can emit an invalid descriptor. ## Traps - The kit is header-only and WASI-clean: no threads, filesystem, iostream, host - SDK linkage, or exceptions across its API. In SDK 0.22 the supported build - product is nevertheless native-only. + SDK linkage, or exceptions across its API. The supported build product is + nevertheless native-only. - Return `pj::Status` / `pj::Expected`; do not throw. `Blob` uses nothrow allocation and protobuf matching is bounded, so allocation failure is a reported data error rather than a process abort or contract strike. diff --git a/experimental/parser_module/CMakeLists.txt b/experimental/parser_module/CMakeLists.txt new file mode 100644 index 00000000..f9f956bd --- /dev/null +++ b/experimental/parser_module/CMakeLists.txt @@ -0,0 +1,242 @@ +# --------------------------------------------------------------------------- +# EXPERIMENTAL: functional parser modules (authoring kit, export ABI, native +# host loader/runtime). Not installed, not packaged, and outside the SDK +# compatibility promise. See README.md. +# --------------------------------------------------------------------------- + +include(cmake/PjParserModule.cmake) +find_package(nlohmann_json REQUIRED) + +# Standalone, header-only authoring kit. Carries include paths and a C++ floor +# only: modules link no SDK library and stay suitable for native and WASI +# reactor builds. pj_add_parser_module() links it by this alias. +add_library(pj_parser_module INTERFACE) +target_include_directories(pj_parser_module INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/pj_base/include +) +target_compile_features(pj_parser_module INTERFACE cxx_std_17) +add_library(plotjuggler_sdk::parser_module ALIAS pj_parser_module) + +# Export-ABI block codecs and the wasm manifest-section codec, shared by the +# host runtime and by test fixtures that must not link the loader. +add_library(pj_parser_module_abi STATIC + src/parser_module_abi.cpp + src/parser_module_manifest.cpp +) +target_include_directories(pj_parser_module_abi PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) +target_compile_options(pj_parser_module_abi PRIVATE ${PJ_WARNING_FLAGS} ${PJ_SANITIZER_FLAGS}) +set_target_properties(pj_parser_module_abi PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_link_libraries(pj_parser_module_abi PUBLIC pj_base) + +# Native module loader, runtime and manifest-to-claims decoding. +add_library(pj_parser_module_host STATIC + src/native_parser_module.cpp + src/parser_module_runtime.cpp + src/parser_module_claims.cpp +) +target_include_directories(pj_parser_module_host PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) +target_compile_options(pj_parser_module_host PRIVATE ${PJ_WARNING_FLAGS}) +set_target_properties(pj_parser_module_host PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_link_libraries(pj_parser_module_host + PUBLIC + pj_parser_module_abi + pj_parser_claim_catalog + PRIVATE + pj_plugin_loader_detail # detail/library_loader.hpp: the plugin loaders' dlopen policy + nlohmann_json::nlohmann_json + ${CMAKE_DL_LIBS} +) + +if(NOT PJ_BUILD_TESTS) + return() +endif() + +foreach(test_name parser_module_abi_test parser_module_manifest_test) + add_executable(${test_name} tests/${test_name}.cpp) + target_link_libraries(${test_name} PRIVATE pj_parser_module_abi GTest::gtest_main) + add_test(NAME ${test_name} COMMAND ${test_name}) +endforeach() + +add_executable(parser_module_claims_test tests/parser_module_claims_test.cpp) +target_compile_options(parser_module_claims_test PRIVATE ${PJ_WARNING_FLAGS}) +target_link_libraries(parser_module_claims_test PRIVATE pj_parser_module_host GTest::gtest_main) +add_test(NAME parser_module_claims_test COMMAND parser_module_claims_test) + +# C++17 standalone authoring-kit gate. This target intentionally does not +# link pj_base; GTest is test infrastructure only. +add_executable(parser_module_kit_test tests/parser_module_kit_test.cpp) +set_target_properties(parser_module_kit_test PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED YES + CXX_EXTENSIONS NO +) +target_compile_options(parser_module_kit_test PRIVATE ${PJ_WARNING_FLAGS}) +target_link_libraries(parser_module_kit_test PRIVATE pj_parser_module GTest::gtest_main) +add_test(NAME parser_module_kit_test COMMAND parser_module_kit_test) + +add_executable(parser_module_object_writer_test tests/parser_module_object_writer_test.cpp) +target_compile_definitions(parser_module_object_writer_test PRIVATE PJ_PARSER_MODULE_CLAIM_COUNT=1) +target_compile_options(parser_module_object_writer_test PRIVATE ${PJ_WARNING_FLAGS}) +target_link_libraries(parser_module_object_writer_test PRIVATE pj_parser_module pj_parser_module_abi GTest::gtest_main) +add_test(NAME parser_module_object_writer_test COMMAND parser_module_object_writer_test) + +# Static wasm ABI conformance. The authoring source is compiled directly by +# wasi-sdk, then a host-side auditor embeds and validates the final module. +set(PJ_WASI_SDK_ROOT "$ENV{PJ_WASI_SDK_ROOT}" CACHE PATH + "wasi-sdk 27 root used for parser-module conformance tests") +set(_pj_wasi_clang "${PJ_WASI_SDK_ROOT}/bin/clang++") +set(_pj_wasi_sysroot "${PJ_WASI_SDK_ROOT}/share/wasi-sysroot") +if(NOT PJ_WASI_SDK_ROOT) + message(STATUS + "Parser-module WASI conformance skipped: PJ_WASI_SDK_ROOT cache/env variable is not set") +elseif(NOT EXISTS "${_pj_wasi_clang}" OR NOT IS_DIRECTORY "${_pj_wasi_sysroot}") + message(STATUS + "Parser-module WASI conformance skipped: wasi-sdk is missing under PJ_WASI_SDK_ROOT=${PJ_WASI_SDK_ROOT}") +else() + file(STRINGS "${PJ_WASI_SDK_ROOT}/VERSION" _pj_wasi_version LIMIT_COUNT 1) + if(NOT _pj_wasi_version MATCHES "^27\\.") + message(FATAL_ERROR + "Parser-module WASI conformance requires wasi-sdk 27; found '${_pj_wasi_version}'") + endif() + message(STATUS + "Parser-module WASI conformance enabled with wasi-sdk ${_pj_wasi_version}: ${PJ_WASI_SDK_ROOT}") + + add_executable(parser_module_wasm_audit tests/parser_module_wasm_audit.cpp) + target_compile_options(parser_module_wasm_audit PRIVATE ${PJ_WARNING_FLAGS}) + target_link_libraries(parser_module_wasm_audit PRIVATE pj_parser_module_abi) + + file(GLOB _pj_parser_module_headers CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/include/pj_base/parser_module/*.hpp") + set(_pj_wasm_dir "${CMAKE_CURRENT_BINARY_DIR}/parser_module_wasm_conformance") + set(_pj_wasm_raw "${_pj_wasm_dir}/toy_cdr_pointcloud.raw.wasm") + set(_pj_wasm_embedded "${_pj_wasm_dir}/toy_cdr_pointcloud.wasm") + set(_pj_wasm_source "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud_module.cpp") + set(_pj_wasm_manifest "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud.module.json") + + add_custom_command( + OUTPUT "${_pj_wasm_raw}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${_pj_wasm_dir}" + COMMAND "${_pj_wasi_clang}" + --target=wasm32-wasip1 + --sysroot=${_pj_wasi_sysroot} + -mexec-model=reactor + -std=c++17 + -fno-exceptions + -fno-rtti + -fvisibility=hidden + -O1 + -Wall -Wextra -Werror + -DPJ_PARSER_MODULE_CLAIM_COUNT=2 + -I${CMAKE_CURRENT_SOURCE_DIR}/include + -I${PROJECT_SOURCE_DIR}/pj_base/include + "${_pj_wasm_source}" + -Wl,--export=pj_module_abi + -Wl,--export=pj_module_create + -Wl,--export=pj_module_destroy + -Wl,--export=pj_module_bind + -Wl,--export=pj_module_parse + -Wl,--export=pj_module_last_error + -Wl,--export=pj_module_alloc + -Wl,--export=pj_module_free + -o "${_pj_wasm_raw}" + DEPENDS "${_pj_wasm_source}" ${_pj_parser_module_headers} + "${PROJECT_SOURCE_DIR}/pj_base/include/pj_base/time_math.hpp" + COMMENT "Compiling C++17 parser-module WASI reactor fixture" + VERBATIM + ) + add_custom_command( + OUTPUT "${_pj_wasm_embedded}" + COMMAND ${CMAKE_COMMAND} -E env ASAN_OPTIONS=detect_leaks=0 + $ + --embed "${_pj_wasm_raw}" "${_pj_wasm_manifest}" "${_pj_wasm_embedded}" + DEPENDS "${_pj_wasm_raw}" "${_pj_wasm_manifest}" parser_module_wasm_audit + COMMENT "Embedding parser-module wasm manifest with the shared codec" + VERBATIM + ) + add_custom_target(parser_module_wasm_conformance_fixture ALL + DEPENDS "${_pj_wasm_embedded}") + add_test(NAME parser_module_wasm_conformance_test + COMMAND parser_module_wasm_audit --audit "${_pj_wasm_embedded}" "${_pj_wasm_manifest}") + set_tests_properties(parser_module_wasm_conformance_test PROPERTIES + LABELS "parser_module;wasi" + REQUIRED_FILES "${_pj_wasm_embedded}") +endif() + +pj_add_parser_module(toy_cdr_pointcloud_module + SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud_module.cpp" + MANIFEST "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud.module.json" + TARGETS native +) + +# --------------------------------------------------------------------------- +# Native functional parser-module fixtures (real shared libraries) +# --------------------------------------------------------------------------- + +# Trailing arguments become PRIVATE compile definitions, each selecting one +# deliberately defective variant of the shared fixture source. +function(pj_add_native_parser_module_fixture target) + add_library(${target} SHARED tests/native_parser_module_fixture.cpp) + target_compile_features(${target} PRIVATE cxx_std_20) + target_compile_options(${target} PRIVATE ${PJ_WARNING_FLAGS}) + set_target_properties(${target} PROPERTIES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN YES + ) + target_link_libraries(${target} PRIVATE pj_parser_module_abi) + if(ARGN) + target_compile_definitions(${target} PRIVATE ${ARGN}) + endif() + if(UNIX AND NOT APPLE) + target_link_options(${target} PRIVATE "LINKER:-z,defs" "LINKER:--exclude-libs,ALL") + endif() +endfunction() + +pj_add_native_parser_module_fixture(native_parser_module_fixture) +pj_add_native_parser_module_fixture(native_parser_module_missing_export PJ_FIXTURE_OMIT_FREE) +pj_add_native_parser_module_fixture(native_parser_module_wrong_abi PJ_FIXTURE_WRONG_ABI) +pj_add_native_parser_module_fixture(native_parser_module_unreadable_manifest PJ_FIXTURE_UNREADABLE_MANIFEST) + +# Integration tests: native functional parser-module loading and execution. +add_executable(native_parser_module_test tests/native_parser_module_test.cpp) +add_dependencies(native_parser_module_test + native_parser_module_fixture + native_parser_module_missing_export + native_parser_module_wrong_abi + native_parser_module_unreadable_manifest +) +target_compile_definitions(native_parser_module_test PRIVATE + PJ_NATIVE_MODULE_FIXTURE_PATH="$" + PJ_NATIVE_MODULE_MISSING_EXPORT_PATH="$" + PJ_NATIVE_MODULE_WRONG_ABI_PATH="$" + PJ_NATIVE_MODULE_UNREADABLE_MANIFEST_PATH="$" +) +target_compile_options(native_parser_module_test PRIVATE ${PJ_WARNING_FLAGS}) +target_link_libraries(native_parser_module_test PRIVATE + pj_parser_module_host GTest::gtest_main +) +add_test(NAME native_parser_module_test COMMAND native_parser_module_test) + +add_executable(parser_module_runtime_test tests/parser_module_runtime_test.cpp) +add_dependencies(parser_module_runtime_test native_parser_module_fixture) +target_compile_definitions(parser_module_runtime_test PRIVATE + PJ_NATIVE_MODULE_FIXTURE_PATH="$" +) +target_compile_options(parser_module_runtime_test PRIVATE ${PJ_WARNING_FLAGS}) +target_link_libraries(parser_module_runtime_test PRIVATE + pj_parser_module_host GTest::gtest_main +) +add_test(NAME parser_module_runtime_test COMMAND parser_module_runtime_test) + +# Keystone integration: a C++17 module authored and built only with the +# header-only kit, then exercised through the production native loader/runtime. +add_executable(parser_module_authoring_e2e_test tests/parser_module_authoring_e2e_test.cpp) +add_dependencies(parser_module_authoring_e2e_test toy_cdr_pointcloud_module) +target_compile_definitions(parser_module_authoring_e2e_test PRIVATE + PJ_TOY_CDR_POINTCLOUD_MODULE_PATH="$" +) +target_compile_options(parser_module_authoring_e2e_test PRIVATE ${PJ_WARNING_FLAGS}) +target_link_libraries(parser_module_authoring_e2e_test PRIVATE + pj_parser_module_host GTest::gtest_main +) +add_test(NAME parser_module_authoring_e2e_test COMMAND parser_module_authoring_e2e_test) diff --git a/experimental/parser_module/README.md b/experimental/parser_module/README.md new file mode 100644 index 00000000..e2e58985 --- /dev/null +++ b/experimental/parser_module/README.md @@ -0,0 +1,144 @@ +# Functional parser modules (experimental) + +**Status: experimental.** Nothing under `experimental/parser_module/` is +installed, packaged, or covered by the SDK compatibility promise. The export +ABI (`PJ_PARSER_MODULE_ABI_VERSION` 1), the manifest schema and the authoring +API may all change without a MAJOR release. No host loads modules yet. + +It is built and tested here when `PJ_BUILD_EXPERIMENTAL` is ON (the default +follows `PJ_BUILD_TESTS`). Headers keep their `pj_base/...` and +`pj_plugins/host/...` include spellings, so promoting the component back into +the installed SDK is a move, not a rewrite. + +| Path | Contents | +|---|---| +| `include/pj_base/parser_module/` | Header-only C++17 authoring kit (`module.hpp` is the umbrella) | +| `include/pj_base/parser_module_abi.h`, `parser_module_manifest.hpp` | Export ABI, block codecs, wasm manifest-section codec | +| `include/pj_plugins/host/` | Native loader, instance runtime, manifest-to-claims decoding | +| `cmake/PjParserModule.cmake` | `pj_add_parser_module()` | +| `AUTHORING.md` | Manifest, schema-locator, splice, lifetime and error contracts | + +Open before the ABI can be called stable: + +- The object output descriptor carries no record timestamp; the scalar output does. +- The parse input copies the whole payload inline on every message. +- A module cannot report a contract violation through a result code, only + through a `pj.parser.contract_violation:` message prefix. +- `decodeParserModuleManifest` accepts exactly one `module_abi` value. + +## Authoring kit + +The kit is standalone: its headers may include others in +`include/pj_base/parser_module/`, C/C++ standard-library headers and the shared +C++17 checked arithmetic in `pj_base/time_math.hpp`. `time.hpp` uses that +arithmetic. + +Headers must remain suitable for a WASI reactor build: no filesystem, threads, +iostreams, host SDK linkage or exceptions crossing public/ABI calls. + +All data received from a host or message is borrowed through `ByteView` / +`PayloadView`. Fallible operations return the local `pj::Status` or +`pj::Expected` types. Output storage is owned by `pj::Blob`, whose allocation +entry points report failure instead of exposing allocation exceptions. + +The umbrella include for module authors is: + +```cpp +#include + +class RawImageParser final : public pj::FunctionalParser { + public: + pj::Status bind(const pj::BindingInfo& info) override { + return info.route() == pj::Route::kObject + ? pj::Status::ok() + : pj::Status::decline("object route only"); + } + + pj::Status parseObject(pj::PayloadView payload, pj::Timestamp timestamp, + pj::ObjectWriter& output) override { + auto image = output.image(); + if (timestamp.has_value) { + if (auto status = image.setTimestamp(timestamp.nanoseconds); + !status.isOk()) { + return status; + } + } + if (auto status = image.setEncoding("mono8"); !status.isOk()) { + return status; + } + return image.setData(payload); + } +}; + +PJ_FUNCTIONAL_PARSER(RawImageParser) +``` + +Override `parseScalars(PayloadView, Timestamp, ScalarWriter&)` for scalar +claims. `ObjectWriter` provides `image`, `pointCloud`, `depthImage`, +`occupancyGrid`, `compressedPointCloud`, `mesh3D`, `videoFrame`, +`occupancyGridUpdate`, `voxelGrid`, and `gridMap` builders. + +Native modules are built with `pj_add_parser_module(... TARGETS native)`. The +target links no SDK library; its include paths expose this kit and the shared +checked time arithmetic. +`TARGETS wasm` is not available. The wasi-sdk gate compiles and +statically audits a reactor fixture, but it does not provide wasm authoring or +execution. + +See [AUTHORING.md](AUTHORING.md) for the manifest, schema-locator, splice, +lifetime, and error contracts. + +## Host loader and runtime + +Functional parser modules use the family-independent export ABI in +`include/pj_base/parser_module_abi.h`, not a plugin-family vtable. The host-side +`NativeParserModule` loader opens each artifact with local, immediate symbol +resolution, resolves every operational and native metadata export, gates ABI +version 1, and copies the embedded manifest for subsequent +`ParserClaimCatalog` admission. Native module mappings have process-session +lifetime in v1 and are never unloaded. + +`NativeParserModuleInstance` owns one create/bind/parse/destroy lifecycle. It +uses the frozen little-endian codecs for input and output, consumes all +module-owned descriptor views before returning, validates output route and +expected object type, and checks a splice against both the canonical object's +eligible field and the original payload bounds. A valid splice is materialized +into that field, so the returned canonical object is complete; its sidecar +retains the original offset and bytes for future zero-copy integration. +Malformed returned descriptors, ineligible/out-of-bounds returned splices, +type/route mismatch, and bad tokens are contract violations; other +module-reported per-message failures are data errors. The separate +non-thread-safe `ParserModuleStrikeTracker` quarantines a module claim on its +third contract violation, permits one same-descriptor recreation, and disables +the claim after a repeated three-strike cycle. Executor placement, generation +ownership, folder scanning, and rescan policy remain application concerns. + +Module authors use the standalone C++17 headers under +`include/pj_base/parser_module/` through the zero-linkage +`plotjuggler_sdk::parser_module` target. The readers compile ROS 2 concatenated +`.msg` bundles or protobuf `FileDescriptorSet` field paths at bind time; the +CDR locator supports fixed arrays, bounded and unbounded sequences, bounded +strings, and string arrays/sequences, while cyclic or over-depth schemas fail at +bind. The typed `ObjectWriter` emits Image, PointCloud, DepthImage, +OccupancyGrid, CompressedPointCloud, Mesh3D, VideoFrame, OccupancyGridUpdate, +and VoxelGrid descriptors, including each type's eligible single-splice form. +Module parse callbacks receive the per-message `pj::Timestamp` alongside the +payload. `BindingInfo` views expire when `bind()` returns; `owningCopy()` is the +fallible retention path. Bulk storage uses nothrow allocation, and protobuf +matching is bounded, so allocation failure is returned as data error rather +than escaping as a trap. `PJ_FUNCTIONAL_PARSER` supplies the complete native +export set, uses synchronized index+generation instance tokens to reject stale +handles, and catches user exceptions at the C boundary. +`pj_add_parser_module(... TARGETS native)` builds a hidden-visibility module and +embeds its JSON manifest behind the native metadata exports. `TARGETS wasm` is +not available. The shared host codec can append and read the exact +JSON bytes in a wasm `pj_parser_module_manifest` custom section; the conditional +wasi-sdk 27 compile gate builds the toy source with C++17 and exceptions +disabled, then statically audits the reactor model and every operational export +signature. This is structural conformance only: no wasm loader or execution +runtime exists. + +Native modules use the same absolute-path normalization, package-scoped +platform open, and defining-module provenance checks as the plugin-family +loaders for every required ABI export (`pj_plugins/src/detail/library_loader.hpp`). +Their narrow path API is explicitly UTF-8 on Windows. diff --git a/cmake/PjParserModule.cmake b/experimental/parser_module/cmake/PjParserModule.cmake similarity index 100% rename from cmake/PjParserModule.cmake rename to experimental/parser_module/cmake/PjParserModule.cmake diff --git a/pj_base/include/pj_base/parser_module/cdr_field_locator.hpp b/experimental/parser_module/include/pj_base/parser_module/cdr_field_locator.hpp similarity index 100% rename from pj_base/include/pj_base/parser_module/cdr_field_locator.hpp rename to experimental/parser_module/include/pj_base/parser_module/cdr_field_locator.hpp diff --git a/pj_base/include/pj_base/parser_module/cdr_reader.hpp b/experimental/parser_module/include/pj_base/parser_module/cdr_reader.hpp similarity index 100% rename from pj_base/include/pj_base/parser_module/cdr_reader.hpp rename to experimental/parser_module/include/pj_base/parser_module/cdr_reader.hpp diff --git a/pj_base/include/pj_base/parser_module/core.hpp b/experimental/parser_module/include/pj_base/parser_module/core.hpp similarity index 100% rename from pj_base/include/pj_base/parser_module/core.hpp rename to experimental/parser_module/include/pj_base/parser_module/core.hpp diff --git a/pj_base/include/pj_base/parser_module/module.hpp b/experimental/parser_module/include/pj_base/parser_module/module.hpp similarity index 100% rename from pj_base/include/pj_base/parser_module/module.hpp rename to experimental/parser_module/include/pj_base/parser_module/module.hpp diff --git a/pj_base/include/pj_base/parser_module/object_writer.hpp b/experimental/parser_module/include/pj_base/parser_module/object_writer.hpp similarity index 100% rename from pj_base/include/pj_base/parser_module/object_writer.hpp rename to experimental/parser_module/include/pj_base/parser_module/object_writer.hpp diff --git a/pj_base/include/pj_base/parser_module/proto_field_locator.hpp b/experimental/parser_module/include/pj_base/parser_module/proto_field_locator.hpp similarity index 100% rename from pj_base/include/pj_base/parser_module/proto_field_locator.hpp rename to experimental/parser_module/include/pj_base/parser_module/proto_field_locator.hpp diff --git a/pj_base/include/pj_base/parser_module/proto_reader.hpp b/experimental/parser_module/include/pj_base/parser_module/proto_reader.hpp similarity index 100% rename from pj_base/include/pj_base/parser_module/proto_reader.hpp rename to experimental/parser_module/include/pj_base/parser_module/proto_reader.hpp diff --git a/pj_base/include/pj_base/parser_module/time.hpp b/experimental/parser_module/include/pj_base/parser_module/time.hpp similarity index 100% rename from pj_base/include/pj_base/parser_module/time.hpp rename to experimental/parser_module/include/pj_base/parser_module/time.hpp diff --git a/pj_base/include/pj_base/parser_module_abi.h b/experimental/parser_module/include/pj_base/parser_module_abi.h similarity index 100% rename from pj_base/include/pj_base/parser_module_abi.h rename to experimental/parser_module/include/pj_base/parser_module_abi.h diff --git a/pj_base/include/pj_base/parser_module_manifest.hpp b/experimental/parser_module/include/pj_base/parser_module_manifest.hpp similarity index 100% rename from pj_base/include/pj_base/parser_module_manifest.hpp rename to experimental/parser_module/include/pj_base/parser_module_manifest.hpp diff --git a/pj_plugins/include/pj_plugins/host/native_parser_module.hpp b/experimental/parser_module/include/pj_plugins/host/native_parser_module.hpp similarity index 100% rename from pj_plugins/include/pj_plugins/host/native_parser_module.hpp rename to experimental/parser_module/include/pj_plugins/host/native_parser_module.hpp diff --git a/experimental/parser_module/include/pj_plugins/host/parser_module_claims.hpp b/experimental/parser_module/include/pj_plugins/host/parser_module_claims.hpp new file mode 100644 index 00000000..9d342d59 --- /dev/null +++ b/experimental/parser_module/include/pj_plugins/host/parser_module_claims.hpp @@ -0,0 +1,41 @@ +#pragma once +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +/** + * @file parser_module_claims.hpp + * @brief Parser-module manifest decoding into ParserClaimCatalog claims. + */ + +#include +#include +#include +#include + +#include "pj_base/diagnostic_sink.hpp" +#include "pj_base/expected.hpp" +#include "pj_plugins/host/parser_claim_catalog.hpp" + +namespace PJ { + +/// Parsed module metadata. Claim order remains manifest order. +struct ParserModuleManifest { + std::string id; + std::string name; + std::string version; + std::vector claims; +}; + +/// Decode and validate one complete parser-module manifest. Provenance is +/// supplied by the host and is never read from JSON. +[[nodiscard]] Expected decodeParserModuleManifest( + std::string_view manifest_json, ParserClaimProvenance provenance); + +/// Decode a complete module manifest and atomically admit its claims into +/// `catalog`. A manifest that fails to decode is reported to `sink` (pass the +/// catalog's own sink); admission failures are reported by the catalog. +[[nodiscard]] Expected ingestParserModuleManifest( + ParserClaimCatalog& catalog, std::string_view manifest_json, ParserClaimProvenance provenance, + uint64_t provider_generation, const DiagnosticSink& sink = {}); + +} // namespace PJ diff --git a/pj_plugins/include/pj_plugins/host/parser_module_runtime.hpp b/experimental/parser_module/include/pj_plugins/host/parser_module_runtime.hpp similarity index 100% rename from pj_plugins/include/pj_plugins/host/parser_module_runtime.hpp rename to experimental/parser_module/include/pj_plugins/host/parser_module_runtime.hpp diff --git a/pj_plugins/src/detail/native_parser_module_loader.hpp b/experimental/parser_module/src/detail/native_parser_module_loader.hpp similarity index 100% rename from pj_plugins/src/detail/native_parser_module_loader.hpp rename to experimental/parser_module/src/detail/native_parser_module_loader.hpp diff --git a/pj_plugins/src/detail/native_parser_module_state.hpp b/experimental/parser_module/src/detail/native_parser_module_state.hpp similarity index 100% rename from pj_plugins/src/detail/native_parser_module_state.hpp rename to experimental/parser_module/src/detail/native_parser_module_state.hpp diff --git a/pj_plugins/src/native_parser_module.cpp b/experimental/parser_module/src/native_parser_module.cpp similarity index 100% rename from pj_plugins/src/native_parser_module.cpp rename to experimental/parser_module/src/native_parser_module.cpp diff --git a/pj_base/src/parser_module_abi.cpp b/experimental/parser_module/src/parser_module_abi.cpp similarity index 100% rename from pj_base/src/parser_module_abi.cpp rename to experimental/parser_module/src/parser_module_abi.cpp diff --git a/experimental/parser_module/src/parser_module_claims.cpp b/experimental/parser_module/src/parser_module_claims.cpp new file mode 100644 index 00000000..c90424bb --- /dev/null +++ b/experimental/parser_module/src/parser_module_claims.cpp @@ -0,0 +1,247 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include "pj_plugins/host/parser_module_claims.hpp" + +#include +#include +#include +#include +#include +#include + +#include "pj_base/parser_module_abi.h" +#include "pj_base/parser_route_claims_protocol.h" +#include "pj_base/sdk/semver.hpp" + +namespace PJ { +namespace { + +[[nodiscard]] Expected requiredString( + const nlohmann::json& object, std::string_view key, std::string_view context) { + const auto it = object.find(std::string(key)); + if (it == object.end() || !it->is_string() || it->get_ref().empty()) { + return unexpected(std::string(context) + " missing required non-empty string key: " + std::string(key)); + } + return it->get(); +} + +[[nodiscard]] Expected requiredPriority(const nlohmann::json& object, size_t claim_index) { + const auto it = object.find("priority"); + if (it == object.end() || !it->is_number_integer()) { + return unexpected("parser module manifest claim " + std::to_string(claim_index) + " missing integer priority"); + } + + if (it->is_number_unsigned()) { + const uint64_t value = it->get(); + if (value > static_cast(std::numeric_limits::max())) { + return unexpected("parser module manifest claim priority does not fit int32"); + } + return static_cast(value); + } + const int64_t value = it->get(); + if (value < std::numeric_limits::min() || value > std::numeric_limits::max()) { + return unexpected("parser module manifest claim priority does not fit int32"); + } + return static_cast(value); +} + +[[nodiscard]] Expected readRouteFlags(const nlohmann::json& claim, size_t claim_index) { + const auto routes = claim.find("routes"); + if (routes == claim.end() || !routes->is_array() || routes->empty()) { + return unexpected( + "parser module manifest claim " + std::to_string(claim_index) + " requires a non-empty routes array"); + } + + uint16_t flags = 0; + for (const auto& route : *routes) { + if (!route.is_string()) { + return unexpected("parser module manifest claim " + std::to_string(claim_index) + " routes must contain strings"); + } + const auto& name = route.get_ref(); + if (name == "scalar") { + flags |= PJ_PARSER_ROUTE_FLAG_SCALAR_V1; + } else if (name == "object") { + flags |= PJ_PARSER_ROUTE_FLAG_OBJECT_V1; + } else { + return unexpected("parser module manifest claim uses unknown route: " + name); + } + } + return flags; +} + +[[nodiscard]] Expected> readSchemaDigests(const nlohmann::json& claim, size_t claim_index) { + std::set digests; + const auto values = claim.find("schema_digests"); + if (values == claim.end()) { + return digests; + } + if (!values->is_array()) { + return unexpected( + "parser module manifest claim " + std::to_string(claim_index) + " schema_digests must be an array"); + } + for (const auto& value : *values) { + if (!value.is_string()) { + return unexpected( + "parser module manifest claim " + std::to_string(claim_index) + " schema_digests must contain strings"); + } + digests.insert(value.get()); + } + return digests; +} + +} // namespace + +Expected decodeParserModuleManifest( + std::string_view manifest_json, ParserClaimProvenance provenance) { + if (manifest_json.empty()) { + return unexpected("parser module manifest is empty"); + } + + nlohmann::json json; + try { + json = nlohmann::json::parse(manifest_json); + } catch (const nlohmann::json::exception& error) { + return unexpected("parser module manifest is invalid JSON: " + std::string(error.what())); + } + if (!json.is_object()) { + return unexpected("parser module manifest must be a JSON object"); + } + if (json.contains("provenance")) { + return unexpected("parser module manifest must not declare host-owned provenance"); + } + + const auto abi = json.find("module_abi"); + if (abi == json.end() || !abi->is_number_integer()) { + return unexpected("parser module manifest missing integer module_abi"); + } + // The host ABI version is positive, so any signed (hence negative) encoding + // of module_abi is a mismatch by construction. + if (!abi->is_number_unsigned() || abi->get() != PJ_PARSER_MODULE_ABI_VERSION) { + return unexpected( + "parser module manifest module_abi does not match host ABI " + std::to_string(PJ_PARSER_MODULE_ABI_VERSION)); + } + + auto id = requiredString(json, "id", "parser module manifest"); + if (!id) { + return unexpected(id.error()); + } + auto name = requiredString(json, "name", "parser module manifest"); + if (!name) { + return unexpected(name.error()); + } + auto version = requiredString(json, "version", "parser module manifest"); + if (!version) { + return unexpected(version.error()); + } + if (auto parsed = SemVer::parse(*version); !parsed) { + return unexpected("parser module manifest version is not valid SemVer: " + parsed.error()); + } + + const auto claims_json = json.find("claims"); + if (claims_json == json.end() || !claims_json->is_array()) { + return unexpected("parser module manifest requires a claims array"); + } + + ParserModuleManifest manifest{.id = *id, .name = *name, .version = *version, .claims = {}}; + manifest.claims.reserve(claims_json->size()); + + for (size_t index = 0; index < claims_json->size(); ++index) { + const auto& claim_json = (*claims_json)[index]; + if (!claim_json.is_object()) { + return unexpected("parser module manifest claim " + std::to_string(index) + " must be an object"); + } + if (claim_json.contains("provenance")) { + return unexpected("parser module manifest claim must not declare host-owned provenance"); + } + + auto claim_id = requiredString(claim_json, "claim_id", "parser module manifest claim"); + if (!claim_id) { + return unexpected(claim_id.error()); + } + auto encoding = requiredString(claim_json, "encoding", "parser module manifest claim"); + if (!encoding) { + return unexpected(encoding.error()); + } + auto type_name = requiredString(claim_json, "type_name", "parser module manifest claim"); + if (!type_name) { + return unexpected(type_name.error()); + } + auto route_flags = readRouteFlags(claim_json, index); + if (!route_flags) { + return unexpected(route_flags.error()); + } + auto priority = requiredPriority(claim_json, index); + if (!priority) { + return unexpected(priority.error()); + } + auto schema_digests = readSchemaDigests(claim_json, index); + if (!schema_digests) { + return unexpected(schema_digests.error()); + } + + std::optional object_type; + const auto object_type_json = claim_json.find("object_type"); + if (object_type_json != claim_json.end()) { + if (!object_type_json->is_string()) { + return unexpected("parser module manifest claim object_type must be a string"); + } + object_type = sdk::parseBuiltinObjectType(object_type_json->get_ref()); + if (!object_type.has_value() || *object_type == sdk::BuiltinObjectType::kNone) { + return unexpected( + "parser module manifest claim uses unknown object_type: " + object_type_json->get()); + } + } + + ParserClaim claim{ + .encoding = *encoding, + .type_name = *type_name, + .route_flags = *route_flags, + .object_type = object_type, + .schema_digests = std::move(*schema_digests), + .provider_id = manifest.id, + .claim_id = *claim_id, + .priority = *priority, + .provenance = provenance, + }; + manifest.claims.push_back(std::move(claim)); + } + + // Validate and normalize exactly as admission does, without widening the + // catalog's public API: admit into a scratch catalog and read the claims back. + ParserClaimCatalog scratch; + if (auto admitted = scratch.admitClaims(std::move(manifest.claims), provenance, 0); !admitted) { + return unexpected(admitted.error()); + } + manifest.claims.clear(); + for (const auto& entry : scratch.claims()) { + manifest.claims.push_back(entry.claim); + } + return manifest; +} + +Expected ingestParserModuleManifest( + ParserClaimCatalog& catalog, std::string_view manifest_json, ParserClaimProvenance provenance, + uint64_t provider_generation, const DiagnosticSink& sink) { + auto manifest = decodeParserModuleManifest(manifest_json, provenance); + if (!manifest) { + if (sink) { + sink( + Diagnostic{ + .level = DiagnosticLevel::kError, + .source = "ParserModuleManifest", + .id = {}, + .message = manifest.error(), + .timestamp = std::chrono::system_clock::now(), + }); + } + return unexpected(manifest.error()); + } + auto admission = catalog.admitClaims(manifest->claims, provenance, provider_generation); + if (!admission) { + return unexpected(admission.error()); + } + return manifest; +} + +} // namespace PJ diff --git a/pj_base/src/parser_module_manifest.cpp b/experimental/parser_module/src/parser_module_manifest.cpp similarity index 100% rename from pj_base/src/parser_module_manifest.cpp rename to experimental/parser_module/src/parser_module_manifest.cpp diff --git a/pj_plugins/src/parser_module_runtime.cpp b/experimental/parser_module/src/parser_module_runtime.cpp similarity index 100% rename from pj_plugins/src/parser_module_runtime.cpp rename to experimental/parser_module/src/parser_module_runtime.cpp diff --git a/pj_plugins/tests/native_parser_module_fixture.cpp b/experimental/parser_module/tests/native_parser_module_fixture.cpp similarity index 100% rename from pj_plugins/tests/native_parser_module_fixture.cpp rename to experimental/parser_module/tests/native_parser_module_fixture.cpp diff --git a/pj_plugins/tests/native_parser_module_fixture.hpp b/experimental/parser_module/tests/native_parser_module_fixture.hpp similarity index 100% rename from pj_plugins/tests/native_parser_module_fixture.hpp rename to experimental/parser_module/tests/native_parser_module_fixture.hpp diff --git a/pj_plugins/tests/native_parser_module_test.cpp b/experimental/parser_module/tests/native_parser_module_test.cpp similarity index 96% rename from pj_plugins/tests/native_parser_module_test.cpp rename to experimental/parser_module/tests/native_parser_module_test.cpp index 8604f8be..cb2f0b77 100644 --- a/pj_plugins/tests/native_parser_module_test.cpp +++ b/experimental/parser_module/tests/native_parser_module_test.cpp @@ -13,7 +13,7 @@ #include "native_parser_module_fixture.hpp" #include "pj_base/parser_module_abi.h" -#include "pj_plugins/host/parser_claim_catalog.hpp" +#include "pj_plugins/host/parser_module_claims.hpp" namespace PJ { namespace { @@ -66,7 +66,7 @@ TEST(NativeParserModule, LoadsCompleteAbiAndCopiesManifestForCatalogAdmission) { EXPECT_TRUE(diagnostics.empty()); ParserClaimCatalog catalog; - auto manifest = catalog.ingestModuleManifest(module->manifestJson(), ParserClaimProvenance::kFolderDrop, 7); + auto manifest = ingestParserModuleManifest(catalog, module->manifestJson(), ParserClaimProvenance::kFolderDrop, 7); ASSERT_TRUE(manifest.has_value()) << manifest.error(); EXPECT_EQ(manifest->id, "org.plotjuggler.test.native-module"); EXPECT_EQ(manifest->claims.size(), pj_fixture::kClaimCount); diff --git a/pj_base/tests/parser_module_abi_test.cpp b/experimental/parser_module/tests/parser_module_abi_test.cpp similarity index 83% rename from pj_base/tests/parser_module_abi_test.cpp rename to experimental/parser_module/tests/parser_module_abi_test.cpp index 44b77aae..79230332 100644 --- a/pj_base/tests/parser_module_abi_test.cpp +++ b/experimental/parser_module/tests/parser_module_abi_test.cpp @@ -15,6 +15,44 @@ #include "pj_base/builtin_object_abi.h" +// Export ABI pins: a change here is a module ABI break. +static_assert(PJ_PARSER_MODULE_ABI_VERSION == 1, "parser module ABI version pinned"); +static_assert(PJ_MODULE_OK == 0, "parser module OK result pinned"); +static_assert(PJ_MODULE_DECLINE == 1, "parser module DECLINE result pinned"); +static_assert(PJ_MODULE_ERR_GENERIC == -1, "parser module generic error pinned"); +static_assert(PJ_MODULE_ERR_BAD_TOKEN == -2, "parser module bad-token error pinned"); +static_assert(PJ_MODULE_ERR_MALFORMED_INPUT == -3, "parser module malformed-input error pinned"); +static_assert(PJ_MODULE_ERR_BAD_CLAIM_INDEX == -4, "parser module bad-claim-index error pinned"); +static_assert(PJ_MODULE_ERR_ALLOCATION_FAILURE == -5, "parser module allocation error pinned"); +static_assert(PJ_MODULE_CREATION_ERROR_TOKEN == 0, "parser module creation-error token pinned"); +static_assert(PJ_MODULE_ERROR_BUFFER_SIZE == 512, "parser module error buffer size pinned"); +static_assert(std::string_view(PJ_MODULE_ABI_EXPORT_NAME) == "pj_module_abi", "parser module ABI export name pinned"); +static_assert( + std::string_view(PJ_MODULE_CREATE_EXPORT_NAME) == "pj_module_create", "parser module create export name pinned"); +static_assert( + std::string_view(PJ_MODULE_DESTROY_EXPORT_NAME) == "pj_module_destroy", "parser module destroy export name pinned"); +static_assert( + std::string_view(PJ_MODULE_BIND_EXPORT_NAME) == "pj_module_bind", "parser module bind export name pinned"); +static_assert( + std::string_view(PJ_MODULE_PARSE_EXPORT_NAME) == "pj_module_parse", "parser module parse export name pinned"); +static_assert( + std::string_view(PJ_MODULE_LAST_ERROR_EXPORT_NAME) == "pj_module_last_error", + "parser module last-error export name pinned"); +static_assert( + std::string_view(PJ_MODULE_ALLOC_EXPORT_NAME) == "pj_module_alloc", "parser module alloc export name pinned"); +static_assert( + std::string_view(PJ_MODULE_FREE_EXPORT_NAME) == "pj_module_free", "parser module free export name pinned"); +static_assert( + std::string_view(PJ_MODULE_MANIFEST_ADDR_EXPORT_NAME) == "pj_module_manifest_addr", + "native parser module manifest-address export name pinned"); +static_assert( + std::string_view(PJ_MODULE_MANIFEST_LEN_EXPORT_NAME) == "pj_module_manifest_len", + "native parser module manifest-length export name pinned"); +static_assert( + std::string_view(PJ_PARSER_MODULE_MANIFEST_SECTION_NAME) == "pj_parser_module_manifest", + "wasm parser module manifest section name pinned"); +static_assert(sizeof(PJ_module_parse_fn_t) == sizeof(void*), "parser module function pointer width pinned"); + namespace { PJ::Span bytesOf(std::string_view value) { diff --git a/pj_plugins/tests/parser_module_authoring_e2e_test.cpp b/experimental/parser_module/tests/parser_module_authoring_e2e_test.cpp similarity index 96% rename from pj_plugins/tests/parser_module_authoring_e2e_test.cpp rename to experimental/parser_module/tests/parser_module_authoring_e2e_test.cpp index 1c19dc83..fdff03bf 100644 --- a/pj_plugins/tests/parser_module_authoring_e2e_test.cpp +++ b/experimental/parser_module/tests/parser_module_authoring_e2e_test.cpp @@ -14,7 +14,7 @@ #include "pj_base/builtin_object_abi.h" #include "pj_base/parser_module_abi.h" #include "pj_plugins/host/native_parser_module.hpp" -#include "pj_plugins/host/parser_claim_catalog.hpp" +#include "pj_plugins/host/parser_module_claims.hpp" #include "pj_plugins/host/parser_module_runtime.hpp" namespace PJ { @@ -62,7 +62,7 @@ TEST(ParserModuleAuthoringE2E, LoadsAdmitsBindsAndParsesFullAndSplicedPointCloud auto module = NativeParserModule::load(PJ_TOY_CDR_POINTCLOUD_MODULE_PATH); ASSERT_TRUE(module.has_value()) << module.error(); ParserClaimCatalog catalog; - auto manifest = catalog.ingestModuleManifest(module->manifestJson(), ParserClaimProvenance::kFolderDrop, 12); + auto manifest = ingestParserModuleManifest(catalog, module->manifestJson(), ParserClaimProvenance::kFolderDrop, 12); ASSERT_TRUE(manifest.has_value()) << manifest.error(); ASSERT_EQ(manifest->claims.size(), 2U); diff --git a/experimental/parser_module/tests/parser_module_claims_test.cpp b/experimental/parser_module/tests/parser_module_claims_test.cpp new file mode 100644 index 00000000..d03ba0a4 --- /dev/null +++ b/experimental/parser_module/tests/parser_module_claims_test.cpp @@ -0,0 +1,122 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include "pj_plugins/host/parser_module_claims.hpp" + +#include + +#include +#include +#include + +#include "pj_base/parser_route_claims_protocol.h" + +namespace PJ { +namespace { + +DiagnosticSink recordInto(std::vector& diagnostics) { + return [&diagnostics](const Diagnostic& diagnostic) { diagnostics.push_back(diagnostic); }; +} + +TEST(ParserModuleClaims, ModuleManifestDecodesValidatedClaimsAndMetadata) { + const auto manifest = decodeParserModuleManifest( + R"({ + "module_abi": 1, + "id": "com.example.radar", + "name": "Radar Parsers", + "version": "1.2.3-rc.1+build.7", + "claims": [ + {"claim_id":"object","encoding":"ros2msg","type_name":"radar_msgs/RadarScan", + "routes":["scalar","object"],"object_type":"kPointCloud", + "schema_digests":[ + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],"priority":17}, + {"claim_id":"fallback","encoding":"protobuf","type_name":".example.RadarScan", + "routes":["scalar"],"priority":-2} + ] + })", + ParserClaimProvenance::kMarketplace); + + ASSERT_TRUE(manifest.has_value()) << manifest.error(); + EXPECT_EQ(manifest->id, "com.example.radar"); + EXPECT_EQ(manifest->name, "Radar Parsers"); + EXPECT_EQ(manifest->version, "1.2.3-rc.1+build.7"); + ASSERT_EQ(manifest->claims.size(), 2U); + EXPECT_EQ(manifest->claims[0].type_name, "radar_msgs/msg/RadarScan"); + EXPECT_EQ(manifest->claims[0].route_flags, PJ_PARSER_ROUTE_FLAG_SCALAR_V1 | PJ_PARSER_ROUTE_FLAG_OBJECT_V1); + EXPECT_EQ(manifest->claims[0].object_type, sdk::BuiltinObjectType::kPointCloud); + EXPECT_EQ(manifest->claims[0].schema_digests.size(), 2U); + EXPECT_EQ(manifest->claims[0].provenance, ParserClaimProvenance::kMarketplace); + EXPECT_EQ(manifest->claims[1].type_name, "example.RadarScan"); + EXPECT_FALSE(manifest->claims[1].object_type.has_value()); +} + +TEST(ParserModuleClaims, ModuleManifestRejectsMalformedJsonAndAbiVersion) { + for (const std::string_view manifest : { + "not json", + R"([])", + R"({"module_abi":2,"id":"x","name":"X","version":"1.0.0","claims":[]})", + R"({"id":"x","name":"X","version":"1.0.0","claims":[]})", + }) { + SCOPED_TRACE(manifest); + const auto parsed = decodeParserModuleManifest(manifest, ParserClaimProvenance::kFolderDrop); + EXPECT_FALSE(parsed.has_value()); + } +} + +TEST(ParserModuleClaims, MalformedModuleManifestEmitsOneDiagnosticAndAdmitsNothing) { + std::vector diagnostics; + ParserClaimCatalog catalog(recordInto(diagnostics)); + + const auto result = + ingestParserModuleManifest(catalog, "not json", ParserClaimProvenance::kFolderDrop, 1, recordInto(diagnostics)); + + ASSERT_FALSE(result.has_value()); + ASSERT_EQ(diagnostics.size(), 1U); + EXPECT_EQ(diagnostics.front().level, DiagnosticLevel::kError); + EXPECT_NE(diagnostics.front().message.find("invalid JSON"), std::string::npos); + EXPECT_TRUE(catalog.claims().empty()); +} + +TEST(ParserModuleClaims, ModuleManifestRequiresStableIdentityNameAndThreePartSemver) { + for (const std::string_view manifest : { + R"({"module_abi":1,"name":"X","version":"1.0.0","claims":[]})", + R"({"module_abi":1,"id":"x","version":"1.0.0","claims":[]})", + R"({"module_abi":1,"id":"x","name":"X","version":"1.0","claims":[]})", + }) { + SCOPED_TRACE(manifest); + const auto parsed = decodeParserModuleManifest(manifest, ParserClaimProvenance::kBundled); + EXPECT_FALSE(parsed.has_value()); + } +} + +TEST(ParserModuleClaims, ModuleManifestRejectsDuplicateClaimsAsAWhole) { + std::vector diagnostics; + ParserClaimCatalog catalog(recordInto(diagnostics)); + const auto result = ingestParserModuleManifest( + catalog, + R"({"module_abi":1,"id":"module","name":"Module","version":"1.0.0","claims":[ + {"claim_id":"same","encoding":"protobuf","type_name":"a.Type","routes":["scalar"],"priority":0}, + {"claim_id":"same","encoding":"protobuf","type_name":"b.Type","routes":["scalar"],"priority":0} + ]})", + ParserClaimProvenance::kFolderDrop, 1, recordInto(diagnostics)); + + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().find("duplicate"), std::string::npos); + EXPECT_TRUE(catalog.claims().empty()); + ASSERT_EQ(diagnostics.size(), 1U); +} + +TEST(ParserModuleClaims, ModuleManifestCannotForgeProvenance) { + const auto result = decodeParserModuleManifest( + R"({"module_abi":1,"id":"module","name":"Module","version":"1.0.0", + "provenance":"bundled","claims":[]})", + ParserClaimProvenance::kFolderDrop); + + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().find("provenance"), std::string::npos); +} + +} // namespace +} // namespace PJ diff --git a/pj_base/tests/parser_module_kit_test.cpp b/experimental/parser_module/tests/parser_module_kit_test.cpp similarity index 100% rename from pj_base/tests/parser_module_kit_test.cpp rename to experimental/parser_module/tests/parser_module_kit_test.cpp diff --git a/pj_base/tests/parser_module_manifest_test.cpp b/experimental/parser_module/tests/parser_module_manifest_test.cpp similarity index 100% rename from pj_base/tests/parser_module_manifest_test.cpp rename to experimental/parser_module/tests/parser_module_manifest_test.cpp diff --git a/pj_base/tests/parser_module_object_writer_test.cpp b/experimental/parser_module/tests/parser_module_object_writer_test.cpp similarity index 100% rename from pj_base/tests/parser_module_object_writer_test.cpp rename to experimental/parser_module/tests/parser_module_object_writer_test.cpp diff --git a/pj_plugins/tests/parser_module_runtime_test.cpp b/experimental/parser_module/tests/parser_module_runtime_test.cpp similarity index 100% rename from pj_plugins/tests/parser_module_runtime_test.cpp rename to experimental/parser_module/tests/parser_module_runtime_test.cpp diff --git a/pj_base/tests/parser_module_wasm_audit.cpp b/experimental/parser_module/tests/parser_module_wasm_audit.cpp similarity index 100% rename from pj_base/tests/parser_module_wasm_audit.cpp rename to experimental/parser_module/tests/parser_module_wasm_audit.cpp diff --git a/pj_base/tests/toy_cdr_pointcloud.module.json b/experimental/parser_module/tests/toy_cdr_pointcloud.module.json similarity index 100% rename from pj_base/tests/toy_cdr_pointcloud.module.json rename to experimental/parser_module/tests/toy_cdr_pointcloud.module.json diff --git a/pj_base/tests/toy_cdr_pointcloud_module.cpp b/experimental/parser_module/tests/toy_cdr_pointcloud_module.cpp similarity index 100% rename from pj_base/tests/toy_cdr_pointcloud_module.cpp rename to experimental/parser_module/tests/toy_cdr_pointcloud_module.cpp diff --git a/pj_base/CMakeLists.txt b/pj_base/CMakeLists.txt index cb9de01d..911f0810 100644 --- a/pj_base/CMakeLists.txt +++ b/pj_base/CMakeLists.txt @@ -33,8 +33,6 @@ add_library(pj_base STATIC src/builtin/mesh_map_update_codec.cpp src/number_parse.cpp src/time_format.cpp - src/parser_module_abi.cpp - src/parser_module_manifest.cpp src/semver.cpp src/type_tree.cpp src/data_source_host_views.cpp @@ -74,18 +72,6 @@ if(PJ_ASSERT_THROWS) target_compile_definitions(pj_base PUBLIC PJ_ASSERT_THROWS) endif() -# Standalone, header-only functional parser-module authoring kit. This target -# intentionally carries include paths and a C++ floor only: modules link no SDK -# library and remain suitable for native and WASI reactor builds. -add_library(pj_parser_module INTERFACE) -target_include_directories(pj_parser_module INTERFACE - $ - $ -) -target_compile_features(pj_parser_module INTERFACE cxx_std_17) -set_target_properties(pj_parser_module PROPERTIES EXPORT_NAME parser_module) -add_library(plotjuggler_sdk::parser_module ALIAS pj_parser_module) - # --------------------------------------------------------------------------- # pj_source — the callee side of pj.descriptor_import.v1 for # provider plugins: origin policy, canonical descriptors, the request- @@ -139,7 +125,7 @@ add_library(plotjuggler_sdk::descriptor_import_support ALIAS pj_descriptor_impor # --------------------------------------------------------------------------- if(PJ_INSTALL_SDK) - install(TARGETS pj_base pj_parser_module pj_source pj_descriptor_import_support EXPORT plotjuggler_sdkTargets + install(TARGETS pj_base pj_source pj_descriptor_import_support EXPORT plotjuggler_sdkTargets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) @@ -164,8 +150,6 @@ if(PJ_BUILD_TESTS) tests/number_parse_test.cpp tests/semver_test.cpp tests/plugin_data_api_test.cpp - tests/parser_module_abi_test.cpp - tests/parser_module_manifest_test.cpp tests/playback_viewport_api_test.cpp tests/plot_tabs_api_test.cpp tests/dataset_qualified_name_api_test.cpp @@ -246,109 +230,4 @@ if(PJ_BUILD_TESTS) target_link_libraries(${test_name} PRIVATE pj_source GTest::gtest_main) add_test(NAME ${test_name} COMMAND ${test_name}) endforeach() - - # C++17 standalone authoring-kit gate. This target intentionally does not - # link pj_base; GTest is test infrastructure only. - add_executable(parser_module_kit_test tests/parser_module_kit_test.cpp) - set_target_properties(parser_module_kit_test PROPERTIES - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED YES - CXX_EXTENSIONS NO - ) - target_compile_options(parser_module_kit_test PRIVATE ${PJ_WARNING_FLAGS}) - target_link_libraries(parser_module_kit_test PRIVATE pj_parser_module GTest::gtest_main) - add_test(NAME parser_module_kit_test COMMAND parser_module_kit_test) - - add_executable(parser_module_object_writer_test tests/parser_module_object_writer_test.cpp) - target_compile_definitions(parser_module_object_writer_test PRIVATE PJ_PARSER_MODULE_CLAIM_COUNT=1) - target_compile_options(parser_module_object_writer_test PRIVATE ${PJ_WARNING_FLAGS}) - target_link_libraries(parser_module_object_writer_test PRIVATE pj_parser_module pj_base GTest::gtest_main) - add_test(NAME parser_module_object_writer_test COMMAND parser_module_object_writer_test) - - # Static wasm ABI conformance. The authoring source is compiled directly by - # wasi-sdk, then a host-side auditor embeds and validates the final module. - set(PJ_WASI_SDK_ROOT "$ENV{PJ_WASI_SDK_ROOT}" CACHE PATH - "wasi-sdk 27 root used for parser-module conformance tests") - set(_pj_wasi_clang "${PJ_WASI_SDK_ROOT}/bin/clang++") - set(_pj_wasi_sysroot "${PJ_WASI_SDK_ROOT}/share/wasi-sysroot") - if(NOT PJ_WASI_SDK_ROOT) - message(STATUS - "Parser-module WASI conformance skipped: PJ_WASI_SDK_ROOT cache/env variable is not set") - elseif(NOT EXISTS "${_pj_wasi_clang}" OR NOT IS_DIRECTORY "${_pj_wasi_sysroot}") - message(STATUS - "Parser-module WASI conformance skipped: wasi-sdk is missing under PJ_WASI_SDK_ROOT=${PJ_WASI_SDK_ROOT}") - else() - file(STRINGS "${PJ_WASI_SDK_ROOT}/VERSION" _pj_wasi_version LIMIT_COUNT 1) - if(NOT _pj_wasi_version MATCHES "^27\\.") - message(FATAL_ERROR - "Parser-module WASI conformance requires wasi-sdk 27; found '${_pj_wasi_version}'") - endif() - message(STATUS - "Parser-module WASI conformance enabled with wasi-sdk ${_pj_wasi_version}: ${PJ_WASI_SDK_ROOT}") - - add_executable(parser_module_wasm_audit tests/parser_module_wasm_audit.cpp) - target_compile_options(parser_module_wasm_audit PRIVATE ${PJ_WARNING_FLAGS}) - target_link_libraries(parser_module_wasm_audit PRIVATE pj_base) - - file(GLOB _pj_parser_module_headers CONFIGURE_DEPENDS - "${CMAKE_CURRENT_SOURCE_DIR}/include/pj_base/parser_module/*.hpp") - set(_pj_wasm_dir "${CMAKE_CURRENT_BINARY_DIR}/parser_module_wasm_conformance") - set(_pj_wasm_raw "${_pj_wasm_dir}/toy_cdr_pointcloud.raw.wasm") - set(_pj_wasm_embedded "${_pj_wasm_dir}/toy_cdr_pointcloud.wasm") - set(_pj_wasm_source "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud_module.cpp") - set(_pj_wasm_manifest "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud.module.json") - - add_custom_command( - OUTPUT "${_pj_wasm_raw}" - COMMAND ${CMAKE_COMMAND} -E make_directory "${_pj_wasm_dir}" - COMMAND "${_pj_wasi_clang}" - --target=wasm32-wasip1 - --sysroot=${_pj_wasi_sysroot} - -mexec-model=reactor - -std=c++17 - -fno-exceptions - -fno-rtti - -fvisibility=hidden - -O1 - -Wall -Wextra -Werror - -DPJ_PARSER_MODULE_CLAIM_COUNT=2 - -I${CMAKE_CURRENT_SOURCE_DIR}/include - "${_pj_wasm_source}" - -Wl,--export=pj_module_abi - -Wl,--export=pj_module_create - -Wl,--export=pj_module_destroy - -Wl,--export=pj_module_bind - -Wl,--export=pj_module_parse - -Wl,--export=pj_module_last_error - -Wl,--export=pj_module_alloc - -Wl,--export=pj_module_free - -o "${_pj_wasm_raw}" - DEPENDS "${_pj_wasm_source}" ${_pj_parser_module_headers} - "${CMAKE_CURRENT_SOURCE_DIR}/include/pj_base/time_math.hpp" - COMMENT "Compiling C++17 parser-module WASI reactor fixture" - VERBATIM - ) - add_custom_command( - OUTPUT "${_pj_wasm_embedded}" - COMMAND ${CMAKE_COMMAND} -E env ASAN_OPTIONS=detect_leaks=0 - $ - --embed "${_pj_wasm_raw}" "${_pj_wasm_manifest}" "${_pj_wasm_embedded}" - DEPENDS "${_pj_wasm_raw}" "${_pj_wasm_manifest}" parser_module_wasm_audit - COMMENT "Embedding parser-module wasm manifest with the shared codec" - VERBATIM - ) - add_custom_target(parser_module_wasm_conformance_fixture ALL - DEPENDS "${_pj_wasm_embedded}") - add_test(NAME parser_module_wasm_conformance_test - COMMAND parser_module_wasm_audit --audit "${_pj_wasm_embedded}" "${_pj_wasm_manifest}") - set_tests_properties(parser_module_wasm_conformance_test PROPERTIES - LABELS "parser_module;wasi" - REQUIRED_FILES "${_pj_wasm_embedded}") - endif() - - pj_add_parser_module(toy_cdr_pointcloud_module - SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud_module.cpp" - MANIFEST "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud.module.json" - TARGETS native - ) endif() diff --git a/pj_base/feature_floors.json b/pj_base/feature_floors.json index f3d668fd..73b27922 100644 --- a/pj_base/feature_floors.json +++ b/pj_base/feature_floors.json @@ -59,7 +59,6 @@ "PJ_DATA_PROCESSOR_FLAG_EPHEMERAL", "PJ_DESCRIPTOR_IMPORT_START_FLAGS_V1_MASK", "PJ_DESCRIPTOR_IMPORT_START_FLAG_NONE", - "PJ_MODULE_PARSE_INPUT_FLAG_HAS_TIMESTAMP", "PJ_PARSER_ROUTE_FLAG_OBJECT_V1", "PJ_PARSER_ROUTE_FLAG_SCALAR_V1", "PJ_colormap_registry_vtable_t::register_map", diff --git a/pj_base/include/pj_base/parser_module/README.md b/pj_base/include/pj_base/parser_module/README.md deleted file mode 100644 index 2dca85b4..00000000 --- a/pj_base/include/pj_base/parser_module/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Functional parser-module authoring kit - -This directory provides a standalone, header-only C++17 API. Headers may include -others in this subtree, C/C++ standard-library headers and the shared C++17 -checked arithmetic in `pj_base/time_math.hpp`. `time.hpp` uses that arithmetic. - -Headers must remain suitable for a WASI reactor build: no filesystem, threads, -iostreams, host SDK linkage or exceptions crossing public/ABI calls. - -All data received from a host or message is borrowed through `ByteView` / -`PayloadView`. Fallible operations return the local `pj::Status` or -`pj::Expected` types. Output storage is owned by `pj::Blob`, whose allocation -entry points report failure instead of exposing allocation exceptions. - -The umbrella include for module authors is: - -```cpp -#include - -class RawImageParser final : public pj::FunctionalParser { - public: - pj::Status bind(const pj::BindingInfo& info) override { - return info.route() == pj::Route::kObject - ? pj::Status::ok() - : pj::Status::decline("object route only"); - } - - pj::Status parseObject(pj::PayloadView payload, pj::Timestamp timestamp, - pj::ObjectWriter& output) override { - auto image = output.image(); - if (timestamp.has_value) { - if (auto status = image.setTimestamp(timestamp.nanoseconds); - !status.isOk()) { - return status; - } - } - if (auto status = image.setEncoding("mono8"); !status.isOk()) { - return status; - } - return image.setData(payload); - } -}; - -PJ_FUNCTIONAL_PARSER(RawImageParser) -``` - -Override `parseScalars(PayloadView, Timestamp, ScalarWriter&)` for scalar -claims. `ObjectWriter` provides `image`, `pointCloud`, `depthImage`, -`occupancyGrid`, `compressedPointCloud`, `mesh3D`, `videoFrame`, -`occupancyGridUpdate`, `voxelGrid`, and `gridMap` builders. - -Native modules are built with `pj_add_parser_module(... TARGETS native)`. The -target links no SDK library; its include paths expose this kit and the shared -checked time arithmetic. -`TARGETS wasm` is not available in SDK 0.22. The wasi-sdk gate compiles and -statically audits a reactor fixture, but it does not provide wasm authoring or -execution. - -See -`.claude/skills/plotjuggler-plugin/references/parser-module.md` at the repository -root for the manifest, schema-locator, splice, lifetime, and error contracts. diff --git a/pj_base/tests/abi_layout_sentinels_test.cpp b/pj_base/tests/abi_layout_sentinels_test.cpp index e9852faf..1f784580 100644 --- a/pj_base/tests/abi_layout_sentinels_test.cpp +++ b/pj_base/tests/abi_layout_sentinels_test.cpp @@ -34,7 +34,6 @@ #include "pj_base/descriptor_import_protocol.h" #include "pj_base/message_parser_protocol.h" #include "pj_base/parser_functional_protocol.h" -#include "pj_base/parser_module_abi.h" #include "pj_base/parser_route_claims_protocol.h" #include "pj_base/plugin_data_api.h" #include "pj_base/toolbox_protocol.h" @@ -207,43 +206,6 @@ static_assert(sizeof(PJ_parser_route_claims_v1_t) == 16, "route claims extension static_assert(PJ_PARSER_ROUTE_CLAIMS_V1_MIN_SIZE == 16, "route claims minimum size pinned"); #endif -static_assert(PJ_PARSER_MODULE_ABI_VERSION == 1, "parser module ABI version pinned"); -static_assert(PJ_MODULE_OK == 0, "parser module OK result pinned"); -static_assert(PJ_MODULE_DECLINE == 1, "parser module DECLINE result pinned"); -static_assert(PJ_MODULE_ERR_GENERIC == -1, "parser module generic error pinned"); -static_assert(PJ_MODULE_ERR_BAD_TOKEN == -2, "parser module bad-token error pinned"); -static_assert(PJ_MODULE_ERR_MALFORMED_INPUT == -3, "parser module malformed-input error pinned"); -static_assert(PJ_MODULE_ERR_BAD_CLAIM_INDEX == -4, "parser module bad-claim-index error pinned"); -static_assert(PJ_MODULE_ERR_ALLOCATION_FAILURE == -5, "parser module allocation error pinned"); -static_assert(PJ_MODULE_CREATION_ERROR_TOKEN == 0, "parser module creation-error token pinned"); -static_assert(PJ_MODULE_ERROR_BUFFER_SIZE == 512, "parser module error buffer size pinned"); -static_assert(std::string_view(PJ_MODULE_ABI_EXPORT_NAME) == "pj_module_abi", "parser module ABI export name pinned"); -static_assert( - std::string_view(PJ_MODULE_CREATE_EXPORT_NAME) == "pj_module_create", "parser module create export name pinned"); -static_assert( - std::string_view(PJ_MODULE_DESTROY_EXPORT_NAME) == "pj_module_destroy", "parser module destroy export name pinned"); -static_assert( - std::string_view(PJ_MODULE_BIND_EXPORT_NAME) == "pj_module_bind", "parser module bind export name pinned"); -static_assert( - std::string_view(PJ_MODULE_PARSE_EXPORT_NAME) == "pj_module_parse", "parser module parse export name pinned"); -static_assert( - std::string_view(PJ_MODULE_LAST_ERROR_EXPORT_NAME) == "pj_module_last_error", - "parser module last-error export name pinned"); -static_assert( - std::string_view(PJ_MODULE_ALLOC_EXPORT_NAME) == "pj_module_alloc", "parser module alloc export name pinned"); -static_assert( - std::string_view(PJ_MODULE_FREE_EXPORT_NAME) == "pj_module_free", "parser module free export name pinned"); -static_assert( - std::string_view(PJ_MODULE_MANIFEST_ADDR_EXPORT_NAME) == "pj_module_manifest_addr", - "native parser module manifest-address export name pinned"); -static_assert( - std::string_view(PJ_MODULE_MANIFEST_LEN_EXPORT_NAME) == "pj_module_manifest_len", - "native parser module manifest-length export name pinned"); -static_assert( - std::string_view(PJ_PARSER_MODULE_MANIFEST_SECTION_NAME) == "pj_parser_module_manifest", - "wasm parser module manifest section name pinned"); -static_assert(sizeof(PJ_module_parse_fn_t) == sizeof(void*), "parser module function pointer width pinned"); - static_assert(sizeof(PJ_payload_anchor_t) == 16, "PJ_payload_anchor_t pinned (ctx + release fn ptr)"); static_assert(offsetof(PJ_payload_anchor_t, ctx) == 0, "ctx at offset 0"); static_assert(offsetof(PJ_payload_anchor_t, release) == 8, "release at offset 8"); diff --git a/pj_plugins/CLAUDE.md b/pj_plugins/CLAUDE.md index e93c9cfd..2e80dd45 100644 --- a/pj_plugins/CLAUDE.md +++ b/pj_plugins/CLAUDE.md @@ -15,10 +15,8 @@ order, then read this file or [pj_base/CLAUDE.md](../pj_base/CLAUDE.md) as relev - `include/pj_plugins/host/` — DataSource/MessageParser/Toolbox loaders and RAII handles. `plugin_catalog` scans embedded manifests (`scanPluginDsos` / `inspectPluginDso`). `ParserClaimCatalog` and `ParserRouteResolver` handle - parser claim admission and per-route resolution. Native parser-module - loading/execution uses `NativeParserModule`, `NativeParserModuleInstance` - and `ParserModuleStrikeTracker`. Also contains `ServiceRegistryBuilder` - and `ConfigEnvelope`. + parser claim admission and per-route resolution. Also contains + `ServiceRegistryBuilder` and `ConfigEnvelope`. The DSO catalog that resolves duplicates and composes loaded plugin families is **host policy**. It lives in the app (`pj_runtime`, `PluginRuntimeCatalog`). - `include/pj_plugins/sdk/` — SDK pieces that live here, not in `pj_base`: @@ -44,19 +42,15 @@ order, then read this file or [pj_base/CLAUDE.md](../pj_base/CLAUDE.md) as relev - **The SDK is split across two modules.** `DataSourcePluginBase` / `ToolboxPluginBase` / `data_source_patterns.hpp` live in **`pj_base/sdk/`**; parser/dialog authoring and shared plugin policies live under - `pj_plugins/sdk/`. Functional parser-module authoring headers and the native - CMake helper live under `pj_base`; claim resolution and native module - loading/runtime live here. + `pj_plugins/sdk/`. Parser claim resolution lives here. Functional parser + modules (authoring kit, native loader and runtime) are experimental and live + outside the installed SDK, under `../experimental/parser_module/`. - **Handles keep the DSO mapped.** Every handle holds a `shared_ptr` library token, exposed via `libraryOwner()`. Destroying or hot-reloading the loader therefore cannot `dlclose` a live plugin. A lazy ObjectStore payload anchor can capture the token to stay safe after the handle dies; its `release` function is plugin code. Dialog handles provide non-owning `borrowed()` handles for source/toolbox embedded dialogs. These must not outlive the owning handle. -- **Native parser modules never unload in v1.** `NativeParserModule` resolves - the complete per-handle export set and retains every opened DSO for the - process session, including rejected artifacts. Instance wrappers still call - `pj_module_destroy`. Only the code mapping has session lifetime. ## Read deeper | For | Read | @@ -69,7 +63,6 @@ order, then read this file or [pj_base/CLAUDE.md](../pj_base/CLAUDE.md) as relev | Host loader + factory pattern | `include/pj_plugins/host/data_source_library.hpp`, `…/data_source_handle.hpp` | | Discovery from embedded manifests | `include/pj_plugins/host/plugin_catalog.hpp` (the duplicate-resolution catalog is host-side in `pj_runtime`) | | Parser claim admission and route selection | `include/pj_plugins/host/parser_claim_catalog.hpp`, `parser_route_resolver.hpp` | -| Native functional parser modules | `include/pj_plugins/host/native_parser_module.hpp`, `parser_module_runtime.hpp` | -| Authoring native functional parser modules | `../pj_base/include/pj_base/parser_module/README.md`, `module.hpp`, `../.claude/skills/plotjuggler-plugin/references/parser-module.md` | +| Functional parser modules (experimental, not installed) | `../experimental/parser_module/README.md`, `AUTHORING.md` | | Service wiring into `bind()` | `include/pj_plugins/host/service_registry_builder.hpp` | | Builtin-object ingest policy | `include/pj_plugins/sdk/object_ingest_policy.hpp` | diff --git a/pj_plugins/CMakeLists.txt b/pj_plugins/CMakeLists.txt index 8335737a..bbe21b7e 100644 --- a/pj_plugins/CMakeLists.txt +++ b/pj_plugins/CMakeLists.txt @@ -91,35 +91,6 @@ target_link_libraries(pj_parser_claim_catalog ) add_library(plotjuggler_sdk::parser_claim_catalog ALIAS pj_parser_claim_catalog) -# --------------------------------------------------------------------------- -# pj_parser_module_host — native functional-module loader and runtime -# --------------------------------------------------------------------------- - -add_library(pj_parser_module_host STATIC - src/native_parser_module.cpp - src/parser_module_runtime.cpp -) -target_include_directories(pj_parser_module_host - PUBLIC - $ - $ - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src -) -target_compile_features(pj_parser_module_host PUBLIC cxx_std_20) -target_compile_options(pj_parser_module_host PRIVATE ${PJ_WARNING_FLAGS}) -set_target_properties(pj_parser_module_host PROPERTIES - POSITION_INDEPENDENT_CODE ON - EXPORT_NAME parser_module_host -) -target_link_libraries(pj_parser_module_host - PUBLIC - pj_base - PRIVATE - ${CMAKE_DL_LIBS} -) -add_library(plotjuggler_sdk::parser_module_host ALIAS pj_parser_module_host) - # --------------------------------------------------------------------------- # pj_data_source_host — host-side DataSource library loader # --------------------------------------------------------------------------- @@ -280,9 +251,9 @@ target_link_libraries(pj_toolbox_host # --------------------------------------------------------------------------- # pj_plugin_host — umbrella INTERFACE that bundles every host-side loader. # Hosts (the PlotJuggler app, test rigs) link this single target to get -# discovery, all five plugin-family loaders, and functional parser-module -# execution. The duplicate-resolution catalog is host policy and lives in the -# app (pj_runtime), built on these primitives. +# discovery and all five plugin-family loaders. The duplicate-resolution +# catalog is host policy and lives in the app (pj_runtime), built on these +# primitives. # --------------------------------------------------------------------------- add_library(pj_plugin_host INTERFACE) @@ -294,41 +265,12 @@ target_link_libraries(pj_plugin_host INTERFACE pj_widget_host pj_plugin_catalog pj_parser_claim_catalog - pj_parser_module_host ) set_target_properties(pj_plugin_host PROPERTIES EXPORT_NAME plugin_host) add_library(plotjuggler_sdk::plugin_host ALIAS pj_plugin_host) if(PJ_BUILD_TESTS) -# --------------------------------------------------------------------------- -# Native functional parser-module fixtures (real shared libraries) -# --------------------------------------------------------------------------- - -# Trailing arguments become PRIVATE compile definitions, each selecting one -# deliberately defective variant of the shared fixture source. -function(pj_add_native_parser_module_fixture target) - add_library(${target} SHARED tests/native_parser_module_fixture.cpp) - target_compile_features(${target} PRIVATE cxx_std_20) - target_compile_options(${target} PRIVATE ${PJ_WARNING_FLAGS}) - set_target_properties(${target} PROPERTIES - CXX_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN YES - ) - target_link_libraries(${target} PRIVATE pj_base) - if(ARGN) - target_compile_definitions(${target} PRIVATE ${ARGN}) - endif() - if(UNIX AND NOT APPLE) - target_link_options(${target} PRIVATE "LINKER:-z,defs" "LINKER:--exclude-libs,ALL") - endif() -endfunction() - -pj_add_native_parser_module_fixture(native_parser_module_fixture) -pj_add_native_parser_module_fixture(native_parser_module_missing_export PJ_FIXTURE_OMIT_FREE) -pj_add_native_parser_module_fixture(native_parser_module_wrong_abi PJ_FIXTURE_WRONG_ABI) -pj_add_native_parser_module_fixture(native_parser_module_unreadable_manifest PJ_FIXTURE_UNREADABLE_MANIFEST) - # --------------------------------------------------------------------------- # Mock Toolbox plugin (shared library, for dlopen tests) # --------------------------------------------------------------------------- @@ -654,50 +596,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") endif() endif() -# Integration tests: native functional parser-module loading and execution. -add_executable(native_parser_module_test tests/native_parser_module_test.cpp) -add_dependencies(native_parser_module_test - native_parser_module_fixture - native_parser_module_missing_export - native_parser_module_wrong_abi - native_parser_module_unreadable_manifest -) -target_compile_definitions(native_parser_module_test PRIVATE - PJ_NATIVE_MODULE_FIXTURE_PATH="$" - PJ_NATIVE_MODULE_MISSING_EXPORT_PATH="$" - PJ_NATIVE_MODULE_WRONG_ABI_PATH="$" - PJ_NATIVE_MODULE_UNREADABLE_MANIFEST_PATH="$" -) -target_compile_options(native_parser_module_test PRIVATE ${PJ_WARNING_FLAGS}) -target_link_libraries(native_parser_module_test PRIVATE - pj_parser_module_host pj_parser_claim_catalog GTest::gtest_main -) -add_test(NAME native_parser_module_test COMMAND native_parser_module_test) - -add_executable(parser_module_runtime_test tests/parser_module_runtime_test.cpp) -add_dependencies(parser_module_runtime_test native_parser_module_fixture) -target_compile_definitions(parser_module_runtime_test PRIVATE - PJ_NATIVE_MODULE_FIXTURE_PATH="$" -) -target_compile_options(parser_module_runtime_test PRIVATE ${PJ_WARNING_FLAGS}) -target_link_libraries(parser_module_runtime_test PRIVATE - pj_parser_module_host GTest::gtest_main -) -add_test(NAME parser_module_runtime_test COMMAND parser_module_runtime_test) - -# Keystone integration: a C++17 module authored and built only with the -# header-only kit, then exercised through the production native loader/runtime. -add_executable(parser_module_authoring_e2e_test tests/parser_module_authoring_e2e_test.cpp) -add_dependencies(parser_module_authoring_e2e_test toy_cdr_pointcloud_module) -target_compile_definitions(parser_module_authoring_e2e_test PRIVATE - PJ_TOY_CDR_POINTCLOUD_MODULE_PATH="$" -) -target_compile_options(parser_module_authoring_e2e_test PRIVATE ${PJ_WARNING_FLAGS}) -target_link_libraries(parser_module_authoring_e2e_test PRIVATE - pj_parser_module_host pj_parser_claim_catalog GTest::gtest_main -) -add_test(NAME parser_module_authoring_e2e_test COMMAND parser_module_authoring_e2e_test) - # Unit test: ServiceRegistryBuilder registration rules. PJ_ASSERT_THROWS makes # the convenience overload's invariant observable in every build type — a plain # assert() is compiled away under NDEBUG, which is where it would go unchecked. @@ -807,7 +705,6 @@ if(PJ_INSTALL_SDK) pj_plugin_sdk pj_plugin_catalog pj_parser_claim_catalog - pj_parser_module_host pj_data_source_host pj_message_parser_host pj_toolbox_host diff --git a/pj_plugins/docs/ARCHITECTURE.md b/pj_plugins/docs/ARCHITECTURE.md index 6fb26bce..d186fbd5 100644 --- a/pj_plugins/docs/ARCHITECTURE.md +++ b/pj_plugins/docs/ARCHITECTURE.md @@ -150,55 +150,12 @@ because schema-generic plugins may register their handler while binding. A parser that still implements only legacy `parse()` does not advertise the functional extensions merely because it was rebuilt. -### Functional parser modules - -Functional parser modules use the family-independent export ABI in -`pj_base/parser_module_abi.h`, not a plugin-family vtable. The host-side -`NativeParserModule` loader opens each artifact with local, immediate symbol -resolution, resolves every operational and native metadata export, gates ABI -version 1, and copies the embedded manifest for subsequent -`ParserClaimCatalog` admission. Native module mappings have process-session -lifetime in v1 and are never unloaded. - -`NativeParserModuleInstance` owns one create/bind/parse/destroy lifecycle. It -uses the frozen little-endian codecs for input and output, consumes all -module-owned descriptor views before returning, validates output route and -expected object type, and checks a splice against both the canonical object's -eligible field and the original payload bounds. A valid splice is materialized -into that field, so the returned canonical object is complete; its sidecar -retains the original offset and bytes for future zero-copy integration. -Malformed returned descriptors, ineligible/out-of-bounds returned splices, -type/route mismatch, and bad tokens are contract violations; other -module-reported per-message failures are data errors. The separate -non-thread-safe `ParserModuleStrikeTracker` quarantines a module claim on its -third contract violation, permits one same-descriptor recreation, and disables -the claim after a repeated three-strike cycle. Executor placement, generation -ownership, folder scanning, and rescan policy remain application concerns. - -Module authors use the standalone C++17 headers under -`pj_base/include/pj_base/parser_module/` through the zero-linkage -`plotjuggler_sdk::parser_module` target. The readers compile ROS 2 concatenated -`.msg` bundles or protobuf `FileDescriptorSet` field paths at bind time; the -CDR locator supports fixed arrays, bounded and unbounded sequences, bounded -strings, and string arrays/sequences, while cyclic or over-depth schemas fail at -bind. The typed `ObjectWriter` emits Image, PointCloud, DepthImage, -OccupancyGrid, CompressedPointCloud, Mesh3D, VideoFrame, OccupancyGridUpdate, -and VoxelGrid descriptors, including each type's eligible single-splice form. -Module parse callbacks receive the per-message `pj::Timestamp` alongside the -payload. `BindingInfo` views expire when `bind()` returns; `owningCopy()` is the -fallible retention path. Bulk storage uses nothrow allocation, and protobuf -matching is bounded, so allocation failure is returned as data error rather -than escaping as a trap. `PJ_FUNCTIONAL_PARSER` supplies the complete native -export set, uses synchronized index+generation instance tokens to reject stale -handles, and catches user exceptions at the C boundary. -`pj_add_parser_module(... TARGETS native)` builds a hidden-visibility module and -embeds its JSON manifest behind the native metadata exports. `TARGETS wasm` is -not available in SDK 0.22. The shared host codec can append and read the exact -JSON bytes in a wasm `pj_parser_module_manifest` custom section; the conditional -wasi-sdk 27 compile gate builds the toy source with C++17 and exceptions -disabled, then statically audits the reactor model and every operational export -signature. This is structural conformance only: no wasm loader or execution -runtime ships in this release. +### Functional parser modules (experimental) + +A lighter extension shape, one C++17 source exporting a family-independent ABI, +lives under [`experimental/parser_module/`](../../experimental/parser_module/README.md). +It is built and tested in this repository but is not installed, not packaged and +not covered by the compatibility promise; no host loads modules yet. ## 0. C protocol v4 (current under ABI v5) @@ -394,15 +351,6 @@ pj_base/ message_parser_protocol.h ← C ABI parser_functional_protocol.h ← C ABI: pj.parser_functional.v1/v2 sinks parser_route_claims_protocol.h ← C ABI: exact parser route classification - parser_module_abi.h ← native/wasm module exports + byte codecs - parser_module_manifest.hpp ← wasm manifest custom-section embed/read codec - parser_module/ ← standalone C++17 header-only module authoring kit - module.hpp ← umbrella API + PJ_FUNCTIONAL_PARSER exports - cdr_reader.hpp ← bounded XCDR1 reader - cdr_field_locator.hpp ← ROS 2 .msg field-path compiler/cache - proto_reader.hpp ← bounded protobuf wire reader - proto_field_locator.hpp ← FileDescriptorSet field-path compiler - object_writer.hpp ← ten splice-eligible canonical object builders (including gridMap) toolbox_protocol.h ← C ABI plugin_data_api.h ← shared data-plane ABI (write hosts) descriptor_import_protocol.h ← C ABI: pj.descriptor_import.v1 extension + @@ -455,10 +403,8 @@ pj_plugins/ toolbox_library.hpp toolbox_handle.hpp plugin_catalog.hpp ← embedded-manifest DSO scanner (scanPluginDsos / inspectPluginDso) - parser_claim_catalog.hpp ← parser claims, manifest admission, plugin-claim synthesis + parser_claim_catalog.hpp ← parser claims, admission, plugin-claim synthesis parser_route_resolver.hpp ← ordered per-route selection + probe cache - native_parser_module.hpp ← session-lifetime native module loader - parser_module_runtime.hpp ← module instances, outputs, fault tracking service_registry_builder.hpp ← service wiring into bind() config_envelope.hpp ← versioned config wrapper include/pj_plugins/sdk/ @@ -473,13 +419,8 @@ pj_plugins/ message_parser_library.cpp parser_claim_catalog.cpp parser_route_resolver.cpp - native_parser_module.cpp - parser_module_runtime.cpp toolbox_library.cpp -cmake/ - PjParserModule.cmake ← pj_add_parser_module native target helper - (PlotJuggler application repo — not part of this SDK submodule) pj_datastore/ include/pj_datastore/ @@ -490,8 +431,7 @@ pj_datastore/ **Dependency direction:** Installed plugin authors consume `plotjuggler_sdk::plugin_sdk`, which combines `pj_base` with the MessageParser, -dialog and widget authoring headers from `pj_plugins`. Functional parser modules depend -only on the zero-linkage `plotjuggler_sdk::parser_module` header target. Host +dialog and widget authoring headers from `pj_plugins`. Host libraries in `pj_plugins` depend on `pj_base`. `pj_datastore` — now a module in the PlotJuggler application repo, not part of this SDK — provides the concrete data-host implementations that bridge plugin writes to the columnar storage @@ -581,10 +521,6 @@ DataSource with an embedded dialog). These deferred lookups use the recorded load-time paths, so they remain valid after the candidate file is removed, the process working directory changes, or dyld reports a symlink-resolved filename. -Native functional parser modules use the same absolute-path normalization, -package-scoped platform open, and defining-module provenance checks for every -required ABI export. Their narrow path API is explicitly UTF-8 on Windows. - ### 5.1 Host-side diagnostic propagation Host code that loads plugins accepts an diff --git a/pj_plugins/docs/message-parser-guide.md b/pj_plugins/docs/message-parser-guide.md index 02b82e99..f5a7f0d2 100644 --- a/pj_plugins/docs/message-parser-guide.md +++ b/pj_plugins/docs/message-parser-guide.md @@ -26,11 +26,6 @@ has selected a topic/encoding and supplies payloads to decode. If one `parse()` call naturally yields a batch, the parser write host can accept it through `writeHost().appendArrowStream(...)`. -If a supported encoding already exists and you only need one or a few exact -custom message types rendered as canonical objects or scalars, use a native -**functional parser module** instead of owning or forking the whole encoding. -See `.claude/skills/plotjuggler-plugin/references/parser-module.md`. - A MessageParser is the right choice when: - Each message is independently decodable (JSON line, single Protobuf message, ROS message, Influx line). diff --git a/pj_plugins/include/pj_plugins/host/parser_claim_catalog.hpp b/pj_plugins/include/pj_plugins/host/parser_claim_catalog.hpp index 153cfea3..07b492e0 100644 --- a/pj_plugins/include/pj_plugins/host/parser_claim_catalog.hpp +++ b/pj_plugins/include/pj_plugins/host/parser_claim_catalog.hpp @@ -4,12 +4,12 @@ /** * @file parser_claim_catalog.hpp - * @brief Host-owned parser claims, admission, and manifest decoding. + * @brief Host-owned parser claims and admission. * - * Claims from parser plugins and functional modules share one validated value - * model. Artifact metadata supplies identity and coverage; the host supplies - * provenance and provider generation when it admits a batch. Batch admission - * is transactional so one invalid or duplicate claim rejects the whole batch. + * Every parser provider shares one validated claim model. Artifact metadata + * supplies identity and coverage; the host supplies provenance and provider + * generation when it admits a batch. Batch admission is transactional so one + * invalid or duplicate claim rejects the whole batch. */ #include @@ -53,14 +53,6 @@ struct ParserClaimEntry { uint64_t provider_generation = 0; }; -/// Parsed module metadata. Claim order remains manifest order. -struct ParserModuleManifest { - std::string id; - std::string name; - std::string version; - std::vector claims; -}; - /// One successful route-classification result supplied by a parser-plugin host. struct ParserPluginExactClaim { std::string encoding; @@ -83,11 +75,6 @@ struct ParserPluginExactClaim { /// spelling `*` is preserved for every encoding. [[nodiscard]] Expected normalizeParserTypeName(std::string_view encoding, std::string_view type_name); -/// Decode and validate one complete parser-module manifest. Provenance is -/// supplied by the host and is never read from JSON. -[[nodiscard]] Expected decodeParserModuleManifest( - std::string_view manifest_json, ParserClaimProvenance provenance); - /// Synthesize the frozen wildcard and exact parser-plugin claim identities. /// Declined classifications add no exact claim. Malformed classification /// records reject the complete synthesized batch. @@ -108,10 +95,6 @@ class ParserClaimCatalog { [[nodiscard]] Status admitClaims( std::vector claims, ParserClaimProvenance provenance, uint64_t provider_generation); - /// Decode and atomically admit a complete module manifest. - [[nodiscard]] Expected ingestModuleManifest( - std::string_view manifest_json, ParserClaimProvenance provenance, uint64_t provider_generation); - /// Synthesize and atomically admit all claims for one parser plugin. [[nodiscard]] Status admitParserPlugin( std::string_view provider_id, std::span manifest_encodings, diff --git a/pj_plugins/src/parser_claim_catalog.cpp b/pj_plugins/src/parser_claim_catalog.cpp index ca7e58d8..2fa3a23f 100644 --- a/pj_plugins/src/parser_claim_catalog.cpp +++ b/pj_plugins/src/parser_claim_catalog.cpp @@ -6,15 +6,10 @@ #include #include #include -#include -#include #include #include #include -#include "pj_base/parser_module_abi.h" -#include "pj_base/sdk/semver.hpp" - namespace PJ { namespace { @@ -115,79 +110,6 @@ constexpr std::array kRegisteredEncodings = { return claim; } -[[nodiscard]] Expected requiredString( - const nlohmann::json& object, std::string_view key, std::string_view context) { - const auto it = object.find(std::string(key)); - if (it == object.end() || !it->is_string() || it->get_ref().empty()) { - return unexpected(std::string(context) + " missing required non-empty string key: " + std::string(key)); - } - return it->get(); -} - -[[nodiscard]] Expected requiredPriority(const nlohmann::json& object, size_t claim_index) { - const auto it = object.find("priority"); - if (it == object.end() || !it->is_number_integer()) { - return unexpected("parser module manifest claim " + std::to_string(claim_index) + " missing integer priority"); - } - - if (it->is_number_unsigned()) { - const uint64_t value = it->get(); - if (value > static_cast(std::numeric_limits::max())) { - return unexpected("parser module manifest claim priority does not fit int32"); - } - return static_cast(value); - } - const int64_t value = it->get(); - if (value < std::numeric_limits::min() || value > std::numeric_limits::max()) { - return unexpected("parser module manifest claim priority does not fit int32"); - } - return static_cast(value); -} - -[[nodiscard]] Expected readRouteFlags(const nlohmann::json& claim, size_t claim_index) { - const auto routes = claim.find("routes"); - if (routes == claim.end() || !routes->is_array() || routes->empty()) { - return unexpected( - "parser module manifest claim " + std::to_string(claim_index) + " requires a non-empty routes array"); - } - - uint16_t flags = 0; - for (const auto& route : *routes) { - if (!route.is_string()) { - return unexpected("parser module manifest claim " + std::to_string(claim_index) + " routes must contain strings"); - } - const auto& name = route.get_ref(); - if (name == "scalar") { - flags |= PJ_PARSER_ROUTE_FLAG_SCALAR_V1; - } else if (name == "object") { - flags |= PJ_PARSER_ROUTE_FLAG_OBJECT_V1; - } else { - return unexpected("parser module manifest claim uses unknown route: " + name); - } - } - return flags; -} - -[[nodiscard]] Expected> readSchemaDigests(const nlohmann::json& claim, size_t claim_index) { - std::set digests; - const auto values = claim.find("schema_digests"); - if (values == claim.end()) { - return digests; - } - if (!values->is_array()) { - return unexpected( - "parser module manifest claim " + std::to_string(claim_index) + " schema_digests must be an array"); - } - for (const auto& value : *values) { - if (!value.is_string()) { - return unexpected( - "parser module manifest claim " + std::to_string(claim_index) + " schema_digests must contain strings"); - } - digests.insert(value.get()); - } - return digests; -} - [[nodiscard]] std::string claimIdentity(const ParserClaim& claim) { return claim.provider_id + "/" + claim.claim_id; } @@ -242,132 +164,6 @@ Expected normalizeParserTypeName(std::string_view encoding, std::st return std::string(type_name); } -Expected decodeParserModuleManifest( - std::string_view manifest_json, ParserClaimProvenance provenance) { - if (manifest_json.empty()) { - return unexpected("parser module manifest is empty"); - } - - nlohmann::json json; - try { - json = nlohmann::json::parse(manifest_json); - } catch (const nlohmann::json::exception& error) { - return unexpected("parser module manifest is invalid JSON: " + std::string(error.what())); - } - if (!json.is_object()) { - return unexpected("parser module manifest must be a JSON object"); - } - if (json.contains("provenance")) { - return unexpected("parser module manifest must not declare host-owned provenance"); - } - - const auto abi = json.find("module_abi"); - if (abi == json.end() || !abi->is_number_integer()) { - return unexpected("parser module manifest missing integer module_abi"); - } - // The host ABI version is positive, so any signed (hence negative) encoding - // of module_abi is a mismatch by construction. - if (!abi->is_number_unsigned() || abi->get() != PJ_PARSER_MODULE_ABI_VERSION) { - return unexpected( - "parser module manifest module_abi does not match host ABI " + std::to_string(PJ_PARSER_MODULE_ABI_VERSION)); - } - - auto id = requiredString(json, "id", "parser module manifest"); - if (!id) { - return unexpected(id.error()); - } - auto name = requiredString(json, "name", "parser module manifest"); - if (!name) { - return unexpected(name.error()); - } - auto version = requiredString(json, "version", "parser module manifest"); - if (!version) { - return unexpected(version.error()); - } - if (auto parsed = SemVer::parse(*version); !parsed) { - return unexpected("parser module manifest version is not valid SemVer: " + parsed.error()); - } - - const auto claims_json = json.find("claims"); - if (claims_json == json.end() || !claims_json->is_array()) { - return unexpected("parser module manifest requires a claims array"); - } - - ParserModuleManifest manifest{.id = *id, .name = *name, .version = *version, .claims = {}}; - manifest.claims.reserve(claims_json->size()); - std::set> identities; - - for (size_t index = 0; index < claims_json->size(); ++index) { - const auto& claim_json = (*claims_json)[index]; - if (!claim_json.is_object()) { - return unexpected("parser module manifest claim " + std::to_string(index) + " must be an object"); - } - if (claim_json.contains("provenance")) { - return unexpected("parser module manifest claim must not declare host-owned provenance"); - } - - auto claim_id = requiredString(claim_json, "claim_id", "parser module manifest claim"); - if (!claim_id) { - return unexpected(claim_id.error()); - } - auto encoding = requiredString(claim_json, "encoding", "parser module manifest claim"); - if (!encoding) { - return unexpected(encoding.error()); - } - auto type_name = requiredString(claim_json, "type_name", "parser module manifest claim"); - if (!type_name) { - return unexpected(type_name.error()); - } - auto route_flags = readRouteFlags(claim_json, index); - if (!route_flags) { - return unexpected(route_flags.error()); - } - auto priority = requiredPriority(claim_json, index); - if (!priority) { - return unexpected(priority.error()); - } - auto schema_digests = readSchemaDigests(claim_json, index); - if (!schema_digests) { - return unexpected(schema_digests.error()); - } - - std::optional object_type; - const auto object_type_json = claim_json.find("object_type"); - if (object_type_json != claim_json.end()) { - if (!object_type_json->is_string()) { - return unexpected("parser module manifest claim object_type must be a string"); - } - object_type = sdk::parseBuiltinObjectType(object_type_json->get_ref()); - if (!object_type.has_value() || *object_type == sdk::BuiltinObjectType::kNone) { - return unexpected( - "parser module manifest claim uses unknown object_type: " + object_type_json->get()); - } - } - - ParserClaim claim{ - .encoding = *encoding, - .type_name = *type_name, - .route_flags = *route_flags, - .object_type = object_type, - .schema_digests = std::move(*schema_digests), - .provider_id = manifest.id, - .claim_id = *claim_id, - .priority = *priority, - .provenance = provenance, - }; - auto validated = normalizeAndValidateClaim(std::move(claim)); - if (!validated) { - return unexpected(validated.error()); - } - if (!identities.emplace(validated->provider_id, validated->claim_id).second) { - return unexpected("duplicate parser claim identity: " + claimIdentity(*validated)); - } - manifest.claims.push_back(std::move(*validated)); - } - - return manifest; -} - Expected> synthesizeParserPluginClaims( std::string_view provider_id, std::span manifest_encodings, std::span exact_claims, ParserClaimProvenance provenance) { @@ -503,20 +299,6 @@ Status ParserClaimCatalog::admitClaims( return okStatus(); } -Expected ParserClaimCatalog::ingestModuleManifest( - std::string_view manifest_json, ParserClaimProvenance provenance, uint64_t provider_generation) { - auto manifest = decodeParserModuleManifest(manifest_json, provenance); - if (!manifest) { - report(DiagnosticLevel::kError, {}, manifest.error()); - return unexpected(manifest.error()); - } - auto admission = admitClaims(manifest->claims, provenance, provider_generation); - if (!admission) { - return unexpected(admission.error()); - } - return manifest; -} - Status ParserClaimCatalog::admitParserPlugin( std::string_view provider_id, std::span manifest_encodings, std::span exact_claims, ParserClaimProvenance provenance, diff --git a/pj_plugins/tests/parser_claim_catalog_test.cpp b/pj_plugins/tests/parser_claim_catalog_test.cpp index 18ca47c7..24977942 100644 --- a/pj_plugins/tests/parser_claim_catalog_test.cpp +++ b/pj_plugins/tests/parser_claim_catalog_test.cpp @@ -12,7 +12,6 @@ #include #include -#include "pj_base/parser_module_abi.h" #include "pj_base/parser_route_claims_protocol.h" namespace PJ { @@ -156,104 +155,6 @@ TEST(ParserClaimCatalog, AdmissionNormalizesNamesAndUsesHostProvenance) { EXPECT_EQ(catalog.claims()[0].provider_generation, 42U); } -TEST(ParserClaimCatalog, ModuleManifestDecodesValidatedClaimsAndMetadata) { - const auto manifest = decodeParserModuleManifest( - R"({ - "module_abi": 1, - "id": "com.example.radar", - "name": "Radar Parsers", - "version": "1.2.3-rc.1+build.7", - "claims": [ - {"claim_id":"object","encoding":"ros2msg","type_name":"radar_msgs/RadarScan", - "routes":["scalar","object"],"object_type":"kPointCloud", - "schema_digests":[ - "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"],"priority":17}, - {"claim_id":"fallback","encoding":"protobuf","type_name":".example.RadarScan", - "routes":["scalar"],"priority":-2} - ] - })", - ParserClaimProvenance::kMarketplace); - - ASSERT_TRUE(manifest.has_value()) << manifest.error(); - EXPECT_EQ(manifest->id, "com.example.radar"); - EXPECT_EQ(manifest->name, "Radar Parsers"); - EXPECT_EQ(manifest->version, "1.2.3-rc.1+build.7"); - ASSERT_EQ(manifest->claims.size(), 2U); - EXPECT_EQ(manifest->claims[0].type_name, "radar_msgs/msg/RadarScan"); - EXPECT_EQ(manifest->claims[0].route_flags, PJ_PARSER_ROUTE_FLAG_SCALAR_V1 | PJ_PARSER_ROUTE_FLAG_OBJECT_V1); - EXPECT_EQ(manifest->claims[0].object_type, sdk::BuiltinObjectType::kPointCloud); - EXPECT_EQ(manifest->claims[0].schema_digests.size(), 2U); - EXPECT_EQ(manifest->claims[0].provenance, ParserClaimProvenance::kMarketplace); - EXPECT_EQ(manifest->claims[1].type_name, "example.RadarScan"); - EXPECT_FALSE(manifest->claims[1].object_type.has_value()); -} - -TEST(ParserClaimCatalog, ModuleManifestRejectsMalformedJsonAndAbiVersion) { - for (const std::string_view manifest : { - "not json", - R"([])", - R"({"module_abi":2,"id":"x","name":"X","version":"1.0.0","claims":[]})", - R"({"id":"x","name":"X","version":"1.0.0","claims":[]})", - }) { - SCOPED_TRACE(manifest); - const auto parsed = decodeParserModuleManifest(manifest, ParserClaimProvenance::kFolderDrop); - EXPECT_FALSE(parsed.has_value()); - } -} - -TEST(ParserClaimCatalog, MalformedModuleManifestEmitsOneDiagnosticAndAdmitsNothing) { - std::vector diagnostics; - ParserClaimCatalog catalog(recordInto(diagnostics)); - - const auto result = catalog.ingestModuleManifest("not json", ParserClaimProvenance::kFolderDrop, 1); - - ASSERT_FALSE(result.has_value()); - ASSERT_EQ(diagnostics.size(), 1U); - EXPECT_EQ(diagnostics.front().level, DiagnosticLevel::kError); - EXPECT_NE(diagnostics.front().message.find("invalid JSON"), std::string::npos); - EXPECT_TRUE(catalog.claims().empty()); -} - -TEST(ParserClaimCatalog, ModuleManifestRequiresStableIdentityNameAndThreePartSemver) { - for (const std::string_view manifest : { - R"({"module_abi":1,"name":"X","version":"1.0.0","claims":[]})", - R"({"module_abi":1,"id":"x","version":"1.0.0","claims":[]})", - R"({"module_abi":1,"id":"x","name":"X","version":"1.0","claims":[]})", - }) { - SCOPED_TRACE(manifest); - const auto parsed = decodeParserModuleManifest(manifest, ParserClaimProvenance::kBundled); - EXPECT_FALSE(parsed.has_value()); - } -} - -TEST(ParserClaimCatalog, ModuleManifestRejectsDuplicateClaimsAsAWhole) { - std::vector diagnostics; - ParserClaimCatalog catalog(recordInto(diagnostics)); - const auto result = catalog.ingestModuleManifest( - R"({"module_abi":1,"id":"module","name":"Module","version":"1.0.0","claims":[ - {"claim_id":"same","encoding":"protobuf","type_name":"a.Type","routes":["scalar"],"priority":0}, - {"claim_id":"same","encoding":"protobuf","type_name":"b.Type","routes":["scalar"],"priority":0} - ]})", - ParserClaimProvenance::kFolderDrop, 1); - - ASSERT_FALSE(result.has_value()); - EXPECT_NE(result.error().find("duplicate"), std::string::npos); - EXPECT_TRUE(catalog.claims().empty()); - ASSERT_EQ(diagnostics.size(), 1U); -} - -TEST(ParserClaimCatalog, ModuleManifestCannotForgeProvenance) { - const auto result = decodeParserModuleManifest( - R"({"module_abi":1,"id":"module","name":"Module","version":"1.0.0", - "provenance":"bundled","claims":[]})", - ParserClaimProvenance::kFolderDrop); - - ASSERT_FALSE(result.has_value()); - EXPECT_NE(result.error().find("provenance"), std::string::npos); -} - TEST(ParserClaimCatalog, ParserPluginSynthesisFreezesWildcardAndHandlerIds) { const std::vector encodings{"ros2msg", "protobuf"}; const std::vector exact{ diff --git a/recipe.yaml b/recipe.yaml index 96953ae1..64c74f87 100644 --- a/recipe.yaml +++ b/recipe.yaml @@ -60,7 +60,6 @@ tests: - ${{ "lib" if unix else "Library/lib" }}/cmake/plotjuggler_sdk/plotjuggler_sdkConfigVersion.cmake - ${{ "lib" if unix else "Library/lib" }}/cmake/plotjuggler_sdk/PjPlugin.cmake - ${{ "lib" if unix else "Library/lib" }}/cmake/plotjuggler_sdk/PjCheckElfPluginExports.cmake - - ${{ "lib" if unix else "Library/lib" }}/cmake/plotjuggler_sdk/PjParserModule.cmake - ${{ "include" if unix else "Library/include" }}/pj_base/sdk/version.hpp - ${{ "include" if unix else "Library/include" }}/pj_base/sdk/semver.hpp # Actually exercise the package: build the repo's example consumer, which diff --git a/test_sdk_install.sh b/test_sdk_install.sh index c9ab59bc..eb9c3ec8 100755 --- a/test_sdk_install.sh +++ b/test_sdk_install.sh @@ -76,7 +76,7 @@ cmake --build "$CONSUMER_BUILD_DIR" -j "$(nproc)" echo "" echo "--- Step 4: Smoke-test find_package COMPONENTS ---" -for comp in base parser_module plugin_sdk plugin_host source descriptor_import_support; do +for comp in base plugin_sdk plugin_host source descriptor_import_support; do COMP_DIR="$(mktemp -d)" cat > "$COMP_DIR/CMakeLists.txt" <