From 4f8b6dca9205addbed7fbfe1bc3630cb2e34df55 Mon Sep 17 00:00:00 2001 From: Davide Faconti Date: Sat, 14 Nov 2026 08:08:12 +0100 Subject: [PATCH 1/2] feat(parser-module): WebAssembly loader, session budget and hardening for experimental parser modules Integrates PR #173 (originally 75ca1a0 "wasm parser-module loader, hardening, and authoring target" and 8873b0d "host-driven fault policy, resource-keyed budgets, table caps, in-tree executor") on top of main after the parser-module kit moved to experimental/parser_module/ (#230). - WasmParserModule / WasmParserModuleInstance execute WASI reactors through a statically linked, pinned Wasmer 7.0.1 C API. Admission runs one static audit (validateParserModuleWasmArtifact, shared with the pj-wasm-embed-manifest tool) before compilation: exactly one manifest section, an empty import allow-list, the frozen export ABI, _initialize and no start function, exported memory and tables with bounded declared maxima (table cap 65536 elements). Singlepass is preferred when available. - Execution is store-per-instance with per-call instruction metering (an instruction budget, not a wall-clock deadline). Linear memory is reacquired and every returned range revalidated after each guest call; splices resolve against the original host payload. - The wasm wrapper has the native wrapper's fault contract: traps, metering exhaustion, malformed descriptors and bad splices are returned contract violations; strikes, quarantine, create/bind replay and disabling stay host policy through ParserModuleStrikeTracker, which is now thread-safe and disables a claim whose replay faults. - ParserModuleSessionBudgetTracker (optional, thread-safe) bounds modules, artifact size, claims, active instances and aggregate declared memory under opaque reservation ids; both loaders accept a null budget and then do no accounting. - pj_add_parser_module(... TARGETS native wasm) builds both artifacts from one source with wasi-sdk 27, embeds the manifest and audits post-link; the toy module and adversarial guests (trap, infinite loop, memory growth, table cap, quarantine replay) dogfood it. The parser_module_wasm_audit test tool is replaced by pj-wasm-embed-manifest. - CI installs Wasmer 7.0.1 and runs ctest -L wasmer. The feature lives entirely under experimental/parser_module/: targets pj_wasm_parser_module_host, pj_wasm_embed_manifest and the wasm tests are defined in its CMakeLists.txt (Wasmer gated on PJ_WASMER_ROOT, wasi-sdk on PJ_WASI_SDK_ROOT). Nothing is installed or packaged, so the SDK version, changelog, feature floors and Conan recipe are unchanged and installed packages are Wasmer-free. Documentation is in experimental/parser_module/README.md ("WebAssembly loader") and AUTHORING.md. Co-Authored-By: Claude Fable 5.2 --- .github/workflows/linux-ci.yml | 23 + experimental/parser_module/AUTHORING.md | 22 +- experimental/parser_module/CMakeLists.txt | 200 +++-- experimental/parser_module/README.md | 141 ++- .../parser_module/cmake/PjParserModule.cmake | 249 ++++-- .../cmake/parser_module_wasi_no_io_stubs.cpp | 31 + .../include/pj_base/parser_module_wasm.hpp | 141 +++ .../pj_plugins/host/native_parser_module.hpp | 16 +- .../pj_plugins/host/parser_module_runtime.hpp | 15 +- .../host/parser_module_session_budget.hpp | 105 +++ .../pj_plugins/host/wasm_parser_module.hpp | 107 +++ .../host/wasm_parser_module_runtime.hpp | 82 ++ .../src/detail/native_parser_module_state.hpp | 8 + .../detail/parser_module_result_helpers.hpp | 151 ++++ .../src/detail/wasm_parser_module_state.hpp | 32 + .../src/native_parser_module.cpp | 40 + .../src/parser_module_runtime.cpp | 166 +--- .../src/parser_module_session_budget.cpp | 119 +++ .../parser_module/src/parser_module_wasm.cpp | 666 ++++++++++++++ .../parser_module/src/wasm_parser_module.cpp | 219 +++++ .../src/wasm_parser_module_runtime.cpp | 687 +++++++++++++++ .../tests/adversarial_wasm_parser_module.cpp | 64 ++ ...adversarial_wasm_parser_module.module.json | 16 + .../tests/native_parser_module_test.cpp | 43 + .../tests/parser_module_runtime_test.cpp | 20 + .../parser_module_session_budget_test.cpp | 151 ++++ .../tests/parser_module_wasm_audit.cpp | 554 ------------ .../tests/parser_module_wasm_test.cpp | 114 +++ .../wasm_parser_module_hardening_test.cpp | 275 ++++++ .../tests/wasm_parser_module_test.cpp | 811 ++++++++++++++++++ .../wasmer_shared_module_prototype_test.cpp | 142 +++ .../tools/pj_wasm_embed_manifest.cpp | 145 ++++ 32 files changed, 4720 insertions(+), 835 deletions(-) create mode 100644 experimental/parser_module/cmake/parser_module_wasi_no_io_stubs.cpp create mode 100644 experimental/parser_module/include/pj_base/parser_module_wasm.hpp create mode 100644 experimental/parser_module/include/pj_plugins/host/parser_module_session_budget.hpp create mode 100644 experimental/parser_module/include/pj_plugins/host/wasm_parser_module.hpp create mode 100644 experimental/parser_module/include/pj_plugins/host/wasm_parser_module_runtime.hpp create mode 100644 experimental/parser_module/src/detail/parser_module_result_helpers.hpp create mode 100644 experimental/parser_module/src/detail/wasm_parser_module_state.hpp create mode 100644 experimental/parser_module/src/parser_module_session_budget.cpp create mode 100644 experimental/parser_module/src/parser_module_wasm.cpp create mode 100644 experimental/parser_module/src/wasm_parser_module.cpp create mode 100644 experimental/parser_module/src/wasm_parser_module_runtime.cpp create mode 100644 experimental/parser_module/tests/adversarial_wasm_parser_module.cpp create mode 100644 experimental/parser_module/tests/adversarial_wasm_parser_module.module.json create mode 100644 experimental/parser_module/tests/parser_module_session_budget_test.cpp delete mode 100644 experimental/parser_module/tests/parser_module_wasm_audit.cpp create mode 100644 experimental/parser_module/tests/parser_module_wasm_test.cpp create mode 100644 experimental/parser_module/tests/wasm_parser_module_hardening_test.cpp create mode 100644 experimental/parser_module/tests/wasm_parser_module_test.cpp create mode 100644 experimental/parser_module/tests/wasmer_shared_module_prototype_test.cpp create mode 100644 experimental/parser_module/tools/pj_wasm_embed_manifest.cpp diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index cbf1f071..2922cf17 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -161,6 +161,25 @@ jobs: echo "PJ_WASI_SDK_ROOT=${install_dir}" >> "${GITHUB_ENV}" "${install_dir}/bin/clang++" --version + - name: Install Wasmer C API 7.0.1 + shell: bash + run: | + set -euo pipefail + version="7.0.1" + archive="${RUNNER_TEMP}/wasmer-${version}.tar.gz" + install_dir="${RUNNER_TEMP}/wasmer-${version}" + url="https://github.com/wasmerio/wasmer/releases/download/v${version}/wasmer-linux-amd64.tar.gz" + + curl -fsSL --retry 5 --retry-all-errors "${url}" -o "${archive}" + mkdir -p "${install_dir}" + tar -xzf "${archive}" -C "${install_dir}" + test -f "${install_dir}/include/wasm.h" + test -f "${install_dir}/include/wasmer.h" + test -f "${install_dir}/lib/libwasmer.a" + nm -g --defined-only "${install_dir}/lib/libwasmer.a" > "${RUNNER_TEMP}/wasmer-symbols.txt" + grep -q wasmer_metering_set_remaining_points "${RUNNER_TEMP}/wasmer-symbols.txt" + echo "PJ_WASMER_ROOT=${install_dir}" >> "${GITHUB_ENV}" + - name: Configure ccache # Compiler-output cache for our own C++ — complementary to the Conan # cache (which holds prebuilt third-party packages, not our objects). @@ -223,6 +242,7 @@ jobs: -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DPJ_WASI_SDK_ROOT=${PJ_WASI_SDK_ROOT} + -DPJ_WASMER_ROOT=${PJ_WASMER_ROOT} -DPJ_ENABLE_ABI_CHECK=ON - name: Build @@ -233,6 +253,9 @@ jobs: cmake --build build --target parser_module_wasm_conformance_fixture ctest --test-dir build -L wasi --output-on-failure + - name: Wasmer parser-module execution + run: ctest --test-dir build -L wasmer --output-on-failure + - name: ABI drift gate # Mechanically enforces the Release Versioning policy (CLAUDE.md): a non-MAJOR # change must not break ABI. Diffs the mock_data_source_plugin canary DSO against diff --git a/experimental/parser_module/AUTHORING.md b/experimental/parser_module/AUTHORING.md index 73f117ce..35694e66 100644 --- a/experimental/parser_module/AUTHORING.md +++ b/experimental/parser_module/AUTHORING.md @@ -120,15 +120,19 @@ installed component to `find_package`. pj_add_parser_module(raw_mono_image_parser SOURCE raw_mono_image_parser.cpp MANIFEST raw_mono_image_parser.module.json - TARGETS native + TARGETS native wasm # either, or both from the same source ) ``` The helper embeds the manifest, hides every non-ABI symbol, and exports the -complete native `pj_module_*` set. Only `TARGETS native` is accepted; -requesting `TARGETS wasm` stops configuration with “wasm support arrives with -the SDK wasm loader milestone”. The WASI check is structural -conformance testing, not a wasm authoring or execution target. +complete native `pj_module_*` set. `TARGETS wasm` requires `PJ_WASI_SDK_ROOT` +pointing at wasi-sdk 27: it builds a C++17 WASI reactor with exceptions +disabled, omits the native manifest address/length exports, embeds the manifest +in the `pj_parser_module_manifest` custom section via the in-tree +`pj-wasm-embed-manifest` tool, and audits the export set post-link. Wasm +reactors import nothing and must declare a linear-memory maximum (default +256 MiB; override with `PJ_PARSER_MODULE_WASM_MAX_MEMORY_BYTES`) and a +function-table maximum (wasm-ld emits one; the host caps it at 65536 elements). ## Choose a schema-compatibility strategy @@ -175,8 +179,12 @@ selection before it can emit an invalid descriptor. ## Traps - The kit is header-only and WASI-clean: no threads, filesystem, iostream, host - SDK linkage, or exceptions across its API. The supported build product is - nevertheless native-only. + SDK linkage, or exceptions across its API. The same source builds the native + and the wasm artifact; keep it that way even if you only ship one today. +- Wasm execution is instruction-metered per guest call (an instruction budget, + not a wall-clock deadline) and memory-capped by the declared maximum; a trap + or metering exhaustion is a contract violation the host strikes, not a data + error. Native and wasm instances share one host-side strike/quarantine loop. - Return `pj::Status` / `pj::Expected`; do not throw. `Blob` uses nothrow allocation and protobuf matching is bounded, so allocation failure is a reported data error rather than a process abort or contract strike. diff --git a/experimental/parser_module/CMakeLists.txt b/experimental/parser_module/CMakeLists.txt index f9f956bd..f09d6b21 100644 --- a/experimental/parser_module/CMakeLists.txt +++ b/experimental/parser_module/CMakeLists.txt @@ -1,6 +1,6 @@ # --------------------------------------------------------------------------- # EXPERIMENTAL: functional parser modules (authoring kit, export ABI, native -# host loader/runtime). Not installed, not packaged, and outside the SDK +# and wasm host loader/runtime). Not installed, not packaged, and outside the SDK # compatibility promise. See README.md. # --------------------------------------------------------------------------- @@ -23,16 +23,28 @@ add_library(plotjuggler_sdk::parser_module ALIAS pj_parser_module) add_library(pj_parser_module_abi STATIC src/parser_module_abi.cpp src/parser_module_manifest.cpp + src/parser_module_wasm.cpp ) target_include_directories(pj_parser_module_abi PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) target_compile_options(pj_parser_module_abi PRIVATE ${PJ_WARNING_FLAGS} ${PJ_SANITIZER_FLAGS}) set_target_properties(pj_parser_module_abi PROPERTIES POSITION_INDEPENDENT_CODE ON) target_link_libraries(pj_parser_module_abi PUBLIC pj_base) -# Native module loader, runtime and manifest-to-claims decoding. +# Build-time frontend for embedding exact manifest bytes and running the same +# static wasm audit used by the conformance gate. pj_add_parser_module(TARGETS +# wasm) invokes it by this alias. +add_executable(pj_wasm_embed_manifest tools/pj_wasm_embed_manifest.cpp) +target_compile_features(pj_wasm_embed_manifest PRIVATE cxx_std_20) +target_compile_options(pj_wasm_embed_manifest PRIVATE ${PJ_WARNING_FLAGS}) +target_link_libraries(pj_wasm_embed_manifest PRIVATE pj_parser_module_abi) +set_target_properties(pj_wasm_embed_manifest PROPERTIES OUTPUT_NAME pj-wasm-embed-manifest) +add_executable(plotjuggler_sdk::wasm_embed_manifest ALIAS pj_wasm_embed_manifest) + +# Native module loader, runtime, session budget and manifest-to-claims decoding. add_library(pj_parser_module_host STATIC src/native_parser_module.cpp src/parser_module_runtime.cpp + src/parser_module_session_budget.cpp src/parser_module_claims.cpp ) target_include_directories(pj_parser_module_host PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) @@ -48,11 +60,67 @@ target_link_libraries(pj_parser_module_host ${CMAKE_DL_LIBS} ) +# Wasm module loader/runtime, executed by a statically linked Wasmer 7.0.1 +# C API. Built only when PJ_WASMER_ROOT names that installation. +set(PJ_WASMER_ROOT "$ENV{PJ_WASMER_ROOT}" CACHE PATH + "Wasmer 7.0.1 C-API root used for wasm parser-module execution") +set(_pj_wasmer_include "${PJ_WASMER_ROOT}/include") +find_file(_pj_wasmer_library + NAMES libwasmer.a libwasmer.lib wasmer.lib + PATHS "${PJ_WASMER_ROOT}/lib" + NO_DEFAULT_PATH + NO_CACHE) +set(PJ_WASMER_AVAILABLE OFF) +if(NOT PJ_WASMER_ROOT) + message(STATUS + "Wasm parser-module loader skipped: PJ_WASMER_ROOT cache/env variable is not set") +elseif(NOT EXISTS "${_pj_wasmer_include}/wasm.h" OR + NOT EXISTS "${_pj_wasmer_include}/wasmer.h" OR + NOT _pj_wasmer_library) + message(STATUS + "Wasm parser-module loader skipped: Wasmer C-API is missing under PJ_WASMER_ROOT=${PJ_WASMER_ROOT}") +else() + file(STRINGS "${_pj_wasmer_include}/wasmer.h" _pj_wasmer_version_line + REGEX "^#define WASMER_VERSION ") + if(NOT _pj_wasmer_version_line MATCHES "\"7\\.0\\.1\"") + message(FATAL_ERROR + "Wasm parser-module loader requires Wasmer 7.0.1; found '${_pj_wasmer_version_line}'") + endif() + find_package(Threads REQUIRED) + add_library(pj_wasmer_static UNKNOWN IMPORTED) + set_target_properties(pj_wasmer_static PROPERTIES + IMPORTED_LOCATION "${_pj_wasmer_library}" + INTERFACE_INCLUDE_DIRECTORIES "${_pj_wasmer_include}" + ) + set(PJ_WASMER_AVAILABLE ON) + message(STATUS "Wasm parser-module loader enabled with Wasmer 7.0.1: ${PJ_WASMER_ROOT}") + + add_library(pj_wasm_parser_module_host STATIC + src/wasm_parser_module.cpp + src/wasm_parser_module_runtime.cpp + ) + target_include_directories(pj_wasm_parser_module_host PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + target_compile_features(pj_wasm_parser_module_host PUBLIC cxx_std_20) + target_compile_options(pj_wasm_parser_module_host PRIVATE ${PJ_WARNING_FLAGS}) + set_target_properties(pj_wasm_parser_module_host PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_link_libraries(pj_wasm_parser_module_host + PUBLIC + pj_parser_module_host + PRIVATE + pj_wasmer_static + Threads::Threads + ${CMAKE_DL_LIBS} + ) + if(UNIX) + target_link_libraries(pj_wasm_parser_module_host PRIVATE m) + endif() +endif() + if(NOT PJ_BUILD_TESTS) return() endif() -foreach(test_name parser_module_abi_test parser_module_manifest_test) +foreach(test_name parser_module_abi_test parser_module_manifest_test parser_module_wasm_test) add_executable(${test_name} tests/${test_name}.cpp) target_link_libraries(${test_name} PRIVATE pj_parser_module_abi GTest::gtest_main) add_test(NAME ${test_name} COMMAND ${test_name}) @@ -63,6 +131,11 @@ target_compile_options(parser_module_claims_test PRIVATE ${PJ_WARNING_FLAGS}) target_link_libraries(parser_module_claims_test PRIVATE pj_parser_module_host GTest::gtest_main) add_test(NAME parser_module_claims_test COMMAND parser_module_claims_test) +add_executable(parser_module_session_budget_test tests/parser_module_session_budget_test.cpp) +target_compile_options(parser_module_session_budget_test PRIVATE ${PJ_WARNING_FLAGS}) +target_link_libraries(parser_module_session_budget_test PRIVATE pj_parser_module_host GTest::gtest_main) +add_test(NAME parser_module_session_budget_test COMMAND parser_module_session_budget_test) + # C++17 standalone authoring-kit gate. This target intentionally does not # link pj_base; GTest is test infrastructure only. add_executable(parser_module_kit_test tests/parser_module_kit_test.cpp) @@ -81,8 +154,9 @@ target_compile_options(parser_module_object_writer_test PRIVATE ${PJ_WARNING_FLA target_link_libraries(parser_module_object_writer_test PRIVATE pj_parser_module pj_parser_module_abi GTest::gtest_main) add_test(NAME parser_module_object_writer_test COMMAND parser_module_object_writer_test) -# Static wasm ABI conformance. The authoring source is compiled directly by -# wasi-sdk, then a host-side auditor embeds and validates the final module. +# Static wasm ABI conformance. The production authoring helper compiles the +# fixture through wasi-sdk, embeds its manifest, and audits the result. +set(_pj_build_toy_wasm OFF) set(PJ_WASI_SDK_ROOT "$ENV{PJ_WASI_SDK_ROOT}" CACHE PATH "wasi-sdk 27 root used for parser-module conformance tests") set(_pj_wasi_clang "${PJ_WASI_SDK_ROOT}/bin/clang++") @@ -101,73 +175,71 @@ else() endif() message(STATUS "Parser-module WASI conformance enabled with wasi-sdk ${_pj_wasi_version}: ${PJ_WASI_SDK_ROOT}") + set(_pj_build_toy_wasm ON) +endif() - add_executable(parser_module_wasm_audit tests/parser_module_wasm_audit.cpp) - target_compile_options(parser_module_wasm_audit PRIVATE ${PJ_WARNING_FLAGS}) - target_link_libraries(parser_module_wasm_audit PRIVATE pj_parser_module_abi) - - file(GLOB _pj_parser_module_headers CONFIGURE_DEPENDS - "${CMAKE_CURRENT_SOURCE_DIR}/include/pj_base/parser_module/*.hpp") - set(_pj_wasm_dir "${CMAKE_CURRENT_BINARY_DIR}/parser_module_wasm_conformance") - set(_pj_wasm_raw "${_pj_wasm_dir}/toy_cdr_pointcloud.raw.wasm") - set(_pj_wasm_embedded "${_pj_wasm_dir}/toy_cdr_pointcloud.wasm") - set(_pj_wasm_source "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud_module.cpp") +set(_pj_toy_targets native) +if(_pj_build_toy_wasm) + list(APPEND _pj_toy_targets wasm) +endif() +pj_add_parser_module(toy_cdr_pointcloud_module + SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud_module.cpp" + MANIFEST "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud.module.json" + TARGETS ${_pj_toy_targets} +) +if(_pj_build_toy_wasm) + get_property(_pj_wasm_embedded TARGET toy_cdr_pointcloud_module_wasm PROPERTY PJ_PARSER_MODULE_WASM_PATH) set(_pj_wasm_manifest "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud.module.json") - - add_custom_command( - OUTPUT "${_pj_wasm_raw}" - COMMAND ${CMAKE_COMMAND} -E make_directory "${_pj_wasm_dir}" - COMMAND "${_pj_wasi_clang}" - --target=wasm32-wasip1 - --sysroot=${_pj_wasi_sysroot} - -mexec-model=reactor - -std=c++17 - -fno-exceptions - -fno-rtti - -fvisibility=hidden - -O1 - -Wall -Wextra -Werror - -DPJ_PARSER_MODULE_CLAIM_COUNT=2 - -I${CMAKE_CURRENT_SOURCE_DIR}/include - -I${PROJECT_SOURCE_DIR}/pj_base/include - "${_pj_wasm_source}" - -Wl,--export=pj_module_abi - -Wl,--export=pj_module_create - -Wl,--export=pj_module_destroy - -Wl,--export=pj_module_bind - -Wl,--export=pj_module_parse - -Wl,--export=pj_module_last_error - -Wl,--export=pj_module_alloc - -Wl,--export=pj_module_free - -o "${_pj_wasm_raw}" - DEPENDS "${_pj_wasm_source}" ${_pj_parser_module_headers} - "${PROJECT_SOURCE_DIR}/pj_base/include/pj_base/time_math.hpp" - COMMENT "Compiling C++17 parser-module WASI reactor fixture" - VERBATIM - ) - add_custom_command( - OUTPUT "${_pj_wasm_embedded}" - COMMAND ${CMAKE_COMMAND} -E env ASAN_OPTIONS=detect_leaks=0 - $ - --embed "${_pj_wasm_raw}" "${_pj_wasm_manifest}" "${_pj_wasm_embedded}" - DEPENDS "${_pj_wasm_raw}" "${_pj_wasm_manifest}" parser_module_wasm_audit - COMMENT "Embedding parser-module wasm manifest with the shared codec" - VERBATIM - ) - add_custom_target(parser_module_wasm_conformance_fixture ALL - DEPENDS "${_pj_wasm_embedded}") + add_custom_target(parser_module_wasm_conformance_fixture ALL DEPENDS toy_cdr_pointcloud_module_wasm) add_test(NAME parser_module_wasm_conformance_test - COMMAND parser_module_wasm_audit --audit "${_pj_wasm_embedded}" "${_pj_wasm_manifest}") + COMMAND pj_wasm_embed_manifest verify "${_pj_wasm_embedded}" "${_pj_wasm_manifest}") set_tests_properties(parser_module_wasm_conformance_test PROPERTIES LABELS "parser_module;wasi" REQUIRED_FILES "${_pj_wasm_embedded}") endif() -pj_add_parser_module(toy_cdr_pointcloud_module - SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud_module.cpp" - MANIFEST "${CMAKE_CURRENT_SOURCE_DIR}/tests/toy_cdr_pointcloud.module.json" - TARGETS native -) +# Wasm execution through Wasmer: an isolated Wasmer prototype, the toy module +# end to end, and adversarial guests against the hardening limits. +if(PJ_WASMER_AVAILABLE) + add_executable(wasmer_shared_module_prototype_test tests/wasmer_shared_module_prototype_test.cpp) + target_compile_options(wasmer_shared_module_prototype_test PRIVATE ${PJ_WARNING_FLAGS}) + target_link_libraries(wasmer_shared_module_prototype_test PRIVATE + pj_wasmer_static Threads::Threads ${CMAKE_DL_LIBS} GTest::gtest_main) + if(UNIX) + target_link_libraries(wasmer_shared_module_prototype_test PRIVATE m) + endif() + add_test(NAME wasmer_shared_module_prototype_test COMMAND wasmer_shared_module_prototype_test) + set_tests_properties(wasmer_shared_module_prototype_test PROPERTIES LABELS "parser_module;wasmer") + + if(_pj_build_toy_wasm) + add_executable(wasm_parser_module_test tests/wasm_parser_module_test.cpp) + add_dependencies(wasm_parser_module_test parser_module_wasm_conformance_fixture) + target_compile_definitions(wasm_parser_module_test PRIVATE + PJ_TOY_CDR_POINTCLOUD_WASM_PATH="${_pj_wasm_embedded}") + target_compile_options(wasm_parser_module_test PRIVATE ${PJ_WARNING_FLAGS}) + target_link_libraries(wasm_parser_module_test PRIVATE pj_wasm_parser_module_host GTest::gtest_main) + add_test(NAME wasm_parser_module_test COMMAND wasm_parser_module_test) + set_tests_properties(wasm_parser_module_test PROPERTIES LABELS "parser_module;wasmer;wasi") + + pj_add_parser_module(adversarial_wasm_parser_module + SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/tests/adversarial_wasm_parser_module.cpp" + MANIFEST "${CMAKE_CURRENT_SOURCE_DIR}/tests/adversarial_wasm_parser_module.module.json" + TARGETS wasm + ) + get_property(_pj_adversarial_wasm_path TARGET adversarial_wasm_parser_module_wasm + PROPERTY PJ_PARSER_MODULE_WASM_PATH) + add_executable(wasm_parser_module_hardening_test tests/wasm_parser_module_hardening_test.cpp) + add_dependencies(wasm_parser_module_hardening_test adversarial_wasm_parser_module_wasm) + target_compile_definitions(wasm_parser_module_hardening_test PRIVATE + PJ_ADVERSARIAL_WASM_PATH="${_pj_adversarial_wasm_path}") + target_compile_options(wasm_parser_module_hardening_test PRIVATE ${PJ_WARNING_FLAGS}) + target_link_libraries(wasm_parser_module_hardening_test PRIVATE pj_wasm_parser_module_host GTest::gtest_main) + add_test(NAME wasm_parser_module_hardening_test COMMAND wasm_parser_module_hardening_test) + set_tests_properties(wasm_parser_module_hardening_test PROPERTIES LABELS "parser_module;wasmer;wasi") + else() + message(STATUS "Wasm parser-module E2E skipped: the wasi-sdk fixture target is unavailable") + endif() +endif() # --------------------------------------------------------------------------- # Native functional parser-module fixtures (real shared libraries) diff --git a/experimental/parser_module/README.md b/experimental/parser_module/README.md index e2e58985..f78b3420 100644 --- a/experimental/parser_module/README.md +++ b/experimental/parser_module/README.md @@ -14,8 +14,11 @@ the installed SDK is a move, not a rewrite. |---|---| | `include/pj_base/parser_module/` | Header-only C++17 authoring kit (`module.hpp` is the umbrella) | | `include/pj_base/parser_module_abi.h`, `parser_module_manifest.hpp` | Export ABI, block codecs, wasm manifest-section codec | -| `include/pj_plugins/host/` | Native loader, instance runtime, manifest-to-claims decoding | -| `cmake/PjParserModule.cmake` | `pj_add_parser_module()` | +| `include/pj_base/parser_module_wasm.hpp` | Static wasm artifact audit shared by the loader and the embed tool | +| `include/pj_plugins/host/` | Native and wasm loaders, instance runtimes, session budget, manifest-to-claims decoding | +| `cmake/PjParserModule.cmake` | `pj_add_parser_module()` (native and wasm targets) | +| `cmake/parser_module_wasi_no_io_stubs.cpp` | WASI I/O fallbacks that keep authored reactors import-free | +| `tools/pj_wasm_embed_manifest.cpp` | `pj-wasm-embed-manifest`: embed or verify manifest bytes, run the static audit | | `AUTHORING.md` | Manifest, schema-locator, splice, lifetime and error contracts | Open before the ABI can be called stable: @@ -78,12 +81,19 @@ claims. `ObjectWriter` provides `image`, `pointCloud`, `depthImage`, `occupancyGrid`, `compressedPointCloud`, `mesh3D`, `videoFrame`, `occupancyGridUpdate`, `voxelGrid`, and `gridMap` builders. -Native modules are built with `pj_add_parser_module(... TARGETS native)`. The -target links no SDK library; its include paths expose this kit and the shared -checked time arithmetic. -`TARGETS wasm` is not available. The wasi-sdk gate compiles and -statically audits a reactor fixture, but it does not provide wasm authoring or -execution. +Native and wasm modules are built with `pj_add_parser_module` using `TARGETS +native`, `TARGETS wasm`, or both. Each target links no SDK library; its include +paths expose this kit and the shared checked time arithmetic. Wasm targets +require `PJ_WASI_SDK_ROOT` to select wasi-sdk 27 and are post-linked through +the `pj-wasm-embed-manifest` embed-and-audit frontend. + +WASI reactor modules use the same operational exports and compile with +exceptions disabled. Their manifest is delivered in the +`pj_parser_module_manifest` custom section, so the native-only manifest address +and length exports are omitted automatically when targeting wasm. Authored +reactors have an empty import set, a declared linear-memory maximum (default +256 MiB, configurable with `PJ_PARSER_MODULE_WASM_MAX_MEMORY_BYTES`), and a +declared table maximum (wasm-ld emits one; the host caps it at 65536 elements). See [AUTHORING.md](AUTHORING.md) for the manifest, schema-locator, splice, lifetime, and error contracts. @@ -108,10 +118,13 @@ retains the original offset and bytes for future zero-copy integration. Malformed returned descriptors, ineligible/out-of-bounds returned splices, type/route mismatch, and bad tokens are contract violations; other module-reported per-message failures are data errors. The separate -non-thread-safe `ParserModuleStrikeTracker` quarantines a module claim on its -third contract violation, permits one same-descriptor recreation, and disables -the claim after a repeated three-strike cycle. Executor placement, generation -ownership, folder scanning, and rescan policy remain application concerns. +`ParserModuleStrikeTracker` quarantines a module claim on its third contract +violation, permits one same-descriptor recreation, and disables the claim after +a repeated three-strike cycle or a contract violation during the replay itself. +It is shared by native and wasm instances and is thread-safe, because the scalar +and object routes of one claim run on different threads. Executor placement, +generation ownership, folder scanning, and rescan policy remain application +concerns. Module authors use the standalone C++17 headers under `include/pj_base/parser_module/` through the zero-linkage @@ -129,16 +142,104 @@ matching is bounded, so allocation failure is returned as data error rather than escaping as a trap. `PJ_FUNCTIONAL_PARSER` supplies the complete native export set, uses synchronized index+generation instance tokens to reject stale handles, and catches user exceptions at the C boundary. -`pj_add_parser_module(... TARGETS native)` builds a hidden-visibility module and -embeds its JSON manifest behind the native metadata exports. `TARGETS wasm` is -not available. The shared host codec can append and read the exact -JSON bytes in a wasm `pj_parser_module_manifest` custom section; the conditional -wasi-sdk 27 compile gate builds the toy source with C++17 and exceptions -disabled, then statically audits the reactor model and every operational export -signature. This is structural conformance only: no wasm loader or execution -runtime exists. +`pj_add_parser_module(... TARGETS native wasm)` builds hidden-visibility native +and wasi-sdk 27 reactor artifacts from one source. It embeds the JSON manifest +behind the native metadata exports and in the wasm custom section. Wasm +reactors omit those two metadata exports and carry the exact JSON bytes in the +`pj_parser_module_manifest` custom section. The shared host codec appends and +reads that section; the conditional wasi-sdk 27 compile gate builds the same +toy module with C++17 and exceptions disabled, then statically audits the +reactor model and every operational export signature without executing wasm. Native modules use the same absolute-path normalization, package-scoped platform open, and defining-module provenance checks as the plugin-family loaders for every required ABI export (`pj_plugins/src/detail/library_loader.hpp`). Their narrow path API is explicitly UTF-8 on Windows. + +## WebAssembly loader + +`WasmParserModule` and `WasmParserModuleInstance` (target +`pj_wasm_parser_module_host`) load and execute wasm reactors through a +statically linked Wasmer 7.0.1 C API. Like everything here, the executor is +in-tree only: nothing is installed or packaged, so installed SDK packages stay +Wasmer-free. + +### Requirements and CMake variables + +| Variable | Meaning | +|---|---| +| `PJ_WASI_SDK_ROOT` (cache or environment) | wasi-sdk 27 root. Enables `TARGETS wasm`, the WASI conformance test (label `wasi`) and the wasm fixtures. Any other major version is a configure error. | +| `PJ_WASMER_ROOT` (cache or environment) | Wasmer 7.0.1 C-API root with `include/wasm.h`, `include/wasmer.h` and `lib/libwasmer.a`. Enables `pj_wasm_parser_module_host` and the execution tests (label `wasmer`). Any other version is a configure error. | +| `PJ_PARSER_MODULE_WASM_MAX_MEMORY_BYTES` | Declared linear-memory maximum of authored reactors; default 256 MiB, a positive multiple of 64 KiB. | +| `PJ_PARSER_MODULE_WASM_STACK_SIZE_BYTES` | Shadow-stack size of authored reactors; default 1 MiB, at least 256 KiB. | + +Either root left unset skips its part with a status message. CI installs both +and runs `ctest -L wasi` and `ctest -L wasmer`. + +### Admission + +Before compilation the loader runs the same static audit as the +`pj-wasm-embed-manifest` tool (`validateParserModuleWasmArtifact`): exactly one +manifest section, an empty import set (the frozen v1 allow-list), the frozen +export ABI, `_initialize`, no start function, exported linear memory with a +declared maximum, and tables with a declared maximum. The authoring preset +supplies unreachable WASI I/O fallbacks (`cmake/parser_module_wasi_no_io_stubs.cpp`), +so no fd, path, socket, clock, random, environment or scheduler capability +enters the allow-list. Compilation prefers Wasmer's Singlepass backend (linear +compile time; there is no cancellable compile API) and always runs +synchronously inside `load()`, so the host keeps it off the UI thread. + +### Execution + +One engine-owned compiled module is instantiated in one independent store per +bound instance. Store calls may migrate between threads sequentially, but the +host must serialize calls on an instance. ABI blocks are copied through guest +allocation; the runtime reacquires linear memory after every guest call, +validates every returned range, and resolves splices against the original host +payload. + +### Fault policy + +The wasm wrapper carries exactly the native wrapper's fault contract: guest +traps and metering exhaustion join malformed descriptors and bad offsets as +returned contract violations, and module-reported parse errors are strike-free +data errors. The wrapper only classifies; the host records faults, quarantines, +replays create/bind and disables through the shared `ParserModuleStrikeTracker`. + +### Limits and session budget + +| Limit | Default | Where | +|---|---|---| +| Instructions per guest call (Wasmer metering, reset for every ABI call) | 10,000,000 | `WasmParserModuleLimits::metering_points_per_call` | +| Artifact size | 64 MiB | `WasmParserModuleLimits::maximum_artifact_bytes` | +| Declared linear-memory maximum | 256 MiB | `ParserModuleWasmLimits::maximum_linear_memory_bytes` | +| Declared table elements (host memory outside the linear-memory budget) | 65536 | `ParserModuleWasmLimits::maximum_table_elements` | + +Metering is an instruction budget, not a wall-clock deadline: bulk operators +such as `memory.copy` cost one point regardless of size, and the pinned archive +exposes no public interrupt or epoch API. Native stack depth uses Wasmer's +guarded default. + +The optional, thread-safe `ParserModuleSessionBudgetTracker` bounds a whole +session: module count (64), artifact bytes (64 MiB each), total claims (4096), +active instances (128) and aggregate per-instance declared memory (4 GiB). It +counts resources under opaque reservation ids and never keys on manifest +identity, which remains catalog policy. Both loaders accept a null budget and +then perform no admission accounting; `NativeParserModule::load(path, sink)` +keeps its budget-free behaviour. + +### Wasmer pin (7.0.1, evaluated against 7.2.1 on 2026-08-09) + +- 7.2.x adds nothing the loader needs: `wasm_module_share/obtain` are still + absent from the static archive, the exported metering symbol set is + identical, and the "interruptable computation" work remains internal Rust + surface with no public C interrupt/epoch API. The only C-API additions are + `wasmer_features_*` toggles the loader does not require. +- The WASI-syscall CVEs fixed in 7.2.0 (unbounded host allocation in + `getcwd`/`random_get`, `poll_oneoff`, `sock_recv`/`sock_recv_from`) are + structurally unreachable here: the empty import allow-list rejects any + module importing those syscalls before instantiation. **Re-evaluate the pin + before ever widening the import allow-list**; no release currently combines + those fixes with x86_64-darwin support. +- 7.2.0 dropped the x86_64-darwin target, so moving the pin would end wasm + parser-module support on Intel macOS. diff --git a/experimental/parser_module/cmake/PjParserModule.cmake b/experimental/parser_module/cmake/PjParserModule.cmake index b70d6847..a2e72544 100644 --- a/experimental/parser_module/cmake/PjParserModule.cmake +++ b/experimental/parser_module/cmake/PjParserModule.cmake @@ -1,7 +1,29 @@ # PjParserModule.cmake # -# Native functional parser-module target helper. Wasm manifest embedding and -# toolchain support arrive with the wasm loader milestone. +# Functional parser-module target helper for native modules and wasi-sdk 27 +# reactor modules. Both targets compile the same author source and consume the +# same manifest bytes. + +# The kit spans more than one include root (its own headers plus the pj_base +# headers they include), so every resolvable root is returned. +function(_pj_parser_module_kit_includes OUTPUT) + if(NOT TARGET plotjuggler_sdk::parser_module) + message(FATAL_ERROR "pj_add_parser_module: plotjuggler_sdk::parser_module is unavailable") + endif() + get_target_property(_includes plotjuggler_sdk::parser_module INTERFACE_INCLUDE_DIRECTORIES) + set(_resolved) + foreach(_include IN LISTS _includes) + if(_include MATCHES "^\\$$") + list(APPEND _resolved "${CMAKE_MATCH_1}") + elseif(NOT _include MATCHES "^\\$<" AND IS_DIRECTORY "${_include}") + list(APPEND _resolved "${_include}") + endif() + endforeach() + if(NOT _resolved) + message(FATAL_ERROR "pj_add_parser_module: cannot resolve the parser-module kit include directory") + endif() + set(${OUTPUT} "${_resolved}" PARENT_SCOPE) +endfunction() function(pj_add_parser_module TARGET) set(_options) @@ -20,13 +42,21 @@ function(pj_add_parser_module TARGET) message(FATAL_ERROR "pj_add_parser_module(${TARGET}): MANIFEST is required") endif() if(NOT ARG_TARGETS) - message(FATAL_ERROR "pj_add_parser_module(${TARGET}): TARGETS native is required") + message(FATAL_ERROR "pj_add_parser_module(${TARGET}): TARGETS native and/or wasm is required") endif() + + list(REMOVE_DUPLICATES ARG_TARGETS) + set(_build_native OFF) + set(_build_wasm OFF) foreach(_requested_target IN LISTS ARG_TARGETS) - if(NOT _requested_target STREQUAL "native") + if(_requested_target STREQUAL "native") + set(_build_native ON) + elseif(_requested_target STREQUAL "wasm") + set(_build_wasm ON) + else() message(FATAL_ERROR - "pj_add_parser_module(${TARGET}): TARGETS ${_requested_target} is unavailable; " - "wasm support arrives with the SDK wasm loader milestone") + "pj_add_parser_module(${TARGET}): unsupported TARGETS value '${_requested_target}'; " + "expected native and/or wasm") endif() endforeach() @@ -47,56 +77,169 @@ function(pj_add_parser_module TARGET) "pj_add_parser_module(${TARGET}): MANIFEST must contain a claims array") endif() string(JSON _claim_count LENGTH "${_manifest_json}" claims) - if(_manifest_json MATCHES "\\)PJM\"") - message(FATAL_ERROR - "pj_add_parser_module(${TARGET}): MANIFEST contains the reserved raw-string delimiter") - endif() - set(_generated_dir "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_parser_module_generated") - file(MAKE_DIRECTORY "${_generated_dir}") - set(_manifest_header "${_generated_dir}/${TARGET}_manifest.hpp") - file(WRITE "${_manifest_header}" - "#pragma once\n" - "#define PJ_PARSER_MODULE_HAS_MANIFEST 1\n" - "namespace pj { namespace detail {\n" - "inline constexpr char kBuiltManifest[] = R\"PJM(${_manifest_json})PJM\";\n" - "} }\n" - "#define PJ_PARSER_MODULE_CLAIM_COUNT ${_claim_count}\n") + if(_build_native) + if(_manifest_json MATCHES "\\)PJM\"") + message(FATAL_ERROR + "pj_add_parser_module(${TARGET}): MANIFEST contains the reserved raw-string delimiter") + endif() + + set(_generated_dir "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_parser_module_generated") + file(MAKE_DIRECTORY "${_generated_dir}") + set(_manifest_header "${_generated_dir}/${TARGET}_manifest.hpp") + file(WRITE "${_manifest_header}" + "#pragma once\n" + "#define PJ_PARSER_MODULE_HAS_MANIFEST 1\n" + "namespace pj { namespace detail {\n" + "inline constexpr char kBuiltManifest[] = R\"PJM(${_manifest_json})PJM\";\n" + "} }\n" + "#define PJ_PARSER_MODULE_CLAIM_COUNT ${_claim_count}\n") + + add_library(${TARGET} MODULE "${_module_source}" "${_manifest_header}") + target_link_libraries(${TARGET} PRIVATE plotjuggler_sdk::parser_module) + target_include_directories(${TARGET} PRIVATE "${_generated_dir}") + target_compile_definitions(${TARGET} PRIVATE + PJ_PARSER_MODULE_MANIFEST_HEADER=\"${TARGET}_manifest.hpp\") + set_target_properties(${TARGET} PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED YES + CXX_EXTENSIONS NO + CXX_VISIBILITY_PRESET hidden + C_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN YES + ) + if(DEFINED PJ_WARNING_FLAGS) + target_compile_options(${TARGET} PRIVATE ${PJ_WARNING_FLAGS}) + endif() - add_library(${TARGET} MODULE "${_module_source}" "${_manifest_header}") - target_link_libraries(${TARGET} PRIVATE plotjuggler_sdk::parser_module) - target_include_directories(${TARGET} PRIVATE "${_generated_dir}") - target_compile_definitions(${TARGET} PRIVATE - PJ_PARSER_MODULE_MANIFEST_HEADER=\"${TARGET}_manifest.hpp\") - set_target_properties(${TARGET} PROPERTIES - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED YES - CXX_EXTENSIONS NO - CXX_VISIBILITY_PRESET hidden - C_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN YES - ) - if(DEFINED PJ_WARNING_FLAGS) - target_compile_options(${TARGET} PRIVATE ${PJ_WARNING_FLAGS}) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(_version_script "${_generated_dir}/${TARGET}.map") + file(WRITE "${_version_script}" + "{\n global:\n pj_module_abi;\n pj_module_create;\n pj_module_destroy;\n" + " pj_module_bind;\n pj_module_parse;\n pj_module_last_error;\n" + " pj_module_alloc;\n pj_module_free;\n pj_module_manifest_addr;\n" + " pj_module_manifest_len;\n local: *;\n};\n") + target_link_options(${TARGET} PRIVATE + "LINKER:-z,defs" + "LINKER:--exclude-libs,ALL" + "LINKER:--version-script=${_version_script}") + elseif(APPLE) + set(_exported_symbols "${_generated_dir}/${TARGET}.exports") + file(WRITE "${_exported_symbols}" + "_pj_module_abi\n_pj_module_create\n_pj_module_destroy\n_pj_module_bind\n" + "_pj_module_parse\n_pj_module_last_error\n_pj_module_alloc\n_pj_module_free\n" + "_pj_module_manifest_addr\n_pj_module_manifest_len\n") + target_link_options(${TARGET} PRIVATE "LINKER:-exported_symbols_list,${_exported_symbols}") + endif() endif() - if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - set(_version_script "${_generated_dir}/${TARGET}.map") - file(WRITE "${_version_script}" - "{\n global:\n pj_module_abi;\n pj_module_create;\n pj_module_destroy;\n" - " pj_module_bind;\n pj_module_parse;\n pj_module_last_error;\n" - " pj_module_alloc;\n pj_module_free;\n pj_module_manifest_addr;\n" - " pj_module_manifest_len;\n local: *;\n};\n") - target_link_options(${TARGET} PRIVATE - "LINKER:-z,defs" - "LINKER:--exclude-libs,ALL" - "LINKER:--version-script=${_version_script}") - elseif(APPLE) - set(_exported_symbols "${_generated_dir}/${TARGET}.exports") - file(WRITE "${_exported_symbols}" - "_pj_module_abi\n_pj_module_create\n_pj_module_destroy\n_pj_module_bind\n" - "_pj_module_parse\n_pj_module_last_error\n_pj_module_alloc\n_pj_module_free\n" - "_pj_module_manifest_addr\n_pj_module_manifest_len\n") - target_link_options(${TARGET} PRIVATE "LINKER:-exported_symbols_list,${_exported_symbols}") + if(_build_wasm) + if(NOT DEFINED PJ_WASI_SDK_ROOT OR NOT PJ_WASI_SDK_ROOT) + set(PJ_WASI_SDK_ROOT "$ENV{PJ_WASI_SDK_ROOT}" CACHE PATH + "wasi-sdk 27 root used for parser-module wasm targets") + endif() + if(NOT PJ_WASI_SDK_ROOT) + message(FATAL_ERROR + "pj_add_parser_module(${TARGET}): TARGETS wasm requires PJ_WASI_SDK_ROOT " + "to name a wasi-sdk 27 installation") + endif() + set(_wasi_clang "${PJ_WASI_SDK_ROOT}/bin/clang++") + set(_wasi_sysroot "${PJ_WASI_SDK_ROOT}/share/wasi-sysroot") + if(NOT EXISTS "${_wasi_clang}" OR NOT IS_DIRECTORY "${_wasi_sysroot}" OR + NOT EXISTS "${PJ_WASI_SDK_ROOT}/VERSION") + message(FATAL_ERROR + "pj_add_parser_module(${TARGET}): wasi-sdk is incomplete under " + "PJ_WASI_SDK_ROOT=${PJ_WASI_SDK_ROOT}") + endif() + file(STRINGS "${PJ_WASI_SDK_ROOT}/VERSION" _wasi_version LIMIT_COUNT 1) + if(NOT _wasi_version MATCHES "^27\\.") + message(FATAL_ERROR + "pj_add_parser_module(${TARGET}): TARGETS wasm requires wasi-sdk 27; " + "found '${_wasi_version}'") + endif() + if(NOT TARGET plotjuggler_sdk::wasm_embed_manifest) + message(FATAL_ERROR + "pj_add_parser_module(${TARGET}): the pj-wasm-embed-manifest SDK tool is unavailable") + endif() + + set(PJ_PARSER_MODULE_WASM_MAX_MEMORY_BYTES "268435456" CACHE STRING + "Declared maximum linear memory for authored parser-module wasm reactors") + set(PJ_PARSER_MODULE_WASM_STACK_SIZE_BYTES "1048576" CACHE STRING + "Shadow-stack size for authored parser-module wasm reactors") + if(NOT PJ_PARSER_MODULE_WASM_MAX_MEMORY_BYTES MATCHES "^[0-9]+$") + message(FATAL_ERROR "PJ_PARSER_MODULE_WASM_MAX_MEMORY_BYTES must be an integer byte count") + endif() + math(EXPR _maximum_memory_remainder "${PJ_PARSER_MODULE_WASM_MAX_MEMORY_BYTES} % 65536") + if(PJ_PARSER_MODULE_WASM_MAX_MEMORY_BYTES LESS 65536 OR NOT _maximum_memory_remainder EQUAL 0) + message(FATAL_ERROR + "PJ_PARSER_MODULE_WASM_MAX_MEMORY_BYTES must be a positive multiple of 65536") + endif() + if(NOT PJ_PARSER_MODULE_WASM_STACK_SIZE_BYTES MATCHES "^[0-9]+$" OR + PJ_PARSER_MODULE_WASM_STACK_SIZE_BYTES LESS 262144) + message(FATAL_ERROR + "PJ_PARSER_MODULE_WASM_STACK_SIZE_BYTES must be an integer of at least 262144 bytes") + endif() + + _pj_parser_module_kit_includes(_kit_includes) + list(TRANSFORM _kit_includes PREPEND "-I" OUTPUT_VARIABLE _kit_include_flags) + set(_no_io_stubs "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/parser_module_wasi_no_io_stubs.cpp") + if(NOT EXISTS "${_no_io_stubs}") + message(FATAL_ERROR "pj_add_parser_module(${TARGET}): missing wasm support source ${_no_io_stubs}") + endif() + list(TRANSFORM _kit_includes APPEND "/pj_base/parser_module/*.hpp" OUTPUT_VARIABLE _kit_header_globs) + file(GLOB _parser_module_headers CONFIGURE_DEPENDS ${_kit_header_globs}) + set(_wasm_dir "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_parser_module_wasm") + set(_wasm_raw "${_wasm_dir}/${TARGET}.raw.wasm") + set(_wasm_output "${_wasm_dir}/${TARGET}.wasm") + + add_custom_command( + OUTPUT "${_wasm_raw}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${_wasm_dir}" + COMMAND "${_wasi_clang}" + --target=wasm32-wasip1 + --sysroot=${_wasi_sysroot} + -mexec-model=reactor + -std=c++17 + -fno-exceptions + -fno-rtti + -fvisibility=hidden + -O1 + -Wall -Wextra -Werror + -DPJ_PARSER_MODULE_CLAIM_COUNT=${_claim_count} + ${_kit_include_flags} + "${_module_source}" + "${_no_io_stubs}" + -Wl,--export=pj_module_abi + -Wl,--export=pj_module_create + -Wl,--export=pj_module_destroy + -Wl,--export=pj_module_bind + -Wl,--export=pj_module_parse + -Wl,--export=pj_module_last_error + -Wl,--export=pj_module_alloc + -Wl,--export=pj_module_free + -Wl,--export-memory + -Wl,--max-memory=${PJ_PARSER_MODULE_WASM_MAX_MEMORY_BYTES} + -Wl,-z,stack-size=${PJ_PARSER_MODULE_WASM_STACK_SIZE_BYTES} + -Wl,--stack-first + -o "${_wasm_raw}" + DEPENDS "${_module_source}" "${_no_io_stubs}" ${_parser_module_headers} + COMMENT "Compiling ${TARGET} as a C++17 WASI reactor" + VERBATIM + ) + add_custom_command( + OUTPUT "${_wasm_output}" + COMMAND ${CMAKE_COMMAND} -E env ASAN_OPTIONS=detect_leaks=0 + $ + embed "${_wasm_raw}" "${_module_manifest}" "${_wasm_output}" + COMMAND ${CMAKE_COMMAND} -E env ASAN_OPTIONS=detect_leaks=0 + $ + verify "${_wasm_output}" "${_module_manifest}" + "${PJ_PARSER_MODULE_WASM_MAX_MEMORY_BYTES}" + DEPENDS "${_wasm_raw}" "${_module_manifest}" plotjuggler_sdk::wasm_embed_manifest + COMMENT "Embedding and auditing ${TARGET} parser-module manifest" + VERBATIM + ) + add_custom_target(${TARGET}_wasm ALL DEPENDS "${_wasm_output}") + set_property(TARGET ${TARGET}_wasm PROPERTY PJ_PARSER_MODULE_WASM_PATH "${_wasm_output}") endif() endfunction() diff --git a/experimental/parser_module/cmake/parser_module_wasi_no_io_stubs.cpp b/experimental/parser_module/cmake/parser_module_wasi_no_io_stubs.cpp new file mode 100644 index 00000000..e6f529d5 --- /dev/null +++ b/experimental/parser_module/cmake/parser_module_wasi_no_io_stubs.cpp @@ -0,0 +1,31 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include + +// The authoring kit performs no I/O, but wasi-libc's unreachable abort and +// stdio teardown paths otherwise leave three fd imports in a C++ reactor. +// Resolve them inside each authored module so the frozen host import allow-list +// remains empty. WASI errno 8 is BADF. +extern "C" uint32_t __imported_wasi_snapshot_preview1_fd_close(uint32_t fd) { + (void)fd; + return 8; +} + +extern "C" uint32_t __imported_wasi_snapshot_preview1_fd_seek( + uint32_t fd, uint64_t offset, uint32_t whence, uint32_t new_offset) { + (void)fd; + (void)offset; + (void)whence; + (void)new_offset; + return 8; +} + +extern "C" uint32_t __imported_wasi_snapshot_preview1_fd_write( + uint32_t fd, uint32_t iovecs, uint32_t iovec_count, uint32_t written) { + (void)fd; + (void)iovecs; + (void)iovec_count; + (void)written; + return 8; +} diff --git a/experimental/parser_module/include/pj_base/parser_module_wasm.hpp b/experimental/parser_module/include/pj_base/parser_module_wasm.hpp new file mode 100644 index 00000000..78f15a51 --- /dev/null +++ b/experimental/parser_module/include/pj_base/parser_module_wasm.hpp @@ -0,0 +1,141 @@ +#pragma once +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +/** + * @file parser_module_wasm.hpp + * @brief Bounds-checked static inspection of parser-module wasm binaries. + */ + +#include +#include +#include +#include +#include +#include + +#include "pj_base/expected.hpp" +#include "pj_base/span.hpp" + +namespace PJ::parser_module { + +inline constexpr uint8_t kWasmValueI32 = UINT8_C(0x7F); +inline constexpr uint8_t kWasmValueI64 = UINT8_C(0x7E); +inline constexpr uint64_t kWasmMemoryPageBytes = UINT64_C(65536); + +enum class WasmExternalKind : uint8_t { + kFunction = 0, + kTable = 1, + kMemory = 2, + kGlobal = 3, + kTag = 4, +}; + +struct WasmFunctionSignature { + std::vector parameters; + std::vector results; + + bool operator==(const WasmFunctionSignature&) const = default; +}; + +struct WasmMemoryLimits { + uint32_t minimum_pages = 0; + std::optional maximum_pages; + + bool operator==(const WasmMemoryLimits&) const = default; +}; + +struct WasmTableLimits { + uint32_t minimum_elements = 0; + std::optional maximum_elements; + + bool operator==(const WasmTableLimits&) const = default; +}; + +struct WasmImport { + std::string module; + std::string name; + WasmExternalKind kind = WasmExternalKind::kFunction; + std::optional function_signature; +}; + +struct WasmExport { + std::string name; + WasmExternalKind kind = WasmExternalKind::kFunction; + uint32_t index = 0; + std::optional function_signature; + std::optional memory_limits; +}; + +struct WasmModuleInfo { + std::vector imports; + std::vector exports; + std::vector memories; + std::vector tables; + bool has_start_section = false; + size_t section_count = 0; + size_t function_type_count = 0; + size_t function_count = 0; + + [[nodiscard]] const WasmExport* findExport(std::string_view name) const noexcept; +}; + +/** Inspect the sections needed for parser-module admission. + * + * Every section and variable-length integer is bounds-checked. Function + * signatures are resolved for imported and exported functions. Malformed, + * truncated, duplicate, or unsupported binary constructs return an error. + */ +[[nodiscard]] Expected inspectWasmModule(Span wasm); + +/** Validate the frozen operational exports and reactor constraints. + * + * Import policy and the exported linear memory needed for execution are loader + * policy layered on top of this ABI-only validation. + */ +[[nodiscard]] Expected validateParserModuleWasmAbi(const WasmModuleInfo& module); + +/** Validate bounded linear memory and return its aggregate declared maximum. + * + * Every declared memory must provide a maximum no larger than + * `maximum_bytes`. The operational `memory` export must resolve to one of + * those declarations. Byte arithmetic is overflow-checked. + */ +[[nodiscard]] Expected validateParserModuleWasmMemory(const WasmModuleInfo& module, uint64_t maximum_bytes); + +/** Validate bounded tables and return their aggregate declared maximum. + * + * Table storage is host memory outside linear memory, so every declared table + * (imported or defined) must provide a maximum, and the sum of those maxima + * must not exceed `maximum_elements`. A module without tables returns 0. + */ +[[nodiscard]] Expected validateParserModuleWasmTables( + const WasmModuleInfo& module, uint64_t maximum_elements); + +/// Per-artifact resource caps applied by the static admission audit. +struct ParserModuleWasmLimits { + static constexpr uint64_t kDefaultMaximumLinearMemoryBytes = UINT64_C(256) * 1024U * 1024U; + static constexpr uint64_t kDefaultMaximumTableElements = UINT64_C(65536); + + uint64_t maximum_linear_memory_bytes = kDefaultMaximumLinearMemoryBytes; + uint64_t maximum_table_elements = kDefaultMaximumTableElements; +}; + +/// Result of a passed admission audit. `manifest_json` views the audited bytes. +struct ParserModuleWasmArtifact { + Span manifest_json; + WasmModuleInfo module; + uint64_t declared_linear_memory_maximum = 0; + uint64_t declared_table_elements = 0; +}; + +/** The complete static admission audit shared by the host loader and the + * `pj-wasm-embed-manifest` tool: exactly one manifest section, an empty import + * set (the frozen v1 allow-list), the frozen operational export ABI, bounded + * exported linear memory, and bounded tables. Nothing is compiled or executed. + * The manifest bytes are returned verbatim; JSON validation is the caller's. + */ +[[nodiscard]] Expected validateParserModuleWasmArtifact( + Span wasm, const ParserModuleWasmLimits& limits); + +} // namespace PJ::parser_module diff --git a/experimental/parser_module/include/pj_plugins/host/native_parser_module.hpp b/experimental/parser_module/include/pj_plugins/host/native_parser_module.hpp index 2930decf..8c10456c 100644 --- a/experimental/parser_module/include/pj_plugins/host/native_parser_module.hpp +++ b/experimental/parser_module/include/pj_plugins/host/native_parser_module.hpp @@ -6,10 +6,11 @@ * @file native_parser_module.hpp * @brief Session-lifetime loader for native functional parser modules. * - * Loading resolves the complete frozen module ABI and copies the module's - * embedded manifest. Successfully opened artifacts remain loaded for the - * process session; no module code or manifest pointer is used after unload. - * Manifest admission remains an explicit ParserClaimCatalog caller step. + * Loading resolves the complete frozen module ABI, copies the embedded + * manifest, and validates its identity and claims for budget accounting. + * Successfully opened artifacts remain loaded for the process session; no + * module code or manifest pointer is used after unload. Catalog insertion and + * provenance assignment remain an explicit ParserClaimCatalog caller step. */ #include @@ -18,6 +19,7 @@ #include "pj_base/diagnostic_sink.hpp" #include "pj_base/expected.hpp" +#include "pj_plugins/host/parser_module_session_budget.hpp" namespace PJ { @@ -36,6 +38,12 @@ class NativeParserModule { [[nodiscard]] static Expected load( std::string_view path, DiagnosticSink sink = {}, std::string diagnostic_source = "NativeParserModule"); + /// Load under an application-owned aggregate session budget. A null budget + /// behaves exactly like the overload above: no admission accounting. + [[nodiscard]] static Expected load( + std::string_view path, std::shared_ptr budget, DiagnosticSink sink = {}, + std::string diagnostic_source = "NativeParserModule"); + [[nodiscard]] bool valid() const noexcept { return state_ != nullptr; } diff --git a/experimental/parser_module/include/pj_plugins/host/parser_module_runtime.hpp b/experimental/parser_module/include/pj_plugins/host/parser_module_runtime.hpp index dbccc8e4..3f76d5b4 100644 --- a/experimental/parser_module/include/pj_plugins/host/parser_module_runtime.hpp +++ b/experimental/parser_module/include/pj_plugins/host/parser_module_runtime.hpp @@ -8,14 +8,17 @@ * * The wrapper performs one serialized create/bind/parse/destroy lifecycle. * Module-owned descriptor views are decoded and copied before parse returns. - * The independent strike tracker is deliberately pure, non-thread-safe state - * so a host executor can apply its own scheduling and generation policy. + * The independent strike tracker holds pure per-claim state that a host + * executor drives with its own scheduling and generation policy; it is + * thread-safe because the scalar and object routes of one claim run on + * different threads. */ #include #include #include #include +#include #include #include #include @@ -126,6 +129,7 @@ class NativeParserModuleInstance { parser_module::Route bound_route_ = parser_module::Route::kScalar; uint16_t expected_object_type_ = 0; bool bound_ = false; + bool instance_budget_reserved_ = false; }; struct ParserModuleClaimKey { @@ -147,7 +151,11 @@ struct ParserModuleStrikeState { uint8_t quarantine_count = 0; }; -/// Pure per-(module, claim) contract-fault state. Data errors never mutate it. +/// Per-(module, claim) contract-fault state shared by native and wasm modules. +/// Data errors never mutate it. Three contract violations quarantine the claim; +/// the host then replays create/bind and calls markRecreated(). A contract +/// violation while quarantined (a failed replay) or a second three-strike cycle +/// disables the claim for the session. Thread-safe. class ParserModuleStrikeTracker { public: [[nodiscard]] ParserModuleStrikeState recordFault(const ParserModuleClaimKey& key, ParserModuleFaultKind fault); @@ -159,6 +167,7 @@ class ParserModuleStrikeTracker { [[nodiscard]] ParserModuleStrikeState state(const ParserModuleClaimKey& key) const; private: + mutable std::mutex mutex_; std::map states_; }; diff --git a/experimental/parser_module/include/pj_plugins/host/parser_module_session_budget.hpp b/experimental/parser_module/include/pj_plugins/host/parser_module_session_budget.hpp new file mode 100644 index 00000000..2c4fb509 --- /dev/null +++ b/experimental/parser_module/include/pj_plugins/host/parser_module_session_budget.hpp @@ -0,0 +1,105 @@ +#pragma once +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +/** + * @file parser_module_session_budget.hpp + * @brief Aggregate resource accounting for parser-module admission. + * + * The tracker counts resources, not identities: every accepted module + * reservation is an opaque id, so two loads of one artifact (or native and + * wasm builds of one source) are two reservations. Duplicate-provider policy + * belongs to the claim catalog, not here. + * + * The tracker is thread-safe. Loader wrappers release reservations from their + * destructors, which run on whichever thread drops the wrapper, so callers + * cannot be asked to serialize it. A declined reservation never mutates usage. + */ + +#include +#include +#include +#include + +namespace PJ { + +struct ParserModuleSessionBudgetLimits { + static constexpr uint64_t kDefaultMaximumModules = 64; + static constexpr uint64_t kDefaultMaximumArtifactBytes = UINT64_C(64) * 1024U * 1024U; + static constexpr uint64_t kDefaultMaximumClaims = 4096; + static constexpr uint64_t kDefaultMaximumActiveInstances = 128; + static constexpr uint64_t kDefaultMaximumLinearMemoryBytes = UINT64_C(4) * 1024U * 1024U * 1024U; + + uint64_t maximum_modules = kDefaultMaximumModules; + uint64_t maximum_artifact_bytes = kDefaultMaximumArtifactBytes; + uint64_t maximum_claims = kDefaultMaximumClaims; + uint64_t maximum_active_instances = kDefaultMaximumActiveInstances; + uint64_t maximum_linear_memory_bytes = kDefaultMaximumLinearMemoryBytes; +}; + +enum class ParserModuleBudgetKind : uint8_t { + kNone, + kModuleCount, + kArtifactFileSize, + kTotalClaims, + kActiveInstances, + kTotalLinearMemory, +}; + +/// Outcome of one admission request. `reservation` is nonzero exactly when +/// the request was accepted and is the handle for every later call. +struct ParserModuleAdmissionDecision { + uint64_t reservation = 0; + ParserModuleBudgetKind exhausted_budget = ParserModuleBudgetKind::kNone; + std::string diagnostic; + + [[nodiscard]] bool accepted() const noexcept { + return reservation != 0; + } +}; + +struct ParserModuleSessionBudgetUsage { + uint64_t modules = 0; + uint64_t claims = 0; + uint64_t active_instances = 0; + uint64_t declared_linear_memory_bytes = 0; +}; + +class ParserModuleSessionBudgetTracker { + public: + explicit ParserModuleSessionBudgetTracker(ParserModuleSessionBudgetLimits limits = {}); + + /// Reserve one module before compilation. `artifact_bytes` is a per-file + /// gate; `claim_count` joins the aggregate claim total; + /// `declared_linear_memory_maximum` is charged per admitted instance. + [[nodiscard]] ParserModuleAdmissionDecision admitModule( + uint64_t artifact_bytes, uint64_t claim_count, uint64_t declared_linear_memory_maximum); + + /// Reserve one instance of an admitted module. Every instance owns an + /// independent store, so the module's declared memory is charged again. + [[nodiscard]] ParserModuleAdmissionDecision admitInstance(uint64_t module_reservation); + + /// Releases are idempotent-safe: unknown or already-released ids are ignored. + void releaseInstance(uint64_t module_reservation); + /// A module reservation is released even when instances are still live; + /// their later releases then find no module and are ignored. + void releaseModule(uint64_t module_reservation); + + [[nodiscard]] const ParserModuleSessionBudgetLimits& limits() const noexcept; + [[nodiscard]] ParserModuleSessionBudgetUsage usage() const; + + private: + struct ModuleReservation { + uint64_t claim_count = 0; + uint64_t declared_linear_memory_maximum = 0; + uint64_t active_instances = 0; + }; + + ParserModuleSessionBudgetLimits limits_; + mutable std::mutex mutex_; + ParserModuleSessionBudgetUsage usage_; + uint64_t next_reservation_ = 1; + std::map modules_; +}; + +} // namespace PJ diff --git a/experimental/parser_module/include/pj_plugins/host/wasm_parser_module.hpp b/experimental/parser_module/include/pj_plugins/host/wasm_parser_module.hpp new file mode 100644 index 00000000..d5d3bfb9 --- /dev/null +++ b/experimental/parser_module/include/pj_plugins/host/wasm_parser_module.hpp @@ -0,0 +1,107 @@ +#pragma once +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +/** + * @file wasm_parser_module.hpp + * @brief Wasmer-backed loader for sandboxed functional parser modules. + * + * Admission is passive: the shared pj_base audit validates the manifest + * section, reactor shape, operational signatures, exported memory, bounded + * tables, and imports before Wasmer compiles the artifact. The v1 import + * allow-list is deliberately empty. In particular, every + * `wasi_snapshot_preview1` fd, path, socket, environment, clock, random, + * process, and scheduler import is rejected. + * + * Wasmer 7.0.1's static C archive does not provide the share/obtain symbols + * declared by wasm.h. Its engine-owned `wasmer_module_new` extension is the + * equivalent used here: one compiled module is instantiated in an independent + * store for every bound instance. The exit-criterion prototype also verified + * that a store/instance tolerates sequential calls from a thread other than + * its creator. Instances therefore have no creator-thread affinity, but calls + * on one instance must never overlap. Per-store executor serialization remains + * a host responsibility. + * + * Compilation uses Wasmer's Singlepass backend when the archive provides it: + * its compile time is linear in artifact size, which is the only bound + * available for compiling an untrusted artifact (there is no cancellable + * compile API). Wasmer's metering middleware is enabled for every compiled + * module and each guest call receives a fresh instruction-point allowance. + * That is an instruction budget, not a wall-clock deadline: bulk operators + * such as memory.copy cost one point regardless of size, and the pinned + * archive exports no interrupt or epoch API. Linear memory must declare a + * maximum within the loader cap; Wasmer enforces that maximum at runtime. The + * SDK authoring preset puts a configurable 1 MiB guest shadow stack before + * data segments so overflow traps instead of corrupting them. Native engine + * stack depth relies on Wasmer 7's guarded default because its C API exposes + * no stack-limit setter. + * + * Like the native loader, this loader carries no fault policy: instances + * classify faults and the host feeds them to ParserModuleStrikeTracker. + */ + +#include +#include +#include +#include + +#include "pj_base/diagnostic_sink.hpp" +#include "pj_base/expected.hpp" +#include "pj_base/parser_module_wasm.hpp" +#include "pj_plugins/host/parser_module_session_budget.hpp" + +namespace PJ { + +namespace detail { +struct WasmParserModuleState; +} + +class WasmParserModuleInstance; + +/// Per-artifact caps and the per-call instruction budget. +struct WasmParserModuleLimits { + static constexpr uint64_t kDefaultMaximumArtifactBytes = UINT64_C(64) * 1024U * 1024U; + static constexpr uint64_t kDefaultMeteringPointsPerCall = UINT64_C(10000000); + + uint64_t maximum_artifact_bytes = kDefaultMaximumArtifactBytes; + uint64_t maximum_linear_memory_bytes = parser_module::ParserModuleWasmLimits::kDefaultMaximumLinearMemoryBytes; + uint64_t maximum_table_elements = parser_module::ParserModuleWasmLimits::kDefaultMaximumTableElements; + uint64_t metering_points_per_call = kDefaultMeteringPointsPerCall; +}; + +struct WasmParserModuleLoadOptions { + WasmParserModuleLimits limits; + /// Optional application-owned aggregate session budget. + std::shared_ptr budget; + /// Every rejection emits exactly one error diagnostic when a sink is set. + DiagnosticSink sink; + std::string diagnostic_source = "WasmParserModule"; +}; + +class WasmParserModule { + public: + WasmParserModule() = default; + + /// Read, audit, admit, and compile one wasm parser module. + [[nodiscard]] static Expected load( + std::string_view path, const WasmParserModuleLoadOptions& options = {}); + + [[nodiscard]] bool valid() const noexcept { + return state_ != nullptr; + } + + [[nodiscard]] std::string_view path() const noexcept; + [[nodiscard]] std::string_view manifestJson() const noexcept; + [[nodiscard]] uint64_t artifactSize() const noexcept; + [[nodiscard]] uint64_t declaredLinearMemoryMaximum() const noexcept; + [[nodiscard]] uint64_t declaredTableElements() const noexcept; + + private: + explicit WasmParserModule(std::shared_ptr state); + + std::shared_ptr state_; + + friend class WasmParserModuleInstance; +}; + +} // namespace PJ diff --git a/experimental/parser_module/include/pj_plugins/host/wasm_parser_module_runtime.hpp b/experimental/parser_module/include/pj_plugins/host/wasm_parser_module_runtime.hpp new file mode 100644 index 00000000..d6690d95 --- /dev/null +++ b/experimental/parser_module/include/pj_plugins/host/wasm_parser_module_runtime.hpp @@ -0,0 +1,82 @@ +#pragma once +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +/** + * @file wasm_parser_module_runtime.hpp + * @brief Store-per-instance wasm parser-module lifecycle wrapper. + * + * The wrapper is move-only and not concurrently callable. A host may migrate + * it between threads when calls do not overlap. Every host access to guest + * memory re-acquires the current base and size after the preceding guest call. + * + * Like NativeParserModuleInstance, the wrapper only classifies faults: traps, + * metering exhaustion, malformed descriptors, and bad splices are returned as + * contract violations, module-reported failures as data errors. Recording + * strikes, quarantining a claim, and replaying create/bind are host policy + * driven through ParserModuleStrikeTracker, exactly as for native modules. + */ + +#include +#include +#include +#include + +#include "pj_base/expected.hpp" +#include "pj_base/parser_module_abi.h" +#include "pj_plugins/host/parser_module_runtime.hpp" +#include "pj_plugins/host/wasm_parser_module.hpp" + +namespace PJ { + +namespace detail { +struct WasmParserModuleInstanceState; +} + +enum class WasmParserModuleCreateOutcome : uint8_t { + kError, + kAdmissionDecline, +}; + +struct WasmParserModuleCreateError { + WasmParserModuleCreateOutcome outcome = WasmParserModuleCreateOutcome::kError; + ParserModuleFaultKind fault = ParserModuleFaultKind::kNone; + std::string message; +}; + +class WasmParserModuleInstance { + public: + WasmParserModuleInstance(); + ~WasmParserModuleInstance(); + + WasmParserModuleInstance(WasmParserModuleInstance&& other) noexcept; + WasmParserModuleInstance& operator=(WasmParserModuleInstance&& other) noexcept; + + WasmParserModuleInstance(const WasmParserModuleInstance&) = delete; + WasmParserModuleInstance& operator=(const WasmParserModuleInstance&) = delete; + + /// Instantiate the shared compiled module in a new store, run `_initialize` + /// exactly once, then create the manifest claim at `claim_index`. A trap on + /// that path is returned with `fault == kContractViolation`; a session-budget + /// rejection with `outcome == kAdmissionDecline`. + [[nodiscard]] static Expected create( + const WasmParserModule& module, uint32_t claim_index); + + [[nodiscard]] Expected bind(const parser_module::BindingInfoV1& info); + + /// Parse one message and consume the returned descriptor transactionally. + [[nodiscard]] Expected parse(const parser_module::ParseInputV1& input); + + [[nodiscard]] bool valid() const noexcept; + [[nodiscard]] uint32_t claimIndex() const noexcept; + /// The most recent contract-violation text, including guest `free` faults + /// raised while cleaning up after another failure. + [[nodiscard]] std::string_view lifecycleDiagnostic() const noexcept; + + private: + explicit WasmParserModuleInstance(std::unique_ptr state); + + std::unique_ptr state_; +}; + +} // namespace PJ diff --git a/experimental/parser_module/src/detail/native_parser_module_state.hpp b/experimental/parser_module/src/detail/native_parser_module_state.hpp index 91a7a7e7..bb0c3d9d 100644 --- a/experimental/parser_module/src/detail/native_parser_module_state.hpp +++ b/experimental/parser_module/src/detail/native_parser_module_state.hpp @@ -2,17 +2,25 @@ // Copyright 2026 Davide Faconti // SPDX-License-Identifier: Apache-2.0 +#include +#include #include #include "detail/native_parser_module_loader.hpp" #include "pj_base/parser_module_abi.h" +#include "pj_plugins/host/parser_module_session_budget.hpp" namespace PJ::detail { struct NativeParserModuleState { + ~NativeParserModuleState(); + NativeModuleHandle handle = nullptr; std::string path; std::string manifest_json; + /// Null when the host loaded without a session budget (0.22 behavior). + std::shared_ptr session_budget; + uint64_t module_reservation = 0; PJ_module_abi_fn_t abi = nullptr; PJ_module_create_fn_t create = nullptr; diff --git a/experimental/parser_module/src/detail/parser_module_result_helpers.hpp b/experimental/parser_module/src/detail/parser_module_result_helpers.hpp new file mode 100644 index 00000000..2b6725c6 --- /dev/null +++ b/experimental/parser_module/src/detail/parser_module_result_helpers.hpp @@ -0,0 +1,151 @@ +#pragma once +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pj_base/builtin/builtin_object_codec.hpp" +#include "pj_base/builtin/grid_map_codec.hpp" +#include "pj_base/builtin_object_abi.h" +#include "pj_base/expected.hpp" +#include "pj_base/parser_module_abi.h" +#include "pj_base/span.hpp" +#include "pj_plugins/host/parser_module_runtime.hpp" + +namespace PJ::detail { + +inline ParserModuleParseResult contractViolation(int32_t code, std::string message) { + return ParserModuleParseResult{ + .fault = ParserModuleFaultKind::kContractViolation, + .result_code = code, + .message = std::move(message), + .output = std::nullopt, + }; +} + +inline Expected ownScalarOutput(const parser_module::ScalarOutputV1& scalar) { + ParserModuleScalarOutput owned; + owned.has_timestamp = scalar.has_timestamp; + owned.timestamp_ns = scalar.timestamp_ns; + owned.fields.reserve(scalar.fields.size()); + for (const auto& field : scalar.fields) { + ParserModuleScalarValue value = std::visit( + [](const Value& item) -> ParserModuleScalarValue { + if constexpr (std::is_same_v) { + return std::string(item); + } else { + return item; + } + }, + field.value); + owned.fields.push_back(ParserModuleScalarField{.name = std::string(field.name), .value = std::move(value)}); + } + return owned; +} + +inline Expected ownObjectOutput( + const parser_module::ObjectOutputV1& object, Span input_payload, uint16_t expected_type) { + if (object.object_type != expected_type) { + return unexpected( + "output object type " + std::to_string(object.object_type) + " does not match bound type " + + std::to_string(expected_type)); + } + + const auto type = static_cast(object.object_type); + auto decoded = deserializeBuiltinObject(type, object.wire.data(), object.wire.size()); + if (!decoded) { + return unexpected("output canonical wire is malformed: " + decoded.error()); + } + + ParserModuleObjectOutput owned; + owned.object = std::move(*decoded); + owned.wire.assign(object.wire.begin(), object.wire.end()); + if (object.splice.has_value()) { + uint32_t eligible_field = 0; + if (!pj_builtin_object_splice_field_number_v1(object.object_type, &eligible_field) || + eligible_field != object.splice->field_number) { + return unexpected("output splice field is not eligible for the object type"); + } + const uint64_t payload_size = static_cast(input_payload.size()); + if (object.splice->input_offset > payload_size || + object.splice->input_length > payload_size - object.splice->input_offset) { + return unexpected("output splice range is outside the parse payload"); + } + const auto offset = static_cast(object.splice->input_offset); + const auto length = static_cast(object.splice->input_length); + auto materialized = std::make_shared>( + input_payload.begin() + static_cast(offset), + input_payload.begin() + static_cast(offset + length)); + const auto attach = [&]() -> bool { + auto* typed = owned.object.get(); + if (typed == nullptr) { + return false; + } + typed->data = Span(materialized->data(), materialized->size()); + typed->anchor = materialized; + return true; + }; + bool attached = false; + switch (type) { + case sdk::BuiltinObjectType::kImage: + attached = attach.template operator()(); + break; + case sdk::BuiltinObjectType::kPointCloud: + attached = attach.template operator()(); + break; + case sdk::BuiltinObjectType::kDepthImage: + attached = attach.template operator()(); + break; + case sdk::BuiltinObjectType::kOccupancyGrid: + attached = attach.template operator()(); + break; + case sdk::BuiltinObjectType::kCompressedPointCloud: + attached = attach.template operator()(); + break; + case sdk::BuiltinObjectType::kMesh3D: + attached = attach.template operator()(); + break; + case sdk::BuiltinObjectType::kVideoFrame: + attached = attach.template operator()(); + break; + case sdk::BuiltinObjectType::kOccupancyGridUpdate: + attached = attach.template operator()(); + break; + case sdk::BuiltinObjectType::kVoxelGrid: + attached = attach.template operator()(); + break; + case sdk::BuiltinObjectType::kGridMap: + attached = attach.template operator()(); + break; + default: + break; + } + if (!attached) { + return unexpected("output splice could not be attached to its canonical object"); + } + // GridMap decodes header-only for splices; the data-length check waits + // until the bytes exist, which is now. + if (type == sdk::BuiltinObjectType::kGridMap) { + if (auto valid = validateGridMap(*owned.object.get()); !valid) { + return unexpected("output spliced GridMap layout is invalid: " + valid.error()); + } + } + owned.splice = ParserModuleObjectSplice{ + .field_number = object.splice->field_number, + .input_offset = object.splice->input_offset, + .payload_bytes = *materialized, + }; + } + return owned; +} + +} // namespace PJ::detail diff --git a/experimental/parser_module/src/detail/wasm_parser_module_state.hpp b/experimental/parser_module/src/detail/wasm_parser_module_state.hpp new file mode 100644 index 00000000..56d26e85 --- /dev/null +++ b/experimental/parser_module/src/detail/wasm_parser_module_state.hpp @@ -0,0 +1,32 @@ +#pragma once +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include + +#include "pj_plugins/host/parser_module_session_budget.hpp" + +namespace PJ::detail { + +struct WasmParserModuleState { + ~WasmParserModuleState(); + + std::string path; + std::string manifest_json; + uint64_t claim_count = 0; + uint64_t artifact_size = 0; + uint64_t declared_linear_memory_maximum = 0; + uint64_t declared_table_elements = 0; + uint64_t metering_points_per_call = 0; + wasm_engine_t* engine = nullptr; + wasm_module_t* module = nullptr; + /// Null when the host loaded without a session budget. + std::shared_ptr session_budget; + uint64_t module_reservation = 0; +}; + +} // namespace PJ::detail diff --git a/experimental/parser_module/src/native_parser_module.cpp b/experimental/parser_module/src/native_parser_module.cpp index 0a9a181f..fe20acd9 100644 --- a/experimental/parser_module/src/native_parser_module.cpp +++ b/experimental/parser_module/src/native_parser_module.cpp @@ -13,6 +13,7 @@ #include "detail/native_parser_module_state.hpp" #include "pj_base/parser_module_abi.h" +#include "pj_plugins/host/parser_module_claims.hpp" namespace PJ { namespace { @@ -58,11 +59,27 @@ Expected resolve( } // namespace +namespace detail { + +NativeParserModuleState::~NativeParserModuleState() { + if (session_budget != nullptr) { + session_budget->releaseModule(module_reservation); + } +} + +} // namespace detail + NativeParserModule::NativeParserModule(std::shared_ptr state) : state_(std::move(state)) {} Expected NativeParserModule::load( std::string_view path, DiagnosticSink sink, std::string diagnostic_source) { + return load(path, nullptr, std::move(sink), std::move(diagnostic_source)); +} + +Expected NativeParserModule::load( + std::string_view path, std::shared_ptr budget, DiagnosticSink sink, + std::string diagnostic_source) { detail::LibraryPathIdentity recorded_path; auto handle_result = detail::openNativeParserModule(path, &recorded_path); if (!handle_result) { @@ -113,6 +130,29 @@ Expected NativeParserModule::load( const auto* manifest = reinterpret_cast(static_cast(manifest_addr)); state->manifest_json.assign(manifest, static_cast(manifest_len)); + if (budget != nullptr) { + // Budget accounting needs the claim count, so the manifest is decoded here + // as well as at catalog ingestion. Without a budget the 0.22 contract holds: + // the loader copies the bytes and leaves every manifest decision to the + // catalog. + auto decoded_manifest = decodeParserModuleManifest(state->manifest_json, ParserClaimProvenance::kFolderDrop); + if (!decoded_manifest) { + return rejectLoad( + path, sink, diagnostic_source, "invalid native parser-module manifest: " + decoded_manifest.error()); + } + std::error_code file_error; + const uintmax_t artifact_size = std::filesystem::file_size(std::filesystem::path(path), file_error); + if (file_error || artifact_size > std::numeric_limits::max()) { + return rejectLoad(path, sink, diagnostic_source, "native parser-module artifact file size is unreadable"); + } + auto admission = budget->admitModule(static_cast(artifact_size), decoded_manifest->claims.size(), 0); + if (!admission.accepted()) { + return rejectLoad(path, sink, diagnostic_source, std::move(admission.diagnostic)); + } + state->session_budget = std::move(budget); + state->module_reservation = admission.reservation; + } + return NativeParserModule(std::move(state)); } diff --git a/experimental/parser_module/src/parser_module_runtime.cpp b/experimental/parser_module/src/parser_module_runtime.cpp index 1d713f81..d9afb435 100644 --- a/experimental/parser_module/src/parser_module_runtime.cpp +++ b/experimental/parser_module/src/parser_module_runtime.cpp @@ -9,15 +9,14 @@ #include #include #include +#include #include #include #include #include #include "detail/native_parser_module_state.hpp" -#include "pj_base/builtin/builtin_object_codec.hpp" -#include "pj_base/builtin/grid_map_codec.hpp" -#include "pj_base/builtin_object_abi.h" +#include "detail/parser_module_result_helpers.hpp" #include "pj_base/span.hpp" namespace PJ { @@ -38,130 +37,9 @@ Expected copyLastError(const detail::NativeParserModuleState& modul return std::string(buffer.begin(), terminator); } -ParserModuleParseResult contractViolation(int32_t code, std::string message) { - return ParserModuleParseResult{ - .fault = ParserModuleFaultKind::kContractViolation, - .result_code = code, - .message = std::move(message), - .output = std::nullopt, - }; -} - -Expected ownScalarOutput(const parser_module::ScalarOutputV1& scalar) { - ParserModuleScalarOutput owned; - owned.has_timestamp = scalar.has_timestamp; - owned.timestamp_ns = scalar.timestamp_ns; - owned.fields.reserve(scalar.fields.size()); - for (const auto& field : scalar.fields) { - ParserModuleScalarValue value = std::visit( - [](const Value& item) -> ParserModuleScalarValue { - if constexpr (std::is_same_v) { - return std::string(item); - } else { - return item; - } - }, - field.value); - owned.fields.push_back(ParserModuleScalarField{.name = std::string(field.name), .value = std::move(value)}); - } - return owned; -} - -Expected ownObjectOutput( - const parser_module::ObjectOutputV1& object, Span input_payload, uint16_t expected_type) { - if (object.object_type != expected_type) { - return unexpected( - "output object type " + std::to_string(object.object_type) + " does not match bound type " + - std::to_string(expected_type)); - } - - const auto type = static_cast(object.object_type); - auto decoded = deserializeBuiltinObject(type, object.wire.data(), object.wire.size()); - if (!decoded) { - return unexpected("output canonical wire is malformed: " + decoded.error()); - } - - ParserModuleObjectOutput owned; - owned.object = std::move(*decoded); - owned.wire.assign(object.wire.begin(), object.wire.end()); - if (object.splice.has_value()) { - uint32_t eligible_field = 0; - if (!pj_builtin_object_splice_field_number_v1(object.object_type, &eligible_field) || - eligible_field != object.splice->field_number) { - return unexpected("output splice field is not eligible for the object type"); - } - const uint64_t payload_size = static_cast(input_payload.size()); - if (object.splice->input_offset > payload_size || - object.splice->input_length > payload_size - object.splice->input_offset) { - return unexpected("output splice range is outside the parse payload"); - } - const auto offset = static_cast(object.splice->input_offset); - const auto length = static_cast(object.splice->input_length); - auto materialized = std::make_shared>( - input_payload.begin() + static_cast(offset), - input_payload.begin() + static_cast(offset + length)); - const auto attach = [&]() -> bool { - auto* typed = owned.object.get(); - if (typed == nullptr) { - return false; - } - typed->data = Span(materialized->data(), materialized->size()); - typed->anchor = materialized; - return true; - }; - bool attached = false; - switch (type) { - case sdk::BuiltinObjectType::kImage: - attached = attach.template operator()(); - break; - case sdk::BuiltinObjectType::kPointCloud: - attached = attach.template operator()(); - break; - case sdk::BuiltinObjectType::kDepthImage: - attached = attach.template operator()(); - break; - case sdk::BuiltinObjectType::kOccupancyGrid: - attached = attach.template operator()(); - break; - case sdk::BuiltinObjectType::kCompressedPointCloud: - attached = attach.template operator()(); - break; - case sdk::BuiltinObjectType::kMesh3D: - attached = attach.template operator()(); - break; - case sdk::BuiltinObjectType::kVideoFrame: - attached = attach.template operator()(); - break; - case sdk::BuiltinObjectType::kOccupancyGridUpdate: - attached = attach.template operator()(); - break; - case sdk::BuiltinObjectType::kVoxelGrid: - attached = attach.template operator()(); - break; - case sdk::BuiltinObjectType::kGridMap: - attached = attach.template operator()(); - break; - default: - break; - } - if (!attached) { - return unexpected("output splice could not be attached to its canonical object"); - } - // GridMap decodes header-only for splices; the data-length check waits - // until the bytes exist, which is now. - if (type == sdk::BuiltinObjectType::kGridMap) { - if (auto valid = validateGridMap(*owned.object.get()); !valid) { - return unexpected("output spliced GridMap layout is invalid: " + valid.error()); - } - } - owned.splice = ParserModuleObjectSplice{ - .field_number = object.splice->field_number, - .input_offset = object.splice->input_offset, - .payload_bytes = *materialized, - }; - } - return owned; -} +using detail::contractViolation; +using detail::ownObjectOutput; +using detail::ownScalarOutput; } // namespace @@ -179,7 +57,8 @@ NativeParserModuleInstance::NativeParserModuleInstance(NativeParserModuleInstanc claim_index_(other.claim_index_), bound_route_(other.bound_route_), expected_object_type_(other.expected_object_type_), - bound_(other.bound_) { + bound_(other.bound_), + instance_budget_reserved_(std::exchange(other.instance_budget_reserved_, false)) { other.bound_ = false; } @@ -192,6 +71,7 @@ NativeParserModuleInstance& NativeParserModuleInstance::operator=(NativeParserMo bound_route_ = other.bound_route_; expected_object_type_ = other.expected_object_type_; bound_ = other.bound_; + instance_budget_reserved_ = std::exchange(other.instance_budget_reserved_, false); other.bound_ = false; } return *this; @@ -202,12 +82,24 @@ Expected NativeParserModuleInstance::create( if (!module.valid()) { return unexpected("cannot create an instance from an invalid native parser module"); } + const auto& budget = module.state_->session_budget; + if (budget != nullptr) { + auto admission = budget->admitInstance(module.state_->module_reservation); + if (!admission.accepted()) { + return unexpected(std::move(admission.diagnostic)); + } + } const uint64_t token = module.state_->create(claim_index); if (token == PJ_MODULE_CREATION_ERROR_TOKEN) { + if (budget != nullptr) { + budget->releaseInstance(module.state_->module_reservation); + } auto message = copyLastError(*module.state_, PJ_MODULE_CREATION_ERROR_TOKEN); return unexpected(message ? *message : message.error()); } - return NativeParserModuleInstance(module.state_, token, claim_index); + NativeParserModuleInstance instance(module.state_, token, claim_index); + instance.instance_budget_reserved_ = budget != nullptr; + return instance; } Expected NativeParserModuleInstance::bind(const parser_module::BindingInfoV1& info) { @@ -342,17 +234,29 @@ void NativeParserModuleInstance::reset() noexcept { if (module_ != nullptr && token_ != PJ_MODULE_CREATION_ERROR_TOKEN) { module_->destroy(token_); } + if (module_ != nullptr && instance_budget_reserved_) { + module_->session_budget->releaseInstance(module_->module_reservation); + } token_ = PJ_MODULE_CREATION_ERROR_TOKEN; bound_ = false; + instance_budget_reserved_ = false; module_.reset(); } ParserModuleStrikeState ParserModuleStrikeTracker::recordFault( const ParserModuleClaimKey& key, ParserModuleFaultKind fault) { + const std::scoped_lock lock(mutex_); auto [it, inserted] = states_.try_emplace(key); (void)inserted; auto& state = it->second; - if (fault != ParserModuleFaultKind::kContractViolation || state.health != ParserModuleClaimHealth::kActive) { + if (fault != ParserModuleFaultKind::kContractViolation || state.health == ParserModuleClaimHealth::kDisabled) { + return state; + } + if (state.health == ParserModuleClaimHealth::kQuarantined) { + // The replay itself faulted: that is the repeat the policy disables on. + state.strikes = 0; + ++state.quarantine_count; + state.health = ParserModuleClaimHealth::kDisabled; return state; } @@ -370,6 +274,7 @@ ParserModuleStrikeState ParserModuleStrikeTracker::recordFault( } bool ParserModuleStrikeTracker::markRecreated(const ParserModuleClaimKey& key) { + const std::scoped_lock lock(mutex_); auto it = states_.find(key); if (it == states_.end() || it->second.health != ParserModuleClaimHealth::kQuarantined) { return false; @@ -379,6 +284,7 @@ bool ParserModuleStrikeTracker::markRecreated(const ParserModuleClaimKey& key) { } ParserModuleStrikeState ParserModuleStrikeTracker::state(const ParserModuleClaimKey& key) const { + const std::scoped_lock lock(mutex_); const auto it = states_.find(key); return it == states_.end() ? ParserModuleStrikeState{} : it->second; } diff --git a/experimental/parser_module/src/parser_module_session_budget.cpp b/experimental/parser_module/src/parser_module_session_budget.cpp new file mode 100644 index 00000000..99309aba --- /dev/null +++ b/experimental/parser_module/src/parser_module_session_budget.cpp @@ -0,0 +1,119 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include "pj_plugins/host/parser_module_session_budget.hpp" + +#include +#include +#include +#include + +namespace PJ { +namespace { + +ParserModuleAdmissionDecision declineBudget(ParserModuleBudgetKind kind, std::string_view name) { + return ParserModuleAdmissionDecision{ + .reservation = 0, + .exhausted_budget = kind, + .diagnostic = "parser-module admission DECLINE: " + std::string(name) + " budget exhausted", + }; +} + +bool exceedsAggregate(uint64_t current, uint64_t additional, uint64_t maximum) { + return current > maximum || additional > maximum - current; +} + +} // namespace + +ParserModuleSessionBudgetTracker::ParserModuleSessionBudgetTracker(ParserModuleSessionBudgetLimits limits) + : limits_(limits) {} + +ParserModuleAdmissionDecision ParserModuleSessionBudgetTracker::admitModule( + uint64_t artifact_bytes, uint64_t claim_count, uint64_t declared_linear_memory_maximum) { + const std::scoped_lock lock(mutex_); + if (usage_.modules >= limits_.maximum_modules) { + return declineBudget(ParserModuleBudgetKind::kModuleCount, "module_count"); + } + if (artifact_bytes > limits_.maximum_artifact_bytes) { + return declineBudget(ParserModuleBudgetKind::kArtifactFileSize, "artifact_file_size"); + } + if (exceedsAggregate(usage_.claims, claim_count, limits_.maximum_claims)) { + return declineBudget(ParserModuleBudgetKind::kTotalClaims, "total_claims"); + } + + const uint64_t reservation = next_reservation_++; + modules_.emplace( + reservation, ModuleReservation{ + .claim_count = claim_count, + .declared_linear_memory_maximum = declared_linear_memory_maximum, + .active_instances = 0, + }); + ++usage_.modules; + usage_.claims += claim_count; + return ParserModuleAdmissionDecision{ + .reservation = reservation, .exhausted_budget = ParserModuleBudgetKind::kNone, .diagnostic = {}}; +} + +ParserModuleAdmissionDecision ParserModuleSessionBudgetTracker::admitInstance(uint64_t module_reservation) { + const std::scoped_lock lock(mutex_); + auto module = modules_.find(module_reservation); + if (module == modules_.end()) { + return ParserModuleAdmissionDecision{ + .reservation = 0, + .exhausted_budget = ParserModuleBudgetKind::kNone, + .diagnostic = "parser-module admission DECLINE: module is not admitted", + }; + } + if (usage_.active_instances >= limits_.maximum_active_instances) { + return declineBudget(ParserModuleBudgetKind::kActiveInstances, "active_instances"); + } + if (exceedsAggregate( + usage_.declared_linear_memory_bytes, module->second.declared_linear_memory_maximum, + limits_.maximum_linear_memory_bytes)) { + return declineBudget(ParserModuleBudgetKind::kTotalLinearMemory, "total_linear_memory"); + } + + ++module->second.active_instances; + ++usage_.active_instances; + usage_.declared_linear_memory_bytes += module->second.declared_linear_memory_maximum; + return ParserModuleAdmissionDecision{ + .reservation = module_reservation, .exhausted_budget = ParserModuleBudgetKind::kNone, .diagnostic = {}}; +} + +void ParserModuleSessionBudgetTracker::releaseInstance(uint64_t module_reservation) { + const std::scoped_lock lock(mutex_); + auto module = modules_.find(module_reservation); + if (module == modules_.end() || module->second.active_instances == 0) { + return; + } + --module->second.active_instances; + --usage_.active_instances; + usage_.declared_linear_memory_bytes -= module->second.declared_linear_memory_maximum; +} + +void ParserModuleSessionBudgetTracker::releaseModule(uint64_t module_reservation) { + const std::scoped_lock lock(mutex_); + auto module = modules_.find(module_reservation); + if (module == modules_.end()) { + return; + } + // Live instances of a dropped module still hold their store; their memory + // and instance counts are given back when each of them is released. + usage_.active_instances -= module->second.active_instances; + usage_.declared_linear_memory_bytes -= + module->second.active_instances * module->second.declared_linear_memory_maximum; + --usage_.modules; + usage_.claims -= module->second.claim_count; + modules_.erase(module); +} + +const ParserModuleSessionBudgetLimits& ParserModuleSessionBudgetTracker::limits() const noexcept { + return limits_; +} + +ParserModuleSessionBudgetUsage ParserModuleSessionBudgetTracker::usage() const { + const std::scoped_lock lock(mutex_); + return usage_; +} + +} // namespace PJ diff --git a/experimental/parser_module/src/parser_module_wasm.cpp b/experimental/parser_module/src/parser_module_wasm.cpp new file mode 100644 index 00000000..75300100 --- /dev/null +++ b/experimental/parser_module/src/parser_module_wasm.cpp @@ -0,0 +1,666 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include "pj_base/parser_module_wasm.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pj_base/parser_module_abi.h" +#include "pj_base/parser_module_manifest.hpp" + +namespace PJ::parser_module { +namespace { + +constexpr std::array kWasmPreamble{0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00}; + +class Cursor { + public: + explicit Cursor(Span bytes) : bytes_(bytes) {} + + [[nodiscard]] bool empty() const noexcept { + return position_ == bytes_.size(); + } + + [[nodiscard]] size_t remaining() const noexcept { + return position_ <= bytes_.size() ? bytes_.size() - position_ : 0; + } + + [[nodiscard]] Expected byte() { + if (position_ >= bytes_.size()) { + return unexpected(std::string("truncated wasm byte")); + } + return bytes_[position_++]; + } + + [[nodiscard]] Expected varUint32() { + uint32_t value = 0; + for (size_t index = 0; index < 5; ++index) { + auto next = byte(); + if (!next) { + return unexpected(next.error()); + } + if (index == 4 && (*next & UINT8_C(0xF0)) != 0) { + return unexpected(std::string("wasm varuint32 overflows uint32")); + } + value |= static_cast(*next & UINT8_C(0x7F)) << (index * 7U); + if ((*next & UINT8_C(0x80)) == 0) { + return value; + } + } + return unexpected(std::string("wasm varuint32 exceeds five bytes")); + } + + [[nodiscard]] Expected name() { + auto length = varUint32(); + if (!length) { + return unexpected(length.error()); + } + if (static_cast(*length) > remaining()) { + return unexpected(std::string("wasm name exceeds the remaining section bytes")); + } + const char* begin = reinterpret_cast(bytes_.data() + position_); + position_ += *length; + return std::string(begin, *length); + } + + [[nodiscard]] Expected take(uint32_t size) { + if (static_cast(size) > remaining()) { + return unexpected(std::string("wasm section exceeds the remaining module bytes")); + } + Cursor result(bytes_.subspan(position_, size)); + position_ += size; + return result; + } + + private: + Span bytes_; + size_t position_ = 0; +}; + +struct PendingImport { + WasmImport value; + std::optional function_type; +}; + +struct ModuleBuilder { + std::vector types; + std::vector imported_function_types; + std::vector defined_function_types; + std::vector imported_memories; + std::vector defined_memories; + std::vector tables; + std::vector imports; + std::vector exports; + bool has_start_section = false; + size_t section_count = 0; +}; + +[[nodiscard]] bool validValueType(uint8_t value) { + switch (value) { + case kWasmValueI32: + case kWasmValueI64: + case 0x7D: // f32 + case 0x7C: // f64 + case 0x7B: // v128 + case 0x70: // funcref + case 0x6F: // externref + return true; + default: + return false; + } +} + +[[nodiscard]] Expected requireCountFits( + uint32_t count, size_t remaining, size_t minimum_entry_size, std::string_view section) { + if (minimum_entry_size == 0 || static_cast(count) > remaining / minimum_entry_size) { + return unexpected(std::string("wasm ") + std::string(section) + " count exceeds the remaining section bytes"); + } + return {}; +} + +[[nodiscard]] Expected> readValueTypes(Cursor* cursor) { + auto count = cursor->varUint32(); + if (!count) { + return unexpected(count.error()); + } + auto bounded = requireCountFits(*count, cursor->remaining(), 1, "value-type"); + if (!bounded) { + return unexpected(bounded.error()); + } + std::vector result; + result.reserve(*count); + for (uint32_t index = 0; index < *count; ++index) { + auto value = cursor->byte(); + if (!value) { + return unexpected(value.error()); + } + if (!validValueType(*value)) { + return unexpected(std::string("wasm function type contains an invalid value type")); + } + result.push_back(*value); + } + return result; +} + +[[nodiscard]] Expected requireConsumed(const Cursor& cursor, std::string_view section) { + if (!cursor.empty()) { + return unexpected(std::string(section) + " section contains trailing bytes"); + } + return {}; +} + +[[nodiscard]] Expected parseTypeSection(Cursor cursor, ModuleBuilder* module) { + auto count = cursor.varUint32(); + if (!count) { + return unexpected(count.error()); + } + auto bounded = requireCountFits(*count, cursor.remaining(), 3, "type-section entry"); + if (!bounded) { + return unexpected(bounded.error()); + } + module->types.reserve(*count); + for (uint32_t index = 0; index < *count; ++index) { + auto form = cursor.byte(); + if (!form || *form != UINT8_C(0x60)) { + return unexpected(std::string("wasm type section contains a non-function type")); + } + auto parameters = readValueTypes(&cursor); + auto results = readValueTypes(&cursor); + if (!parameters) { + return unexpected(parameters.error()); + } + if (!results) { + return unexpected(results.error()); + } + module->types.push_back(WasmFunctionSignature{std::move(*parameters), std::move(*results)}); + } + return requireConsumed(cursor, "type"); +} + +[[nodiscard]] Expected readLimits(Cursor* cursor) { + auto flags = cursor->varUint32(); + auto minimum = cursor->varUint32(); + if (!flags || !minimum) { + return unexpected(std::string("truncated wasm limits")); + } + if (*flags > 1) { + return unexpected(std::string("unsupported wasm limits flags")); + } + WasmMemoryLimits limits{.minimum_pages = *minimum, .maximum_pages = std::nullopt}; + if ((*flags & 1U) != 0) { + auto maximum = cursor->varUint32(); + if (!maximum) { + return unexpected(maximum.error()); + } + if (*maximum < *minimum) { + return unexpected(std::string("wasm limits maximum is smaller than its minimum")); + } + limits.maximum_pages = *maximum; + } + return limits; +} + +[[nodiscard]] Expected parseImportSection(Cursor cursor, ModuleBuilder* module) { + auto count = cursor.varUint32(); + if (!count) { + return unexpected(count.error()); + } + auto bounded = requireCountFits(*count, cursor.remaining(), 4, "import-section entry"); + if (!bounded) { + return unexpected(bounded.error()); + } + module->imports.reserve(*count); + for (uint32_t index = 0; index < *count; ++index) { + auto module_name = cursor.name(); + auto field_name = cursor.name(); + auto kind = cursor.byte(); + if (!module_name || !field_name || !kind || *kind > static_cast(WasmExternalKind::kTag)) { + return unexpected(std::string("truncated or invalid wasm import entry")); + } + + PendingImport imported{ + .value = + WasmImport{ + .module = std::move(*module_name), + .name = std::move(*field_name), + .kind = static_cast(*kind), + .function_signature = std::nullopt, + }, + .function_type = std::nullopt, + }; + switch (imported.value.kind) { + case WasmExternalKind::kFunction: { + auto type_index = cursor.varUint32(); + if (!type_index) { + return unexpected(type_index.error()); + } + imported.function_type = *type_index; + module->imported_function_types.push_back(*type_index); + break; + } + case WasmExternalKind::kTable: { + auto element_type = cursor.byte(); + if (!element_type || (*element_type != UINT8_C(0x70) && *element_type != UINT8_C(0x6F))) { + return unexpected(std::string("invalid wasm table import")); + } + auto limits = readLimits(&cursor); + if (!limits) { + return unexpected(limits.error()); + } + module->tables.push_back( + WasmTableLimits{.minimum_elements = limits->minimum_pages, .maximum_elements = limits->maximum_pages}); + break; + } + case WasmExternalKind::kMemory: { + auto limits = readLimits(&cursor); + if (!limits) { + return unexpected(limits.error()); + } + module->imported_memories.push_back(*limits); + break; + } + case WasmExternalKind::kGlobal: { + auto value_type = cursor.byte(); + auto mutability = cursor.byte(); + if (!value_type || !mutability || !validValueType(*value_type) || *mutability > 1) { + return unexpected(std::string("invalid wasm global import")); + } + break; + } + case WasmExternalKind::kTag: { + auto attribute = cursor.varUint32(); + auto type_index = cursor.varUint32(); + if (!attribute || !type_index) { + return unexpected(std::string("truncated wasm tag import")); + } + break; + } + } + module->imports.push_back(std::move(imported)); + } + return requireConsumed(cursor, "import"); +} + +[[nodiscard]] Expected parseFunctionSection(Cursor cursor, ModuleBuilder* module) { + auto count = cursor.varUint32(); + if (!count) { + return unexpected(count.error()); + } + auto bounded = requireCountFits(*count, cursor.remaining(), 1, "function-section entry"); + if (!bounded) { + return unexpected(bounded.error()); + } + module->defined_function_types.reserve(*count); + for (uint32_t index = 0; index < *count; ++index) { + auto type_index = cursor.varUint32(); + if (!type_index) { + return unexpected(type_index.error()); + } + module->defined_function_types.push_back(*type_index); + } + return requireConsumed(cursor, "function"); +} + +[[nodiscard]] Expected parseTableSection(Cursor cursor, ModuleBuilder* module) { + auto count = cursor.varUint32(); + if (!count) { + return unexpected(count.error()); + } + auto bounded = requireCountFits(*count, cursor.remaining(), 3, "table-section entry"); + if (!bounded) { + return unexpected(bounded.error()); + } + module->tables.reserve(module->tables.size() + *count); + for (uint32_t index = 0; index < *count; ++index) { + auto reference_type = cursor.byte(); + if (!reference_type) { + return unexpected(reference_type.error()); + } + if (*reference_type != UINT8_C(0x70) && *reference_type != UINT8_C(0x6F)) { + // 0x40 introduces the table-with-initializer form; neither it nor typed + // references are part of the frozen v1 module shape. + return unexpected(std::string("unsupported wasm table form")); + } + auto limits = readLimits(&cursor); + if (!limits) { + return unexpected(limits.error()); + } + module->tables.push_back( + WasmTableLimits{.minimum_elements = limits->minimum_pages, .maximum_elements = limits->maximum_pages}); + } + return requireConsumed(cursor, "table"); +} + +[[nodiscard]] Expected parseMemorySection(Cursor cursor, ModuleBuilder* module) { + auto count = cursor.varUint32(); + if (!count) { + return unexpected(count.error()); + } + auto bounded = requireCountFits(*count, cursor.remaining(), 2, "memory-section entry"); + if (!bounded) { + return unexpected(bounded.error()); + } + module->defined_memories.reserve(*count); + for (uint32_t index = 0; index < *count; ++index) { + auto limits = readLimits(&cursor); + if (!limits) { + return unexpected(limits.error()); + } + module->defined_memories.push_back(*limits); + } + return requireConsumed(cursor, "memory"); +} + +[[nodiscard]] Expected parseExportSection(Cursor cursor, ModuleBuilder* module) { + auto count = cursor.varUint32(); + if (!count) { + return unexpected(count.error()); + } + auto bounded = requireCountFits(*count, cursor.remaining(), 3, "export-section entry"); + if (!bounded) { + return unexpected(bounded.error()); + } + module->exports.reserve(*count); + for (uint32_t index = 0; index < *count; ++index) { + auto name = cursor.name(); + auto kind = cursor.byte(); + auto item_index = cursor.varUint32(); + if (!name || !kind || !item_index || *kind > static_cast(WasmExternalKind::kTag)) { + return unexpected(std::string("truncated or invalid wasm export entry")); + } + if (std::any_of(module->exports.begin(), module->exports.end(), [&](const WasmExport& item) { + return item.name == *name; + })) { + return unexpected(std::string("duplicate wasm export name: ") + *name); + } + module->exports.push_back( + WasmExport{ + .name = std::move(*name), + .kind = static_cast(*kind), + .index = *item_index, + .function_signature = std::nullopt, + .memory_limits = std::nullopt, + }); + } + return requireConsumed(cursor, "export"); +} + +[[nodiscard]] Expected finishModule(ModuleBuilder builder) { + std::vector function_types; + function_types.reserve(builder.imported_function_types.size() + builder.defined_function_types.size()); + function_types.insert( + function_types.end(), builder.imported_function_types.begin(), builder.imported_function_types.end()); + function_types.insert( + function_types.end(), builder.defined_function_types.begin(), builder.defined_function_types.end()); + for (const uint32_t type_index : function_types) { + if (type_index >= builder.types.size()) { + return unexpected(std::string("wasm function references an invalid type index")); + } + } + + WasmModuleInfo result; + result.has_start_section = builder.has_start_section; + result.section_count = builder.section_count; + result.function_type_count = builder.types.size(); + result.function_count = function_types.size(); + result.memories.reserve(builder.imported_memories.size() + builder.defined_memories.size()); + result.memories.insert(result.memories.end(), builder.imported_memories.begin(), builder.imported_memories.end()); + result.memories.insert(result.memories.end(), builder.defined_memories.begin(), builder.defined_memories.end()); + result.tables = std::move(builder.tables); + result.imports.reserve(builder.imports.size()); + for (auto& imported : builder.imports) { + if (imported.function_type.has_value()) { + if (*imported.function_type >= builder.types.size()) { + return unexpected(std::string("wasm function import references an invalid type index")); + } + imported.value.function_signature = builder.types[*imported.function_type]; + } + result.imports.push_back(std::move(imported.value)); + } + result.exports.reserve(builder.exports.size()); + for (auto& exported : builder.exports) { + if (exported.kind == WasmExternalKind::kFunction) { + if (exported.index >= function_types.size()) { + return unexpected(std::string("wasm function export references an invalid function index")); + } + exported.function_signature = builder.types[function_types[exported.index]]; + } else if (exported.kind == WasmExternalKind::kMemory) { + if (exported.index >= result.memories.size()) { + return unexpected(std::string("wasm memory export references an invalid memory index")); + } + exported.memory_limits = result.memories[exported.index]; + } + result.exports.push_back(std::move(exported)); + } + return result; +} + +[[nodiscard]] Expected inspectWasmModuleImpl(Span wasm) { + if (wasm.size() < kWasmPreamble.size() || !std::equal(kWasmPreamble.begin(), kWasmPreamble.end(), wasm.begin())) { + return unexpected(std::string("invalid wasm preamble")); + } + + ModuleBuilder module; + Cursor cursor(wasm.subspan(kWasmPreamble.size())); + std::array seen{}; + while (!cursor.empty()) { + auto section_id = cursor.byte(); + auto section_size = cursor.varUint32(); + if (!section_id || !section_size) { + return unexpected(std::string("truncated wasm section header")); + } + if (*section_id > 12) { + return unexpected(std::string("unknown wasm section id")); + } + auto section = cursor.take(*section_size); + if (!section) { + return unexpected(section.error()); + } + ++module.section_count; + if (*section_id != 0) { + if (seen[*section_id]) { + return unexpected(std::string("duplicate standard wasm section")); + } + seen[*section_id] = true; + } + + Expected parsed; + switch (*section_id) { + case 1: + parsed = parseTypeSection(*section, &module); + break; + case 2: + parsed = parseImportSection(*section, &module); + break; + case 3: + parsed = parseFunctionSection(*section, &module); + break; + case 4: + parsed = parseTableSection(*section, &module); + break; + case 5: + parsed = parseMemorySection(*section, &module); + break; + case 7: + parsed = parseExportSection(*section, &module); + break; + case 8: + module.has_start_section = true; + break; + default: + break; + } + if (!parsed) { + return unexpected(parsed.error()); + } + } + return finishModule(std::move(module)); +} + +struct ExpectedExport { + std::string_view name; + WasmFunctionSignature signature; +}; + +} // namespace + +const WasmExport* WasmModuleInfo::findExport(std::string_view name) const noexcept { + const auto found = + std::find_if(exports.begin(), exports.end(), [&](const WasmExport& item) { return item.name == name; }); + return found == exports.end() ? nullptr : &*found; +} + +Expected inspectWasmModule(Span wasm) { + try { + return inspectWasmModuleImpl(wasm); + } catch (const std::bad_alloc&) { + return unexpected(std::string("allocation failed while inspecting the wasm module")); + } catch (...) { + return unexpected(std::string("unexpected failure while inspecting the wasm module")); + } +} + +Expected validateParserModuleWasmAbi(const WasmModuleInfo& module) { + if (module.has_start_section) { + return unexpected(std::string("wasm reactor contains a forbidden start section")); + } + if (module.findExport("_start") != nullptr) { + return unexpected(std::string("wasm reactor exports forbidden _start")); + } + if (module.findExport(PJ_MODULE_MANIFEST_ADDR_EXPORT_NAME) != nullptr || + module.findExport(PJ_MODULE_MANIFEST_LEN_EXPORT_NAME) != nullptr) { + return unexpected(std::string("wasm reactor exports native-only manifest metadata")); + } + + const std::array expected{{ + {PJ_MODULE_ABI_EXPORT_NAME, {{}, {kWasmValueI32}}}, + {PJ_MODULE_CREATE_EXPORT_NAME, {{kWasmValueI32}, {kWasmValueI64}}}, + {PJ_MODULE_DESTROY_EXPORT_NAME, {{kWasmValueI64}, {}}}, + {PJ_MODULE_BIND_EXPORT_NAME, {{kWasmValueI64, kWasmValueI64, kWasmValueI64}, {kWasmValueI32}}}, + {PJ_MODULE_PARSE_EXPORT_NAME, + {{kWasmValueI64, kWasmValueI64, kWasmValueI64, kWasmValueI64, kWasmValueI64}, {kWasmValueI32}}}, + {PJ_MODULE_LAST_ERROR_EXPORT_NAME, {{kWasmValueI64, kWasmValueI64, kWasmValueI64}, {kWasmValueI64}}}, + {PJ_MODULE_ALLOC_EXPORT_NAME, {{kWasmValueI64}, {kWasmValueI64}}}, + {PJ_MODULE_FREE_EXPORT_NAME, {{kWasmValueI64, kWasmValueI64}, {}}}, + {"_initialize", {{}, {}}}, + }}; + for (const auto& entry : expected) { + const WasmExport* exported = module.findExport(entry.name); + if (exported == nullptr) { + return unexpected(std::string("missing function export: ") + std::string(entry.name)); + } + if (exported->kind != WasmExternalKind::kFunction || !exported->function_signature.has_value()) { + return unexpected(std::string("export is not a valid function: ") + std::string(entry.name)); + } + if (*exported->function_signature != entry.signature) { + return unexpected(std::string("function export has the wrong wasm signature: ") + std::string(entry.name)); + } + } + + for (const auto& exported : module.exports) { + if (exported.name.rfind("pj_module_", 0) != 0) { + continue; + } + const auto found = std::find_if( + expected.begin(), expected.end(), [&](const ExpectedExport& entry) { return entry.name == exported.name; }); + if (found == expected.end()) { + return unexpected(std::string("unexpected parser-module export: ") + exported.name); + } + } + return {}; +} + +Expected validateParserModuleWasmMemory(const WasmModuleInfo& module, uint64_t maximum_bytes) { + const WasmExport* memory_export = module.findExport("memory"); + if (memory_export == nullptr || memory_export->kind != WasmExternalKind::kMemory || + !memory_export->memory_limits.has_value()) { + return unexpected(std::string("wasm parser module must export declared linear memory as 'memory'")); + } + if (module.memories.empty()) { + return unexpected(std::string("wasm parser module declares no linear memory")); + } + + uint64_t aggregate_maximum = 0; + for (const auto& memory : module.memories) { + if (!memory.maximum_pages.has_value()) { + return unexpected(std::string("wasm parser-module memory has no declared maximum")); + } + const uint64_t pages = *memory.maximum_pages; + if (pages > std::numeric_limits::max() / kWasmMemoryPageBytes) { + return unexpected(std::string("wasm parser-module memory maximum overflows bytes")); + } + const uint64_t bytes = pages * kWasmMemoryPageBytes; + if (bytes > maximum_bytes) { + return unexpected( + "wasm parser-module memory maximum " + std::to_string(bytes) + " exceeds configured cap " + + std::to_string(maximum_bytes)); + } + if (bytes > std::numeric_limits::max() - aggregate_maximum) { + return unexpected(std::string("aggregate wasm parser-module memory maximum overflows bytes")); + } + aggregate_maximum += bytes; + } + return aggregate_maximum; +} + +Expected validateParserModuleWasmTables(const WasmModuleInfo& module, uint64_t maximum_elements) { + uint64_t aggregate_maximum = 0; + for (const auto& table : module.tables) { + if (!table.maximum_elements.has_value()) { + return unexpected(std::string("wasm parser-module table has no declared maximum")); + } + const uint64_t elements = *table.maximum_elements; + if (elements > maximum_elements || elements > maximum_elements - aggregate_maximum) { + return unexpected( + "wasm parser-module table maximum " + std::to_string(elements) + " exceeds configured cap " + + std::to_string(maximum_elements) + " elements"); + } + aggregate_maximum += elements; + } + return aggregate_maximum; +} + +Expected validateParserModuleWasmArtifact( + Span wasm, const ParserModuleWasmLimits& limits) { + auto manifest = readManifestSection(wasm); + if (!manifest) { + return unexpected("invalid wasm parser-module manifest: " + manifest.error()); + } + auto inspected = inspectWasmModule(wasm); + if (!inspected) { + return unexpected("invalid wasm parser module: " + inspected.error()); + } + if (!inspected->imports.empty()) { + const auto& imported = inspected->imports.front(); + return unexpected("wasm parser module uses disallowed import '" + imported.module + "." + imported.name + "'"); + } + auto abi = validateParserModuleWasmAbi(*inspected); + if (!abi) { + return unexpected(abi.error()); + } + auto memory_maximum = validateParserModuleWasmMemory(*inspected, limits.maximum_linear_memory_bytes); + if (!memory_maximum) { + return unexpected(memory_maximum.error()); + } + auto table_maximum = validateParserModuleWasmTables(*inspected, limits.maximum_table_elements); + if (!table_maximum) { + return unexpected(table_maximum.error()); + } + return ParserModuleWasmArtifact{ + .manifest_json = *manifest, + .module = std::move(*inspected), + .declared_linear_memory_maximum = *memory_maximum, + .declared_table_elements = *table_maximum, + }; +} + +} // namespace PJ::parser_module diff --git a/experimental/parser_module/src/wasm_parser_module.cpp b/experimental/parser_module/src/wasm_parser_module.cpp new file mode 100644 index 00000000..0085c966 --- /dev/null +++ b/experimental/parser_module/src/wasm_parser_module.cpp @@ -0,0 +1,219 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include "pj_plugins/host/wasm_parser_module.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "detail/wasm_parser_module_state.hpp" +#include "pj_base/parser_module_manifest.hpp" +#include "pj_base/parser_module_wasm.hpp" +#include "pj_plugins/host/parser_module_claims.hpp" + +namespace PJ { +namespace { + +Expected> readFile(std::string_view path, uint64_t maximum_bytes) { + std::ifstream input(std::string(path), std::ios::binary | std::ios::ate); + if (!input) { + return unexpected(std::string("cannot open wasm parser module: ") + std::string(path)); + } + const std::streamoff end = input.tellg(); + if (end < 0 || static_cast(end) > std::numeric_limits::max() || + end > std::numeric_limits::max()) { + return unexpected(std::string("wasm parser-module file size is invalid")); + } + if (static_cast(end) > maximum_bytes) { + return unexpected( + "parser-module admission DECLINE: artifact_file_size budget exhausted (size " + + std::to_string(static_cast(end)) + ", limit " + std::to_string(maximum_bytes) + ")"); + } + std::vector bytes(static_cast(end)); + input.seekg(0); + if (!bytes.empty()) { + input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + } + if (!input) { + return unexpected(std::string("cannot read complete wasm parser-module file")); + } + return bytes; +} + +std::string wasmerLastError() { + const int length = wasmer_last_error_length(); + if (length <= 0) { + return "Wasmer supplied no diagnostic"; + } + std::string message(static_cast(length), '\0'); + const int written = wasmer_last_error_message(message.data(), length); + if (written <= 0) { + return "Wasmer diagnostic retrieval failed"; + } + if (!message.empty() && message.back() == '\0') { + message.pop_back(); + } + return message; +} + +std::string wasmerFailure(std::string_view action) { + return std::string(action) + ": " + wasmerLastError(); +} + +uint64_t unitMeteringCost(wasmer_parser_operator_t) { + return 1; +} + +Expected createMeteredEngine(uint64_t points_per_call) { + wasm_config_t* config = wasm_config_new(); + if (config == nullptr) { + return unexpected(wasmerFailure("failed to create Wasmer configuration")); + } + if (wasmer_is_backend_available(SINGLEPASS)) { + wasm_config_set_backend(config, SINGLEPASS); + } + wasmer_metering_t* metering = wasmer_metering_new(points_per_call, &unitMeteringCost); + if (metering == nullptr) { + wasm_config_delete(config); + return unexpected(wasmerFailure("failed to create Wasmer metering middleware")); + } + wasmer_middleware_t* middleware = wasmer_metering_as_middleware(metering); + if (middleware == nullptr) { + wasmer_metering_delete(metering); + wasm_config_delete(config); + return unexpected(wasmerFailure("failed to adapt Wasmer metering middleware")); + } + wasm_config_push_middleware(config, middleware); + wasm_engine_t* engine = wasm_engine_new_with_config(config); + if (engine == nullptr) { + wasm_config_delete(config); + return unexpected(wasmerFailure("failed to create metered Wasmer engine")); + } + return engine; +} + +} // namespace + +namespace detail { + +WasmParserModuleState::~WasmParserModuleState() { + if (module != nullptr) { + wasm_module_delete(module); + } + if (engine != nullptr) { + wasm_engine_delete(engine); + } + if (session_budget != nullptr) { + session_budget->releaseModule(module_reservation); + } +} + +} // namespace detail + +WasmParserModule::WasmParserModule(std::shared_ptr state) + : state_(std::move(state)) {} + +Expected WasmParserModule::load(std::string_view path, const WasmParserModuleLoadOptions& options) { + // Every rejection emits exactly one error diagnostic and returns the same + // text to the caller. + const auto reject = [&](std::string message) -> Expected { + if (options.sink) { + options.sink( + Diagnostic{ + .level = DiagnosticLevel::kError, + .source = options.diagnostic_source, + .id = std::string(path), + .message = message, + }); + } + return unexpected(std::move(message)); + }; + + const WasmParserModuleLimits& limits = options.limits; + if (limits.maximum_artifact_bytes == 0 || limits.maximum_linear_memory_bytes == 0 || + limits.metering_points_per_call == 0) { + return reject("wasm parser-module limits must all be nonzero"); + } + auto bytes = readFile(path, limits.maximum_artifact_bytes); + if (!bytes) { + return reject(bytes.error()); + } + + auto audited = parser_module::validateParserModuleWasmArtifact( + *bytes, parser_module::ParserModuleWasmLimits{ + .maximum_linear_memory_bytes = limits.maximum_linear_memory_bytes, + .maximum_table_elements = limits.maximum_table_elements, + }); + if (!audited) { + return reject(audited.error()); + } + const char* manifest_data = + audited->manifest_json.empty() ? "" : reinterpret_cast(audited->manifest_json.data()); + const std::string_view manifest_json(manifest_data, audited->manifest_json.size()); + auto decoded_manifest = decodeParserModuleManifest(manifest_json, ParserClaimProvenance::kFolderDrop); + if (!decoded_manifest) { + return reject("invalid wasm parser-module manifest: " + decoded_manifest.error()); + } + + auto state = std::make_shared(); + state->path = path; + state->manifest_json.assign(manifest_json); + state->claim_count = decoded_manifest->claims.size(); + state->artifact_size = bytes->size(); + state->declared_linear_memory_maximum = audited->declared_linear_memory_maximum; + state->declared_table_elements = audited->declared_table_elements; + state->metering_points_per_call = limits.metering_points_per_call; + if (options.budget != nullptr) { + auto admission = options.budget->admitModule( + bytes->size(), decoded_manifest->claims.size(), audited->declared_linear_memory_maximum); + if (!admission.accepted()) { + return reject(std::move(admission.diagnostic)); + } + state->session_budget = options.budget; + state->module_reservation = admission.reservation; + } + auto engine = createMeteredEngine(limits.metering_points_per_call); + if (!engine) { + return reject(engine.error()); + } + state->engine = *engine; + const wasm_byte_vec_t binary{ + .size = bytes->size(), + .data = reinterpret_cast(bytes->data()), + }; + state->module = wasmer_module_new(state->engine, &binary); + if (state->module == nullptr) { + return reject(wasmerFailure("Wasmer rejected parser module")); + } + return WasmParserModule(std::move(state)); +} + +std::string_view WasmParserModule::path() const noexcept { + return state_ == nullptr ? std::string_view{} : std::string_view(state_->path); +} + +std::string_view WasmParserModule::manifestJson() const noexcept { + return state_ == nullptr ? std::string_view{} : std::string_view(state_->manifest_json); +} + +uint64_t WasmParserModule::artifactSize() const noexcept { + return state_ == nullptr ? 0 : state_->artifact_size; +} + +uint64_t WasmParserModule::declaredLinearMemoryMaximum() const noexcept { + return state_ == nullptr ? 0 : state_->declared_linear_memory_maximum; +} + +uint64_t WasmParserModule::declaredTableElements() const noexcept { + return state_ == nullptr ? 0 : state_->declared_table_elements; +} + +} // namespace PJ diff --git a/experimental/parser_module/src/wasm_parser_module_runtime.cpp b/experimental/parser_module/src/wasm_parser_module_runtime.cpp new file mode 100644 index 00000000..94c1a05e --- /dev/null +++ b/experimental/parser_module/src/wasm_parser_module_runtime.cpp @@ -0,0 +1,687 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include "pj_plugins/host/wasm_parser_module_runtime.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "detail/parser_module_result_helpers.hpp" +#include "detail/wasm_parser_module_state.hpp" +#include "pj_base/parser_module_wasm.hpp" +#include "pj_base/span.hpp" + +namespace PJ { +namespace { + +using detail::contractViolation; + +/// pj_module_parse writes the output descriptor address into the first host +/// slot and its length into the second. Both slots are i64. +constexpr size_t kOutputSlotBytes = 8; +constexpr size_t kOutputBlockBytes = 2 * kOutputSlotBytes; + +std::string trapMessage(wasm_trap_t* trap) { + wasm_message_t message = WASM_EMPTY_VEC; + wasm_trap_message(trap, &message); + size_t size = message.size; + if (size != 0 && message.data[size - 1] == '\0') { + --size; + } + std::string result; + if (size != 0 && message.data != nullptr) { + result.assign(message.data, size); + } + wasm_byte_vec_delete(&message); + wasm_trap_delete(trap); + return result.empty() ? "wasm trap without a message" : "wasm trap: " + result; +} + +std::string meteringExhaustedMessage(std::string_view function_name, uint64_t points_per_call) { + return "wasm metering exhausted during " + std::string(function_name) + " (instruction-point limit " + + std::to_string(points_per_call) + ")"; +} + +/// Every guest ABI argument is an i64 token, address, or byte count. +wasm_val_t wasmI64(uint64_t value) { + const wasm_val_t argument = WASM_I64_VAL(static_cast(value)); + return argument; +} + +/// Call one guest export with a fresh instruction-point allowance. A trap and +/// an exhausted allowance both fail the call; exhaustion is reported in +/// preference to the trap message because it names the enforced deadline. +Expected callGuest( + wasm_instance_t* instance, uint64_t points_per_call, std::string_view function_name, wasm_func_t* function, + Span arguments, wasm_val_vec_t* results) { + wasmer_metering_set_remaining_points(instance, points_per_call); + const wasm_val_vec_t args{.size = arguments.size(), .data = arguments.data()}; + if (wasm_trap_t* trap = wasm_func_call(function, &args, results)) { + const bool exhausted = wasmer_metering_points_are_exhausted(instance); + std::string message = trapMessage(trap); + if (exhausted) { + return unexpected(meteringExhaustedMessage(function_name, points_per_call)); + } + return unexpected(std::move(message)); + } + if (wasmer_metering_points_are_exhausted(instance)) { + return unexpected(meteringExhaustedMessage(function_name, points_per_call)); + } + return {}; +} + +std::string exportName(const wasm_exporttype_t* exported) { + const wasm_name_t* name = wasm_exporttype_name(exported); + return std::string(name->data, name->size); +} + +uint64_t decodeU64(Span bytes) { + uint64_t value = 0; + for (size_t index = 0; index < kOutputSlotBytes; ++index) { + value |= static_cast(bytes[index]) << (index * 8U); + } + return value; +} + +ParserModuleBindResult bindContractViolation(int32_t code, std::string message) { + return ParserModuleBindResult{ + .outcome = ParserModuleBindOutcome::kError, + .fault = ParserModuleFaultKind::kContractViolation, + .result_code = code, + .message = std::move(message), + }; +} + +} // namespace + +namespace detail { + +struct WasmParserModuleInstanceState { + ~WasmParserModuleInstanceState() { + if (token != PJ_MODULE_CREATION_ERROR_TOKEN && destroy != nullptr) { + // Best-effort teardown: the store is deleted right after, so a destroy + // trap has nothing left to corrupt and nobody left to report to. + wasm_val_t arguments[1] = {wasmI64(token)}; + (void)callVoid(PJ_MODULE_DESTROY_EXPORT_NAME, destroy, arguments); + } + if (exports_initialized) { + wasm_extern_vec_delete(&exports); + } + if (instance != nullptr) { + wasm_instance_delete(instance); + } + if (store != nullptr) { + wasm_store_delete(store); + } + if (instance_budget_reserved && module->session_budget != nullptr) { + module->session_budget->releaseInstance(module->module_reservation); + } + } + + void noteContractViolation(std::string message) { + lifecycle_diagnostic = std::move(message); + } + + [[nodiscard]] Expected callVoid( + std::string_view function_name, wasm_func_t* function, Span arguments) const { + wasm_val_vec_t results = WASM_EMPTY_VEC; + return callGuest(instance, module->metering_points_per_call, function_name, function, arguments, &results); + } + + [[nodiscard]] Expected callI32( + std::string_view function_name, wasm_func_t* function, Span arguments) const { + wasm_val_t values[1] = {WASM_INIT_VAL}; + wasm_val_vec_t results = WASM_ARRAY_VEC(values); + auto called = callGuest(instance, module->metering_points_per_call, function_name, function, arguments, &results); + if (!called) { + return unexpected(called.error()); + } + if (values[0].kind != WASM_I32) { + return unexpected(std::string("wasm function returned a non-i32 result")); + } + return values[0].of.i32; + } + + [[nodiscard]] Expected callI64( + std::string_view function_name, wasm_func_t* function, Span arguments) const { + wasm_val_t values[1] = {WASM_INIT_VAL}; + wasm_val_vec_t results = WASM_ARRAY_VEC(values); + auto called = callGuest(instance, module->metering_points_per_call, function_name, function, arguments, &results); + if (!called) { + return unexpected(called.error()); + } + if (values[0].kind != WASM_I64) { + return unexpected(std::string("wasm function returned a non-i64 result")); + } + return values[0].of.i64; + } + + [[nodiscard]] Expected> memoryRange(uint64_t address, uint64_t length) const { + if (memory == nullptr || address > std::numeric_limits::max() || + length > std::numeric_limits::max()) { + return unexpected(std::string("guest memory range exceeds the host address range")); + } + const size_t offset = static_cast(address); + const size_t size = static_cast(length); + const size_t memory_size = wasm_memory_data_size(memory); + if (offset > memory_size || size > memory_size - offset) { + return unexpected(std::string("guest memory range is outside current linear memory")); + } + auto* base = reinterpret_cast(wasm_memory_data(memory)); + if (base == nullptr && memory_size != 0) { + return unexpected(std::string("Wasmer returned a null linear-memory base")); + } + return Span(base == nullptr ? nullptr : base + offset, size); + } + + [[nodiscard]] Expected allocate(uint64_t size) const { + wasm_val_t arguments[1] = {wasmI64(size)}; + auto result = callI64(PJ_MODULE_ALLOC_EXPORT_NAME, alloc, arguments); + if (!result) { + return unexpected(result.error()); + } + const uint64_t address = static_cast(*result); + if (address == 0) { + return unexpected(std::string("pj_module_alloc returned token zero")); + } + return address; + } + + [[nodiscard]] Expected freeAllocation(uint64_t address, uint64_t size) const { + wasm_val_t arguments[2] = {wasmI64(address), wasmI64(size)}; + return callVoid(PJ_MODULE_FREE_EXPORT_NAME, free, arguments); + } + + /// Read the guest error string into an already-allocated guest buffer. The + /// caller owns that buffer and releases it whatever the outcome here. + [[nodiscard]] Expected readLastError(uint64_t error_token, uint64_t buffer_address) const { + wasm_val_t arguments[3] = { + wasmI64(error_token), + wasmI64(buffer_address), + wasmI64(PJ_MODULE_ERROR_BUFFER_SIZE), + }; + auto written_result = callI64(PJ_MODULE_LAST_ERROR_EXPORT_NAME, last_error, arguments); + if (!written_result) { + return unexpected("pj_module_last_error trapped: " + written_result.error()); + } + const uint64_t written = static_cast(*written_result); + if (written > PJ_MODULE_ERROR_BUFFER_SIZE) { + return unexpected(std::string("pj_module_last_error returned a length larger than its buffer")); + } + auto bytes = memoryRange(buffer_address, written); + if (!bytes) { + return unexpected(bytes.error()); + } + const auto terminator = std::find(bytes->begin(), bytes->end(), uint8_t{0}); + return std::string(bytes->begin(), terminator); + } + + [[nodiscard]] Expected copyLastError(uint64_t error_token) const { + auto address = allocate(PJ_MODULE_ERROR_BUFFER_SIZE); + if (!address) { + return unexpected("cannot allocate the guest error buffer: " + address.error()); + } + auto message = readLastError(error_token, *address); + auto released = freeAllocation(*address, PJ_MODULE_ERROR_BUFFER_SIZE); + if (!message) { + return unexpected(message.error()); + } + if (!released) { + return unexpected("pj_module_free trapped after last_error: " + released.error()); + } + return message; + } + + std::shared_ptr module; + wasm_store_t* store = nullptr; + wasm_instance_t* instance = nullptr; + wasm_extern_vec_t exports = WASM_EMPTY_VEC; + bool exports_initialized = false; + wasm_memory_t* memory = nullptr; + wasm_func_t* initialize = nullptr; + wasm_func_t* abi = nullptr; + wasm_func_t* create = nullptr; + wasm_func_t* destroy = nullptr; + wasm_func_t* bind = nullptr; + wasm_func_t* parse = nullptr; + wasm_func_t* last_error = nullptr; + wasm_func_t* alloc = nullptr; + wasm_func_t* free = nullptr; + uint64_t token = PJ_MODULE_CREATION_ERROR_TOKEN; + uint32_t claim_index = 0; + parser_module::Route bound_route = parser_module::Route::kScalar; + uint16_t expected_object_type = 0; + bool bound = false; + bool instance_budget_reserved = false; + std::string lifecycle_diagnostic; +}; + +} // namespace detail + +namespace { + +/// Owns one `pj_module_alloc` region for the duration of a host operation. +/// The destructor releases it and notes a contract fault if guest free traps; +/// `release()` frees it early and returns that fault directly. +class GuestAllocation { + public: + GuestAllocation(detail::WasmParserModuleInstanceState& state, uint64_t address, uint64_t size) + : state_(&state), address_(address), size_(size) {} + + GuestAllocation(const GuestAllocation&) = delete; + GuestAllocation& operator=(const GuestAllocation&) = delete; + + ~GuestAllocation() { + auto released = release(); + if (!released) { + state_->noteContractViolation("pj_module_free failed during cleanup: " + released.error()); + } + } + + [[nodiscard]] uint64_t address() const noexcept { + return address_; + } + + /// Free now instead of at scope exit. A released allocation stays released. + [[nodiscard]] Expected release() { + if (address_ == 0) { + return {}; + } + return state_->freeAllocation(std::exchange(address_, UINT64_C(0)), size_); + } + + private: + detail::WasmParserModuleInstanceState* state_; + uint64_t address_; + uint64_t size_; +}; + +/// Release both parse allocations, output slots first, and report the first +/// guest free that failed. +Expected releaseParseBuffers(GuestAllocation& slots, GuestAllocation& input) { + auto released_slots = slots.release(); + auto released_input = input.release(); + if (!released_slots) { + return unexpected("pj_module_free failed: " + released_slots.error()); + } + if (!released_input) { + return unexpected("pj_module_free failed: " + released_input.error()); + } + return {}; +} + +Expected bindRuntimeExports(detail::WasmParserModuleInstanceState* state) { + wasm_exporttype_vec_t declarations = WASM_EMPTY_VEC; + wasm_module_exports(state->module->module, &declarations); + wasm_instance_exports(state->instance, &state->exports); + state->exports_initialized = true; + if (declarations.size != state->exports.size) { + wasm_exporttype_vec_delete(&declarations); + return unexpected(std::string("Wasmer returned an export count inconsistent with the compiled module")); + } + + for (size_t index = 0; index < declarations.size; ++index) { + const std::string name = exportName(declarations.data[index]); + wasm_extern_t* external = state->exports.data[index]; + if (name == "memory") { + state->memory = wasm_extern_as_memory(external); + } else if (name == "_initialize") { + state->initialize = wasm_extern_as_func(external); + } else if (name == PJ_MODULE_ABI_EXPORT_NAME) { + state->abi = wasm_extern_as_func(external); + } else if (name == PJ_MODULE_CREATE_EXPORT_NAME) { + state->create = wasm_extern_as_func(external); + } else if (name == PJ_MODULE_DESTROY_EXPORT_NAME) { + state->destroy = wasm_extern_as_func(external); + } else if (name == PJ_MODULE_BIND_EXPORT_NAME) { + state->bind = wasm_extern_as_func(external); + } else if (name == PJ_MODULE_PARSE_EXPORT_NAME) { + state->parse = wasm_extern_as_func(external); + } else if (name == PJ_MODULE_LAST_ERROR_EXPORT_NAME) { + state->last_error = wasm_extern_as_func(external); + } else if (name == PJ_MODULE_ALLOC_EXPORT_NAME) { + state->alloc = wasm_extern_as_func(external); + } else if (name == PJ_MODULE_FREE_EXPORT_NAME) { + state->free = wasm_extern_as_func(external); + } + } + wasm_exporttype_vec_delete(&declarations); + + if (state->memory == nullptr || state->initialize == nullptr || state->abi == nullptr || state->create == nullptr || + state->destroy == nullptr || state->bind == nullptr || state->parse == nullptr || state->last_error == nullptr || + state->alloc == nullptr || state->free == nullptr) { + return unexpected(std::string("Wasmer instance is missing a statically validated runtime export")); + } + return {}; +} + +} // namespace + +WasmParserModuleInstance::WasmParserModuleInstance(std::unique_ptr state) + : state_(std::move(state)) {} + +WasmParserModuleInstance::WasmParserModuleInstance() = default; +WasmParserModuleInstance::~WasmParserModuleInstance() = default; +WasmParserModuleInstance::WasmParserModuleInstance(WasmParserModuleInstance&& other) noexcept = default; +WasmParserModuleInstance& WasmParserModuleInstance::operator=(WasmParserModuleInstance&& other) noexcept = default; + +Expected WasmParserModuleInstance::create( + const WasmParserModule& module, uint32_t claim_index) { + const auto reject = + [](std::string message, ParserModuleFaultKind fault = ParserModuleFaultKind::kNone, + WasmParserModuleCreateOutcome outcome = + WasmParserModuleCreateOutcome::kError) -> Expected { + return unexpected( + WasmParserModuleCreateError{ + .outcome = outcome, + .fault = fault, + .message = std::move(message), + }); + }; + if (!module.valid()) { + return reject("cannot create an instance from an invalid wasm parser module"); + } + if (claim_index >= module.state_->claim_count) { + return reject("claim index is outside the wasm parser-module manifest"); + } + auto state = std::make_unique(); + state->module = module.state_; + state->claim_index = claim_index; + if (module.state_->session_budget != nullptr) { + auto admission = module.state_->session_budget->admitInstance(module.state_->module_reservation); + if (!admission.accepted()) { + return reject( + std::move(admission.diagnostic), ParserModuleFaultKind::kNone, + WasmParserModuleCreateOutcome::kAdmissionDecline); + } + state->instance_budget_reserved = true; + } + state->store = wasm_store_new(state->module->engine); + if (state->store == nullptr) { + return reject("failed to create a Wasmer store"); + } + + const wasm_extern_vec_t imports = WASM_EMPTY_VEC; + wasm_trap_t* instantiation_trap = nullptr; + state->instance = wasm_instance_new(state->store, state->module->module, &imports, &instantiation_trap); + if (state->instance == nullptr) { + if (instantiation_trap != nullptr) { + return reject( + "wasm instantiation failed: " + trapMessage(instantiation_trap), ParserModuleFaultKind::kContractViolation); + } + return reject("Wasmer failed to instantiate the parser module"); + } + auto exports = bindRuntimeExports(state.get()); + if (!exports) { + return reject(exports.error(), ParserModuleFaultKind::kContractViolation); + } + auto initialized = state->callVoid("_initialize", state->initialize, {}); + if (!initialized) { + return reject( + "parser-module _initialize failed: " + initialized.error(), ParserModuleFaultKind::kContractViolation); + } + auto abi = state->callI32(PJ_MODULE_ABI_EXPORT_NAME, state->abi, {}); + if (!abi) { + return reject("pj_module_abi failed: " + abi.error(), ParserModuleFaultKind::kContractViolation); + } + if (static_cast(*abi) != PJ_PARSER_MODULE_ABI_VERSION) { + return reject( + "wasm parser module ABI mismatch (expected " + std::to_string(PJ_PARSER_MODULE_ABI_VERSION) + ", got " + + std::to_string(static_cast(*abi)) + ")", + ParserModuleFaultKind::kContractViolation); + } + + wasm_val_t arguments[1] = {WASM_I32_VAL(static_cast(claim_index))}; + auto token = state->callI64(PJ_MODULE_CREATE_EXPORT_NAME, state->create, arguments); + if (!token) { + return reject("pj_module_create failed: " + token.error(), ParserModuleFaultKind::kContractViolation); + } + state->token = static_cast(*token); + if (state->token == PJ_MODULE_CREATION_ERROR_TOKEN) { + auto message = state->copyLastError(PJ_MODULE_CREATION_ERROR_TOKEN); + if (!message) { + return reject(message.error(), ParserModuleFaultKind::kContractViolation); + } + return reject( + message->empty() ? std::string("pj_module_create returned the creation-error token without a diagnostic") + : std::move(*message)); + } + return WasmParserModuleInstance(std::move(state)); +} + +Expected WasmParserModuleInstance::bind(const parser_module::BindingInfoV1& info) { + if (!valid()) { + return unexpected(std::string("cannot bind an invalid wasm parser-module instance")); + } + if (info.claim_index != state_->claim_index) { + return unexpected(std::string("BindingInfo claim_index does not match the created wasm instance")); + } + auto encoded = parser_module::writeBindingInfoV1(info); + if (!encoded) { + return unexpected(encoded.error()); + } + auto address = state_->allocate(encoded->size()); + if (!address) { + state_->noteContractViolation(address.error()); + return bindContractViolation(PJ_MODULE_ERR_ALLOCATION_FAILURE, address.error()); + } + GuestAllocation input_buffer(*state_, *address, encoded->size()); + auto guest_input = state_->memoryRange(input_buffer.address(), encoded->size()); + if (!guest_input) { + state_->noteContractViolation(guest_input.error()); + return bindContractViolation(PJ_MODULE_ERR_GENERIC, guest_input.error()); + } + std::copy(encoded->begin(), encoded->end(), guest_input->begin()); + + wasm_val_t arguments[3] = { + wasmI64(state_->token), + wasmI64(input_buffer.address()), + wasmI64(encoded->size()), + }; + auto code_result = state_->callI32(PJ_MODULE_BIND_EXPORT_NAME, state_->bind, arguments); + auto released = input_buffer.release(); + if (!code_result) { + state_->bound = false; + state_->noteContractViolation(code_result.error()); + return bindContractViolation(PJ_MODULE_ERR_GENERIC, code_result.error()); + } + if (!released) { + state_->bound = false; + const std::string message = "pj_module_free failed after bind: " + released.error(); + state_->noteContractViolation(message); + return bindContractViolation(PJ_MODULE_ERR_GENERIC, message); + } + + const int32_t code = *code_result; + ParserModuleBindResult result{ + .outcome = ParserModuleBindOutcome::kError, + .fault = ParserModuleFaultKind::kNone, + .result_code = code, + .message = {}, + }; + if (code == PJ_MODULE_OK) { + result.outcome = ParserModuleBindOutcome::kAccept; + state_->bound_route = info.route; + state_->expected_object_type = info.expected_object_type; + state_->bound = true; + return result; + } + + state_->bound = false; + if (code == PJ_MODULE_DECLINE) { + result.outcome = ParserModuleBindOutcome::kDecline; + } else if (code < 0) { + result.outcome = ParserModuleBindOutcome::kError; + if (code == PJ_MODULE_ERR_BAD_TOKEN) { + result.fault = ParserModuleFaultKind::kContractViolation; + } + } else { + const std::string message = "pj_module_bind returned an out-of-contract positive result"; + state_->noteContractViolation(message); + return bindContractViolation(code, message); + } + + auto message = state_->copyLastError(state_->token); + if (!message) { + result.fault = ParserModuleFaultKind::kContractViolation; + result.message = message.error(); + } else { + result.message = std::move(*message); + } + if (result.fault == ParserModuleFaultKind::kContractViolation) { + state_->noteContractViolation(result.message); + } + return result; +} + +Expected WasmParserModuleInstance::parse(const parser_module::ParseInputV1& input) { + if (!valid()) { + return unexpected(std::string("cannot parse with an invalid wasm parser-module instance")); + } + if (!state_->bound) { + return unexpected(std::string("cannot parse before an accepted wasm module bind")); + } + + const auto parse_once = [&]() -> Expected { + auto encoded = parser_module::writeParseInputV1(input); + if (!encoded) { + return unexpected(encoded.error()); + } + + auto input_address = state_->allocate(encoded->size()); + if (!input_address) { + return contractViolation(PJ_MODULE_ERR_ALLOCATION_FAILURE, input_address.error()); + } + GuestAllocation input_buffer(*state_, *input_address, encoded->size()); + auto slots_address = state_->allocate(kOutputBlockBytes); + if (!slots_address) { + return contractViolation(PJ_MODULE_ERR_ALLOCATION_FAILURE, slots_address.error()); + } + GuestAllocation slots_buffer(*state_, *slots_address, kOutputBlockBytes); + + auto guest_input = state_->memoryRange(input_buffer.address(), encoded->size()); + auto guest_slots = state_->memoryRange(slots_buffer.address(), kOutputBlockBytes); + if (!guest_input || !guest_slots) { + return contractViolation(PJ_MODULE_ERR_GENERIC, guest_input ? guest_slots.error() : guest_input.error()); + } + std::copy(encoded->begin(), encoded->end(), guest_input->begin()); + std::fill(guest_slots->begin(), guest_slots->end(), uint8_t{0}); + + wasm_val_t arguments[5] = { + wasmI64(state_->token), + wasmI64(input_buffer.address()), + wasmI64(encoded->size()), + wasmI64(slots_buffer.address()), + wasmI64(slots_buffer.address() + kOutputSlotBytes), + }; + auto code_result = state_->callI32(PJ_MODULE_PARSE_EXPORT_NAME, state_->parse, arguments); + if (!code_result) { + return contractViolation(PJ_MODULE_ERR_GENERIC, code_result.error()); + } + + const int32_t code = *code_result; + if (code < 0) { + auto message = state_->copyLastError(state_->token); + auto released = releaseParseBuffers(slots_buffer, input_buffer); + if (!message || !released) { + return contractViolation(code, !message ? message.error() : released.error()); + } + return ParserModuleParseResult{ + .fault = code == PJ_MODULE_ERR_BAD_TOKEN ? ParserModuleFaultKind::kContractViolation + : ParserModuleFaultKind::kDataError, + .result_code = code, + .message = std::move(*message), + .output = std::nullopt, + }; + } + if (code != PJ_MODULE_OK) { + return contractViolation(code, "pj_module_parse returned a nonzero non-error result"); + } + + // parse may grow or relocate memory. Re-acquire before reading both return + // slots, then re-acquire again for the module-owned descriptor itself. + auto returned_slots = state_->memoryRange(slots_buffer.address(), kOutputBlockBytes); + if (!returned_slots) { + return contractViolation(code, returned_slots.error()); + } + const uint64_t output_address = decodeU64(returned_slots->first(kOutputSlotBytes)); + const uint64_t output_length = decodeU64(returned_slots->subspan(kOutputSlotBytes, kOutputSlotBytes)); + if (output_address == 0 || output_length == 0) { + return contractViolation(code, "pj_module_parse returned an unreadable output descriptor"); + } + auto guest_output = state_->memoryRange(output_address, output_length); + if (!guest_output) { + return contractViolation(code, guest_output.error()); + } + const std::vector descriptor_bytes(guest_output->begin(), guest_output->end()); + + auto released = releaseParseBuffers(slots_buffer, input_buffer); + if (!released) { + return contractViolation(code, released.error()); + } + + auto descriptor = parser_module::readOutputDescriptorV1(descriptor_bytes); + if (!descriptor) { + return contractViolation(code, "malformed output descriptor: " + descriptor.error()); + } + + std::optional output; + if (state_->bound_route == parser_module::Route::kScalar) { + const auto* scalar = std::get_if(&*descriptor); + if (scalar == nullptr) { + return contractViolation(code, "output descriptor route does not match the scalar binding"); + } + auto owned = detail::ownScalarOutput(*scalar); + if (!owned) { + return contractViolation(code, owned.error()); + } + output = ParserModuleOutput(std::move(*owned)); + } else { + const auto* object = std::get_if(&*descriptor); + if (object == nullptr) { + return contractViolation(code, "output descriptor route does not match the object binding"); + } + auto owned = detail::ownObjectOutput(*object, input.payload, state_->expected_object_type); + if (!owned) { + return contractViolation(code, owned.error()); + } + output = ParserModuleOutput(std::move(*owned)); + } + return ParserModuleParseResult{ + .fault = ParserModuleFaultKind::kNone, + .result_code = code, + .message = {}, + .output = std::move(output), + }; + }; + + auto result = parse_once(); + if (result && result->fault == ParserModuleFaultKind::kContractViolation) { + state_->noteContractViolation(result->message); + } + return result; +} + +bool WasmParserModuleInstance::valid() const noexcept { + return state_ != nullptr && state_->token != PJ_MODULE_CREATION_ERROR_TOKEN; +} + +uint32_t WasmParserModuleInstance::claimIndex() const noexcept { + return state_ == nullptr ? 0 : state_->claim_index; +} + +std::string_view WasmParserModuleInstance::lifecycleDiagnostic() const noexcept { + return state_ == nullptr ? std::string_view{} : std::string_view(state_->lifecycle_diagnostic); +} + +} // namespace PJ diff --git a/experimental/parser_module/tests/adversarial_wasm_parser_module.cpp b/experimental/parser_module/tests/adversarial_wasm_parser_module.cpp new file mode 100644 index 00000000..b0edc01d --- /dev/null +++ b/experimental/parser_module/tests/adversarial_wasm_parser_module.cpp @@ -0,0 +1,64 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include "pj_base/parser_module/module.hpp" + +class AdversarialWasmParser : public pj::FunctionalParser { + public: + pj::Status bind(const pj::BindingInfo&) override { + return pj::Status::ok(); + } + + pj::Status parseObject(pj::PayloadView payload, pj::Timestamp, pj::ObjectWriter& output) override { + if (payload.size == 0 || payload.data == nullptr) { + return pj::Status::error("adversarial fixture requires a behavior byte"); + } + switch (payload.data[0]) { + case 0: + __builtin_trap(); + return pj::Status::error("unreachable returned"); + case 1: { + volatile uint64_t progress = 0; + while (true) { + ++progress; + } + } + case 2: { + const size_t before = __builtin_wasm_memory_size(0); + const size_t result = __builtin_wasm_memory_grow(0, std::numeric_limits::max()); + const size_t after = __builtin_wasm_memory_size(0); + if (result != std::numeric_limits::max() || before != after) { + return pj::Status::error("memory growth escaped the declared maximum"); + } + return pj::Status::error("memory growth rejected by declared maximum"); + } + default: + break; + } + + auto cloud = output.pointCloud(); + if (auto status = cloud.setWidth(1); !status.isOk()) { + return status; + } + if (auto status = cloud.setHeight(1); !status.isOk()) { + return status; + } + if (auto status = cloud.setPointStep(1); !status.isOk()) { + return status; + } + if (auto status = cloud.setRowStep(1); !status.isOk()) { + return status; + } + if (auto status = cloud.setDense(true); !status.isOk()) { + return status; + } + const uint8_t data = 42; + return cloud.setData(pj::PayloadView(&data, 1)); + } +}; + +PJ_FUNCTIONAL_PARSER(AdversarialWasmParser) diff --git a/experimental/parser_module/tests/adversarial_wasm_parser_module.module.json b/experimental/parser_module/tests/adversarial_wasm_parser_module.module.json new file mode 100644 index 00000000..4ac31574 --- /dev/null +++ b/experimental/parser_module/tests/adversarial_wasm_parser_module.module.json @@ -0,0 +1,16 @@ +{ + "module_abi": 1, + "id": "org.plotjuggler.test.adversarial-wasm", + "name": "Adversarial wasm parser-module fixture", + "version": "1.0.0", + "claims": [ + { + "claim_id": "adversarial", + "encoding": "ros2msg", + "type_name": "test_msgs/msg/Adversarial", + "routes": ["object"], + "object_type": "kPointCloud", + "priority": 0 + } + ] +} diff --git a/experimental/parser_module/tests/native_parser_module_test.cpp b/experimental/parser_module/tests/native_parser_module_test.cpp index cb2f0b77..028b8a2f 100644 --- a/experimental/parser_module/tests/native_parser_module_test.cpp +++ b/experimental/parser_module/tests/native_parser_module_test.cpp @@ -8,12 +8,15 @@ #include #include #include +#include #include #include #include "native_parser_module_fixture.hpp" #include "pj_base/parser_module_abi.h" #include "pj_plugins/host/parser_module_claims.hpp" +#include "pj_plugins/host/parser_module_runtime.hpp" +#include "pj_plugins/host/parser_module_session_budget.hpp" namespace PJ { namespace { @@ -130,5 +133,45 @@ TEST(NativeParserModule, ReportsSpecificLoaderFailureCauses) { EXPECT_NE(unreadable.error().find("manifest is unreadable"), std::string::npos); } +TEST(NativeParserModule, EnforcesAggregateBudgetsAtLoadAndCreate) { + const uint64_t artifact_size = std::filesystem::file_size(PJ_NATIVE_MODULE_FIXTURE_PATH); + const auto make_budget = [&](ParserModuleSessionBudgetLimits limits) { + return std::make_shared(limits); + }; + ParserModuleSessionBudgetLimits limits; + limits.maximum_modules = 0; + auto module_budget = make_budget(limits); + auto module_decline = NativeParserModule::load(PJ_NATIVE_MODULE_FIXTURE_PATH, module_budget); + ASSERT_FALSE(module_decline.has_value()); + EXPECT_NE(module_decline.error().find("module_count"), std::string::npos); + EXPECT_EQ(module_budget->usage().modules, 0U); + + limits = {}; + limits.maximum_artifact_bytes = artifact_size - 1; + auto artifact_budget = make_budget(limits); + auto artifact = NativeParserModule::load(PJ_NATIVE_MODULE_FIXTURE_PATH, artifact_budget); + ASSERT_FALSE(artifact.has_value()); + EXPECT_NE(artifact.error().find("artifact_file_size"), std::string::npos); + EXPECT_EQ(artifact_budget->usage().modules, 0U); + + limits = {}; + limits.maximum_claims = pj_fixture::kClaimCount - 1; + auto claim_budget = make_budget(limits); + auto claims = NativeParserModule::load(PJ_NATIVE_MODULE_FIXTURE_PATH, claim_budget); + ASSERT_FALSE(claims.has_value()); + EXPECT_NE(claims.error().find("total_claims"), std::string::npos); + EXPECT_EQ(claim_budget->usage().modules, 0U); + + limits = {}; + limits.maximum_active_instances = 0; + auto instance_budget = make_budget(limits); + auto module = NativeParserModule::load(PJ_NATIVE_MODULE_FIXTURE_PATH, instance_budget); + ASSERT_TRUE(module.has_value()) << module.error(); + auto instance = NativeParserModuleInstance::create(*module, 0); + ASSERT_FALSE(instance.has_value()); + EXPECT_NE(instance.error().find("active_instances"), std::string::npos); + EXPECT_EQ(instance_budget->usage().active_instances, 0U); +} + } // namespace } // namespace PJ diff --git a/experimental/parser_module/tests/parser_module_runtime_test.cpp b/experimental/parser_module/tests/parser_module_runtime_test.cpp index 9fe880b2..89eed516 100644 --- a/experimental/parser_module/tests/parser_module_runtime_test.cpp +++ b/experimental/parser_module/tests/parser_module_runtime_test.cpp @@ -243,5 +243,25 @@ TEST(ParserModuleRuntime, StrikeTrackerQuarantinesReplaysAndThenDisables) { EXPECT_EQ(tracker.state(key).quarantine_count, 2U); } +TEST(ParserModuleRuntime, StrikeTrackerDisablesWhenTheReplayItselfFaults) { + const ParserModuleClaimKey key{"org.plotjuggler.test.native-module", "malformed"}; + ParserModuleStrikeTracker tracker; + for (int strike = 1; strike <= 3; ++strike) { + (void)tracker.recordFault(key, ParserModuleFaultKind::kContractViolation); + } + ASSERT_EQ(tracker.state(key).health, ParserModuleClaimHealth::kQuarantined); + + // A data error during replay is still not a strike. + EXPECT_EQ(tracker.recordFault(key, ParserModuleFaultKind::kDataError).health, ParserModuleClaimHealth::kQuarantined); + // A contract violation during replay (create/bind trapping again) is the + // repeat the policy disables on; the claim can never be stuck quarantined. + const auto disabled = tracker.recordFault(key, ParserModuleFaultKind::kContractViolation); + EXPECT_EQ(disabled.health, ParserModuleClaimHealth::kDisabled); + EXPECT_EQ(disabled.quarantine_count, 2U); + EXPECT_FALSE(tracker.markRecreated(key)); + EXPECT_EQ( + tracker.recordFault(key, ParserModuleFaultKind::kContractViolation).health, ParserModuleClaimHealth::kDisabled); +} + } // namespace } // namespace PJ diff --git a/experimental/parser_module/tests/parser_module_session_budget_test.cpp b/experimental/parser_module/tests/parser_module_session_budget_test.cpp new file mode 100644 index 00000000..8931530f --- /dev/null +++ b/experimental/parser_module/tests/parser_module_session_budget_test.cpp @@ -0,0 +1,151 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include "pj_plugins/host/parser_module_session_budget.hpp" + +#include + +#include +#include +#include +#include + +namespace PJ { +namespace { + +void expectDecline( + const ParserModuleAdmissionDecision& decision, ParserModuleBudgetKind budget, std::string_view diagnostic_name) { + EXPECT_FALSE(decision.accepted()); + EXPECT_EQ(decision.reservation, 0U); + EXPECT_EQ(decision.exhausted_budget, budget); + EXPECT_NE(decision.diagnostic.find(diagnostic_name), std::string::npos); +} + +TEST(ParserModuleSessionBudget, RejectsEachModuleAdmissionBudgetWithoutMutation) { + const ParserModuleSessionBudgetLimits limits{ + .maximum_modules = 1, + .maximum_artifact_bytes = 100, + .maximum_claims = 2, + .maximum_active_instances = 2, + .maximum_linear_memory_bytes = 400, + }; + + { + ParserModuleSessionBudgetTracker tracker(limits); + expectDecline(tracker.admitModule(101, 1, 100), ParserModuleBudgetKind::kArtifactFileSize, "artifact_file_size"); + EXPECT_EQ(tracker.usage().modules, 0U); + } + { + ParserModuleSessionBudgetTracker tracker(limits); + expectDecline(tracker.admitModule(100, 3, 100), ParserModuleBudgetKind::kTotalClaims, "total_claims"); + EXPECT_EQ(tracker.usage().claims, 0U); + } + { + ParserModuleSessionBudgetTracker tracker(limits); + ASSERT_TRUE(tracker.admitModule(100, 2, 100).accepted()); + expectDecline(tracker.admitModule(1, 0, 1), ParserModuleBudgetKind::kModuleCount, "module_count"); + EXPECT_EQ(tracker.usage().modules, 1U); + EXPECT_EQ(tracker.usage().claims, 2U); + } +} + +TEST(ParserModuleSessionBudget, CountsResourcesNotIdentities) { + // Two loads of one artifact (or native and wasm builds of one source) are + // two reservations; duplicate-provider policy belongs to the catalog. + ParserModuleSessionBudgetLimits limits; + limits.maximum_modules = 2; + ParserModuleSessionBudgetTracker tracker(limits); + const auto first = tracker.admitModule(10, 1, 0); + const auto second = tracker.admitModule(10, 1, 0); + ASSERT_TRUE(first.accepted()); + ASSERT_TRUE(second.accepted()); + EXPECT_NE(first.reservation, second.reservation); + EXPECT_EQ(tracker.usage().modules, 2U); + expectDecline(tracker.admitModule(10, 1, 0), ParserModuleBudgetKind::kModuleCount, "module_count"); +} + +TEST(ParserModuleSessionBudget, AppliesInstanceAndDeclaredMemoryBudgetsIndependently) { + const ParserModuleSessionBudgetLimits limits{ + .maximum_modules = 2, + .maximum_artifact_bytes = 100, + .maximum_claims = 4, + .maximum_active_instances = 2, + .maximum_linear_memory_bytes = 300, + }; + ParserModuleSessionBudgetTracker tracker(limits); + const auto small = tracker.admitModule(100, 1, 100); + const auto large = tracker.admitModule(100, 1, 250); + ASSERT_TRUE(small.accepted()); + ASSERT_TRUE(large.accepted()); + ASSERT_TRUE(tracker.admitInstance(large.reservation).accepted()); + + expectDecline( + tracker.admitInstance(small.reservation), ParserModuleBudgetKind::kTotalLinearMemory, "total_linear_memory"); + EXPECT_EQ(tracker.usage().active_instances, 1U); + tracker.releaseInstance(large.reservation); + ASSERT_TRUE(tracker.admitInstance(small.reservation).accepted()); + ASSERT_TRUE(tracker.admitInstance(small.reservation).accepted()); + expectDecline(tracker.admitInstance(small.reservation), ParserModuleBudgetKind::kActiveInstances, "active_instances"); + + expectDecline(tracker.admitInstance(0), ParserModuleBudgetKind::kNone, "not admitted"); +} + +TEST(ParserModuleSessionBudget, ReleasesAreIdempotentAndRestoreCapacity) { + ParserModuleSessionBudgetTracker tracker( + ParserModuleSessionBudgetLimits{ + .maximum_modules = 1, + .maximum_artifact_bytes = 10, + .maximum_claims = 1, + .maximum_active_instances = 1, + .maximum_linear_memory_bytes = 20, + }); + const auto module = tracker.admitModule(10, 1, 20); + ASSERT_TRUE(module.accepted()); + ASSERT_TRUE(tracker.admitInstance(module.reservation).accepted()); + tracker.releaseInstance(module.reservation); + tracker.releaseInstance(module.reservation); // ignored + EXPECT_EQ(tracker.usage().active_instances, 0U); + EXPECT_EQ(tracker.usage().declared_linear_memory_bytes, 0U); + + ASSERT_TRUE(tracker.admitInstance(module.reservation).accepted()); + tracker.releaseModule(module.reservation); // gives back the live instance too + tracker.releaseModule(module.reservation); // ignored + tracker.releaseInstance(module.reservation); // ignored: module is gone + EXPECT_EQ(tracker.usage().modules, 0U); + EXPECT_EQ(tracker.usage().claims, 0U); + EXPECT_EQ(tracker.usage().active_instances, 0U); + EXPECT_EQ(tracker.usage().declared_linear_memory_bytes, 0U); + ASSERT_TRUE(tracker.admitModule(10, 1, 20).accepted()); +} + +TEST(ParserModuleSessionBudget, TracksConcurrentAdmissionAndReleaseFromManyThreads) { + // Wrapper destructors release reservations on whatever thread drops them, + // so the tracker must synchronize itself. + ParserModuleSessionBudgetLimits limits; + limits.maximum_modules = 1000; + limits.maximum_claims = 100000; + ParserModuleSessionBudgetTracker tracker(limits); + std::vector threads; + for (int worker = 0; worker < 8; ++worker) { + threads.emplace_back([&tracker] { + for (int round = 0; round < 200; ++round) { + const auto module = tracker.admitModule(1, 1, 1); + ASSERT_TRUE(module.accepted()); + const auto instance = tracker.admitInstance(module.reservation); + ASSERT_TRUE(instance.accepted()); + tracker.releaseInstance(module.reservation); + tracker.releaseModule(module.reservation); + } + }); + } + for (auto& thread : threads) { + thread.join(); + } + EXPECT_EQ(tracker.usage().modules, 0U); + EXPECT_EQ(tracker.usage().claims, 0U); + EXPECT_EQ(tracker.usage().active_instances, 0U); + EXPECT_EQ(tracker.usage().declared_linear_memory_bytes, 0U); +} + +} // namespace +} // namespace PJ diff --git a/experimental/parser_module/tests/parser_module_wasm_audit.cpp b/experimental/parser_module/tests/parser_module_wasm_audit.cpp deleted file mode 100644 index d299db50..00000000 --- a/experimental/parser_module/tests/parser_module_wasm_audit.cpp +++ /dev/null @@ -1,554 +0,0 @@ -// Copyright 2026 Davide Faconti -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "pj_base/parser_module_manifest.hpp" - -namespace { - -using PJ::Expected; -using PJ::Span; -using PJ::unexpected; - -constexpr std::array kWasmPreamble{0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00}; -constexpr uint8_t kI32 = 0x7F; -constexpr uint8_t kI64 = 0x7E; - -class Cursor { - public: - explicit Cursor(Span bytes) : bytes_(bytes) {} - - [[nodiscard]] bool empty() const noexcept { - return position_ == bytes_.size(); - } - - [[nodiscard]] size_t remaining() const noexcept { - return position_ <= bytes_.size() ? bytes_.size() - position_ : 0; - } - - [[nodiscard]] Expected byte() { - if (position_ >= bytes_.size()) { - return unexpected(std::string("truncated wasm byte")); - } - return bytes_[position_++]; - } - - [[nodiscard]] Expected varUint32() { - uint32_t value = 0; - for (size_t index = 0; index < 5; ++index) { - auto next = byte(); - if (!next) { - return unexpected(next.error()); - } - if (index == 4 && (*next & UINT8_C(0xF0)) != 0) { - return unexpected(std::string("wasm varuint32 overflows uint32")); - } - value |= static_cast(*next & UINT8_C(0x7F)) << (index * 7U); - if ((*next & UINT8_C(0x80)) == 0) { - return value; - } - } - return unexpected(std::string("wasm varuint32 exceeds five bytes")); - } - - [[nodiscard]] Expected name() { - auto length = varUint32(); - if (!length) { - return unexpected(length.error()); - } - if (static_cast(*length) > remaining()) { - return unexpected(std::string("wasm name exceeds the remaining section bytes")); - } - const char* begin = reinterpret_cast(bytes_.data() + position_); - position_ += *length; - return std::string(begin, *length); - } - - [[nodiscard]] Expected take(uint32_t size) { - if (static_cast(size) > remaining()) { - return unexpected(std::string("wasm section exceeds the remaining module bytes")); - } - Cursor result(bytes_.subspan(position_, size)); - position_ += size; - return result; - } - - private: - Span bytes_; - size_t position_ = 0; -}; - -struct FunctionType { - std::vector parameters; - std::vector results; -}; - -struct Export { - std::string name; - uint8_t kind = 0; - uint32_t index = 0; -}; - -struct ModuleInfo { - std::vector types; - std::vector function_types; - std::vector exports; - bool has_start_section = false; - size_t section_count = 0; -}; - -[[nodiscard]] bool validValueType(uint8_t value) { - switch (value) { - case 0x7F: // i32 - case 0x7E: // i64 - case 0x7D: // f32 - case 0x7C: // f64 - case 0x7B: // v128 - case 0x70: // funcref - case 0x6F: // externref - return true; - default: - return false; - } -} - -[[nodiscard]] Expected> readValueTypes(Cursor* cursor) { - auto count = cursor->varUint32(); - if (!count) { - return unexpected(count.error()); - } - try { - std::vector result; - result.reserve(*count); - for (uint32_t index = 0; index < *count; ++index) { - auto value = cursor->byte(); - if (!value) { - return unexpected(value.error()); - } - if (!validValueType(*value)) { - return unexpected(std::string("wasm function type contains an invalid value type")); - } - result.push_back(*value); - } - return result; - } catch (const std::bad_alloc&) { - return unexpected(std::string("allocation failed while reading wasm value types")); - } -} - -[[nodiscard]] Expected requireConsumed(const Cursor& cursor, std::string_view section) { - if (!cursor.empty()) { - return unexpected(std::string(section) + " section contains trailing bytes"); - } - return {}; -} - -[[nodiscard]] Expected parseTypeSection(Cursor cursor, ModuleInfo* module) { - auto count = cursor.varUint32(); - if (!count) { - return unexpected(count.error()); - } - try { - module->types.reserve(*count); - for (uint32_t index = 0; index < *count; ++index) { - auto form = cursor.byte(); - if (!form || *form != UINT8_C(0x60)) { - return unexpected(std::string("wasm type section contains a non-function type")); - } - auto parameters = readValueTypes(&cursor); - auto results = readValueTypes(&cursor); - if (!parameters) { - return unexpected(parameters.error()); - } - if (!results) { - return unexpected(results.error()); - } - module->types.push_back(FunctionType{std::move(*parameters), std::move(*results)}); - } - } catch (const std::bad_alloc&) { - return unexpected(std::string("allocation failed while reading the wasm type section")); - } - return requireConsumed(cursor, "type"); -} - -[[nodiscard]] Expected readLimits(Cursor* cursor) { - auto flags = cursor->varUint32(); - auto minimum = cursor->varUint32(); - if (!flags || !minimum) { - return unexpected(std::string("truncated wasm limits")); - } - if (*flags > 1) { - return unexpected(std::string("unsupported wasm limits flags")); - } - if ((*flags & 1U) != 0) { - auto maximum = cursor->varUint32(); - if (!maximum) { - return unexpected(maximum.error()); - } - } - return {}; -} - -[[nodiscard]] Expected parseImportSection(Cursor cursor, ModuleInfo* module) { - auto count = cursor.varUint32(); - if (!count) { - return unexpected(count.error()); - } - for (uint32_t index = 0; index < *count; ++index) { - auto module_name = cursor.name(); - auto field_name = cursor.name(); - auto kind = cursor.byte(); - if (!module_name || !field_name || !kind) { - return unexpected(std::string("truncated wasm import entry")); - } - switch (*kind) { - case 0: { - auto type_index = cursor.varUint32(); - if (!type_index) { - return unexpected(type_index.error()); - } - module->function_types.push_back(*type_index); - break; - } - case 1: { - auto element_type = cursor.byte(); - if (!element_type || (*element_type != UINT8_C(0x70) && *element_type != UINT8_C(0x6F))) { - return unexpected(std::string("invalid wasm table import")); - } - auto limits = readLimits(&cursor); - if (!limits) { - return unexpected(limits.error()); - } - break; - } - case 2: { - auto limits = readLimits(&cursor); - if (!limits) { - return unexpected(limits.error()); - } - break; - } - case 3: { - auto value_type = cursor.byte(); - auto mutability = cursor.byte(); - if (!value_type || !mutability || !validValueType(*value_type) || *mutability > 1) { - return unexpected(std::string("invalid wasm global import")); - } - break; - } - case 4: { - auto attribute = cursor.varUint32(); - auto type_index = cursor.varUint32(); - if (!attribute || !type_index) { - return unexpected(std::string("truncated wasm tag import")); - } - break; - } - default: - return unexpected(std::string("unknown wasm import kind")); - } - } - return requireConsumed(cursor, "import"); -} - -[[nodiscard]] Expected parseFunctionSection(Cursor cursor, ModuleInfo* module) { - auto count = cursor.varUint32(); - if (!count) { - return unexpected(count.error()); - } - try { - module->function_types.reserve(module->function_types.size() + *count); - for (uint32_t index = 0; index < *count; ++index) { - auto type_index = cursor.varUint32(); - if (!type_index) { - return unexpected(type_index.error()); - } - module->function_types.push_back(*type_index); - } - } catch (const std::bad_alloc&) { - return unexpected(std::string("allocation failed while reading the wasm function section")); - } - return requireConsumed(cursor, "function"); -} - -[[nodiscard]] Expected parseExportSection(Cursor cursor, ModuleInfo* module) { - auto count = cursor.varUint32(); - if (!count) { - return unexpected(count.error()); - } - try { - module->exports.reserve(*count); - for (uint32_t index = 0; index < *count; ++index) { - auto name = cursor.name(); - auto kind = cursor.byte(); - auto item_index = cursor.varUint32(); - if (!name || !kind || !item_index) { - return unexpected(std::string("truncated wasm export entry")); - } - if (std::any_of( - module->exports.begin(), module->exports.end(), [&](const Export& item) { return item.name == *name; })) { - return unexpected(std::string("duplicate wasm export name: ") + *name); - } - module->exports.push_back(Export{std::move(*name), *kind, *item_index}); - } - } catch (const std::bad_alloc&) { - return unexpected(std::string("allocation failed while reading the wasm export section")); - } - return requireConsumed(cursor, "export"); -} - -[[nodiscard]] Expected inspectModule(Span wasm) { - if (wasm.size() < kWasmPreamble.size() || !std::equal(kWasmPreamble.begin(), kWasmPreamble.end(), wasm.begin())) { - return unexpected(std::string("invalid wasm preamble")); - } - - ModuleInfo module; - Cursor cursor(wasm.subspan(kWasmPreamble.size())); - std::array seen{}; - while (!cursor.empty()) { - auto section_id = cursor.byte(); - auto section_size = cursor.varUint32(); - if (!section_id || !section_size) { - return unexpected(std::string("truncated wasm section header")); - } - if (*section_id > 12) { - return unexpected(std::string("unknown wasm section id")); - } - auto section = cursor.take(*section_size); - if (!section) { - return unexpected(section.error()); - } - ++module.section_count; - if (*section_id != 0) { - if (seen[*section_id]) { - return unexpected(std::string("duplicate standard wasm section")); - } - seen[*section_id] = true; - } - Expected parsed; - switch (*section_id) { - case 1: - parsed = parseTypeSection(*section, &module); - break; - case 2: - parsed = parseImportSection(*section, &module); - break; - case 3: - parsed = parseFunctionSection(*section, &module); - break; - case 7: - parsed = parseExportSection(*section, &module); - break; - case 8: - module.has_start_section = true; - break; - default: - break; - } - if (!parsed) { - return unexpected(parsed.error()); - } - } - if (!seen[1] || !seen[3] || !seen[7]) { - return unexpected(std::string("wasm is missing a type, function, or export section")); - } - for (const uint32_t type_index : module.function_types) { - if (type_index >= module.types.size()) { - return unexpected(std::string("wasm function references an invalid type index")); - } - } - return module; -} - -[[nodiscard]] Expected> readFile(const std::string& path) { - std::ifstream input(path, std::ios::binary | std::ios::ate); - if (!input) { - return unexpected(std::string("cannot open file: ") + path); - } - const std::streamoff end = input.tellg(); - if (end < 0 || static_cast(end) > std::numeric_limits::max() || - end > std::numeric_limits::max()) { - return unexpected(std::string("file size is invalid: ") + path); - } - std::vector bytes(static_cast(end)); - input.seekg(0); - if (!bytes.empty()) { - input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); - } - if (!input) { - return unexpected(std::string("cannot read complete file: ") + path); - } - return bytes; -} - -[[nodiscard]] Expected writeFile(const std::string& path, Span bytes) { - std::ofstream output(path, std::ios::binary | std::ios::trunc); - if (!output) { - return unexpected(std::string("cannot create file: ") + path); - } - if (!bytes.empty()) { - output.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); - } - if (!output) { - return unexpected(std::string("cannot write complete file: ") + path); - } - return {}; -} - -[[nodiscard]] const Export* findExport(const ModuleInfo& module, std::string_view name) { - const Export* result = nullptr; - for (const auto& item : module.exports) { - if (item.name == name) { - if (result != nullptr) { - return nullptr; - } - result = &item; - } - } - return result; -} - -/// One mandatory reactor export and its exact wasm function signature. -struct ExpectedExport { - std::string_view name; - std::vector parameters; - std::vector results; -}; - -[[nodiscard]] Expected requireFunction( - const ModuleInfo& module, std::string_view name, const std::vector& parameters, - const std::vector& results) { - const Export* item = findExport(module, name); - if (item == nullptr) { - return unexpected(std::string("missing or duplicate function export: ") + std::string(name)); - } - if (item->kind != 0 || item->index >= module.function_types.size()) { - return unexpected(std::string("export is not a valid function: ") + std::string(name)); - } - const FunctionType& type = module.types[module.function_types[item->index]]; - if (type.parameters != parameters || type.results != results) { - return unexpected(std::string("function export has the wrong wasm signature: ") + std::string(name)); - } - return {}; -} - -[[nodiscard]] Expected audit(const std::string& wasm_path, const std::string& manifest_path) { - auto wasm = readFile(wasm_path); - auto manifest = readFile(manifest_path); - if (!wasm) { - return unexpected(wasm.error()); - } - if (!manifest) { - return unexpected(manifest.error()); - } - auto embedded = PJ::parser_module::readManifestSection(*wasm); - if (!embedded) { - return unexpected(embedded.error()); - } - if (embedded->size() != manifest->size() || !std::equal(embedded->begin(), embedded->end(), manifest->begin())) { - return unexpected(std::string("embedded parser-module manifest bytes do not match the source file")); - } - - auto module = inspectModule(*wasm); - if (!module) { - return unexpected(module.error()); - } - if (module->has_start_section) { - return unexpected(std::string("wasm reactor contains a forbidden start section")); - } - if (findExport(*module, "_start") != nullptr) { - return unexpected(std::string("wasm reactor exports forbidden _start")); - } - if (findExport(*module, PJ_MODULE_MANIFEST_ADDR_EXPORT_NAME) != nullptr || - findExport(*module, PJ_MODULE_MANIFEST_LEN_EXPORT_NAME) != nullptr) { - return unexpected(std::string("wasm reactor exports native-only manifest metadata")); - } - - const std::array expected{{ - {PJ_MODULE_ABI_EXPORT_NAME, {}, {kI32}}, - {PJ_MODULE_CREATE_EXPORT_NAME, {kI32}, {kI64}}, - {PJ_MODULE_DESTROY_EXPORT_NAME, {kI64}, {}}, - {PJ_MODULE_BIND_EXPORT_NAME, {kI64, kI64, kI64}, {kI32}}, - {PJ_MODULE_PARSE_EXPORT_NAME, {kI64, kI64, kI64, kI64, kI64}, {kI32}}, - {PJ_MODULE_LAST_ERROR_EXPORT_NAME, {kI64, kI64, kI64}, {kI64}}, - {PJ_MODULE_ALLOC_EXPORT_NAME, {kI64}, {kI64}}, - {PJ_MODULE_FREE_EXPORT_NAME, {kI64, kI64}, {}}, - {"_initialize", {}, {}}, - }}; - for (const auto& entry : expected) { - auto valid = requireFunction(*module, entry.name, entry.parameters, entry.results); - if (!valid) { - return unexpected(valid.error()); - } - } - - for (const auto& item : module->exports) { - if (item.name.rfind("pj_module_", 0) != 0) { - continue; - } - const auto found = std::find_if( - expected.begin(), expected.end(), [&](const ExpectedExport& entry) { return entry.name == item.name; }); - if (found == expected.end()) { - return unexpected(std::string("unexpected parser-module export: ") + item.name); - } - } - - std::cout << "WASM parser-module ABI conformance: PASS\n" - << " sections enumerated: " << module->section_count << '\n' - << " function types: " << module->types.size() << ", functions: " << module->function_types.size() - << ", exports: " << module->exports.size() << '\n' - << " operational exports: 8 exact signatures verified\n" - << " reactor: _initialize exported; _start/start section absent\n" - << " native-only metadata exports: absent\n" - << " manifest section: exactly one, " << embedded->size() << " exact bytes\n"; - return {}; -} - -[[nodiscard]] Expected embed( - const std::string& input_path, const std::string& manifest_path, const std::string& output_path) { - auto wasm = readFile(input_path); - auto manifest = readFile(manifest_path); - if (!wasm) { - return unexpected(wasm.error()); - } - if (!manifest) { - return unexpected(manifest.error()); - } - auto output = PJ::parser_module::appendManifestSection(*wasm, *manifest); - if (!output) { - return unexpected(output.error()); - } - return writeFile(output_path, *output); -} - -} // namespace - -int main(int argc, char** argv) { - Expected result = unexpected(std::string("invalid arguments")); - if (argc == 5 && std::string_view(argv[1]) == "--embed") { - result = embed(argv[2], argv[3], argv[4]); - } else if (argc == 4 && std::string_view(argv[1]) == "--audit") { - result = audit(argv[2], argv[3]); - } else { - std::cerr << "usage: parser_module_wasm_audit --embed INPUT.wasm MANIFEST.json OUTPUT.wasm\n" - " or: parser_module_wasm_audit --audit MODULE.wasm MANIFEST.json\n"; - return 2; - } - if (!result) { - std::cerr << "parser-module wasm audit failed: " << result.error() << '\n'; - return 1; - } - return 0; -} diff --git a/experimental/parser_module/tests/parser_module_wasm_test.cpp b/experimental/parser_module/tests/parser_module_wasm_test.cpp new file mode 100644 index 00000000..140e3cd3 --- /dev/null +++ b/experimental/parser_module/tests/parser_module_wasm_test.cpp @@ -0,0 +1,114 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include "pj_base/parser_module_wasm.hpp" + +#include + +#include +#include +#include +#include + +namespace PJ::parser_module { +namespace { + +std::vector module() { + return {0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00}; +} + +void appendSection(std::vector* wasm, uint8_t id, const std::vector& payload) { + ASSERT_LT(payload.size(), 128U); + wasm->push_back(id); + wasm->push_back(static_cast(payload.size())); + wasm->insert(wasm->end(), payload.begin(), payload.end()); +} + +void expectRejected(std::vector wasm, std::string expected) { + auto inspected = inspectWasmModule(wasm); + ASSERT_FALSE(inspected.has_value()); + EXPECT_NE(inspected.error().find(expected), std::string::npos) << inspected.error(); +} + +TEST(ParserModuleWasm, RejectsMalformedSyntheticSections) { + auto truncated = module(); + truncated.insert(truncated.end(), {1, 4, 0}); + expectRejected(std::move(truncated), "section exceeds"); + + auto overflowing_varuint = module(); + overflowing_varuint.insert(overflowing_varuint.end(), {1, 0x80, 0x80, 0x80, 0x80, 0x10}); + expectRejected(std::move(overflowing_varuint), "section header"); + + auto duplicate = module(); + appendSection(&duplicate, 1, {0}); + appendSection(&duplicate, 1, {0}); + expectRejected(std::move(duplicate), "duplicate standard"); + + auto trailing = module(); + appendSection(&trailing, 1, {0, 0}); + expectRejected(std::move(trailing), "trailing bytes"); + + auto invalid_value_type = module(); + appendSection(&invalid_value_type, 1, {1, 0x60, 1, 0x01, 0}); + expectRejected(std::move(invalid_value_type), "invalid value type"); + + auto unknown_section = module(); + appendSection(&unknown_section, 13, {}); + expectRejected(std::move(unknown_section), "unknown wasm section id"); +} + +TEST(ParserModuleWasm, RejectsInvalidIndicesAndUnboundedVectorCounts) { + auto invalid_type_index = module(); + appendSection(&invalid_type_index, 1, {1, 0x60, 0, 0}); + appendSection(&invalid_type_index, 3, {1, 1}); + expectRejected(std::move(invalid_type_index), "invalid type index"); + + auto invalid_function_index = module(); + appendSection(&invalid_function_index, 1, {1, 0x60, 0, 0}); + appendSection(&invalid_function_index, 3, {1, 0}); + appendSection(&invalid_function_index, 7, {1, 1, 'f', 0, 1}); + expectRejected(std::move(invalid_function_index), "invalid function index"); + + auto hostile_count = module(); + appendSection(&hostile_count, 1, {0xFF, 0xFF, 0xFF, 0xFF, 0x0F}); + expectRejected(std::move(hostile_count), "count exceeds the remaining section bytes"); +} + +TEST(ParserModuleWasm, ParsesAndBoundsTableSections) { + auto unsupported_form = module(); + appendSection(&unsupported_form, 4, {1, 0x40, 0x00, 0x70, 0, 1}); + expectRejected(std::move(unsupported_form), "unsupported wasm table form"); + + auto unbounded = module(); + appendSection(&unbounded, 4, {1, 0x70, 0, 25}); + auto inspected_unbounded = inspectWasmModule(unbounded); + ASSERT_TRUE(inspected_unbounded.has_value()) << inspected_unbounded.error(); + ASSERT_EQ(inspected_unbounded->tables.size(), 1U); + EXPECT_EQ(inspected_unbounded->tables.front().minimum_elements, 25U); + EXPECT_FALSE(inspected_unbounded->tables.front().maximum_elements.has_value()); + auto unbounded_tables = validateParserModuleWasmTables(*inspected_unbounded, 1000); + ASSERT_FALSE(unbounded_tables.has_value()); + EXPECT_NE(unbounded_tables.error().find("no declared maximum"), std::string::npos); + + auto bounded = module(); + appendSection(&bounded, 4, {2, 0x70, 1, 25, 100, 0x6F, 1, 0, 50}); + auto inspected_bounded = inspectWasmModule(bounded); + ASSERT_TRUE(inspected_bounded.has_value()) << inspected_bounded.error(); + ASSERT_EQ(inspected_bounded->tables.size(), 2U); + auto within_cap = validateParserModuleWasmTables(*inspected_bounded, 150); + ASSERT_TRUE(within_cap.has_value()) << within_cap.error(); + EXPECT_EQ(*within_cap, 150U); + auto over_cap = validateParserModuleWasmTables(*inspected_bounded, 149); + ASSERT_FALSE(over_cap.has_value()); + EXPECT_NE(over_cap.error().find("exceeds configured cap"), std::string::npos); + + auto none = module(); + auto inspected_none = inspectWasmModule(none); + ASSERT_TRUE(inspected_none.has_value()); + auto no_tables = validateParserModuleWasmTables(*inspected_none, 0); + ASSERT_TRUE(no_tables.has_value()); + EXPECT_EQ(*no_tables, 0U); +} + +} // namespace +} // namespace PJ::parser_module diff --git a/experimental/parser_module/tests/wasm_parser_module_hardening_test.cpp b/experimental/parser_module/tests/wasm_parser_module_hardening_test.cpp new file mode 100644 index 00000000..1e32bf96 --- /dev/null +++ b/experimental/parser_module/tests/wasm_parser_module_hardening_test.cpp @@ -0,0 +1,275 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pj_base/builtin/point_cloud.hpp" +#include "pj_base/builtin_object_abi.h" +#include "pj_plugins/host/parser_module_runtime.hpp" +#include "pj_plugins/host/parser_module_session_budget.hpp" +#include "pj_plugins/host/wasm_parser_module.hpp" +#include "pj_plugins/host/wasm_parser_module_runtime.hpp" + +namespace PJ { +namespace { + +const ParserModuleClaimKey kClaim{"org.plotjuggler.test.adversarial-wasm", "adversarial"}; + +Span bytes(std::string_view text) { + return {reinterpret_cast(text.data()), text.size()}; +} + +parser_module::BindingInfoV1 binding() { + return parser_module::BindingInfoV1{ + .route = parser_module::Route::kObject, + .claim_index = 0, + .expected_object_type = PJ_BUILTIN_OBJECT_TYPE_POINTCLOUD, + .encoding = bytes("ros2msg"), + .type_name = bytes("test_msgs/msg/Adversarial"), + .schema = bytes("uint8 behavior\n"), + .claim_id = bytes("adversarial"), + .config_json = bytes("{}"), + .schema_digest = {}, + }; +} + +Expected createBound(const WasmParserModule& module) { + auto instance = WasmParserModuleInstance::create(module, 0); + if (!instance) { + return unexpected(instance.error().message); + } + auto result = instance->bind(binding()); + if (!result) { + return unexpected(result.error()); + } + if (result->outcome != ParserModuleBindOutcome::kAccept) { + return unexpected("adversarial fixture bind was not accepted: " + result->message); + } + return std::move(*instance); +} + +WasmParserModuleLoadOptions budgeted(std::shared_ptr budget) { + WasmParserModuleLoadOptions options; + options.budget = std::move(budget); + return options; +} + +ParserModuleParseResult parseBehavior(WasmParserModuleInstance& instance, uint8_t behavior) { + const std::array payload{behavior}; + auto result = instance.parse(parser_module::ParseInputV1{.payload = payload}); + EXPECT_TRUE(result.has_value()) << result.error(); + return result ? std::move(*result) : ParserModuleParseResult{}; +} + +void expectPointCloud42(const ParserModuleParseResult& result) { + ASSERT_EQ(result.fault, ParserModuleFaultKind::kNone) << result.message; + const auto* object = std::get_if(&*result.output); + ASSERT_NE(object, nullptr); + const auto* cloud = object->object.get(); + ASSERT_NE(cloud, nullptr); + ASSERT_EQ(cloud->data.size(), 1U); + EXPECT_EQ(cloud->data[0], 42U); +} + +TEST(WasmParserModuleHardening, EnforcesArtifactAndDeclaredMemoryAdmissionCaps) { + const uint64_t file_size = std::filesystem::file_size(PJ_ADVERSARIAL_WASM_PATH); + std::vector diagnostics; + WasmParserModuleLoadOptions options; + options.sink = [&](const Diagnostic& diagnostic) { diagnostics.push_back(diagnostic); }; + + options.limits.maximum_artifact_bytes = file_size - 1; + auto oversized = WasmParserModule::load(PJ_ADVERSARIAL_WASM_PATH, options); + ASSERT_FALSE(oversized.has_value()); + ASSERT_EQ(diagnostics.size(), 1U); + EXPECT_NE(oversized.error().find("artifact_file_size budget exhausted"), std::string::npos); + + options.limits = WasmParserModuleLimits{}; + options.limits.maximum_linear_memory_bytes = UINT64_C(128) * 1024U * 1024U; + diagnostics.clear(); + auto memory_bomb = WasmParserModule::load(PJ_ADVERSARIAL_WASM_PATH, options); + ASSERT_FALSE(memory_bomb.has_value()); + ASSERT_EQ(diagnostics.size(), 1U); + EXPECT_NE(memory_bomb.error().find("exceeds configured cap"), std::string::npos); + + options.limits = WasmParserModuleLimits{}; + options.limits.maximum_table_elements = 0; + diagnostics.clear(); + auto table_bomb = WasmParserModule::load(PJ_ADVERSARIAL_WASM_PATH, options); + ASSERT_FALSE(table_bomb.has_value()); + ASSERT_EQ(diagnostics.size(), 1U); + EXPECT_NE(table_bomb.error().find("table maximum"), std::string::npos); +} + +TEST(WasmParserModuleHardening, EnforcesAggregateBudgetsAtActualAdmissionBoundaries) { + const uint64_t file_size = std::filesystem::file_size(PJ_ADVERSARIAL_WASM_PATH); + const auto load_with = [](ParserModuleSessionBudgetLimits limits) { + auto budget = std::make_shared(limits); + auto loaded = WasmParserModule::load(PJ_ADVERSARIAL_WASM_PATH, budgeted(budget)); + return std::pair(std::move(budget), std::move(loaded)); + }; + + ParserModuleSessionBudgetLimits limits; + limits.maximum_modules = 0; + auto [module_budget, module_decline] = load_with(limits); + ASSERT_FALSE(module_decline.has_value()); + EXPECT_NE(module_decline.error().find("module_count"), std::string::npos); + EXPECT_EQ(module_budget->usage().modules, 0U); + + limits = {}; + limits.maximum_artifact_bytes = file_size - 1; + auto [artifact_budget, artifact_decline] = load_with(limits); + ASSERT_FALSE(artifact_decline.has_value()); + EXPECT_NE(artifact_decline.error().find("artifact_file_size"), std::string::npos); + EXPECT_EQ(artifact_budget->usage().modules, 0U); + + limits = {}; + limits.maximum_claims = 0; + auto [claim_budget, claim_decline] = load_with(limits); + ASSERT_FALSE(claim_decline.has_value()); + EXPECT_NE(claim_decline.error().find("total_claims"), std::string::npos); + EXPECT_EQ(claim_budget->usage().modules, 0U); + + limits = {}; + limits.maximum_active_instances = 0; + auto [instance_budget, active_module] = load_with(limits); + ASSERT_TRUE(active_module.has_value()) << active_module.error(); + auto active_decline = WasmParserModuleInstance::create(*active_module, 0); + ASSERT_FALSE(active_decline.has_value()); + EXPECT_EQ(active_decline.error().outcome, WasmParserModuleCreateOutcome::kAdmissionDecline); + EXPECT_NE(active_decline.error().message.find("active_instances"), std::string::npos); + EXPECT_EQ(instance_budget->usage().active_instances, 0U); + + limits = {}; + limits.maximum_linear_memory_bytes = UINT64_C(128) * 1024U * 1024U; + auto [memory_budget, memory_module] = load_with(limits); + ASSERT_TRUE(memory_module.has_value()) << memory_module.error(); + auto memory_decline = WasmParserModuleInstance::create(*memory_module, 0); + ASSERT_FALSE(memory_decline.has_value()); + EXPECT_EQ(memory_decline.error().outcome, WasmParserModuleCreateOutcome::kAdmissionDecline); + EXPECT_NE(memory_decline.error().message.find("total_linear_memory"), std::string::npos); + EXPECT_EQ(memory_budget->usage().declared_linear_memory_bytes, 0U); + + // Reservations are released by the wrappers themselves, module last. + limits = {}; + limits.maximum_modules = 1; + limits.maximum_active_instances = 1; + auto [release_budget, released_load] = load_with(limits); + ASSERT_TRUE(released_load.has_value()) << released_load.error(); + std::optional released_module(std::move(*released_load)); + { + auto instance = createBound(*released_module); + ASSERT_TRUE(instance.has_value()) << instance.error(); + EXPECT_EQ(release_budget->usage().active_instances, 1U); + released_module.reset(); + EXPECT_EQ(release_budget->usage().modules, 1U) << "module reservation outlives its instances"; + } + EXPECT_EQ(release_budget->usage().modules, 0U); + EXPECT_EQ(release_budget->usage().active_instances, 0U); + EXPECT_EQ(release_budget->usage().declared_linear_memory_bytes, 0U); +} + +TEST(WasmParserModuleHardening, MetersInfiniteLoopAsDistinctContractViolation) { + WasmParserModuleLoadOptions options; + options.limits.metering_points_per_call = UINT64_C(1000000); + auto module = WasmParserModule::load(PJ_ADVERSARIAL_WASM_PATH, options); + ASSERT_TRUE(module.has_value()) << module.error(); + auto instance = createBound(*module); + ASSERT_TRUE(instance.has_value()) << instance.error(); + + const ParserModuleParseResult result = parseBehavior(*instance, 1); + EXPECT_EQ(result.fault, ParserModuleFaultKind::kContractViolation); + EXPECT_NE(result.message.find("wasm metering exhausted during pj_module_parse"), std::string::npos); + EXPECT_NE(result.message.find("instruction-point limit 1000000"), std::string::npos); +} + +TEST(WasmParserModuleHardening, EngineRejectsRuntimeGrowthPastDeclaredMaximum) { + auto module = WasmParserModule::load(PJ_ADVERSARIAL_WASM_PATH); + ASSERT_TRUE(module.has_value()) << module.error(); + EXPECT_EQ(module->declaredLinearMemoryMaximum(), UINT64_C(256) * 1024U * 1024U); + auto instance = createBound(*module); + ASSERT_TRUE(instance.has_value()) << instance.error(); + + const ParserModuleParseResult result = parseBehavior(*instance, 2); + EXPECT_EQ(result.fault, ParserModuleFaultKind::kDataError); + EXPECT_NE(result.message.find("memory growth rejected by declared maximum"), std::string::npos); +} + +TEST(WasmParserModuleHardening, HostDrivenQuarantineReplaysBindingThenDisablesOnRepeat) { + // The wasm wrapper only classifies; this is the host loop, identical to the + // native one: record faults, replay create/bind on quarantine, mark the + // recreation, and stop creating instances once the claim is disabled. + auto module = WasmParserModule::load(PJ_ADVERSARIAL_WASM_PATH); + ASSERT_TRUE(module.has_value()) << module.error(); + ParserModuleStrikeTracker tracker; + auto instance = createBound(*module); + ASSERT_TRUE(instance.has_value()) << instance.error(); + + for (uint8_t strike = 1; strike <= 3; ++strike) { + const ParserModuleParseResult result = parseBehavior(*instance, 0); + ASSERT_EQ(result.fault, ParserModuleFaultKind::kContractViolation); + EXPECT_NE(result.message.find("wasm trap"), std::string::npos); + EXPECT_EQ(instance->lifecycleDiagnostic(), result.message); + const ParserModuleStrikeState state = tracker.recordFault(kClaim, result.fault); + EXPECT_EQ(state.health, strike < 3 ? ParserModuleClaimHealth::kActive : ParserModuleClaimHealth::kQuarantined); + } + EXPECT_EQ(tracker.state(kClaim).quarantine_count, 1U); + + instance = createBound(*module); + ASSERT_TRUE(instance.has_value()) << instance.error(); + ASSERT_TRUE(tracker.markRecreated(kClaim)); + expectPointCloud42(parseBehavior(*instance, 3)); + + for (uint8_t strike = 0; strike < 3; ++strike) { + const ParserModuleParseResult result = parseBehavior(*instance, 0); + ASSERT_EQ(result.fault, ParserModuleFaultKind::kContractViolation); + (void)tracker.recordFault(kClaim, result.fault); + } + const ParserModuleStrikeState disabled = tracker.state(kClaim); + EXPECT_EQ(disabled.health, ParserModuleClaimHealth::kDisabled); + EXPECT_EQ(disabled.quarantine_count, 2U); + EXPECT_FALSE(tracker.markRecreated(kClaim)); +} + +TEST(WasmParserModuleHardening, IndependentInstancesRunConcurrentlyUnderOneTracker) { + // Scalar and object routes of one claim run on different threads in the + // host; the stores are independent and the shared tracker is synchronized. + auto budget = std::make_shared(); + auto module = WasmParserModule::load(PJ_ADVERSARIAL_WASM_PATH, budgeted(budget)); + ASSERT_TRUE(module.has_value()) << module.error(); + ParserModuleStrikeTracker tracker; + + std::vector threads; + for (int worker = 0; worker < 4; ++worker) { + threads.emplace_back([&] { + auto instance = createBound(*module); + ASSERT_TRUE(instance.has_value()) << instance.error(); + for (int round = 0; round < 25; ++round) { + expectPointCloud42(parseBehavior(*instance, 3)); + const ParserModuleParseResult trap = parseBehavior(*instance, 2); // data error, never a strike + EXPECT_EQ(trap.fault, ParserModuleFaultKind::kDataError); + (void)tracker.recordFault(kClaim, trap.fault); + } + }); + } + for (auto& thread : threads) { + thread.join(); + } + EXPECT_EQ(tracker.state(kClaim).strikes, 0U); + EXPECT_EQ(tracker.state(kClaim).health, ParserModuleClaimHealth::kActive); + EXPECT_EQ(budget->usage().active_instances, 0U); + EXPECT_EQ(budget->usage().modules, 1U); +} + +} // namespace +} // namespace PJ diff --git a/experimental/parser_module/tests/wasm_parser_module_test.cpp b/experimental/parser_module/tests/wasm_parser_module_test.cpp new file mode 100644 index 00000000..2b1f98ed --- /dev/null +++ b/experimental/parser_module/tests/wasm_parser_module_test.cpp @@ -0,0 +1,811 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include "pj_plugins/host/wasm_parser_module.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pj_base/builtin/point_cloud.hpp" +#include "pj_base/builtin_object_abi.h" +#include "pj_base/parser_module_abi.h" +#include "pj_base/parser_module_manifest.hpp" +#include "pj_base/parser_module_wasm.hpp" +#include "pj_plugins/host/parser_module_claims.hpp" +#include "pj_plugins/host/wasm_parser_module_runtime.hpp" + +namespace PJ { +namespace { + +constexpr std::string_view kSchema = "uint32 width\nstring frame_id\nuint8[] data\n"; + +Span bytes(std::string_view text) { + return {reinterpret_cast(text.data()), text.size()}; +} + +std::vector readFile(std::string_view path) { + std::ifstream input(std::string(path), std::ios::binary | std::ios::ate); + EXPECT_TRUE(input.good()); + const std::streamoff size = input.tellg(); + EXPECT_GT(size, 0); + std::vector result(static_cast(size)); + input.seekg(0); + input.read(reinterpret_cast(result.data()), static_cast(result.size())); + EXPECT_TRUE(input.good()); + return result; +} + +std::optional readVarUint32(const std::vector& wasm, size_t* position) { + uint32_t value = 0; + for (size_t index = 0; index < 5; ++index) { + if (*position >= wasm.size()) { + return std::nullopt; + } + const uint8_t next = wasm[(*position)++]; + if (index == 4 && (next & UINT8_C(0xF0)) != 0) { + return std::nullopt; + } + value |= static_cast(next & UINT8_C(0x7F)) << (index * 7U); + if ((next & UINT8_C(0x80)) == 0) { + return value; + } + } + return std::nullopt; +} + +std::vector encodeVarUint32(uint32_t value) { + std::vector output; + do { + uint8_t next = static_cast(value & UINT32_C(0x7F)); + value >>= 7U; + if (value != 0) { + next |= UINT8_C(0x80); + } + output.push_back(next); + } while (value != 0); + return output; +} + +template +void append(std::vector* output, const Bytes& bytes) { + output->insert(output->end(), std::begin(bytes), std::end(bytes)); +} + +/// Encode one wasm section: id byte, varuint32 payload size, payload. +std::vector encodeSection(uint8_t id, const std::vector& payload) { + std::vector section{id}; + append(§ion, encodeVarUint32(static_cast(payload.size()))); + append(§ion, payload); + return section; +} + +struct SectionRange { + uint8_t id = 0; + size_t begin = 0; + size_t size_begin = 0; + size_t size_end = 0; + size_t payload_begin = 0; + size_t end = 0; +}; + +std::vector sections(const std::vector& wasm) { + std::vector result; + size_t position = 8; + while (position < wasm.size()) { + const size_t begin = position; + const uint8_t id = wasm[position++]; + const size_t size_begin = position; + auto size = readVarUint32(wasm, &position); + if (!size || static_cast(*size) > wasm.size() - position) { + ADD_FAILURE() << "malformed wasm fixture section"; + return {}; + } + const size_t end = position + *size; + result.push_back( + SectionRange{ + .id = id, + .begin = begin, + .size_begin = size_begin, + .size_end = position, + .payload_begin = position, + .end = end, + }); + position = end; + } + return result; +} + +std::optional findSection(const std::vector& wasm, uint8_t id) { + const auto ranges = sections(wasm); + const auto found = + std::find_if(ranges.begin(), ranges.end(), [&](const SectionRange& section) { return section.id == id; }); + return found == ranges.end() ? std::nullopt : std::optional(*found); +} + +std::optional findManifestSection(const std::vector& wasm) { + for (const auto& section : sections(wasm)) { + if (section.id != 0) { + continue; + } + size_t position = section.payload_begin; + auto name_length = readVarUint32(wasm, &position); + if (!name_length || static_cast(*name_length) > section.end - position) { + return std::nullopt; + } + const std::string_view name( + reinterpret_cast(wasm.data() + position), static_cast(*name_length)); + if (name == PJ_PARSER_MODULE_MANIFEST_SECTION_NAME) { + return section; + } + } + return std::nullopt; +} + +struct ExportLocation { + std::string name; + size_t name_begin = 0; + uint8_t kind = 0; + uint32_t index = 0; + size_t index_begin = 0; + size_t index_end = 0; +}; + +std::vector exportLocations(const std::vector& wasm) { + const auto section = findSection(wasm, 7); + if (!section) { + return {}; + } + size_t position = section->payload_begin; + auto count = readVarUint32(wasm, &position); + if (!count) { + return {}; + } + std::vector result; + for (uint32_t index = 0; index < *count; ++index) { + auto name_length = readVarUint32(wasm, &position); + if (!name_length || static_cast(*name_length) > section->end - position) { + return {}; + } + const size_t name_begin = position; + std::string name(reinterpret_cast(wasm.data() + position), static_cast(*name_length)); + position += *name_length; + if (position >= section->end) { + return {}; + } + const uint8_t kind = wasm[position++]; + const size_t item_begin = position; + auto item_index = readVarUint32(wasm, &position); + if (!item_index) { + return {}; + } + result.push_back( + ExportLocation{ + .name = std::move(name), + .name_begin = name_begin, + .kind = kind, + .index = *item_index, + .index_begin = item_begin, + .index_end = position, + }); + } + return result; +} + +std::optional findExportLocation(const std::vector& wasm, std::string_view name) { + const auto exports = exportLocations(wasm); + const auto found = + std::find_if(exports.begin(), exports.end(), [&](const ExportLocation& item) { return item.name == name; }); + return found == exports.end() ? std::nullopt : std::optional(*found); +} + +void insertSectionAfter(std::vector* wasm, uint8_t preceding_id, const std::vector& section) { + const auto preceding = findSection(*wasm, preceding_id); + ASSERT_TRUE(preceding.has_value()); + wasm->insert(wasm->begin() + static_cast(preceding->end), section.begin(), section.end()); +} + +std::vector withoutManifest(std::vector wasm) { + const auto manifest = findManifestSection(wasm); + EXPECT_TRUE(manifest.has_value()); + if (manifest) { + wasm.erase( + wasm.begin() + static_cast(manifest->begin), wasm.begin() + static_cast(manifest->end)); + } + return wasm; +} + +std::vector withDuplicateManifest(std::vector wasm) { + const auto manifest = findManifestSection(wasm); + EXPECT_TRUE(manifest.has_value()); + if (manifest) { + const std::vector duplicate( + wasm.begin() + static_cast(manifest->begin), wasm.begin() + static_cast(manifest->end)); + wasm.insert(wasm.end(), duplicate.begin(), duplicate.end()); + } + return wasm; +} + +std::vector withMalformedManifest(std::vector wasm) { + wasm = withoutManifest(std::move(wasm)); + constexpr std::string_view kMalformed = "{"; + auto embedded = parser_module::appendManifestSection(wasm, bytes(kMalformed)); + EXPECT_TRUE(embedded.has_value()) << embedded.error(); + return embedded ? std::move(*embedded) : std::move(wasm); +} + +std::vector withStartSection(std::vector wasm) { + const auto initialize = findExportLocation(wasm, "_initialize"); + EXPECT_TRUE(initialize.has_value()); + if (initialize) { + insertSectionAfter(&wasm, 7, encodeSection(8, encodeVarUint32(initialize->index))); + } + return wasm; +} + +std::vector withStartExport(std::vector wasm) { + const auto memory = findExportLocation(wasm, "memory"); + EXPECT_TRUE(memory.has_value()); + if (memory) { + constexpr std::string_view kStart = "_start"; + static_assert(kStart.size() == 6); + std::copy(kStart.begin(), kStart.end(), wasm.begin() + static_cast(memory->name_begin)); + } + return wasm; +} + +std::vector withoutOperationalExport(std::vector wasm) { + const auto exported = findExportLocation(wasm, PJ_MODULE_FREE_EXPORT_NAME); + EXPECT_TRUE(exported.has_value()); + if (exported) { + wasm[exported->name_begin] = 'x'; + } + return wasm; +} + +std::vector withWrongExportSignature(std::vector wasm) { + const auto abi = findExportLocation(wasm, PJ_MODULE_ABI_EXPORT_NAME); + const auto create = findExportLocation(wasm, PJ_MODULE_CREATE_EXPORT_NAME); + EXPECT_TRUE(abi.has_value()); + EXPECT_TRUE(create.has_value()); + if (abi && create) { + const auto encoded_index = encodeVarUint32(create->index); + EXPECT_EQ(encoded_index.size(), abi->index_end - abi->index_begin); + if (encoded_index.size() == abi->index_end - abi->index_begin) { + std::copy(encoded_index.begin(), encoded_index.end(), wasm.begin() + static_cast(abi->index_begin)); + } + } + return wasm; +} + +std::vector withDisallowedImport(std::vector wasm) { + constexpr std::string_view kModule = "wasi_snapshot_preview1"; + constexpr std::string_view kName = "fd_write"; + std::vector payload{1, static_cast(kModule.size())}; + append(&payload, kModule); + payload.push_back(static_cast(kName.size())); + append(&payload, kName); + payload.push_back(0); // function import + payload.push_back(0); // function type zero + insertSectionAfter(&wasm, 1, encodeSection(2, payload)); + return wasm; +} + +std::vector withoutMemoryMaximum(std::vector wasm) { + const auto memory = findSection(wasm, 5); + EXPECT_TRUE(memory.has_value()); + if (!memory) { + return wasm; + } + size_t position = memory->payload_begin; + const auto count = readVarUint32(wasm, &position); + const auto flags = readVarUint32(wasm, &position); + const auto minimum = readVarUint32(wasm, &position); + const auto maximum = readVarUint32(wasm, &position); + EXPECT_EQ(count, 1U); + EXPECT_EQ(flags, 1U); + EXPECT_TRUE(minimum.has_value()); + EXPECT_TRUE(maximum.has_value()); + EXPECT_EQ(position, memory->end); + if (!count || *count != 1 || !flags || *flags != 1 || !minimum || !maximum || position != memory->end) { + return wasm; + } + + std::vector payload{1, 0}; + append(&payload, encodeVarUint32(*minimum)); + const auto replacement = encodeSection(5, payload); + wasm.erase(wasm.begin() + static_cast(memory->begin), wasm.begin() + static_cast(memory->end)); + wasm.insert(wasm.begin() + static_cast(memory->begin), replacement.begin(), replacement.end()); + return wasm; +} + +/// Rewrite the fixture's single funcref table limits. A null maximum drops +/// the maximum flag entirely. +std::vector withTableLimits(std::vector wasm, std::optional maximum) { + const auto table = findSection(wasm, 4); + EXPECT_TRUE(table.has_value()); + if (!table) { + return wasm; + } + size_t position = table->payload_begin; + const auto count = readVarUint32(wasm, &position); + const uint8_t reference_type = wasm[position++]; + const auto flags = readVarUint32(wasm, &position); + const auto minimum = readVarUint32(wasm, &position); + EXPECT_EQ(count, 1U); + EXPECT_EQ(reference_type, 0x70); + EXPECT_EQ(flags, 1U); + EXPECT_TRUE(minimum.has_value()); + if (!count || *count != 1 || !flags || *flags != 1 || !minimum) { + return wasm; + } + + std::vector payload{1, 0x70, static_cast(maximum ? 1 : 0)}; + append(&payload, encodeVarUint32(*minimum)); + if (maximum) { + append(&payload, encodeVarUint32(std::max(*maximum, *minimum))); + } + const auto replacement = encodeSection(4, payload); + wasm.erase(wasm.begin() + static_cast(table->begin), wasm.begin() + static_cast(table->end)); + wasm.insert(wasm.begin() + static_cast(table->begin), replacement.begin(), replacement.end()); + return wasm; +} + +/// Replace the embedded manifest with one declaring three claims while the +/// compiled module still knows two, so the guest's own creation-error path +/// (token zero) is reachable through a claim index the host accepts. +std::vector withThreeClaimManifest(std::vector wasm) { + wasm = withoutManifest(std::move(wasm)); + constexpr std::string_view kManifest = R"({ + "module_abi": 1, + "id": "org.plotjuggler.test.kit-cdr-pointcloud", + "name": "Authoring kit CDR PointCloud fixture", + "version": "1.0.0", + "claims": [ + {"claim_id": "full-wire", "encoding": "ros2msg", "type_name": "toy_msgs/msg/Cloud", + "routes": ["object"], "object_type": "kPointCloud", "priority": 0}, + {"claim_id": "spliced", "encoding": "ros2msg", "type_name": "toy_msgs/msg/CloudSplice", + "routes": ["object"], "object_type": "kPointCloud", "priority": 0}, + {"claim_id": "phantom", "encoding": "ros2msg", "type_name": "toy_msgs/msg/Phantom", + "routes": ["object"], "object_type": "kPointCloud", "priority": 0} + ] +})"; + auto embedded = parser_module::appendManifestSection(wasm, bytes(kManifest)); + EXPECT_TRUE(embedded.has_value()) << embedded.error(); + return embedded ? std::move(*embedded) : std::move(wasm); +} + +std::vector withFunctionFirstOpcode(std::vector wasm, std::string_view export_name, uint8_t opcode) { + const auto function = findExportLocation(wasm, export_name); + const auto code = findSection(wasm, 10); + EXPECT_TRUE(function.has_value()); + EXPECT_TRUE(code.has_value()); + if (!function || !code) { + return wasm; + } + size_t position = code->payload_begin; + auto function_count = readVarUint32(wasm, &position); + EXPECT_TRUE(function_count.has_value()); + if (!function_count || function->index >= *function_count) { + ADD_FAILURE() << export_name << " export does not identify a defined fixture function"; + return wasm; + } + for (uint32_t function_index = 0; function_index < *function_count; ++function_index) { + auto body_size = readVarUint32(wasm, &position); + if (!body_size || static_cast(*body_size) > code->end - position) { + ADD_FAILURE() << "malformed fixture function body"; + return wasm; + } + const size_t body_end = position + *body_size; + auto local_group_count = readVarUint32(wasm, &position); + if (!local_group_count) { + return wasm; + } + for (uint32_t local = 0; local < *local_group_count; ++local) { + auto count = readVarUint32(wasm, &position); + if (!count || position >= body_end) { + return wasm; + } + ++position; // local value type + } + if (function_index == function->index) { + EXPECT_LT(position, body_end); + if (position < body_end) { + wasm[position] = opcode; + } + return wasm; + } + position = body_end; + } + return wasm; +} + +std::vector withInsertedFunctionFirstOpcode( + std::vector wasm, std::string_view export_name, uint8_t opcode) { + const auto function = findExportLocation(wasm, export_name); + const auto code = findSection(wasm, 10); + EXPECT_TRUE(function.has_value()); + EXPECT_TRUE(code.has_value()); + if (!function || !code) { + return wasm; + } + + std::vector payload( + wasm.begin() + static_cast(code->payload_begin), wasm.begin() + static_cast(code->end)); + size_t position = 0; + const auto function_count = readVarUint32(payload, &position); + EXPECT_TRUE(function_count.has_value()); + if (!function_count || function->index >= *function_count) { + ADD_FAILURE() << export_name << " export does not identify a defined fixture function"; + return wasm; + } + + for (uint32_t function_index = 0; function_index < *function_count; ++function_index) { + const size_t body_size_begin = position; + const auto body_size = readVarUint32(payload, &position); + const size_t body_size_end = position; + if (!body_size || static_cast(*body_size) > payload.size() - position) { + ADD_FAILURE() << "malformed fixture function body"; + return wasm; + } + const size_t body_end = position + *body_size; + const auto local_group_count = readVarUint32(payload, &position); + if (!local_group_count) { + return wasm; + } + for (uint32_t local = 0; local < *local_group_count; ++local) { + const auto count = readVarUint32(payload, &position); + if (!count || position >= body_end) { + return wasm; + } + ++position; // local value type + } + if (function_index == function->index) { + const auto encoded_body_size = encodeVarUint32(*body_size + 1U); + payload.erase( + payload.begin() + static_cast(body_size_begin), + payload.begin() + static_cast(body_size_end)); + payload.insert( + payload.begin() + static_cast(body_size_begin), encoded_body_size.begin(), + encoded_body_size.end()); + const size_t adjusted_position = position - (body_size_end - body_size_begin) + encoded_body_size.size(); + payload.insert(payload.begin() + static_cast(adjusted_position), opcode); + + const auto replacement = encodeSection(10, payload); + wasm.erase(wasm.begin() + static_cast(code->begin), wasm.begin() + static_cast(code->end)); + wasm.insert(wasm.begin() + static_cast(code->begin), replacement.begin(), replacement.end()); + return wasm; + } + position = body_end; + } + return wasm; +} + +std::vector withTrappingParse(std::vector wasm) { + return withFunctionFirstOpcode(std::move(wasm), PJ_MODULE_PARSE_EXPORT_NAME, 0x00); // unreachable +} + +std::vector withTrappingCreate(std::vector wasm) { + return withFunctionFirstOpcode(std::move(wasm), PJ_MODULE_CREATE_EXPORT_NAME, 0x00); // unreachable +} + +std::vector withTrappingDestroy(std::vector wasm) { + return withFunctionFirstOpcode(std::move(wasm), PJ_MODULE_DESTROY_EXPORT_NAME, 0x00); // unreachable +} + +std::vector withTrappingFree(std::vector wasm) { + return withInsertedFunctionFirstOpcode(std::move(wasm), PJ_MODULE_FREE_EXPORT_NAME, 0x00); // unreachable +} + +std::vector withInvalidParseOpcode(std::vector wasm) { + return withFunctionFirstOpcode(std::move(wasm), PJ_MODULE_PARSE_EXPORT_NAME, 0xFF); +} + +class TemporaryWasm { + public: + explicit TemporaryWasm(const std::vector& bytes) { + static std::atomic sequence{0}; + const uint64_t nonce = static_cast(std::chrono::steady_clock::now().time_since_epoch().count()); + path_ = std::filesystem::temp_directory_path() / + ("pj-wasm-parser-module-" + std::to_string(nonce) + "-" + std::to_string(sequence.fetch_add(1)) + ".wasm"); + std::ofstream output(path_, std::ios::binary | std::ios::trunc); + output.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + EXPECT_TRUE(output.good()); + } + + ~TemporaryWasm() { + std::error_code error; + std::filesystem::remove(path_, error); + } + + [[nodiscard]] std::string string() const { + return path_.string(); + } + + private: + std::filesystem::path path_; +}; + +void appendU32(std::vector& output, uint32_t value) { + const size_t relative = output.size() - 4; + output.insert(output.end(), (4 - (relative % 4)) % 4, 0); + for (size_t index = 0; index < 4; ++index) { + output.push_back(static_cast(value >> (index * 8U))); + } +} + +std::vector toyPayload() { + std::vector output{0, 1, 0, 0}; + appendU32(output, 2); + appendU32(output, 4); + output.insert(output.end(), {'m', 'a', 'p', 0}); + appendU32(output, 8); + output.insert(output.end(), {1, 2, 3, 4, 5, 6, 7, 8}); + return output; +} + +WasmParserModuleLoadOptions collectingInto(std::vector* diagnostics) { + WasmParserModuleLoadOptions options; + options.sink = [diagnostics](const Diagnostic& diagnostic) { diagnostics->push_back(diagnostic); }; + return options; +} + +parser_module::BindingInfoV1 binding(uint32_t claim_index, std::string_view schema) { + return parser_module::BindingInfoV1{ + .route = parser_module::Route::kObject, + .claim_index = claim_index, + .expected_object_type = PJ_BUILTIN_OBJECT_TYPE_POINTCLOUD, + .encoding = bytes("ros2msg"), + .type_name = bytes(claim_index == 0 ? "toy_msgs/msg/Cloud" : "toy_msgs/msg/CloudSplice"), + .schema = bytes(schema), + .claim_id = bytes(claim_index == 0 ? "full-wire" : "spliced"), + .config_json = bytes("{}"), + .schema_digest = {}, + }; +} + +TEST(WasmParserModule, LoadsValidatesAndAdmitsManifestWithoutInstantiation) { + std::vector diagnostics; + auto module = WasmParserModule::load(PJ_TOY_CDR_POINTCLOUD_WASM_PATH, collectingInto(&diagnostics)); + ASSERT_TRUE(module.has_value()) << module.error(); + EXPECT_TRUE(module->valid()); + EXPECT_TRUE(diagnostics.empty()); + EXPECT_EQ(module->declaredLinearMemoryMaximum(), UINT64_C(256) * 1024U * 1024U); + EXPECT_GT(module->declaredTableElements(), 0U); + EXPECT_LE(module->declaredTableElements(), parser_module::ParserModuleWasmLimits::kDefaultMaximumTableElements); + + ParserClaimCatalog catalog; + auto manifest = ingestParserModuleManifest(catalog, module->manifestJson(), ParserClaimProvenance::kFolderDrop, 31); + ASSERT_TRUE(manifest.has_value()) << manifest.error(); + EXPECT_EQ(manifest->id, "org.plotjuggler.test.kit-cdr-pointcloud"); + EXPECT_EQ(manifest->claims.size(), 2U); +} + +TEST(WasmParserModule, RejectsLoaderViolationsWithOneDiagnostic) { + const auto valid = readFile(PJ_TOY_CDR_POINTCLOUD_WASM_PATH); + const std::vector, std::string>> cases{ + {withoutManifest(valid), "no parser-module manifest"}, + {withDuplicateManifest(valid), "multiple parser-module manifest"}, + {withMalformedManifest(valid), "manifest is invalid JSON"}, + {withStartSection(valid), "forbidden start section"}, + {withStartExport(valid), "forbidden _start"}, + {withoutOperationalExport(valid), PJ_MODULE_FREE_EXPORT_NAME}, + {withWrongExportSignature(valid), "wrong wasm signature"}, + {withDisallowedImport(valid), "wasi_snapshot_preview1.fd_write"}, + {withoutMemoryMaximum(valid), "memory has no declared maximum"}, + {withTableLimits(valid, std::nullopt), "table has no declared maximum"}, + {withTableLimits(valid, UINT32_C(1) << 20U), "table maximum 1048576 exceeds configured cap"}, + {withInvalidParseOpcode(valid), "Wasmer rejected parser module"}, + }; + + for (const auto& [artifact, expected] : cases) { + TemporaryWasm file(artifact); + std::vector diagnostics; + auto module = WasmParserModule::load(file.string(), collectingInto(&diagnostics)); + EXPECT_FALSE(module.has_value()) << expected; + ASSERT_EQ(diagnostics.size(), 1U) << expected; + EXPECT_EQ(diagnostics.front().level, DiagnosticLevel::kError); + EXPECT_EQ(diagnostics.front().message, module.error()); + EXPECT_NE(module.error().find(expected), std::string::npos) << module.error(); + } +} + +TEST(WasmParserModule, RunsFullWireAndSplicedPointCloudLifecycles) { + auto module = WasmParserModule::load(PJ_TOY_CDR_POINTCLOUD_WASM_PATH); + ASSERT_TRUE(module.has_value()) << module.error(); + const auto payload = toyPayload(); + const parser_module::ParseInputV1 input{ + .has_timestamp = false, + .timestamp_ns = 0, + .payload = payload, + }; + + auto full = WasmParserModuleInstance::create(*module, 0); + ASSERT_TRUE(full.has_value()) << full.error().message; + auto full_bind = full->bind(binding(0, kSchema)); + ASSERT_TRUE(full_bind.has_value()) << full_bind.error(); + ASSERT_EQ(full_bind->outcome, ParserModuleBindOutcome::kAccept); + auto full_result = full->parse(input); + ASSERT_TRUE(full_result.has_value()) << full_result.error(); + ASSERT_EQ(full_result->fault, ParserModuleFaultKind::kNone) << full_result->message; + const auto* full_object = std::get_if(&*full_result->output); + ASSERT_NE(full_object, nullptr); + EXPECT_FALSE(full_object->splice.has_value()); + const auto* full_cloud = full_object->object.get(); + ASSERT_NE(full_cloud, nullptr); + EXPECT_EQ(full_cloud->width, 2U); + EXPECT_EQ(full_cloud->frame_id, "map"); + const std::array expected_data{1, 2, 3, 4, 5, 6, 7, 8}; + EXPECT_TRUE(std::equal(full_cloud->data.begin(), full_cloud->data.end(), expected_data.begin(), expected_data.end())); + + auto spliced = WasmParserModuleInstance::create(*module, 1); + ASSERT_TRUE(spliced.has_value()) << spliced.error().message; + auto splice_bind = spliced->bind(binding(1, kSchema)); + ASSERT_TRUE(splice_bind.has_value()) << splice_bind.error(); + ASSERT_EQ(splice_bind->outcome, ParserModuleBindOutcome::kAccept); + auto splice_result = spliced->parse(input); + ASSERT_TRUE(splice_result.has_value()) << splice_result.error(); + ASSERT_EQ(splice_result->fault, ParserModuleFaultKind::kNone) << splice_result->message; + const auto* splice_object = std::get_if(&*splice_result->output); + ASSERT_NE(splice_object, nullptr); + ASSERT_TRUE(splice_object->splice.has_value()); + EXPECT_EQ(splice_object->splice->field_number, 9U); + EXPECT_EQ(splice_object->splice->input_offset, 20U); + EXPECT_EQ(splice_object->splice->payload_bytes, (std::vector{1, 2, 3, 4, 5, 6, 7, 8})); + const auto* splice_cloud = splice_object->object.get(); + ASSERT_NE(splice_cloud, nullptr); + EXPECT_TRUE( + std::equal(splice_cloud->data.begin(), splice_cloud->data.end(), expected_data.begin(), expected_data.end())); +} + +TEST(WasmParserModule, SurfacesBindDecline) { + auto module = WasmParserModule::load(PJ_TOY_CDR_POINTCLOUD_WASM_PATH); + ASSERT_TRUE(module.has_value()) << module.error(); + auto instance = WasmParserModuleInstance::create(*module, 0); + ASSERT_TRUE(instance.has_value()) << instance.error().message; + auto result = instance->bind(binding(0, "uint32 width\nstring frame_id\n")); + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->outcome, ParserModuleBindOutcome::kDecline); + EXPECT_EQ(result->fault, ParserModuleFaultKind::kNone); + EXPECT_NE(result->message.find("unsupported toy schema revision"), std::string::npos); +} + +TEST(WasmParserModule, DeepSchemaReturnsDepthErrorWithConfiguredShadowStack) { + std::string schema = "T1 next\n"; + for (size_t depth = 1; depth <= 64; ++depth) { + schema += "MSG: T" + std::to_string(depth) + "\n"; + schema += depth < 64 ? "T" + std::to_string(depth + 1) + " next\n" : "uint32 value\n"; + } + + auto module = WasmParserModule::load(PJ_TOY_CDR_POINTCLOUD_WASM_PATH); + ASSERT_TRUE(module.has_value()) << module.error(); + auto instance = WasmParserModuleInstance::create(*module, 0); + ASSERT_TRUE(instance.has_value()) << instance.error().message; + auto result = instance->bind(binding(0, schema)); + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->outcome, ParserModuleBindOutcome::kDecline); + EXPECT_EQ(result->fault, ParserModuleFaultKind::kNone); + EXPECT_NE(result->message.find("nesting depth exceeds 64"), std::string::npos); +} + +TEST(WasmParserModule, RejectsClaimIndexOutsideTheManifestBeforeCallingTheGuest) { + auto module = WasmParserModule::load(PJ_TOY_CDR_POINTCLOUD_WASM_PATH); + ASSERT_TRUE(module.has_value()) << module.error(); + auto instance = WasmParserModuleInstance::create(*module, 2); + ASSERT_FALSE(instance.has_value()); + EXPECT_EQ(instance.error().outcome, WasmParserModuleCreateOutcome::kError); + EXPECT_EQ(instance.error().fault, ParserModuleFaultKind::kNone); + EXPECT_NE(instance.error().message.find("claim index is outside the wasm parser-module manifest"), std::string::npos); +} + +TEST(WasmParserModule, CopiesTokenZeroCreationError) { + TemporaryWasm artifact(withThreeClaimManifest(readFile(PJ_TOY_CDR_POINTCLOUD_WASM_PATH))); + auto module = WasmParserModule::load(artifact.string()); + ASSERT_TRUE(module.has_value()) << module.error(); + // The host accepts index 2; the guest (built with two claims) returns the + // creation-error token and the message is read back through + // pj_module_last_error(0, ...). + auto instance = WasmParserModuleInstance::create(*module, 2); + ASSERT_FALSE(instance.has_value()); + EXPECT_EQ(instance.error().outcome, WasmParserModuleCreateOutcome::kError); + EXPECT_EQ(instance.error().fault, ParserModuleFaultKind::kNone); + EXPECT_EQ(instance.error().message, "claim index is outside the module manifest"); +} + +TEST(WasmParserModule, ClassifiesModuleParseErrorAsStrikeFreeDataError) { + auto module = WasmParserModule::load(PJ_TOY_CDR_POINTCLOUD_WASM_PATH); + ASSERT_TRUE(module.has_value()) << module.error(); + auto instance = WasmParserModuleInstance::create(*module, 0); + ASSERT_TRUE(instance.has_value()) << instance.error().message; + auto bound = instance->bind(binding(0, kSchema)); + ASSERT_TRUE(bound.has_value()) << bound.error(); + ASSERT_EQ(bound->outcome, ParserModuleBindOutcome::kAccept); + + const std::array truncated_payload{0, 1, 0, 0}; + auto result = instance->parse(parser_module::ParseInputV1{.payload = truncated_payload}); + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->fault, ParserModuleFaultKind::kDataError); + EXPECT_FALSE(result->output.has_value()); + + ParserModuleStrikeTracker tracker; + const ParserModuleClaimKey key{"org.plotjuggler.test.kit-cdr-pointcloud", "full-wire"}; + EXPECT_EQ(tracker.recordFault(key, result->fault).strikes, 0U); + EXPECT_EQ(tracker.state(key).health, ParserModuleClaimHealth::kActive); +} + +TEST(WasmParserModule, ClassifiesGuestTrapAsContractViolation) { + TemporaryWasm artifact(withTrappingParse(readFile(PJ_TOY_CDR_POINTCLOUD_WASM_PATH))); + auto module = WasmParserModule::load(artifact.string()); + ASSERT_TRUE(module.has_value()) << module.error(); + auto instance = WasmParserModuleInstance::create(*module, 0); + ASSERT_TRUE(instance.has_value()) << instance.error().message; + auto bound = instance->bind(binding(0, kSchema)); + ASSERT_TRUE(bound.has_value()) << bound.error(); + ASSERT_EQ(bound->outcome, ParserModuleBindOutcome::kAccept); + const auto payload = toyPayload(); + auto result = instance->parse(parser_module::ParseInputV1{.payload = payload}); + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->fault, ParserModuleFaultKind::kContractViolation); + EXPECT_NE(result->message.find("wasm trap"), std::string::npos); + + ParserModuleStrikeTracker tracker; + const ParserModuleClaimKey key{"org.plotjuggler.test.kit-cdr-pointcloud", "full-wire"}; + EXPECT_EQ(tracker.recordFault(key, result->fault).strikes, 1U); +} + +TEST(WasmParserModule, TypesCreationTrapsAsContractViolations) { + TemporaryWasm artifact(withTrappingCreate(readFile(PJ_TOY_CDR_POINTCLOUD_WASM_PATH))); + auto module = WasmParserModule::load(artifact.string()); + ASSERT_TRUE(module.has_value()) << module.error(); + auto instance = WasmParserModuleInstance::create(*module, 0); + ASSERT_FALSE(instance.has_value()); + EXPECT_EQ(instance.error().outcome, WasmParserModuleCreateOutcome::kError); + EXPECT_EQ(instance.error().fault, ParserModuleFaultKind::kContractViolation); + EXPECT_NE(instance.error().message.find("pj_module_create failed: wasm trap"), std::string::npos); + + // The host feeds the classified fault to the same tracker native uses. + ParserModuleStrikeTracker tracker; + const ParserModuleClaimKey key{"org.plotjuggler.test.kit-cdr-pointcloud", "full-wire"}; + EXPECT_EQ(tracker.recordFault(key, instance.error().fault).strikes, 1U); +} + +TEST(WasmParserModule, SurvivesDestroyTrapsAndReportsGuestFreeTraps) { + { + TemporaryWasm artifact(withTrappingDestroy(readFile(PJ_TOY_CDR_POINTCLOUD_WASM_PATH))); + auto module = WasmParserModule::load(artifact.string()); + ASSERT_TRUE(module.has_value()) << module.error(); + auto instance = WasmParserModuleInstance::create(*module, 0); + ASSERT_TRUE(instance.has_value()) << instance.error().message; + // Teardown is best-effort: a trapping destroy must not escape the wrapper. + *instance = WasmParserModuleInstance{}; + EXPECT_FALSE(instance->valid()); + } + + TemporaryWasm artifact(withTrappingFree(readFile(PJ_TOY_CDR_POINTCLOUD_WASM_PATH))); + auto module = WasmParserModule::load(artifact.string()); + ASSERT_TRUE(module.has_value()) << module.error(); + auto instance = WasmParserModuleInstance::create(*module, 0); + ASSERT_TRUE(instance.has_value()) << instance.error().message; + auto result = instance->bind(binding(0, kSchema)); + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->fault, ParserModuleFaultKind::kContractViolation); + EXPECT_NE(result->message.find("pj_module_free failed after bind"), std::string::npos); + EXPECT_NE(instance->lifecycleDiagnostic().find("pj_module_free failed after bind"), std::string_view::npos); +} + +} // namespace +} // namespace PJ diff --git a/experimental/parser_module/tests/wasmer_shared_module_prototype_test.cpp b/experimental/parser_module/tests/wasmer_shared_module_prototype_test.cpp new file mode 100644 index 00000000..8bfbebda --- /dev/null +++ b/experimental/parser_module/tests/wasmer_shared_module_prototype_test.cpp @@ -0,0 +1,142 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include + +namespace PJ { +namespace { + +// (module +// (global (mut i32) (i32.const 0)) +// (func (export "next_value") (result i32) +// global.get 0 i32.const 1 i32.add global.set 0 global.get 0)) +constexpr std::array kCounterModule{ + 0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, // preamble + 0x01, 0x05, 0x01, 0x60, 0x00, 0x01, 0x7F, // type + 0x03, 0x02, 0x01, 0x00, // function + 0x06, 0x06, 0x01, 0x7F, 0x01, 0x41, 0x00, 0x0B, // mutable global + 0x07, 0x0E, 0x01, 0x0A, 'n', 'e', 'x', 't', '_', 'v', // export + 'a', 'l', 'u', 'e', 0x00, 0x00, 0x0A, 0x0D, 0x01, 0x0B, 0x00, 0x23, 0x00, 0x41, 0x01, 0x6A, // code + 0x24, 0x00, 0x23, 0x00, 0x0B, +}; + +struct StoreInstance { + ~StoreInstance() { + if (exports_initialized) { + wasm_extern_vec_delete(&exports); + } + if (instance != nullptr) { + wasm_instance_delete(instance); + } + if (store != nullptr) { + wasm_store_delete(store); + } + } + + wasm_store_t* store = nullptr; + wasm_instance_t* instance = nullptr; + wasm_extern_vec_t exports = WASM_EMPTY_VEC; + bool exports_initialized = false; + wasm_func_t* next = nullptr; +}; + +bool instantiate(wasm_engine_t* engine, wasm_module_t* module, StoreInstance* output) { + output->store = wasm_store_new(engine); + if (output->store == nullptr) { + return false; + } + const wasm_extern_vec_t imports = WASM_EMPTY_VEC; + output->instance = wasm_instance_new(output->store, module, &imports, nullptr); + if (output->instance == nullptr) { + return false; + } + wasm_instance_exports(output->instance, &output->exports); + output->exports_initialized = true; + if (output->exports.size != 1) { + return false; + } + output->next = wasm_extern_as_func(output->exports.data[0]); + return output->next != nullptr; +} + +bool nextValue(wasm_func_t* function, int32_t* output) { + const wasm_val_vec_t arguments = WASM_EMPTY_VEC; + wasm_val_t result_values[1] = {WASM_INIT_VAL}; + wasm_val_vec_t results = WASM_ARRAY_VEC(result_values); + wasm_trap_t* trap = wasm_func_call(function, &arguments, &results); + if (trap != nullptr) { + wasm_trap_delete(trap); + return false; + } + if (result_values[0].kind != WASM_I32) { + return false; + } + *output = result_values[0].of.i32; + return true; +} + +TEST(WasmerSharedModulePrototype, CompilesOnceAndInstantiatesIntoIndependentStores) { + wasm_engine_t* engine = wasm_engine_new(); + ASSERT_NE(engine, nullptr); + const wasm_byte_vec_t binary{ + .size = kCounterModule.size(), + .data = reinterpret_cast(const_cast(kCounterModule.data())), + }; + wasm_module_t* module = wasmer_module_new(engine, &binary); + ASSERT_NE(module, nullptr); + + { + StoreInstance first; + StoreInstance second; + ASSERT_TRUE(instantiate(engine, module, &first)); + ASSERT_TRUE(instantiate(engine, module, &second)); + int32_t first_value = 0; + int32_t second_value = 0; + EXPECT_TRUE(nextValue(first.next, &first_value)); + EXPECT_TRUE(nextValue(second.next, &second_value)); + EXPECT_EQ(first_value, 1); + EXPECT_EQ(second_value, 1); + } + + wasm_module_delete(module); + wasm_engine_delete(engine); +} + +TEST(WasmerSharedModulePrototype, AllowsSequentialCallFromNonCreatorThread) { + wasm_engine_t* engine = wasm_engine_new(); + ASSERT_NE(engine, nullptr); + const wasm_byte_vec_t binary{ + .size = kCounterModule.size(), + .data = reinterpret_cast(const_cast(kCounterModule.data())), + }; + wasm_module_t* module = wasmer_module_new(engine, &binary); + ASSERT_NE(module, nullptr); + { + StoreInstance instance; + ASSERT_TRUE(instantiate(engine, module, &instance)); + + int32_t creator_value = 0; + ASSERT_TRUE(nextValue(instance.next, &creator_value)); + int32_t other_thread_value = 0; + bool other_thread_ok = false; + std::thread caller([&] { other_thread_ok = nextValue(instance.next, &other_thread_value); }); + caller.join(); + int32_t returned_value = 0; + EXPECT_TRUE(nextValue(instance.next, &returned_value)); + EXPECT_TRUE(other_thread_ok); + EXPECT_EQ(creator_value, 1); + EXPECT_EQ(other_thread_value, 2); + EXPECT_EQ(returned_value, 3); + } + + wasm_module_delete(module); + wasm_engine_delete(engine); +} + +} // namespace +} // namespace PJ diff --git a/experimental/parser_module/tools/pj_wasm_embed_manifest.cpp b/experimental/parser_module/tools/pj_wasm_embed_manifest.cpp new file mode 100644 index 00000000..dda84bc4 --- /dev/null +++ b/experimental/parser_module/tools/pj_wasm_embed_manifest.cpp @@ -0,0 +1,145 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pj_base/parser_module_manifest.hpp" +#include "pj_base/parser_module_wasm.hpp" + +namespace { + +using PJ::Expected; +using PJ::Span; +using PJ::unexpected; + +[[nodiscard]] Expected> readFile(const std::string& path) { + std::ifstream input(path, std::ios::binary | std::ios::ate); + if (!input) { + return unexpected(std::string("cannot open file: ") + path); + } + const std::streamoff end = input.tellg(); + if (end < 0 || static_cast(end) > std::numeric_limits::max() || + end > std::numeric_limits::max()) { + return unexpected(std::string("file size is invalid: ") + path); + } + std::vector bytes(static_cast(end)); + input.seekg(0); + if (!bytes.empty()) { + input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + } + if (!input) { + return unexpected(std::string("cannot read complete file: ") + path); + } + return bytes; +} + +[[nodiscard]] Expected writeFile(const std::string& path, Span bytes) { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output) { + return unexpected(std::string("cannot create file: ") + path); + } + if (!bytes.empty()) { + output.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + } + if (!output) { + return unexpected(std::string("cannot write complete file: ") + path); + } + return {}; +} + +[[nodiscard]] Expected verify( + const std::string& wasm_path, const std::string& manifest_path, uint64_t maximum_memory_bytes) { + auto wasm = readFile(wasm_path); + auto manifest = readFile(manifest_path); + if (!wasm) { + return unexpected(wasm.error()); + } + if (!manifest) { + return unexpected(manifest.error()); + } + PJ::parser_module::ParserModuleWasmLimits limits; + limits.maximum_linear_memory_bytes = maximum_memory_bytes; + auto artifact = PJ::parser_module::validateParserModuleWasmArtifact(*wasm, limits); + if (!artifact) { + return unexpected(artifact.error()); + } + const auto& embedded = artifact->manifest_json; + if (embedded.size() != manifest->size() || !std::equal(embedded.begin(), embedded.end(), manifest->begin())) { + return unexpected(std::string("embedded parser-module manifest bytes do not match the source file")); + } + + const auto& module = artifact->module; + std::cout << "WASM parser-module ABI conformance: PASS\n" + << " sections enumerated: " << module.section_count << '\n' + << " function types: " << module.function_type_count << ", functions: " << module.function_count + << ", exports: " << module.exports.size() << '\n' + << " operational exports: 8 exact signatures verified\n" + << " reactor: _initialize exported; _start/start section absent\n" + << " native-only metadata exports: absent\n" + << " imports: empty frozen allow-list verified\n" + << " declared linear-memory maximum: " << artifact->declared_linear_memory_maximum << " bytes\n" + << " declared table elements: " << artifact->declared_table_elements << " (cap " + << limits.maximum_table_elements << ")\n" + << " manifest section: exactly one, " << embedded.size() << " exact bytes\n"; + return {}; +} + +[[nodiscard]] Expected embed( + const std::string& input_path, const std::string& manifest_path, const std::string& output_path) { + auto wasm = readFile(input_path); + auto manifest = readFile(manifest_path); + if (!wasm) { + return unexpected(wasm.error()); + } + if (!manifest) { + return unexpected(manifest.error()); + } + auto output = PJ::parser_module::appendManifestSection(*wasm, *manifest); + if (!output) { + return unexpected(output.error()); + } + return writeFile(output_path, *output); +} + +void usage() { + std::cerr << "usage: pj-wasm-embed-manifest embed INPUT.wasm MANIFEST.json OUTPUT.wasm\n" + " or: pj-wasm-embed-manifest verify MODULE.wasm MANIFEST.json [MAX_MEMORY_BYTES]\n"; +} + +} // namespace + +int main(int argc, char** argv) { + Expected result = unexpected(std::string("invalid arguments")); + if (argc == 5 && std::string_view(argv[1]) == "embed") { + result = embed(argv[2], argv[3], argv[4]); + } else if ((argc == 4 || argc == 5) && std::string_view(argv[1]) == "verify") { + uint64_t maximum_memory_bytes = PJ::parser_module::ParserModuleWasmLimits::kDefaultMaximumLinearMemoryBytes; + if (argc == 5) { + const std::string_view text(argv[4]); + const auto parsed = std::from_chars(text.data(), text.data() + text.size(), maximum_memory_bytes); + if (parsed.ec != std::errc{} || parsed.ptr != text.data() + text.size() || maximum_memory_bytes == 0) { + std::cerr << "parser-module wasm operation failed: MAX_MEMORY_BYTES must be a positive integer\n"; + return 1; + } + } + result = verify(argv[2], argv[3], maximum_memory_bytes); + } else { + usage(); + return 2; + } + if (!result) { + std::cerr << "parser-module wasm operation failed: " << result.error() << '\n'; + return 1; + } + return 0; +} From 1cd95ee158004ceec3e565a8a0233d4629da36e8 Mon Sep 17 00:00:00 2001 From: Davide Faconti Date: Sat, 14 Nov 2026 08:26:24 +0100 Subject: [PATCH 2/2] fix(parser-module): bound what the host allocates while reading untrusted module metadata A Codex review found that the wasm loader bounds what the guest may use but not what the host allocates or computes while reading a module's metadata. - Scalar output descriptors: field names may alias, so 4096 fields pointing at one 1 MiB string passed every range check and then cost ~4 GiB of copies. The shared decoder now caps the field count (2^20) and the cumulative bytes of names and string values (64 MiB) before any owned copy, on the native and wasm paths. A bad_alloc in wasm parse() is now a classified data error instead of escaping the wrapper. - Static inspection copied the resolved signature into every export. Exports and imports now hold a type index and resolve through the type table. - Duplicate export names were found with a quadratic scan; it is now a hash set, and type, import and export counts are capped at the WebAssembly JS-API limits. - Manifest decoding built claims (each copying the module id) before any bound. It now bounds the JSON size (4 MiB), every string (1024 bytes), the claim count (4096, the session budget's cap) and digests per claim (64) first. load() contains allocation failure. - Session budget: two instances of one module shared the module's reservation id, so releasing one twice freed the other's charge. Each instance now gets its own id, consumed once. - The wasm build did not depend on headers outside the kit glob (time_math.hpp), so wasm artifacts could go stale. It now uses a compiler-generated depfile. Each code fix has a regression test, seen failing before the fix. Co-Authored-By: Claude Fable 5.2 --- experimental/parser_module/README.md | 20 ++- .../parser_module/cmake/PjParserModule.cmake | 31 +++-- .../include/pj_base/parser_module_abi.h | 11 ++ .../include/pj_base/parser_module_wasm.hpp | 24 +++- .../pj_plugins/host/parser_module_claims.hpp | 15 +++ .../pj_plugins/host/parser_module_runtime.hpp | 2 +- .../host/parser_module_session_budget.hpp | 14 ++- .../parser_module/src/parser_module_abi.cpp | 14 +++ .../src/parser_module_claims.cpp | 41 +++++- .../src/parser_module_runtime.cpp | 16 +-- .../src/parser_module_session_budget.cpp | 19 ++- .../parser_module/src/parser_module_wasm.cpp | 70 ++++++----- .../parser_module/src/wasm_parser_module.cpp | 117 ++++++++++-------- .../src/wasm_parser_module_runtime.cpp | 24 +++- .../tests/native_parser_module_fixture.cpp | 32 ++++- .../tests/native_parser_module_fixture.hpp | 3 +- .../tests/parser_module_abi_test.cpp | 47 +++++++ .../tests/parser_module_claims_test.cpp | 81 ++++++++++++ .../tests/parser_module_runtime_test.cpp | 12 ++ .../parser_module_session_budget_test.cpp | 58 +++++++-- .../tests/parser_module_wasm_test.cpp | 67 ++++++++++ 21 files changed, 571 insertions(+), 147 deletions(-) diff --git a/experimental/parser_module/README.md b/experimental/parser_module/README.md index f78b3420..5dcfe806 100644 --- a/experimental/parser_module/README.md +++ b/experimental/parser_module/README.md @@ -214,6 +214,21 @@ replays create/bind and disables through the shared `ParserModuleStrikeTracker`. | Artifact size | 64 MiB | `WasmParserModuleLimits::maximum_artifact_bytes` | | Declared linear-memory maximum | 256 MiB | `ParserModuleWasmLimits::maximum_linear_memory_bytes` | | Declared table elements (host memory outside the linear-memory budget) | 65536 | `ParserModuleWasmLimits::maximum_table_elements` | +| Type / import / export entries inspected (WebAssembly JS-API limits) | 1,000,000 / 100,000 / 100,000 | `parser_module::kWasmMaximumTypes`, `kWasmMaximumImports`, `kWasmMaximumExports` | +| Manifest JSON size | 4 MiB | `kMaximumParserModuleManifestBytes` | +| Claims per manifest | 4096 | `kMaximumParserModuleManifestClaims` | +| Bytes in each manifest string (`id`, `name`, `version`, `claim_id`, `encoding`, `type_name`, each schema digest) | 1024 | `kMaximumParserModuleManifestStringBytes` | +| Schema digests per claim | 64 | `kMaximumParserModuleClaimSchemaDigests` | +| Fields in one scalar output | 1,048,576 | `parser_module::kMaximumScalarOutputFields` | +| Field-name plus string-value bytes in one scalar output (names may alias) | 64 MiB | `parser_module::kMaximumScalarOutputBytes` | + +The fixed limits bound what the host allocates while reading untrusted +metadata, which guest memory caps and metering do not cover. The entry counts +apply to wasm inspection; the manifest bounds apply to every manifest decode, +native or wasm, as do the scalar output bounds (`readOutputDescriptorV1` rejects an over-limit descriptor, so it +is a contract violation). Host allocation failure inside the limits is not a +module fault: `load()` rejects, and a wasm `parse()` returns a strike-free data +error with `PJ_MODULE_ERR_ALLOCATION_FAILURE`. Metering is an instruction budget, not a wall-clock deadline: bulk operators such as `memory.copy` cost one point regardless of size, and the pinned archive @@ -223,8 +238,9 @@ guarded default. The optional, thread-safe `ParserModuleSessionBudgetTracker` bounds a whole session: module count (64), artifact bytes (64 MiB each), total claims (4096), active instances (128) and aggregate per-instance declared memory (4 GiB). It -counts resources under opaque reservation ids and never keys on manifest -identity, which remains catalog policy. Both loaders accept a null budget and +counts resources under opaque reservation ids (one per module and one per +instance, each consumed once on release) and never keys on manifest identity, +which remains catalog policy. Both loaders accept a null budget and then perform no admission accounting; `NativeParserModule::load(path, sink)` keeps its budget-free behaviour. diff --git a/experimental/parser_module/cmake/PjParserModule.cmake b/experimental/parser_module/cmake/PjParserModule.cmake index a2e72544..f77ca69b 100644 --- a/experimental/parser_module/cmake/PjParserModule.cmake +++ b/experimental/parser_module/cmake/PjParserModule.cmake @@ -191,22 +191,30 @@ function(pj_add_parser_module TARGET) set(_wasm_dir "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_parser_module_wasm") set(_wasm_raw "${_wasm_dir}/${TARGET}.raw.wasm") set(_wasm_output "${_wasm_dir}/${TARGET}.wasm") + set(_wasm_depfile "${_wasm_dir}/${TARGET}.raw.wasm.d") + set(_wasm_compile_flags + --target=wasm32-wasip1 + --sysroot=${_wasi_sysroot} + -std=c++17 + -fno-exceptions + -fno-rtti + -fvisibility=hidden + -O1 + -Wall -Wextra -Werror + -DPJ_PARSER_MODULE_CLAIM_COUNT=${_claim_count} + ${_kit_include_flags}) + # The depfile covers every header the author source includes (kit, the + # shared pj_base/time_math.hpp, and the author's own). It comes from a + # separate -M pass because clang keeps only the last input's dependencies + # when one -MD command compiles two sources. add_custom_command( OUTPUT "${_wasm_raw}" COMMAND ${CMAKE_COMMAND} -E make_directory "${_wasm_dir}" - COMMAND "${_wasi_clang}" - --target=wasm32-wasip1 - --sysroot=${_wasi_sysroot} + COMMAND "${_wasi_clang}" ${_wasm_compile_flags} + -M -MF "${_wasm_depfile}" -MT "${_wasm_raw}" "${_module_source}" + COMMAND "${_wasi_clang}" ${_wasm_compile_flags} -mexec-model=reactor - -std=c++17 - -fno-exceptions - -fno-rtti - -fvisibility=hidden - -O1 - -Wall -Wextra -Werror - -DPJ_PARSER_MODULE_CLAIM_COUNT=${_claim_count} - ${_kit_include_flags} "${_module_source}" "${_no_io_stubs}" -Wl,--export=pj_module_abi @@ -223,6 +231,7 @@ function(pj_add_parser_module TARGET) -Wl,--stack-first -o "${_wasm_raw}" DEPENDS "${_module_source}" "${_no_io_stubs}" ${_parser_module_headers} + DEPFILE "${_wasm_depfile}" COMMENT "Compiling ${TARGET} as a C++17 WASI reactor" VERBATIM ) diff --git a/experimental/parser_module/include/pj_base/parser_module_abi.h b/experimental/parser_module/include/pj_base/parser_module_abi.h index 61a39463..54cc3783 100644 --- a/experimental/parser_module/include/pj_base/parser_module_abi.h +++ b/experimental/parser_module/include/pj_base/parser_module_abi.h @@ -100,6 +100,17 @@ namespace PJ::parser_module { inline constexpr uint16_t kBindingInfoVersionV1 = PJ_MODULE_BINDING_INFO_VERSION_V1; inline constexpr uint16_t kOutputDescriptorVersionV1 = PJ_MODULE_OUTPUT_DESCRIPTOR_VERSION_V1; +/** Host limits on one decoded scalar output; readOutputDescriptorV1 rejects a + * descriptor that exceeds either as malformed. Field names may alias a single + * range of the block, so the bytes the host copies are bounded here, not by the + * block size. A legitimate message flattens to thousands of fields (array + * clamping defaults to 500 elements) with names under a hundred bytes; both + * limits sit orders of magnitude above that. + */ +inline constexpr uint32_t kMaximumScalarOutputFields = UINT32_C(1) << 20U; +/// Cumulative bytes of field names plus string values in one scalar output. +inline constexpr uint64_t kMaximumScalarOutputBytes = UINT64_C(64) * 1024U * 1024U; + enum class Route : uint8_t { kScalar = PJ_MODULE_ROUTE_SCALAR, kObject = PJ_MODULE_ROUTE_OBJECT, diff --git a/experimental/parser_module/include/pj_base/parser_module_wasm.hpp b/experimental/parser_module/include/pj_base/parser_module_wasm.hpp index 78f15a51..d3db7672 100644 --- a/experimental/parser_module/include/pj_base/parser_module_wasm.hpp +++ b/experimental/parser_module/include/pj_base/parser_module_wasm.hpp @@ -23,6 +23,13 @@ inline constexpr uint8_t kWasmValueI32 = UINT8_C(0x7F); inline constexpr uint8_t kWasmValueI64 = UINT8_C(0x7E); inline constexpr uint64_t kWasmMemoryPageBytes = UINT64_C(65536); +/// Entry counts inspection accepts per section: the WebAssembly JS-API +/// implementation limits, which every module a browser can load already meets. +/// They bound the host structures built from a 64 MiB artifact. +inline constexpr uint32_t kWasmMaximumTypes = 1000000; +inline constexpr uint32_t kWasmMaximumImports = 100000; +inline constexpr uint32_t kWasmMaximumExports = 100000; + enum class WasmExternalKind : uint8_t { kFunction = 0, kTable = 1, @@ -52,22 +59,26 @@ struct WasmTableLimits { bool operator==(const WasmTableLimits&) const = default; }; +// Functions refer to their signature by index into +// WasmModuleInfo::function_types: one type can back any number of imports and +// exports, so copying it into each would let a small artifact expand. struct WasmImport { std::string module; std::string name; WasmExternalKind kind = WasmExternalKind::kFunction; - std::optional function_signature; + std::optional function_type; }; struct WasmExport { std::string name; WasmExternalKind kind = WasmExternalKind::kFunction; uint32_t index = 0; - std::optional function_signature; + std::optional function_type; std::optional memory_limits; }; struct WasmModuleInfo { + std::vector function_types; std::vector imports; std::vector exports; std::vector memories; @@ -78,13 +89,16 @@ struct WasmModuleInfo { size_t function_count = 0; [[nodiscard]] const WasmExport* findExport(std::string_view name) const noexcept; + /// The resolved signature of a function export, or nullptr for other kinds. + [[nodiscard]] const WasmFunctionSignature* functionSignature(const WasmExport& exported) const noexcept; }; /** Inspect the sections needed for parser-module admission. * - * Every section and variable-length integer is bounds-checked. Function - * signatures are resolved for imported and exported functions. Malformed, - * truncated, duplicate, or unsupported binary constructs return an error. + * Every section and variable-length integer is bounds-checked. Imported and + * exported functions carry a validated index into `function_types`. Malformed, + * truncated, duplicate, over-limit, or unsupported binary constructs return an + * error. */ [[nodiscard]] Expected inspectWasmModule(Span wasm); diff --git a/experimental/parser_module/include/pj_plugins/host/parser_module_claims.hpp b/experimental/parser_module/include/pj_plugins/host/parser_module_claims.hpp index 9d342d59..d8b7cbd5 100644 --- a/experimental/parser_module/include/pj_plugins/host/parser_module_claims.hpp +++ b/experimental/parser_module/include/pj_plugins/host/parser_module_claims.hpp @@ -7,6 +7,7 @@ * @brief Parser-module manifest decoding into ParserClaimCatalog claims. */ +#include #include #include #include @@ -15,9 +16,23 @@ #include "pj_base/diagnostic_sink.hpp" #include "pj_base/expected.hpp" #include "pj_plugins/host/parser_claim_catalog.hpp" +#include "pj_plugins/host/parser_module_session_budget.hpp" namespace PJ { +/// Manifest bounds, checked before any claim is built: every claim copies the +/// module id, so unbounded metadata would expand in host memory before any +/// session budget sees it. A real manifest is a few kilobytes. +/// Manifest JSON bytes: about 1 KiB for each of the maximum number of claims. +inline constexpr size_t kMaximumParserModuleManifestBytes = size_t{4} * 1024U * 1024U; +/// Claims per manifest: a module with more could never fit the default session budget. +inline constexpr size_t kMaximumParserModuleManifestClaims = ParserModuleSessionBudgetLimits::kDefaultMaximumClaims; +/// Each of id, name, version, claim_id, encoding, type_name and schema digest +/// (reverse-DNS ids, type names and `sha256:<64-hex>` digests are under 256). +inline constexpr size_t kMaximumParserModuleManifestStringBytes = 1024; +/// schema_digests per claim: one per supported schema revision of the type. +inline constexpr size_t kMaximumParserModuleClaimSchemaDigests = 64; + /// Parsed module metadata. Claim order remains manifest order. struct ParserModuleManifest { std::string id; diff --git a/experimental/parser_module/include/pj_plugins/host/parser_module_runtime.hpp b/experimental/parser_module/include/pj_plugins/host/parser_module_runtime.hpp index 3f76d5b4..1952323f 100644 --- a/experimental/parser_module/include/pj_plugins/host/parser_module_runtime.hpp +++ b/experimental/parser_module/include/pj_plugins/host/parser_module_runtime.hpp @@ -129,7 +129,7 @@ class NativeParserModuleInstance { parser_module::Route bound_route_ = parser_module::Route::kScalar; uint16_t expected_object_type_ = 0; bool bound_ = false; - bool instance_budget_reserved_ = false; + uint64_t instance_reservation_ = 0; // nonzero while a session-budget instance is held }; struct ParserModuleClaimKey { diff --git a/experimental/parser_module/include/pj_plugins/host/parser_module_session_budget.hpp b/experimental/parser_module/include/pj_plugins/host/parser_module_session_budget.hpp index 2c4fb509..a71af2f1 100644 --- a/experimental/parser_module/include/pj_plugins/host/parser_module_session_budget.hpp +++ b/experimental/parser_module/include/pj_plugins/host/parser_module_session_budget.hpp @@ -76,13 +76,15 @@ class ParserModuleSessionBudgetTracker { uint64_t artifact_bytes, uint64_t claim_count, uint64_t declared_linear_memory_maximum); /// Reserve one instance of an admitted module. Every instance owns an - /// independent store, so the module's declared memory is charged again. + /// independent store, so the module's declared memory is charged again. The + /// returned reservation is a new id naming this instance alone. [[nodiscard]] ParserModuleAdmissionDecision admitInstance(uint64_t module_reservation); - /// Releases are idempotent-safe: unknown or already-released ids are ignored. - void releaseInstance(uint64_t module_reservation); - /// A module reservation is released even when instances are still live; - /// their later releases then find no module and are ignored. + /// Releases are idempotent-safe: each id is consumed once, and unknown or + /// already-released ids (including module ids) are ignored. + void releaseInstance(uint64_t instance_reservation); + /// A module reservation is released even when instances are still live: it + /// gives back their charges too, and their later releases are ignored. void releaseModule(uint64_t module_reservation); [[nodiscard]] const ParserModuleSessionBudgetLimits& limits() const noexcept; @@ -100,6 +102,8 @@ class ParserModuleSessionBudgetTracker { ParserModuleSessionBudgetUsage usage_; uint64_t next_reservation_ = 1; std::map modules_; + /// Live instance reservation -> its module reservation. + std::map instances_; }; } // namespace PJ diff --git a/experimental/parser_module/src/parser_module_abi.cpp b/experimental/parser_module/src/parser_module_abi.cpp index c4cce005..a0ed1cde 100644 --- a/experimental/parser_module/src/parser_module_abi.cpp +++ b/experimental/parser_module/src/parser_module_abi.cpp @@ -512,12 +512,19 @@ Expected readOutputDescriptorV1(Span bytes) { if (field_count > reader.remaining() / (kScalarFieldPrefixSize + 1)) { return malformed("scalar output field count exceeds the block"); } + if (field_count > kMaximumScalarOutputFields) { + return malformed( + "scalar output has more than " + std::to_string(kMaximumScalarOutputFields) + " fields"); + } struct DecodedField { uint32_t name_offset; uint32_t name_length; ScalarValueV1 value; }; + // Names may alias one range, so their sum is bounded explicitly: it is what + // the host copies when it takes ownership of the output. + uint64_t copied_bytes = 0; std::vector decoded_fields; decoded_fields.reserve(field_count); for (uint32_t index = 0; index < field_count; ++index) { @@ -568,6 +575,7 @@ Expected readOutputDescriptorV1(Span bytes) { if (!reader.u32(&length) || !reader.take(length, &decoded)) { return malformed("truncated scalar string value"); } + copied_bytes += length; // string values never alias, so this stays within the block size value = asStringView(decoded); break; } @@ -589,6 +597,12 @@ Expected readOutputDescriptorV1(Span bytes) { if (name_offset > bytes.size() || name_length > bytes.size() - name_offset) { return malformed("scalar field name range is outside the block"); } + // At most 2^20 names of 2^32 bytes each, so the uint64 sum cannot overflow. + copied_bytes += name_length; + if (copied_bytes > kMaximumScalarOutputBytes) { + return malformed( + "scalar output names and strings exceed " + std::to_string(kMaximumScalarOutputBytes) + " bytes"); + } scalar.fields.push_back( ScalarFieldV1{ .name = asStringView(bytes.subspan(name_offset, name_length)), diff --git a/experimental/parser_module/src/parser_module_claims.cpp b/experimental/parser_module/src/parser_module_claims.cpp index c90424bb..bdbd9f60 100644 --- a/experimental/parser_module/src/parser_module_claims.cpp +++ b/experimental/parser_module/src/parser_module_claims.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,11 @@ namespace { if (it == object.end() || !it->is_string() || it->get_ref().empty()) { return unexpected(std::string(context) + " missing required non-empty string key: " + std::string(key)); } + if (it->get_ref().size() > kMaximumParserModuleManifestStringBytes) { + return unexpected( + std::string(context) + " " + std::string(key) + " exceeds " + + std::to_string(kMaximumParserModuleManifestStringBytes) + " bytes"); + } return it->get(); } @@ -80,23 +86,33 @@ namespace { return unexpected( "parser module manifest claim " + std::to_string(claim_index) + " schema_digests must be an array"); } + if (values->size() > kMaximumParserModuleClaimSchemaDigests) { + return unexpected( + "parser module manifest claim " + std::to_string(claim_index) + " schema_digests has more than " + + std::to_string(kMaximumParserModuleClaimSchemaDigests) + " entries"); + } for (const auto& value : *values) { - if (!value.is_string()) { + if (!value.is_string() || value.get_ref().size() > kMaximumParserModuleManifestStringBytes) { return unexpected( - "parser module manifest claim " + std::to_string(claim_index) + " schema_digests must contain strings"); + "parser module manifest claim " + std::to_string(claim_index) + + " schema_digests must contain strings of at most " + std::to_string(kMaximumParserModuleManifestStringBytes) + + " bytes"); } digests.insert(value.get()); } return digests; } -} // namespace - -Expected decodeParserModuleManifest( +// Every bound runs before the claims are built: each claim copies the module +// id, and the scratch-catalog validation copies the claims again. +[[nodiscard]] Expected decodeBoundedManifest( std::string_view manifest_json, ParserClaimProvenance provenance) { if (manifest_json.empty()) { return unexpected("parser module manifest is empty"); } + if (manifest_json.size() > kMaximumParserModuleManifestBytes) { + return unexpected("parser module manifest exceeds " + std::to_string(kMaximumParserModuleManifestBytes) + " bytes"); + } nlohmann::json json; try { @@ -142,6 +158,10 @@ Expected decodeParserModuleManifest( if (claims_json == json.end() || !claims_json->is_array()) { return unexpected("parser module manifest requires a claims array"); } + if (claims_json->size() > kMaximumParserModuleManifestClaims) { + return unexpected( + "parser module manifest has more than " + std::to_string(kMaximumParserModuleManifestClaims) + " claims"); + } ParserModuleManifest manifest{.id = *id, .name = *name, .version = *version, .claims = {}}; manifest.claims.reserve(claims_json->size()); @@ -220,6 +240,17 @@ Expected decodeParserModuleManifest( return manifest; } +} // namespace + +Expected decodeParserModuleManifest( + std::string_view manifest_json, ParserClaimProvenance provenance) { + try { + return decodeBoundedManifest(manifest_json, provenance); + } catch (const std::bad_alloc&) { + return unexpected("allocation failed while decoding the parser module manifest"); + } +} + Expected ingestParserModuleManifest( ParserClaimCatalog& catalog, std::string_view manifest_json, ParserClaimProvenance provenance, uint64_t provider_generation, const DiagnosticSink& sink) { diff --git a/experimental/parser_module/src/parser_module_runtime.cpp b/experimental/parser_module/src/parser_module_runtime.cpp index d9afb435..eccc68cb 100644 --- a/experimental/parser_module/src/parser_module_runtime.cpp +++ b/experimental/parser_module/src/parser_module_runtime.cpp @@ -58,7 +58,7 @@ NativeParserModuleInstance::NativeParserModuleInstance(NativeParserModuleInstanc bound_route_(other.bound_route_), expected_object_type_(other.expected_object_type_), bound_(other.bound_), - instance_budget_reserved_(std::exchange(other.instance_budget_reserved_, false)) { + instance_reservation_(std::exchange(other.instance_reservation_, 0)) { other.bound_ = false; } @@ -71,7 +71,7 @@ NativeParserModuleInstance& NativeParserModuleInstance::operator=(NativeParserMo bound_route_ = other.bound_route_; expected_object_type_ = other.expected_object_type_; bound_ = other.bound_; - instance_budget_reserved_ = std::exchange(other.instance_budget_reserved_, false); + instance_reservation_ = std::exchange(other.instance_reservation_, 0); other.bound_ = false; } return *this; @@ -83,22 +83,24 @@ Expected NativeParserModuleInstance::create( return unexpected("cannot create an instance from an invalid native parser module"); } const auto& budget = module.state_->session_budget; + uint64_t instance_reservation = 0; if (budget != nullptr) { auto admission = budget->admitInstance(module.state_->module_reservation); if (!admission.accepted()) { return unexpected(std::move(admission.diagnostic)); } + instance_reservation = admission.reservation; } const uint64_t token = module.state_->create(claim_index); if (token == PJ_MODULE_CREATION_ERROR_TOKEN) { if (budget != nullptr) { - budget->releaseInstance(module.state_->module_reservation); + budget->releaseInstance(instance_reservation); } auto message = copyLastError(*module.state_, PJ_MODULE_CREATION_ERROR_TOKEN); return unexpected(message ? *message : message.error()); } NativeParserModuleInstance instance(module.state_, token, claim_index); - instance.instance_budget_reserved_ = budget != nullptr; + instance.instance_reservation_ = instance_reservation; return instance; } @@ -234,12 +236,12 @@ void NativeParserModuleInstance::reset() noexcept { if (module_ != nullptr && token_ != PJ_MODULE_CREATION_ERROR_TOKEN) { module_->destroy(token_); } - if (module_ != nullptr && instance_budget_reserved_) { - module_->session_budget->releaseInstance(module_->module_reservation); + if (module_ != nullptr && instance_reservation_ != 0) { + module_->session_budget->releaseInstance(instance_reservation_); } token_ = PJ_MODULE_CREATION_ERROR_TOKEN; bound_ = false; - instance_budget_reserved_ = false; + instance_reservation_ = 0; module_.reset(); } diff --git a/experimental/parser_module/src/parser_module_session_budget.cpp b/experimental/parser_module/src/parser_module_session_budget.cpp index 99309aba..675b38f3 100644 --- a/experimental/parser_module/src/parser_module_session_budget.cpp +++ b/experimental/parser_module/src/parser_module_session_budget.cpp @@ -4,6 +4,7 @@ #include "pj_plugins/host/parser_module_session_budget.hpp" #include +#include #include #include #include @@ -73,19 +74,24 @@ ParserModuleAdmissionDecision ParserModuleSessionBudgetTracker::admitInstance(ui return declineBudget(ParserModuleBudgetKind::kTotalLinearMemory, "total_linear_memory"); } + const uint64_t reservation = next_reservation_++; + instances_.emplace(reservation, module_reservation); ++module->second.active_instances; ++usage_.active_instances; usage_.declared_linear_memory_bytes += module->second.declared_linear_memory_maximum; return ParserModuleAdmissionDecision{ - .reservation = module_reservation, .exhausted_budget = ParserModuleBudgetKind::kNone, .diagnostic = {}}; + .reservation = reservation, .exhausted_budget = ParserModuleBudgetKind::kNone, .diagnostic = {}}; } -void ParserModuleSessionBudgetTracker::releaseInstance(uint64_t module_reservation) { +void ParserModuleSessionBudgetTracker::releaseInstance(uint64_t instance_reservation) { const std::scoped_lock lock(mutex_); - auto module = modules_.find(module_reservation); - if (module == modules_.end() || module->second.active_instances == 0) { + const auto instance = instances_.find(instance_reservation); + if (instance == instances_.end()) { return; } + // releaseModule erases its instances, so a live instance's module exists. + auto module = modules_.find(instance->second); + instances_.erase(instance); --module->second.active_instances; --usage_.active_instances; usage_.declared_linear_memory_bytes -= module->second.declared_linear_memory_maximum; @@ -97,8 +103,9 @@ void ParserModuleSessionBudgetTracker::releaseModule(uint64_t module_reservation if (module == modules_.end()) { return; } - // Live instances of a dropped module still hold their store; their memory - // and instance counts are given back when each of them is released. + // Live instances give back their charges with the module; erasing their + // reservations makes their later releases no-ops. + std::erase_if(instances_, [module_reservation](const auto& entry) { return entry.second == module_reservation; }); usage_.active_instances -= module->second.active_instances; usage_.declared_linear_memory_bytes -= module->second.active_instances * module->second.declared_linear_memory_maximum; diff --git a/experimental/parser_module/src/parser_module_wasm.cpp b/experimental/parser_module/src/parser_module_wasm.cpp index 75300100..8b4c2629 100644 --- a/experimental/parser_module/src/parser_module_wasm.cpp +++ b/experimental/parser_module/src/parser_module_wasm.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -86,11 +87,6 @@ class Cursor { size_t position_ = 0; }; -struct PendingImport { - WasmImport value; - std::optional function_type; -}; - struct ModuleBuilder { std::vector types; std::vector imported_function_types; @@ -98,7 +94,7 @@ struct ModuleBuilder { std::vector imported_memories; std::vector defined_memories; std::vector tables; - std::vector imports; + std::vector imports; std::vector exports; bool has_start_section = false; size_t section_count = 0; @@ -120,10 +116,15 @@ struct ModuleBuilder { } [[nodiscard]] Expected requireCountFits( - uint32_t count, size_t remaining, size_t minimum_entry_size, std::string_view section) { + uint32_t count, size_t remaining, size_t minimum_entry_size, std::string_view section, + uint32_t maximum_count = std::numeric_limits::max()) { if (minimum_entry_size == 0 || static_cast(count) > remaining / minimum_entry_size) { return unexpected(std::string("wasm ") + std::string(section) + " count exceeds the remaining section bytes"); } + if (count > maximum_count) { + return unexpected( + std::string("wasm ") + std::string(section) + " count exceeds the limit of " + std::to_string(maximum_count)); + } return {}; } @@ -163,7 +164,7 @@ struct ModuleBuilder { if (!count) { return unexpected(count.error()); } - auto bounded = requireCountFits(*count, cursor.remaining(), 3, "type-section entry"); + auto bounded = requireCountFits(*count, cursor.remaining(), 3, "type-section entry", kWasmMaximumTypes); if (!bounded) { return unexpected(bounded.error()); } @@ -214,7 +215,7 @@ struct ModuleBuilder { if (!count) { return unexpected(count.error()); } - auto bounded = requireCountFits(*count, cursor.remaining(), 4, "import-section entry"); + auto bounded = requireCountFits(*count, cursor.remaining(), 4, "import-section entry", kWasmMaximumImports); if (!bounded) { return unexpected(bounded.error()); } @@ -227,17 +228,13 @@ struct ModuleBuilder { return unexpected(std::string("truncated or invalid wasm import entry")); } - PendingImport imported{ - .value = - WasmImport{ - .module = std::move(*module_name), - .name = std::move(*field_name), - .kind = static_cast(*kind), - .function_signature = std::nullopt, - }, + WasmImport imported{ + .module = std::move(*module_name), + .name = std::move(*field_name), + .kind = static_cast(*kind), .function_type = std::nullopt, }; - switch (imported.value.kind) { + switch (imported.kind) { case WasmExternalKind::kFunction: { auto type_index = cursor.varUint32(); if (!type_index) { @@ -365,11 +362,13 @@ struct ModuleBuilder { if (!count) { return unexpected(count.error()); } - auto bounded = requireCountFits(*count, cursor.remaining(), 3, "export-section entry"); + auto bounded = requireCountFits(*count, cursor.remaining(), 3, "export-section entry", kWasmMaximumExports); if (!bounded) { return unexpected(bounded.error()); } module->exports.reserve(*count); + std::unordered_set names; + names.reserve(*count); for (uint32_t index = 0; index < *count; ++index) { auto name = cursor.name(); auto kind = cursor.byte(); @@ -377,9 +376,7 @@ struct ModuleBuilder { if (!name || !kind || !item_index || *kind > static_cast(WasmExternalKind::kTag)) { return unexpected(std::string("truncated or invalid wasm export entry")); } - if (std::any_of(module->exports.begin(), module->exports.end(), [&](const WasmExport& item) { - return item.name == *name; - })) { + if (!names.insert(*name).second) { return unexpected(std::string("duplicate wasm export name: ") + *name); } module->exports.push_back( @@ -387,7 +384,7 @@ struct ModuleBuilder { .name = std::move(*name), .kind = static_cast(*kind), .index = *item_index, - .function_signature = std::nullopt, + .function_type = std::nullopt, .memory_limits = std::nullopt, }); } @@ -408,9 +405,10 @@ struct ModuleBuilder { } WasmModuleInfo result; + result.function_types = std::move(builder.types); result.has_start_section = builder.has_start_section; result.section_count = builder.section_count; - result.function_type_count = builder.types.size(); + result.function_type_count = result.function_types.size(); result.function_count = function_types.size(); result.memories.reserve(builder.imported_memories.size() + builder.defined_memories.size()); result.memories.insert(result.memories.end(), builder.imported_memories.begin(), builder.imported_memories.end()); @@ -418,13 +416,10 @@ struct ModuleBuilder { result.tables = std::move(builder.tables); result.imports.reserve(builder.imports.size()); for (auto& imported : builder.imports) { - if (imported.function_type.has_value()) { - if (*imported.function_type >= builder.types.size()) { - return unexpected(std::string("wasm function import references an invalid type index")); - } - imported.value.function_signature = builder.types[*imported.function_type]; + if (imported.function_type.has_value() && *imported.function_type >= result.function_types.size()) { + return unexpected(std::string("wasm function import references an invalid type index")); } - result.imports.push_back(std::move(imported.value)); + result.imports.push_back(std::move(imported)); } result.exports.reserve(builder.exports.size()); for (auto& exported : builder.exports) { @@ -432,7 +427,7 @@ struct ModuleBuilder { if (exported.index >= function_types.size()) { return unexpected(std::string("wasm function export references an invalid function index")); } - exported.function_signature = builder.types[function_types[exported.index]]; + exported.function_type = function_types[exported.index]; } else if (exported.kind == WasmExternalKind::kMemory) { if (exported.index >= result.memories.size()) { return unexpected(std::string("wasm memory export references an invalid memory index")); @@ -519,6 +514,14 @@ const WasmExport* WasmModuleInfo::findExport(std::string_view name) const noexce return found == exports.end() ? nullptr : &*found; } +const WasmFunctionSignature* WasmModuleInfo::functionSignature(const WasmExport& exported) const noexcept { + if (exported.kind != WasmExternalKind::kFunction || !exported.function_type.has_value() || + *exported.function_type >= function_types.size()) { + return nullptr; + } + return &function_types[*exported.function_type]; +} + Expected inspectWasmModule(Span wasm) { try { return inspectWasmModuleImpl(wasm); @@ -558,10 +561,11 @@ Expected validateParserModuleWasmAbi(const WasmModuleInfo& module) { if (exported == nullptr) { return unexpected(std::string("missing function export: ") + std::string(entry.name)); } - if (exported->kind != WasmExternalKind::kFunction || !exported->function_signature.has_value()) { + const WasmFunctionSignature* signature = module.functionSignature(*exported); + if (signature == nullptr) { return unexpected(std::string("export is not a valid function: ") + std::string(entry.name)); } - if (*exported->function_signature != entry.signature) { + if (*signature != entry.signature) { return unexpected(std::string("function export has the wrong wasm signature: ") + std::string(entry.name)); } } diff --git a/experimental/parser_module/src/wasm_parser_module.cpp b/experimental/parser_module/src/wasm_parser_module.cpp index 0085c966..d66507ec 100644 --- a/experimental/parser_module/src/wasm_parser_module.cpp +++ b/experimental/parser_module/src/wasm_parser_module.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -137,63 +138,69 @@ Expected WasmParserModule::load(std::string_view path, const W return unexpected(std::move(message)); }; - const WasmParserModuleLimits& limits = options.limits; - if (limits.maximum_artifact_bytes == 0 || limits.maximum_linear_memory_bytes == 0 || - limits.metering_points_per_call == 0) { - return reject("wasm parser-module limits must all be nonzero"); - } - auto bytes = readFile(path, limits.maximum_artifact_bytes); - if (!bytes) { - return reject(bytes.error()); - } - - auto audited = parser_module::validateParserModuleWasmArtifact( - *bytes, parser_module::ParserModuleWasmLimits{ - .maximum_linear_memory_bytes = limits.maximum_linear_memory_bytes, - .maximum_table_elements = limits.maximum_table_elements, - }); - if (!audited) { - return reject(audited.error()); - } - const char* manifest_data = - audited->manifest_json.empty() ? "" : reinterpret_cast(audited->manifest_json.data()); - const std::string_view manifest_json(manifest_data, audited->manifest_json.size()); - auto decoded_manifest = decodeParserModuleManifest(manifest_json, ParserClaimProvenance::kFolderDrop); - if (!decoded_manifest) { - return reject("invalid wasm parser-module manifest: " + decoded_manifest.error()); - } - - auto state = std::make_shared(); - state->path = path; - state->manifest_json.assign(manifest_json); - state->claim_count = decoded_manifest->claims.size(); - state->artifact_size = bytes->size(); - state->declared_linear_memory_maximum = audited->declared_linear_memory_maximum; - state->declared_table_elements = audited->declared_table_elements; - state->metering_points_per_call = limits.metering_points_per_call; - if (options.budget != nullptr) { - auto admission = options.budget->admitModule( - bytes->size(), decoded_manifest->claims.size(), audited->declared_linear_memory_maximum); - if (!admission.accepted()) { - return reject(std::move(admission.diagnostic)); + // Every host allocation is bounded by the limits and the manifest bounds, + // but the process may still be short of memory: report it as a rejection. + try { + const WasmParserModuleLimits& limits = options.limits; + if (limits.maximum_artifact_bytes == 0 || limits.maximum_linear_memory_bytes == 0 || + limits.metering_points_per_call == 0) { + return reject("wasm parser-module limits must all be nonzero"); } - state->session_budget = options.budget; - state->module_reservation = admission.reservation; - } - auto engine = createMeteredEngine(limits.metering_points_per_call); - if (!engine) { - return reject(engine.error()); - } - state->engine = *engine; - const wasm_byte_vec_t binary{ - .size = bytes->size(), - .data = reinterpret_cast(bytes->data()), - }; - state->module = wasmer_module_new(state->engine, &binary); - if (state->module == nullptr) { - return reject(wasmerFailure("Wasmer rejected parser module")); + auto bytes = readFile(path, limits.maximum_artifact_bytes); + if (!bytes) { + return reject(bytes.error()); + } + + auto audited = parser_module::validateParserModuleWasmArtifact( + *bytes, parser_module::ParserModuleWasmLimits{ + .maximum_linear_memory_bytes = limits.maximum_linear_memory_bytes, + .maximum_table_elements = limits.maximum_table_elements, + }); + if (!audited) { + return reject(audited.error()); + } + const char* manifest_data = + audited->manifest_json.empty() ? "" : reinterpret_cast(audited->manifest_json.data()); + const std::string_view manifest_json(manifest_data, audited->manifest_json.size()); + auto decoded_manifest = decodeParserModuleManifest(manifest_json, ParserClaimProvenance::kFolderDrop); + if (!decoded_manifest) { + return reject("invalid wasm parser-module manifest: " + decoded_manifest.error()); + } + + auto state = std::make_shared(); + state->path = path; + state->manifest_json.assign(manifest_json); + state->claim_count = decoded_manifest->claims.size(); + state->artifact_size = bytes->size(); + state->declared_linear_memory_maximum = audited->declared_linear_memory_maximum; + state->declared_table_elements = audited->declared_table_elements; + state->metering_points_per_call = limits.metering_points_per_call; + if (options.budget != nullptr) { + auto admission = options.budget->admitModule( + bytes->size(), decoded_manifest->claims.size(), audited->declared_linear_memory_maximum); + if (!admission.accepted()) { + return reject(std::move(admission.diagnostic)); + } + state->session_budget = options.budget; + state->module_reservation = admission.reservation; + } + auto engine = createMeteredEngine(limits.metering_points_per_call); + if (!engine) { + return reject(engine.error()); + } + state->engine = *engine; + const wasm_byte_vec_t binary{ + .size = bytes->size(), + .data = reinterpret_cast(bytes->data()), + }; + state->module = wasmer_module_new(state->engine, &binary); + if (state->module == nullptr) { + return reject(wasmerFailure("Wasmer rejected parser module")); + } + return WasmParserModule(std::move(state)); + } catch (const std::bad_alloc&) { + return reject("allocation failed while loading the wasm parser module"); } - return WasmParserModule(std::move(state)); } std::string_view WasmParserModule::path() const noexcept { diff --git a/experimental/parser_module/src/wasm_parser_module_runtime.cpp b/experimental/parser_module/src/wasm_parser_module_runtime.cpp index 94c1a05e..e32da917 100644 --- a/experimental/parser_module/src/wasm_parser_module_runtime.cpp +++ b/experimental/parser_module/src/wasm_parser_module_runtime.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -123,8 +124,8 @@ struct WasmParserModuleInstanceState { if (store != nullptr) { wasm_store_delete(store); } - if (instance_budget_reserved && module->session_budget != nullptr) { - module->session_budget->releaseInstance(module->module_reservation); + if (instance_reservation != 0 && module->session_budget != nullptr) { + module->session_budget->releaseInstance(instance_reservation); } } @@ -262,7 +263,7 @@ struct WasmParserModuleInstanceState { parser_module::Route bound_route = parser_module::Route::kScalar; uint16_t expected_object_type = 0; bool bound = false; - bool instance_budget_reserved = false; + uint64_t instance_reservation = 0; // nonzero while a session-budget instance is held std::string lifecycle_diagnostic; }; @@ -404,7 +405,7 @@ Expected WasmParserModule std::move(admission.diagnostic), ParserModuleFaultKind::kNone, WasmParserModuleCreateOutcome::kAdmissionDecline); } - state->instance_budget_reserved = true; + state->instance_reservation = admission.reservation; } state->store = wasm_store_new(state->module->engine); if (state->store == nullptr) { @@ -665,7 +666,20 @@ Expected WasmParserModuleInstance::parse(const parser_m }; }; - auto result = parse_once(); + // Descriptor limits bound what the host copies, so running out of memory + // within them is the host's condition, not a module fault: a strike-free + // data error, like the authoring kit's own allocation failures. + Expected result = ParserModuleParseResult{}; + try { + result = parse_once(); + } catch (const std::bad_alloc&) { + result = ParserModuleParseResult{ + .fault = ParserModuleFaultKind::kDataError, + .result_code = PJ_MODULE_ERR_ALLOCATION_FAILURE, + .message = "host allocation failed while consuming the parse output", + .output = std::nullopt, + }; + } if (result && result->fault == ParserModuleFaultKind::kContractViolation) { state_->noteContractViolation(result->message); } diff --git a/experimental/parser_module/tests/native_parser_module_fixture.cpp b/experimental/parser_module/tests/native_parser_module_fixture.cpp index 2e36d746..905bc7d0 100644 --- a/experimental/parser_module/tests/native_parser_module_fixture.cpp +++ b/experimental/parser_module/tests/native_parser_module_fixture.cpp @@ -48,7 +48,8 @@ constexpr char kManifest[] = R"({ {"claim_id":"route-mismatch","encoding":"protobuf","type_name":"fixture.RouteMismatch","routes":["scalar"],"priority":0}, {"claim_id":"type-mismatch","encoding":"protobuf","type_name":"fixture.TypeMismatch","routes":["object"],"object_type":"kPointCloud","priority":0}, {"claim_id":"splice-grid-map","encoding":"protobuf","type_name":"fixture.SpliceGridMap","routes":["object"],"object_type":"kGridMap","priority":0}, - {"claim_id":"splice-grid-map-short","encoding":"protobuf","type_name":"fixture.SpliceGridMapShort","routes":["object"],"object_type":"kGridMap","priority":0} + {"claim_id":"splice-grid-map-short","encoding":"protobuf","type_name":"fixture.SpliceGridMapShort","routes":["object"],"object_type":"kGridMap","priority":0}, + {"claim_id":"aliased-scalar-names","encoding":"protobuf","type_name":"fixture.AliasedScalarNames","routes":["scalar"],"priority":0} ] })"; @@ -96,6 +97,33 @@ int32_t storeOutput(Instance& instance, PJ::parser_module::OutputDescriptorV1 de return PJ_MODULE_OK; } +// A scalar descriptor whose 4096 field names all alias one 64 KiB range: a +// ~100 KiB block that names 256 MiB of host copies. writeOutputDescriptorV1 +// cannot alias names, so the bytes are laid out by hand. +void writeAliasedScalarNames(std::vector& output) { + constexpr size_t kFields = 4096; + constexpr size_t kNameBytes = size_t{64} * 1024U; + constexpr size_t kFieldBytes = 10; // name offset, name length, kind, bool value + constexpr size_t kNameOffset = 24 + kFields * kFieldBytes; + output.assign(kNameOffset + kNameBytes, uint8_t{'n'}); + const auto put = [&output](size_t at, uint64_t value, size_t width) { + for (size_t index = 0; index < width; ++index) { + output[at + index] = static_cast(value >> (8U * index)); + } + }; + put(0, PJ_MODULE_OUTPUT_DESCRIPTOR_VERSION_V1, 2); + put(2, PJ_MODULE_ROUTE_SCALAR, 1); + put(3, 0, 17); // reserved, has_timestamp, padding and timestamp + put(20, kFields, 4); + for (size_t field = 0; field < kFields; ++field) { + const size_t at = 24 + field * kFieldBytes; + put(at, kNameOffset, 4); + put(at + 4, kNameBytes, 4); + put(at + 8, PJ_MODULE_SCALAR_VALUE_BOOL, 1); + put(at + 9, 0, 1); + } +} + } // namespace extern "C" { @@ -182,6 +210,8 @@ PJ_FIXTURE_EXPORT int32_t pj_module_parse( if (instance->claim_index == kMalformed) { // Deliberately not a decodable output descriptor. instance->output = {1, 2, 3}; + } else if (instance->claim_index == kAliasedScalarNames) { + writeAliasedScalarNames(instance->output); } else { // ObjectOutputV1::wire is a borrowed span, so its storage must outlive the // storeOutput call that serializes it. diff --git a/experimental/parser_module/tests/native_parser_module_fixture.hpp b/experimental/parser_module/tests/native_parser_module_fixture.hpp index 51b18a4f..a98a88b6 100644 --- a/experimental/parser_module/tests/native_parser_module_fixture.hpp +++ b/experimental/parser_module/tests/native_parser_module_fixture.hpp @@ -24,7 +24,8 @@ enum ClaimIndex : uint32_t { kTypeMismatch = 11, kSpliceGridMap = 12, kSpliceGridMapShort = 13, - kClaimCount = 14, + kAliasedScalarNames = 14, + kClaimCount = 15, }; } // namespace pj_fixture diff --git a/experimental/parser_module/tests/parser_module_abi_test.cpp b/experimental/parser_module/tests/parser_module_abi_test.cpp index 79230332..209c12fb 100644 --- a/experimental/parser_module/tests/parser_module_abi_test.cpp +++ b/experimental/parser_module/tests/parser_module_abi_test.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -281,6 +282,52 @@ TEST(ParserModuleAbi, OutputReaderRejectsTruncationAndMalformedDescriptors) { EXPECT_FALSE(PJ::parser_module::readOutputDescriptorV1(malformed_scalar)); } +// A scalar descriptor of `field_count` bool fields whose names all alias one +// `name_bytes` range; writeOutputDescriptorV1 cannot alias names. +std::vector aliasedScalarOutput(size_t field_count, size_t name_bytes) { + const size_t name_offset = 24 + field_count * 10; + std::vector block(name_offset + name_bytes, uint8_t{'n'}); + const auto put = [&block](size_t at, uint64_t value, size_t width) { + for (size_t index = 0; index < width; ++index) { + block[at + index] = static_cast(value >> (8U * index)); + } + }; + put(0, PJ_MODULE_OUTPUT_DESCRIPTOR_VERSION_V1, 2); + put(2, PJ_MODULE_ROUTE_SCALAR, 1); + put(3, 0, 17); + put(20, field_count, 4); + for (size_t field = 0; field < field_count; ++field) { + const size_t at = 24 + field * 10; + put(at, name_offset, 4); + put(at + 4, name_bytes, 4); + put(at + 8, PJ_MODULE_SCALAR_VALUE_BOOL, 1); + put(at + 9, 0, 1); + } + return block; +} + +TEST(ParserModuleAbi, OutputReaderBoundsScalarFieldsAndCopiedBytes) { + using PJ::parser_module::kMaximumScalarOutputBytes; + using PJ::parser_module::kMaximumScalarOutputFields; + constexpr size_t kAliases = 64; + constexpr size_t kNameAtLimit = kMaximumScalarOutputBytes / kAliases; + + const auto at_byte_limit = PJ::parser_module::readOutputDescriptorV1(aliasedScalarOutput(kAliases, kNameAtLimit)); + ASSERT_TRUE(at_byte_limit) << at_byte_limit.error(); + const auto over_byte_limit = + PJ::parser_module::readOutputDescriptorV1(aliasedScalarOutput(kAliases, kNameAtLimit + 1)); + ASSERT_FALSE(over_byte_limit); + EXPECT_NE(over_byte_limit.error().find("bytes"), std::string::npos) << over_byte_limit.error(); + + const auto at_field_limit = + PJ::parser_module::readOutputDescriptorV1(aliasedScalarOutput(kMaximumScalarOutputFields, 1)); + ASSERT_TRUE(at_field_limit) << at_field_limit.error(); + const auto over_field_limit = + PJ::parser_module::readOutputDescriptorV1(aliasedScalarOutput(size_t{kMaximumScalarOutputFields} + 1, 1)); + ASSERT_FALSE(over_field_limit); + EXPECT_NE(over_field_limit.error().find("fields"), std::string::npos) << over_field_limit.error(); +} + TEST(BuiltinObjectSpliceTable, ContainsOnlyUnambiguousTopLevelBulkByteFields) { const std::array expected{ PJ_builtin_object_splice_field_v1_t{PJ_BUILTIN_OBJECT_TYPE_IMAGE, 0, 7}, diff --git a/experimental/parser_module/tests/parser_module_claims_test.cpp b/experimental/parser_module/tests/parser_module_claims_test.cpp index d03ba0a4..6573a621 100644 --- a/experimental/parser_module/tests/parser_module_claims_test.cpp +++ b/experimental/parser_module/tests/parser_module_claims_test.cpp @@ -118,5 +118,86 @@ TEST(ParserModuleClaims, ModuleManifestCannotForgeProvenance) { EXPECT_NE(result.error().find("provenance"), std::string::npos); } +std::string claimJson(size_t index) { + return R"({"claim_id":"c)" + std::to_string(index) + R"(","encoding":"protobuf","type_name":"t.T)" + + std::to_string(index) + R"(","routes":["scalar"],"priority":0})"; +} + +std::string manifestJson(std::string_view id, const std::vector& claims) { + std::string json = R"({"module_abi":1,"id":")" + std::string(id) + R"(","name":"N","version":"1.0.0","claims":[)"; + for (size_t index = 0; index < claims.size(); ++index) { + json += (index == 0 ? "" : ",") + claims[index]; + } + return json + "]}"; +} + +void expectBoundRejected(const std::string& json, std::string_view expected) { + const auto decoded = decodeParserModuleManifest(json, ParserClaimProvenance::kFolderDrop); + ASSERT_FALSE(decoded.has_value()); + EXPECT_NE(decoded.error().find(expected), std::string::npos) << decoded.error(); +} + +TEST(ParserModuleClaims, ModuleManifestBoundsMetadataBeforeExpandingClaims) { + const std::string long_string(kMaximumParserModuleManifestStringBytes, 'x'); + const std::string digest = "sha256:" + std::string(64, 'a'); + + // At every limit at once: maximum claims, maximum-length id, claim id and + // type name, and the maximum number of schema digests. + std::vector claims; + for (size_t index = 0; index + 1 < kMaximumParserModuleManifestClaims; ++index) { + claims.push_back(claimJson(index)); + } + std::string digests; + for (size_t index = 0; index < kMaximumParserModuleClaimSchemaDigests; ++index) { + std::string unique = digest; + unique.replace(unique.size() - 4, 4, std::to_string(1000 + index)); + digests += (index == 0 ? "\"" : ",\"") + unique + "\""; + } + const std::string long_type = "t." + std::string(kMaximumParserModuleManifestStringBytes - 2, 'T'); + claims.push_back( + R"({"claim_id":")" + long_string + R"(","encoding":"protobuf","type_name":")" + long_type + + R"(","routes":["scalar"],"priority":0,"schema_digests":[)" + digests + "]}"); + const std::string at_limits = manifestJson(long_string, claims); + ASSERT_LE(at_limits.size(), kMaximumParserModuleManifestBytes); + const auto accepted = decodeParserModuleManifest(at_limits, ParserClaimProvenance::kFolderDrop); + ASSERT_TRUE(accepted.has_value()) << accepted.error(); + EXPECT_EQ(accepted->claims.size(), kMaximumParserModuleManifestClaims); + EXPECT_EQ(accepted->claims.back().schema_digests.size(), kMaximumParserModuleClaimSchemaDigests); + + const std::string too_long = long_string + "x"; + expectBoundRejected(manifestJson(too_long, {claimJson(0)}), "id exceeds"); + + claims.push_back(claimJson(kMaximumParserModuleManifestClaims)); + expectBoundRejected(manifestJson("m", claims), "claims"); + + expectBoundRejected( + manifestJson( + "m", {R"({"claim_id":")" + too_long + + R"(","encoding":"protobuf","type_name":"t.T",)" + R"("routes":["scalar"],"priority":0})"}), + "claim_id exceeds"); + expectBoundRejected( + manifestJson( + "m", {R"({"claim_id":"c","encoding":")" + too_long + + R"(","type_name":"t.T",)" + R"("routes":["scalar"],"priority":0})"}), + "encoding exceeds"); + expectBoundRejected( + manifestJson( + "m", {R"({"claim_id":"c","encoding":"protobuf","type_name":"t.)" + too_long + + R"(","routes":["scalar"],"priority":0})"}), + "type_name exceeds"); + expectBoundRejected( + manifestJson( + "m", {R"({"claim_id":"c","encoding":"protobuf","type_name":"t.T","routes":["scalar"],)" + R"("priority":0,"schema_digests":[)" + + digests + ",\"" + digest + "\"]}"}), + "schema_digests"); + + std::string oversized = manifestJson("m", {claimJson(0)}); + oversized.insert(1, kMaximumParserModuleManifestBytes, ' '); + expectBoundRejected(oversized, "bytes"); +} + } // namespace } // namespace PJ diff --git a/experimental/parser_module/tests/parser_module_runtime_test.cpp b/experimental/parser_module/tests/parser_module_runtime_test.cpp index 89eed516..96714478 100644 --- a/experimental/parser_module/tests/parser_module_runtime_test.cpp +++ b/experimental/parser_module/tests/parser_module_runtime_test.cpp @@ -148,6 +148,18 @@ TEST(ParserModuleRuntime, ClassifiesDataErrorsSeparatelyFromContractViolations) EXPECT_NE(mismatch_result->message.find("does not match"), std::string::npos); } +TEST(ParserModuleRuntime, RejectsScalarNamesAliasedPastTheHostCopyLimit) { + // Every field name passes its own range check, but together they would make + // the host copy 256 MiB from a ~100 KiB descriptor. + auto module = loadFixture(); + auto instance = createBound(module, kAliasedScalarNames, parser_module::Route::kScalar); + auto result = instance.parse(input()); + ASSERT_TRUE(result.has_value()) << result.error(); + EXPECT_EQ(result->fault, ParserModuleFaultKind::kContractViolation); + EXPECT_FALSE(result->output.has_value()); + EXPECT_NE(result->message.find("malformed output descriptor"), std::string::npos) << result->message; +} + TEST(ParserModuleRuntime, AcceptsEligibleSpliceAndRejectsInvalidReferences) { auto module = loadFixture(); diff --git a/experimental/parser_module/tests/parser_module_session_budget_test.cpp b/experimental/parser_module/tests/parser_module_session_budget_test.cpp index 8931530f..6d9b2c15 100644 --- a/experimental/parser_module/tests/parser_module_session_budget_test.cpp +++ b/experimental/parser_module/tests/parser_module_session_budget_test.cpp @@ -77,12 +77,13 @@ TEST(ParserModuleSessionBudget, AppliesInstanceAndDeclaredMemoryBudgetsIndepende const auto large = tracker.admitModule(100, 1, 250); ASSERT_TRUE(small.accepted()); ASSERT_TRUE(large.accepted()); - ASSERT_TRUE(tracker.admitInstance(large.reservation).accepted()); + const auto large_instance = tracker.admitInstance(large.reservation); + ASSERT_TRUE(large_instance.accepted()); expectDecline( tracker.admitInstance(small.reservation), ParserModuleBudgetKind::kTotalLinearMemory, "total_linear_memory"); EXPECT_EQ(tracker.usage().active_instances, 1U); - tracker.releaseInstance(large.reservation); + tracker.releaseInstance(large_instance.reservation); ASSERT_TRUE(tracker.admitInstance(small.reservation).accepted()); ASSERT_TRUE(tracker.admitInstance(small.reservation).accepted()); expectDecline(tracker.admitInstance(small.reservation), ParserModuleBudgetKind::kActiveInstances, "active_instances"); @@ -101,16 +102,18 @@ TEST(ParserModuleSessionBudget, ReleasesAreIdempotentAndRestoreCapacity) { }); const auto module = tracker.admitModule(10, 1, 20); ASSERT_TRUE(module.accepted()); - ASSERT_TRUE(tracker.admitInstance(module.reservation).accepted()); - tracker.releaseInstance(module.reservation); - tracker.releaseInstance(module.reservation); // ignored + const auto instance = tracker.admitInstance(module.reservation); + ASSERT_TRUE(instance.accepted()); + tracker.releaseInstance(instance.reservation); + tracker.releaseInstance(instance.reservation); // ignored EXPECT_EQ(tracker.usage().active_instances, 0U); EXPECT_EQ(tracker.usage().declared_linear_memory_bytes, 0U); - ASSERT_TRUE(tracker.admitInstance(module.reservation).accepted()); - tracker.releaseModule(module.reservation); // gives back the live instance too - tracker.releaseModule(module.reservation); // ignored - tracker.releaseInstance(module.reservation); // ignored: module is gone + const auto live = tracker.admitInstance(module.reservation); + ASSERT_TRUE(live.accepted()); + tracker.releaseModule(module.reservation); // gives back the live instance too + tracker.releaseModule(module.reservation); // ignored + tracker.releaseInstance(live.reservation); // ignored: module is gone EXPECT_EQ(tracker.usage().modules, 0U); EXPECT_EQ(tracker.usage().claims, 0U); EXPECT_EQ(tracker.usage().active_instances, 0U); @@ -118,6 +121,41 @@ TEST(ParserModuleSessionBudget, ReleasesAreIdempotentAndRestoreCapacity) { ASSERT_TRUE(tracker.admitModule(10, 1, 20).accepted()); } +TEST(ParserModuleSessionBudget, InstanceReleaseConsumesOnlyItsOwnReservation) { + ParserModuleSessionBudgetTracker tracker( + ParserModuleSessionBudgetLimits{ + .maximum_modules = 1, + .maximum_artifact_bytes = 10, + .maximum_claims = 1, + .maximum_active_instances = 2, + .maximum_linear_memory_bytes = 40, + }); + const auto module = tracker.admitModule(10, 1, 20); + ASSERT_TRUE(module.accepted()); + const auto a = tracker.admitInstance(module.reservation); + const auto b = tracker.admitInstance(module.reservation); + ASSERT_TRUE(a.accepted()); + ASSERT_TRUE(b.accepted()); + EXPECT_NE(a.reservation, b.reservation); + + tracker.releaseInstance(a.reservation); + tracker.releaseInstance(a.reservation); // ignored: must not release B + EXPECT_EQ(tracker.usage().active_instances, 1U); + EXPECT_EQ(tracker.usage().declared_linear_memory_bytes, 20U); + // B still holds its charge, so only one more instance fits both caps. + const auto c = tracker.admitInstance(module.reservation); + ASSERT_TRUE(c.accepted()); + expectDecline( + tracker.admitInstance(module.reservation), ParserModuleBudgetKind::kActiveInstances, "active_instances"); + tracker.releaseInstance(module.reservation); // a module id is not an instance id + EXPECT_EQ(tracker.usage().active_instances, 2U); + + tracker.releaseInstance(b.reservation); + tracker.releaseInstance(c.reservation); + EXPECT_EQ(tracker.usage().active_instances, 0U); + EXPECT_EQ(tracker.usage().declared_linear_memory_bytes, 0U); +} + TEST(ParserModuleSessionBudget, TracksConcurrentAdmissionAndReleaseFromManyThreads) { // Wrapper destructors release reservations on whatever thread drops them, // so the tracker must synchronize itself. @@ -133,7 +171,7 @@ TEST(ParserModuleSessionBudget, TracksConcurrentAdmissionAndReleaseFromManyThrea ASSERT_TRUE(module.accepted()); const auto instance = tracker.admitInstance(module.reservation); ASSERT_TRUE(instance.accepted()); - tracker.releaseInstance(module.reservation); + tracker.releaseInstance(instance.reservation); tracker.releaseModule(module.reservation); } }); diff --git a/experimental/parser_module/tests/parser_module_wasm_test.cpp b/experimental/parser_module/tests/parser_module_wasm_test.cpp index 140e3cd3..687d15e1 100644 --- a/experimental/parser_module/tests/parser_module_wasm_test.cpp +++ b/experimental/parser_module/tests/parser_module_wasm_test.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -24,6 +25,20 @@ void appendSection(std::vector* wasm, uint8_t id, const std::vectorinsert(wasm->end(), payload.begin(), payload.end()); } +void appendVarUint32(std::vector* bytes, uint32_t value) { + do { + const auto low = static_cast(value & 0x7FU); + value >>= 7U; + bytes->push_back(value == 0 ? low : static_cast(low | 0x80U)); + } while (value != 0); +} + +void appendLargeSection(std::vector* wasm, uint8_t id, const std::vector& payload) { + wasm->push_back(id); + appendVarUint32(wasm, static_cast(payload.size())); + wasm->insert(wasm->end(), payload.begin(), payload.end()); +} + void expectRejected(std::vector wasm, std::string expected) { auto inspected = inspectWasmModule(wasm); ASSERT_FALSE(inspected.has_value()); @@ -110,5 +125,57 @@ TEST(ParserModuleWasm, ParsesAndBoundsTableSections) { EXPECT_EQ(*no_tables, 0U); } +TEST(ParserModuleWasm, SharesOneSignatureAcrossEveryExportOfAFunction) { + // One 64 KiB-parameter signature exported under 4096 names: copying it per + // export would expand a ~100 KiB artifact into 256 MiB during inspection. + constexpr uint32_t kParameters = 64 * 1024; + constexpr uint32_t kExports = 4096; + std::vector types{1, 0x60}; + appendVarUint32(&types, kParameters); + types.insert(types.end(), kParameters, kWasmValueI32); + types.push_back(0); + std::vector exports; + appendVarUint32(&exports, kExports); + for (uint32_t index = 0; index < kExports; ++index) { + const std::string name = "e" + std::to_string(index); + appendVarUint32(&exports, static_cast(name.size())); + exports.insert(exports.end(), name.begin(), name.end()); + exports.insert(exports.end(), {0x00, 0x00}); // function 0 + } + auto wasm = module(); + appendLargeSection(&wasm, 1, types); + appendSection(&wasm, 3, {1, 0}); + appendLargeSection(&wasm, 7, exports); + + auto inspected = inspectWasmModule(wasm); + ASSERT_TRUE(inspected.has_value()) << inspected.error(); + ASSERT_EQ(inspected->exports.size(), kExports); + const WasmFunctionSignature* first = inspected->functionSignature(inspected->exports.front()); + ASSERT_NE(first, nullptr); + EXPECT_EQ(first->parameters.size(), kParameters); + EXPECT_EQ(inspected->functionSignature(inspected->exports.back()), first); +} + +TEST(ParserModuleWasm, RejectsDuplicateExportNamesAndOverLimitEntryCounts) { + auto duplicate = module(); + appendSection(&duplicate, 1, {1, 0x60, 0, 0}); + appendSection(&duplicate, 3, {1, 0}); + appendSection(&duplicate, 7, {2, 1, 'f', 0, 0, 1, 'f', 0, 0}); + expectRejected(std::move(duplicate), "duplicate wasm export name: f"); + + // The count is checked before any entry, so filler bytes suffice. + for (const auto& [id, limit, minimum_entry_bytes] : + {std::tuple{1, kWasmMaximumTypes, 3}, + {2, kWasmMaximumImports, 4}, + {7, kWasmMaximumExports, 3}}) { + std::vector payload; + appendVarUint32(&payload, limit + 1); + payload.resize(payload.size() + (size_t{limit} + 1) * minimum_entry_bytes, 0); + auto over_limit = module(); + appendLargeSection(&over_limit, id, payload); + expectRejected(std::move(over_limit), "count exceeds the limit of " + std::to_string(limit)); + } +} + } // namespace } // namespace PJ::parser_module