diff --git a/.golangci.yml b/.golangci.yml index eee7fbd0336ae..1d38a9fb11978 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -35,6 +35,13 @@ linters: - $all - '!$test' - '!pkg/iceberg/adapter/iceberggo/**/*.go' + # Arrow IPC is isolated to the ingestion bridge and its external + # scan reader. Keep the dependency out of all other production + # packages, including the Iceberg-facing layers. + - '!**/pkg/container/arrowbridge/*.go' + - '!**/pkg/sql/colexec/external/arrowio/*.go' + - '!**/pkg/sql/colexec/external/reader_arrow.go' + - '!**/pkg/sql/compile/compile.go' deny: - pkg: github.com/apache/iceberg-go desc: Iceberg dependency must stay behind pkg/iceberg/adapter/iceberggo diff --git a/cmd/mo-service/config.go b/cmd/mo-service/config.go index a28570fa313ea..228412712533f 100644 --- a/cmd/mo-service/config.go +++ b/cmd/mo-service/config.go @@ -177,7 +177,8 @@ func NewConfig() *Config { CN: cnservice.Config{ AutomaticUpgrade: true, Frontend: config.FrontendParameters{ - MongoDB: *config.NewMongoDBParameters(), + MongoDB: *config.NewMongoDBParameters(), + ArrowLoad: *config.NewArrowLoadParameters(), }, }, } diff --git a/cmd/mo-service/config_test.go b/cmd/mo-service/config_test.go index 0beb6483f1e4f..70ee4f6509702 100644 --- a/cmd/mo-service/config_test.go +++ b/cmd/mo-service/config_test.go @@ -386,6 +386,46 @@ func TestMongoDBEnablementConfigDefaults(t *testing.T) { } } +func TestArrowLoadConfigDefaults(t *testing.T) { + for _, test := range []struct { + name string + input string + enabled bool + s3Enabled bool + distributedEnabled bool + }{ + {name: "omitted"}, + { + name: "S3 opt in", input: "[cn.frontend.arrow-load]\nenabled = true\ns3-enabled = true\n", + enabled: true, s3Enabled: true, + }, + { + name: "all opt in", input: `[cn.frontend.arrow-load] + enabled = true +s3-enabled = true +distributed-enabled = true +`, + enabled: true, s3Enabled: true, distributedEnabled: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + cfg := NewConfig() + require.NoError(t, parseFromString(test.input, cfg)) + require.NoError(t, cfg.setDefaultValue()) + require.Equal(t, test.enabled, cfg.CN.Frontend.ArrowLoad.Enabled) + require.Equal(t, test.s3Enabled, cfg.CN.Frontend.ArrowLoad.S3Enabled) + require.Equal(t, test.distributedEnabled, cfg.CN.Frontend.ArrowLoad.DistributedEnabled) + + // CN construction validates the frontend configuration again. Keep + // every explicit rollback value stable across that second pass. + cfg.CN.SetDefaultValue() + require.Equal(t, test.enabled, cfg.CN.Frontend.ArrowLoad.Enabled) + require.Equal(t, test.s3Enabled, cfg.CN.Frontend.ArrowLoad.S3Enabled) + require.Equal(t, test.distributedEnabled, cfg.CN.Frontend.ArrowLoad.DistributedEnabled) + }) + } +} + func TestClockOffsetBoundRetainedWhenMonitoringDisabled(t *testing.T) { validators := []struct { name string diff --git a/docs/design/23684_arrow_load_design.md b/docs/design/23684_arrow_load_design.md new file mode 100644 index 0000000000000..58c1c4e26ab80 --- /dev/null +++ b/docs/design/23684_arrow_load_design.md @@ -0,0 +1,122 @@ +# #23684 Arrow LOAD design + +Status: approved for fail-closed implementation delivery; production rollout +approval is pending. The independent implementation approval is recorded at +[PR review #5127791633](https://github.com/matrixorigin/matrixone/pull/28145#pullrequestreview-5127791633) +for reviewed revision `53af58d64c2e1d928445cd8104511346a5a156a3`. It approves +merging the fail-closed implementation and expressly does not authorize remote +or distributed production enablement. The release-readiness matrix is maintained in +[`evidence/23684_arrow_load_release_readiness.md`](evidence/23684_arrow_load_release_readiness.md). +The separately versioned +[`implementation delivery decision`](evidence/23684_arrow_load_delivery_decision.md) +defines the mergeable shared substrate and deliberately separates it from +deployment enablement. + +## Problem and scope + +`LOAD DATA` previously accepted only text-oriented formats. Arrow IPC File and +Stream inputs need a bounded, transactional ingestion path that preserves SQL +mapping and error semantics. This design adds Arrow only to `LOAD DATA`; it +does not make Arrow a general external-table, result-scan, or Flight protocol. + +The invariant is: every accepted Arrow input is decoded against the planned +schema, charged to the executing statement, and either becomes one ordinary +transactional LOAD result or leaves no visible partial result. A remote source +or distributed worker is admitted only when **that executing CN** has enabled +the applicable rollout gate before it opens I/O. + +Non-goals are aggregate cluster quota/range planning, automatic cloud-provider +enablement, and a new long-lived cache or controller. Those are deliberately +separate release gates, not implied by local Arrow support. + +## Contract and alternatives + +The input contract follows Apache Arrow IPC (File and Stream container framing) +as implemented by Arrow-Go v18. File footer/message metadata and decoded-size +claims are checked before allocation. Schema binding is compile-time and the +execution reader rejects schema/type drift. Arrow fields map by name by +default, or by the explicit positional option; normal LOAD target rules still +own casts, generated columns, constraints, and transaction commit. + +We considered (1) materializing every Arrow value, (2) accepting Arrow only +through a conversion sidecar, and (3) bounded borrowing with materialization +fallback. Always materializing is simpler but makes large varlen loads pay an +avoidable copy; a sidecar adds an operational data-plane boundary and retry +contract. The selected option borrows only reference-counted Arrow backing +that passes a pin-amplification bound, otherwise materializes. `force- +materialize` is an explicit rollback/diagnostic switch, not a different SQL +semantics mode. + +## Configuration, rollout, and compatibility + +`frontend.arrow-load.enabled`, `s3-enabled`, and `distributed-enabled` all +default to false. Planning samples the +settings for its scope, but every `External.Prepare` repeats the gate using the +worker CN's `ParameterUnit` before constructing an Arrow reader: + +| Source/execution | Required worker settings | Default | +| --- | --- | --- | +| local File or Stream | `enabled` | rejected | +| direct S3 or S3-backed stage | `enabled`, `s3-enabled` | rejected | +| distributed Arrow scope | `enabled`, `distributed-enabled` | rejected | +| distributed S3 scope | all three | rejected | + +This prevents a coordinator's stale or more-permissive configuration from +bypassing a worker's fail-closed policy during a rolling change. The execution +scope remains a positive compile authorization. Arrow fanout additionally +serializes `arrow_distributed_execution`; it is independent of the requested +`Parallel` flag because already-planned shard scopes must clear that flag to +avoid a second split. `External.Prepare` applies the worker's distributed gate +to this execution fact before it opens I/O. MORPC v56 remains the receiver +compatibility gate. v56 is `up/main` v55 plus one on the delivery +rebase; older peers reject the additive Arrow pipeline fields, so mixed-version +deployments must drain or keep remote Arrow modes disabled. Downgrade is safe +under the same gates because local Arrow does not advertise a remote capability. + +## Ownership and failure model + +The compiler owns scope construction and immutable object identity snapshots. +`External.Prepare` owns worker admission; `ArrowReader` owns one open IPC +reader; FileService owns provider response closure; and the statement allocation +account owns capacity reservations. A range is published only after bytes are +read **and** the conditional provider `Close` succeeds. Read, probe, close, +parse, conversion, cancellation, and commit failure all abort the reservation; +the released lease returns its exact charge once. + +For a record batch, the reader validates immutable shape and validity once at +record admission, including dictionary values. It then budgets and converts +windows. Budgeting performs only O(columns) structural validation; each +conversion checks its selected indices, validity window, and cancellation +checkpoint without rescanning immutable dictionary values. Thus a record split +into K output batches has linear total validity work rather than K full-record +scans. The +statement account bounds retained range and vector capacity; pin amplification +forces materialization instead of retaining an oversized source allocation. + +Terminal ownership is explicit: EOF calls `ArrowReader.Close`, which releases +IPC references and returns the underlying stream close result. `External.Call` +propagates that terminal error rather than reporting successful completion. +`Reset`/`Free` remain best-effort cleanup only after the execution result is +already determined. + +## Validation and acceptance + +Focused unit coverage proves worker-side local/S3/distributed gates, positive +compile scope propagation, conditional read success plus close-only +`ErrObjectChanged` with zero committed capacity, terminal external close-error +propagation, malformed IPC/schema/null cases, and budget behavior. Existing +Arrow File/Stream, identity, MinIO, multi-CN, rollback, and SQL BVT cases cover +the consumer and public paths. The immediate predecessor compatibility test +is retained with the MORPC v56 gate. + +Before remote production enablement, the readiness record requires a bounded +cross-worker aggregate admission design, real-provider evidence, exact-release +mixed-version validation, deployment A/B, and independent SQL/execution/ +resource/FileService/storage/security-release decisions. Until then the +defaults above are the conservative rollback and failure-containment plan. + +## Open decisions + +Independent owners must approve the proposed remote rollout only after the +listed acceptance evidence exists. No product choice is needed to ship this +documented fail-closed default: every mode requires explicit deployment opt-in. diff --git a/docs/design/evidence/23684_arrow_bridge_benchmark.md b/docs/design/evidence/23684_arrow_bridge_benchmark.md new file mode 100644 index 0000000000000..573bc2acf54f0 --- /dev/null +++ b/docs/design/evidence/23684_arrow_bridge_benchmark.md @@ -0,0 +1,64 @@ +# #23684 Arrow bridge A/B benchmark + +This record compares identical Arrow records with ordinary borrow policy and +`ForceMaterialize=true`. It contains both a bridge-level attribution benchmark +and a local end-to-end SQL LOAD benchmark; neither is a cloud-provider or +deployment acceptance benchmark. + +Command, run on 2026-09-03 on Darwin/arm64 Apple M4: + +```text +.agents/skills/mo-dev/scripts/mo-cgo-test -run '^$' \ + -bench '^BenchmarkArrowBridgeMaterializeAB$' -benchmem \ + -benchtime=500ms -count=3 ./pkg/container/arrowbridge +``` + +## Results + +| Fixture | Mode | ns/op, three runs | Borrowed / copied payload | Interpretation | +| --- | --- | --- | --- | --- | +| numeric + decimal | borrow | 982 / 991 / 990 | 98,304 / 0 bytes | exact fixed layouts retain the Arrow buffers | +| numeric + decimal | materialize | 19,951 / 20,053 / 20,171 | 0 / 98,304 bytes | forced mode performs the attributed payload copy | +| timestamp + short string | borrow | 346,201 / 346,379 / 345,458 | 0 / 73,728 bytes copied | no payload is borrow-eligible; temporal conversion and canonical inline strings dominate | +| timestamp + short string | materialize | 346,088 / 363,055 / 361,882 | 0 / 73,728 bytes copied | modes are intentionally equivalent for an ineligible schema | +| long binary | borrow | 78,241 / 76,979 / 77,033 | 1,048,576 / 0 bytes | long varlen payload is retained, with descriptor work still performed | +| long binary | materialize | 560,541 / 607,918 / 545,651 | 0 / 1,048,576 bytes | forced mode copies the full payload | + +The counters prove attribution: eligible fixed/long-varlen cases report the +same logical payload in either mode, while ownership changes from borrowed to +copied. The short-string/temporal case correctly reports no eligible bytes and +therefore no artificial zero-copy win. + +## End-to-end LOAD A/B + +The same candidate binary was also exercised through the MySQL frontend, +parser/planner, External reader, conversion, transaction commit, storage, and +result acknowledgement. Each sample loaded 100,000 rows; table truncation was +outside the timer. The benchmark asserts the expected borrowed/copy counters so +the two modes cannot silently collapse to the same policy. + +```text +.agents/skills/mo-dev/scripts/mo-cgo-test -p=1 -run '^$' \ + -bench '^BenchmarkArrowLoadEndToEndMaterializeAB$' -benchmem \ + -benchtime=3x -count=3 -timeout=2400s ./pkg/tests/arrowload +``` + +| Mode | ns/op, three runs | rows/s, three runs | B/op | allocs/op | +| --- | --- | --- | --- | --- | +| borrow | 99,915,126 / 110,736,000 / 77,651,930 | 1,000,849 / 903,049 / 1,287,798 | 18,472,338 / 21,284,285 / 16,848,141 | 149,529 / 220,473 / 139,554 | +| materialize | 86,487,403 / 99,493,569 / 94,644,931 | 1,156,238 / 1,005,090 / 1,056,581 | 16,791,154 / 16,896,400 / 16,789,093 | 139,463 / 139,928 / 139,789 | + +On the final rebased candidate, the median forced-materialize run was 5.3% +faster and delivered 5.6% more rows/s than the median borrow run. Both modes +showed substantial run-to-run noise, including one borrow sample with elevated +allocation counts, so this result proves the two policies execute and remain +within the local reference gate; it does not prove a repeatable performance win +for either policy. Materialize remains a correctness diagnostic and emergency +fallback because it deliberately gives up eligible zero-copy ownership. + +## Remaining deployment evidence + +The local A/B does not replace provider-specific object-store measurements, +ordinary Parquet/INSERT controls on customer data, cache/pin pressure testing, +or production CPU/RSS profiles. Deployment owners must run those controls on +the exact release artifact before enabling S3 or distributed execution. diff --git a/docs/design/evidence/23684_arrow_go_supply_chain.md b/docs/design/evidence/23684_arrow_go_supply_chain.md new file mode 100644 index 0000000000000..e142e014740f9 --- /dev/null +++ b/docs/design/evidence/23684_arrow_go_supply_chain.md @@ -0,0 +1,74 @@ +# #23684 Arrow-Go supply-chain review + +Review date: 2026-09-04. Candidate base: `up/main@bf63172c06`. The dependency is +pinned to `github.com/apache/arrow-go/v18 v18.7.0`; no floating version is used. + +## Arrow runtime closure + +`go list -deps` for the imported `arrow/array` and `arrow/ipc` packages reports +eight non-standard modules: + +| Module | Version | Detected license | +| --- | --- | --- | +| `github.com/apache/arrow-go/v18` | `v18.7.0` | Apache-2.0; upstream NOTICE present | +| `github.com/goccy/go-json` | `v0.10.6` | MIT | +| `github.com/google/flatbuffers` | `v25.12.19+incompatible` | Apache-2.0 | +| `github.com/klauspost/compress` | `v1.19.0` | BSD-3-Clause | +| `github.com/pierrec/lz4/v4` | `v4.1.27` | BSD-2-Clause | +| `github.com/zeebo/xxh3` | `v1.1.0` | BSD-2-Clause | +| `golang.org/x/exp` | `7ab1446f8b90` | BSD-3-Clause | +| `golang.org/x/sys` | `v0.47.0` | BSD-3-Clause | + +No copyleft or source-availability license was found in this closure. Final +NOTICE aggregation remains a release-packaging owner action. + +## SBOM and size + +CycloneDX `cyclonedx-gomod v1.12.0` generated a 1.6 binary SBOM from the local +Darwin/arm64 `mo-service`: 219 components and 220 dependency entries. The Arrow +component and all eight closure modules were present. Reproducible command: + +```text +cyclonedx-gomod bin -json -std -version \ + -output mo-service.cdx.json ./mo-service +``` + +The final local SBOM SHA-256 is +`ab787ad79e00c4f048de8390ebfaf7b91ab9ff5bfd13e3be2fd2ce02bd531282`. +The candidate and identically built baseline binary SHA-256 values are +`7db369ce401480e4dcca98d1463646cdebb86db3d3dfd438adf37568c291640c` and +`308cb694d524206e68ff6957cd54cf74ab60e5f9d778968524397660b7e1f9d8`, +respectively. These hashes identify local evidence artifacts, not signed +release artifacts. + +The local candidate binary was 261,955,890 bytes (249.82 MiB), versus +254,982,450 bytes (243.17 MiB) for an identically built `up/main` binary. The +delta is 6,973,440 bytes, or 2.73%. Packaging owners must decide whether that +increase is acceptable for the release image. + +## Vulnerability review + +`govulncheck -mode=binary` found 17 reachable advisories in the candidate and +18 in the identical `up/main` baseline. The candidate introduced no new +advisory ID and removed the baseline's `GO-2026-4762` result through its module +graph update. The remaining results are repository/toolchain debt in the Go +1.26.4 standard library, gRPC, `x/net`, `x/text`, AWS SDK v2, Avro, and pgx. + +A source scan of `arrowbridge`, `arrowipc`, and `external/arrowio` still reaches +`GO-2026-5764` through the existing FileService AWS SDK (`service/s3 v1.68.0`, +fixed in `v1.97.3`). Arrow-Go itself was not named by a reachable advisory. +Because the full candidate scan is non-zero, security approval is **blocked** +until the repository/toolchain findings are upgraded, waived by the security +owner with scope and rationale, or proven unreachable in the release build. + +## Platforms and compatibility + +MatrixOne's native build matrix names Linux amd64 and Linux arm64; the local +host is Darwin arm64. The imported Arrow `array` and `ipc` packages compile for +all three targets with `CGO_ENABLED=0`. The complete Darwin/arm64 `mo-service` +build and affected CGo test closure pass. Linux image builds remain an exact +release-CI artifact requirement because MatrixOne's native dependencies cannot +be proven by a pure-Go cross compile alone. + +This review is complete as evidence, but its result is not an approval: CVE, +binary-size, Linux artifact, NOTICE, and owner sign-off gates remain explicit. diff --git a/docs/design/evidence/23684_arrow_load_consumer_inventory.md b/docs/design/evidence/23684_arrow_load_consumer_inventory.md new file mode 100644 index 0000000000000..619035a85deea --- /dev/null +++ b/docs/design/evidence/23684_arrow_load_consumer_inventory.md @@ -0,0 +1,34 @@ +# #23684 Arrow LOAD consumer inventory + +This inventory closes the LOAD-reachable borrowed-vector surface. It does not +claim that arbitrary external-table SELECT or every stateful operator can +consume borrowed backing. + +| Reachable boundary | Borrowed-input behavior | Ownership or copy boundary | Evidence | +| --- | --- | --- | --- | +| External Arrow reader | Publishes a generation-scoped batch; fixed and long-varlen payloads may be borrowed | each output vector retains its Arrow ArrayData/range backing; the reader may advance after publication | `pkg/sql/colexec/external/reader_arrow.go`, Arrow reader lifecycle tests | +| PreInsert | Reads source columns synchronously; generated, auto, const, row-id, and transformed columns are copied with `UnionBatch` | destination vectors are MPool-owned; no source lease is detached | `pkg/sql/colexec/preinsert/preinsert.go`, vector `UnionBatch` borrowed-source tests | +| Constraint, dedup, and lock operators | Read input during the call and retain no untracked source pointer | mutation routes through vector materialization; terminal batch owner keeps the generation alive | vector borrowed/COW tests and affected operator package tests | +| Conditional local dispatch / pSpool | Async fan-out is reachable when one source feeds multiple writers | `RetainedReadonlyViewWithMP` copies descriptors where required and retains immutable payload owners per slot; abort and late release are idempotent | `pkg/container/pSpool/copy.go`, `pkg/container/pSpool/sender_test.go` | +| Insert / MultiUpdate | Builds owned write batches with `UnionBatch`, or synchronously delegates a unique batch | copied destinations own MPool data; the direct S3 handoff is accepted only after an owned check | `pkg/sql/colexec/multi_update/insert.go`, multi-update tests | +| Sinker direct stage | Rejects borrowed or `NeedDup` vectors from `WriteOwned` | callers use copying `Write` when ownership cannot move | `pkg/objectio/ioutil/sinker.go`, `pkg/objectio/ioutil/sinker_test.go` | +| Vector mutation | A borrowed vector is readonly until materialized | mutators call `MaterializeOwned`; reserve/allocate/convert/swap is transactional | `pkg/container/vector/buffer_lease.go`, buffer-lease and allocation-account tests | +| Remote marshal / object encoding | Borrowed process pointers never cross the wire or storage boundary | canonical marshal copies fixed data and compacts/rebases only referenced varlen area | `pkg/container/vector/vector.go`, borrowed varlen marshal tests | + +## Required invariants + +- a next `External.Call` may release the reader's record only after every async + consumer has retained a readonly view or made an owned copy; +- `NeedDup` is not a lease carrier and cannot authorize ownership transfer; +- `WriteOwned` is move-only and rejects borrowed backing; +- every materialization is charged to the statement allocation account; +- every retained backing remains charged by physical capacity until its final + release, including cancellation and late consumer cleanup; +- unsupported or unknown mutators materialize before write. + +## Deliberate exclusions + +General external-table SELECT, arbitrary joins/aggregations, spill paths not +reachable from LOAD, and future Python UDF operators are outside this closure. +They must perform their own ownership audit before accepting borrowed Arrow +backing. diff --git a/docs/design/evidence/23684_arrow_load_delivery_decision.md b/docs/design/evidence/23684_arrow_load_delivery_decision.md new file mode 100644 index 0000000000000..7de472bb61349 --- /dev/null +++ b/docs/design/evidence/23684_arrow_load_delivery_decision.md @@ -0,0 +1,88 @@ +# #23684 Arrow LOAD implementation delivery decision + +**Status:** implemented with a fail-closed default. This is a versioned +technical decision for the implementation delivery; it is not an approval to +enable Arrow LOAD in a production deployment. Independent rollout and owner +approvals remain pending as recorded in +[`23684_arrow_load_release_readiness.md`](23684_arrow_load_release_readiness.md). + +## Decision + +The mergeable scope is the static Arrow IPC File/Stream `LOAD DATA` +implementation plus the minimal reusable safety substrate it requires: + +- bounded IPC metadata validation in `arrowipc`; +- reference-counted Arrow buffer and range-lease ownership, borrowed-vector + COW/materialization fallback, and statement-capacity accounting; +- transactional LOAD binding/conversion, FileService conditional reads, and + worker-side admission before I/O; and +- an additive, MORPC v56-gated remote pipeline representation. + +This scope does not authorize a deployment to turn on the feature. The shipped +configuration defaults `enabled`, `s3-enabled`, and `distributed-enabled` to +`false`; a CN without an explicit setting rejects the corresponding Arrow LOAD +before I/O. Every participating worker rechecks its own setting, so a +coordinator's more-permissive or stale configuration cannot grant remote work. +Keeping the implementation present but unreachable by default lets normal +release integration validate its contracts without silently broadening a +deployment's data-plane surface. + +## Independent implementation approval record + +The independent approval for this mergeable, fail-closed implementation is +recorded in [PR review #5127791633](https://github.com/matrixorigin/matrixone/pull/28145#pullrequestreview-5127791633), submitted against exact revision +`53af58d64c2e1d928445cd8104511346a5a156a3`. Its decision is `APPROVED` for +merging the fail-closed implementation, while expressly retaining provider, +aggregate-admission, mixed-version, and rollout-owner gates. This record does +not claim approval to enable a deployment: those separately deferred decisions +remain governed by the readiness matrix below. + +## Contract preserved by this decision + +The implementation remains LOAD-only: it does not add Arrow external tables, +result scanning, a Flight listener, or a Python-UDF ABI. A statement either +commits through the normal LOAD transaction path or releases its ranges, +capacity, Arrow backing, and vectors without publishing partial data. A +conditional object range is admitted only after both reading and the provider +`Close` complete successfully. The remote payload is sent and accepted only by +v56 peers; v55 and older peers reject it. + +The shared substrate is deliberately bounded to these consumers. It does not +confer policy authority on `arrowipc` or `arrowbridge`: FileService retains +object identity and provider closure, `External.Prepare` owns worker admission, +and the statement allocation account owns capacity. The detailed package and +ownership boundaries are in +[`23684_arrow_shared_substrate.md`](23684_arrow_shared_substrate.md). + +## Deferred, separately approved rollout decisions + +The following are not implied by this implementation decision and remain +blocked until their named evidence and owner decisions are recorded: + +1. enabling local Arrow LOAD for a deployment; +2. enabling S3/stage or distributed execution, which additionally requires a + cross-worker aggregate admission/range-pressure design and real-provider + evidence; +3. enabling remote execution during an upgrade, which requires exact-release + mixed-version validation and an explicit supported order; and +4. deployment A/B, security/release, and SQL/execution/resource/FileService/ + storage owner acceptance. + +These separations are intentional: an operator can always retain the +conservative rollback by leaving the gates unset, or stop new admissions by +turning them off on every CN. `force-materialize=true` remains a diagnostic +rollback for borrowed backing without changing SQL semantics. + +## Acceptance evidence + +The exact behavioral contract and tests are defined in the versioned +[`Arrow LOAD design`](../23684_arrow_load_design.md): configuration/planner and +worker gate coverage, v55/v56 predecessor/current protocol coverage, +conditional-close failure cleanup, record/dictionary window bounds, public +File/Stream and multi-CN paths, rollback, and cancellation. The release +readiness record carries the remaining evidence matrix rather than presenting +it as complete. + +Any future change that makes one of the gates default-on, broadens the shared +substrate to a new transport or ABI, or changes the remote compatibility +contract requires a new decision revision and its own acceptance evidence. diff --git a/docs/design/evidence/23684_arrow_load_release_readiness.md b/docs/design/evidence/23684_arrow_load_release_readiness.md new file mode 100644 index 0000000000000..380ea1fe557d3 --- /dev/null +++ b/docs/design/evidence/23684_arrow_load_release_readiness.md @@ -0,0 +1,106 @@ +# #23684 Arrow LOAD release-readiness evidence + +Review date: 2026-09-06. Rebased base: `up/main@f3e1599c7268`. The versioned +[Arrow LOAD design](../23684_arrow_load_design.md) defines the protocol, +ownership, rollout, and acceptance contracts. This record covers +the local release rehearsal; it does not claim cloud-provider or human-owner +approval. + +## Gate status + +| Gate | Local result | Release status | +| --- | --- | --- | +| F-031 through F-040 | fixed, tested, committed in the branch history | complete | +| Rebase | branch rebased onto the stated `up/main` base | recheck immediately before delivery | +| Default admission and flag rollback | no-config Arrow LOAD rejection plus explicit enable/disable/drain/restart coverage | fail-closed pending approval | +| S3/stage and distributed admission | explicit per-CN opt-in is required | fail-closed pending aggregate quota, provider, and owner gates | +| Mixed-version upgrade | default-disabled Arrow does not advertise the new remote pipeline capability | rerun required before enabling distributed execution | +| Commit failure/CN shutdown/cancellation | deterministic commit fault injection, cluster lifecycle, and blocked S3 request cancellation passed | local complete | +| Aggregate pin quota/range planner/deployment stress | deliberately deferred | blocker for S3/distributed production | +| Real AWS/OSS/COS | delegated to provider test owners | external blocker | +| A/B, alerts, rollout/rollback | local E2E A/B and reference gates recorded in runbook | deployment acceptance pending | +| Arrow-Go supply chain | license/SBOM/size/platform/CVE review recorded | security and packaging blockers remain | +| Formal owner approval | packet below prepared | pending human approval | + +## Default fail-closed and mixed-version status + +The current product policy keeps every Arrow surface fail-closed: a candidate CN +using an existing configuration with no Arrow section rejects Arrow LOAD before +I/O. Local File/Stream requires `enabled=true`; direct S3-compatible sources, +S3-backed stages, and distributed record-batch fanout additionally require +explicit `s3-enabled=true` and/or `distributed-enabled=true` configuration on +every participating CN. `TestArrowLoadBVT` and `TestArrowLoadMultiCN` exercise +these opt-in paths; configuration, planner, and public-path gate tests prove +that omitted settings keep them closed. + +The earlier two-binary rehearsal remains evidence that the old binary rejects +Arrow syntax. Before distributed execution can be enabled in a release artifact, +the exact artifact must repeat the mixed-version upgrade test, including routing +a parallel statement while an old CN is present and documenting the supported +upgrade order. + +`TestArrowLoadRolloutRollbackDrain` separately stops a cluster only after an +active large LOAD is visible. On restart with every gate disabled, the table is +either fully committed when shutdown drained the statement or empty when it +canceled; a partial commit is forbidden. A missing-file LOAD proves rejection +occurs before I/O. Re-enabling local LOAD while distributed execution remains +off makes `parallel 'true'` fall back to serial execution and commit all rows. + +## Failure and cancellation evidence + +- `CommitPhaseFailureRollback` injects failure after workspace dump and before + commit visibility. It leaves only the seed row, then succeeds on retry. +- Client-context cancellation blocks an in-flight conditional S3 range request, + then verifies request-context cancellation, statement failure, and zero + committed rows. The prior 2-CN processlist observer is not evidence: the + local fixture can complete before an observer establishes a stable point. +- The former 2-CN processlist-based worker-shutdown observer is not evidence: + the local fixture can complete before an observer establishes a stable point. + `TestArrowLoadRolloutRollbackDrain` instead exercises cluster lifecycle with + an admitted statement, and its complete-or-empty assertion preserves the + transaction-boundary contract. +- Existing File/Stream, transaction/isolation, malformed input, object-change, + MinIO, race, fuzz, and formal distributed SQL cases remain part of the branch + evidence described by the design and shared-substrate records. + +## Performance and operational acceptance + +The end-to-end materialization A/B and raw results are in +`23684_arrow_bridge_benchmark.md`. The runbook defines local reference gates: +zero correctness/leak failures, immediate internal-error escalation, bounded +error-rate escalation, p99 within 20%, throughput at least 90% of the accepted +control, and pinned bytes back to baseline within 60 seconds. No dedicated +Arrow Grafana dashboard is introduced; signals belong in existing FileService +and Pipeline views. + +On the final rebased candidate, the three-run median materialize control was +5.3% lower latency and 5.6% higher throughput than borrow. The broad sample +spread means neither policy has a demonstrated repeatable performance lead; +the local comparison passed the reference regression gate but deployment A/B +must still use representative data and exact release artifacts. + +These values make local rehearsal deterministic. They do not replace workload, +provider, topology, cache-pressure, or exact-release-binary acceptance by the +deployment owner. + +## Owner approval packet + +| Owner | Review surface | Evidence | Decision | +| --- | --- | --- | --- | +| SQL/Planner/Compile | LOAD-only syntax, binding, shard plan, additive protobuf, mixed version | planner/compile UT, remote roundtrip, mixed binary rehearsal | pending owner | +| Execution | External lifecycle, fanout, cancellation, shutdown | Arrow E2E, deterministic lifecycle/cancellation, race/fuzz | pending owner | +| Container/Resource | leases, borrowed vectors/nulls, COW, accounting | owning-package UT/race and consumer inventory | pending owner | +| FileService/S3 | conditional range, cache pin, identity, credentials, request policy | provider UT and local MinIO; aggregate quota/provider cloud gaps explicit | pending owner | +| Transaction/Storage | statement atomicity, retry, encoding boundary | transaction BVT and post-workspace-dump commit fault | pending owner | +| Security/Release | licenses, SBOM/CVE, binary size, platform artifacts | supply-chain review | blocked/pending owner | + +An approval must name the owner, reviewed commit/artifact, decision, date, and +any accepted exception. Author self-review cannot substitute for these entries. + +## Release decision + +Arrow LOAD is disabled by default. Every mode requires explicit deployment +opt-in. Deferred aggregate pin quota/range-planner pressure work, real-provider +testing, deployment A/B, exact Linux artifacts, mixed-version rerun, and formal +owner approval remain release-readiness gates and must not be inferred complete +from local validation. diff --git a/docs/design/evidence/23684_arrow_shared_substrate.md b/docs/design/evidence/23684_arrow_shared_substrate.md new file mode 100644 index 0000000000000..2a93f3da52aff --- /dev/null +++ b/docs/design/evidence/23684_arrow_shared_substrate.md @@ -0,0 +1,110 @@ +# #23684 shared Arrow substrate + +This note records the reusable Arrow boundary introduced while implementing +Arrow IPC LOAD. It is intentionally narrower than either the static-file LOAD +protocol or the Python UDF/Flight design. + +## Package boundaries + +| Package | Owns | Must not own | +| --- | --- | --- | +| `pkg/container/arrowipc` | bounded IPC metadata framing, FlatBuffers graph validation, schema vector/depth/string limits, record node/buffer ranges, compression declarations, decoded-size limits | object listing, FileService identity, Flight invocation sequence, SQL type policy, authorization | +| `pkg/container/arrowbridge` | Arrow ArrayData lifetime retention, borrowed MO vector construction, transactional materialization, COW-compatible ownership, LOAD binding and conversion kernels, copy/pin statistics | file/range acquisition, Flight tokens, transaction completion, credentials, Python SDK metadata | +| `pkg/sql/colexec/external/arrowio` | IPC File/Stream discovery, footer and block planning, dictionary replay, FileService range leases, object identity | MO target-column binding, SQL write semantics, Flight protocol | +| `pkg/sql/compile/sidecarflight` | Sirius capability negotiation, metadata-version and exact result schema checks, Flight message sequencing, Sirius result conversion | generic IPC structural limits, static-file planning, Python UDF semantics | + +`arrowipc` is the first trust pass for both file and Flight messages. A +consumer must still validate its envelope and protocol state after that pass. +In particular, structural success says nothing about an expected message kind, +metadata version, row cardinality, invocation epoch, sequence number, or SQL +type identity. + +`arrowbridge.BindLoad` is explicitly an ingestion policy. LOAD permits checked +integer/float widening and selected temporal conversion. The Python UDF v1 ABI +requires an exact, versioned logical type descriptor, including timezone and +logical metadata. A future UDF binder must validate that descriptor before it +constructs a conversion plan; it must not call `BindLoad` as an ABI shortcut. + +## Ownership contract + +1. A transport validates framing and structure before Arrow-Go or a local + decoder allocates from untrusted lengths. +2. The transport publishes immutable Arrow buffers with one physical lifetime + root. File IPC attaches its `RangeLease` below the Arrow object graph. +3. `arrowbridge` retains `arrow.ArrayData`, never a FileService implementation. + That retain transitively keeps the range and its statement capacity charge + alive. +4. A borrowed MO vector owns exactly one retained backing per data, area, or + validity component. Cleanup releases those owners exactly once. +5. Mutation and owned-only handoff materialize transactionally under the + caller's allocation account. Failure leaves the borrowed source readable. + +## Reuse status + +The shared IPC validator is used by Arrow File/Stream LOAD and the existing +Sirius `sidecarflight` decoder. The container bridge is used by LOAD. This is +shared infrastructure, not a Python UDF implementation: the Function Catalog, +UDF operator, MO-to-Arrow encoder, exact SDK v1 binder, Flight invocation +envelope, trusted Supervisor, TOCTOU-safe output publication, and sandbox +runtime remain separate work. + +## Local evidence + +The final worktree was validated with the following owning-package and real +consumer tests: + +```text +GOWORK=off go test -mod=readonly -count=1 ./pkg/container/arrowipc +GOWORK=off go test -mod=readonly -race -count=1 -timeout=180s \ + ./pkg/container/arrowipc +.agents/skills/mo-dev/scripts/mo-cgo-test -count=1 -timeout=180s \ + ./pkg/container/arrowbridge +.agents/skills/mo-dev/scripts/mo-cgo-test -race -count=1 -timeout=240s \ + ./pkg/container/arrowbridge +.agents/skills/mo-dev/scripts/mo-cgo-test -count=1 -timeout=180s \ + ./pkg/sql/compile/sidecarflight +.agents/skills/mo-dev/scripts/mo-cgo-test -race -count=1 -timeout=240s \ + ./pkg/sql/compile/sidecarflight +.agents/skills/mo-dev/scripts/mo-cgo-test -count=1 -timeout=240s \ + ./pkg/sql/colexec/external/arrowio +.agents/skills/mo-dev/scripts/mo-cgo-test -race -count=1 -timeout=300s \ + ./pkg/sql/colexec/external/arrowio +.agents/skills/mo-dev/scripts/mo-cgo-test -count=1 -timeout=120s -run='^$' \ + -fuzz='^FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak$' -fuzztime=20s \ + ./pkg/sql/colexec/external/arrowio +``` + +All completed successfully. The final fuzz run loaded 47 baseline inputs and +executed about 372,000 mutations. The full `pkg/sql/colexec/external` consumer +suite, including its local MinIO object-change path, also passed. The formal +distributed case `test/distributed/cases/load_data/load_data_arrow.sql` passed +50/50 twice on one clean instance built from the exact worktree. + +The public MySQL-protocol suite in `pkg/tests/arrowload` now starts ephemeral +embedded clusters and, when the `minio` binary is available, its own ephemeral +MinIO server. It covers disk and S3 File/Stream input, direct S3 and S3-backed +stage syntax, multi-object success, corrupt-object rollback followed by reader +reuse, same-key object replacement, conditional-range object-change rejection, +client request cancellation, `KILL QUERY`, 2-CN fanout, and committed-data +persistence across a complete local cluster restart. No cloud credentials or +external services are needed. The added scenarios passed the full package, +focused race runs, and a same-process two-run 1-CN BVT repetition: + +```text +.agents/skills/mo-dev/scripts/mo-cgo-test -p=1 -count=1 -timeout=1200s \ + ./pkg/tests/arrowload +.agents/skills/mo-dev/scripts/mo-cgo-test -race -count=1 -timeout=1200s \ + -run '^TestArrowLoadBVT/(CorruptInputRollback|LocalMinIO)$' \ + ./pkg/tests/arrowload +.agents/skills/mo-dev/scripts/mo-cgo-test -race -count=1 -timeout=1200s \ + -run '^TestArrowLoadMultiCN/(CancelMidLoad|ClientContextCancel)$' \ + ./pkg/tests/arrowload +.agents/skills/mo-dev/scripts/mo-cgo-test -p=1 -count=2 -timeout=1200s \ + -run '^TestArrowLoadBVT$' ./pkg/tests/arrowload +go vet ./pkg/tests/arrowload +``` + +The self-review added deterministic regressions for duplicate output indices +when attribute names are empty, metadata limits that cannot be raised by a +consumer, and cleanup of unpublished vectors when statement admission rejects +their first backing allocation. diff --git a/docs/observability/arrow-load-runbook.md b/docs/observability/arrow-load-runbook.md new file mode 100644 index 0000000000000..462ef796d3d38 --- /dev/null +++ b/docs/observability/arrow-load-runbook.md @@ -0,0 +1,132 @@ +# Arrow LOAD observability and rollback + +Arrow LOAD uses the existing MatrixOne metric registry and operational views. +Do not create a dedicated Arrow dashboard. Put storage/range panels in the +existing FileService view and conversion/publication panels in the existing +Pipeline view when dashboard integration is deployed. + +## Availability and rollback controls + +Arrow LOAD is disabled by default. Local/shared FileService paths require +`enabled=true`; S3/stage sources and distributed execution require their +additional explicit opt-ins in every participating CN. + +The settings are availability gates; `enabled` is also a deployment kill switch. +Specify the explicit admission setting needed for a local-file rollout: + +```toml +[cn.frontend.arrow-load] +enabled = true +``` + +`s3-enabled=true` admits direct S3-compatible and S3-backed stage sources. +`distributed-enabled=true` admits distributed execution; otherwise a requested +parallel Arrow LOAD executes serially. `force-materialize=true` retains Arrow +LOAD but disables the borrowed Arrow backing optimization. Omitted `s3-enabled` +and `distributed-enabled` fields default to `false`; explicit opt-ins survive +repeated configuration validation and service restart. Set `enabled=false` to +roll back admission. + +The current implementation charges every raw range, cache pin, and decoded +Arrow allocation to the shared statement account and limits each cache pin to +4x amplification. The design's additional cross-worker aggregate pin quota is +still a release blocker. `mo_arrow_load_pinned_bytes` observes process usage; it +is not an admission controller and must not be treated as that missing quota. + +Deployments may opt in to a source or execution mode only after accepting its +release gates. During rollback, stop admitting new Arrow statements by disabling +`enabled`; allow already admitted statements to drain or cancel them through the +normal query lifecycle. Do not switch an executing statement to a different +object generation or conversion policy. + +`force-materialize=true` leaves Arrow LOAD enabled but disables borrowed Arrow +backing for statements compiled after the setting is applied. Use it to isolate +ownership/pinning incidents or as a temporary rollback from the borrow path. +The compile snapshot is carried to remote scopes, so a statement never mixes +borrow and forced-materialize policy. Restart or otherwise reload every CN with +the same setting before using this control in a distributed cluster. + +## Metrics + +All labels are bounded and contain no object path, credential, account name, +query text, or user data. + +| Signal | Meaning | +| --- | --- | +| `mo_arrow_load_objects_total{outcome}` | object or object-shard open attempts | +| `mo_arrow_load_shards_total` | successfully opened record shards | +| `mo_arrow_load_records_total` | accepted non-empty IPC record batches | +| `mo_arrow_load_batches_total`, `mo_arrow_load_rows_total` | published MO batches and rows | +| `mo_arrow_load_payload_bytes_total{kind}` | eligible, borrowed, and retained-capacity bytes | +| `mo_arrow_load_copy_bytes_total{layer}` | materialized Arrow-to-MO payload bytes | +| `mo_arrow_load_conversion_columns_total{mode}` | borrowed/materialized column decisions | +| `mo_arrow_load_fallbacks_total{reason}` | pin-amplification or alignment fallback | +| `mo_arrow_load_errors_total{category}` | stable error category | +| `mo_arrow_load_phase_duration_seconds{phase,outcome}` | `open`, `next_record`, `convert`, `wire_budget`, and `publish` latency | +| `mo_arrow_load_pinned_bytes` | current live range/decoded capacity | +| `mo_arrow_load_pinned_bytes_high_water` | process-lifetime pinned high-water mark | + +Useful PromQL: + +```promql +sum(rate(mo_arrow_load_errors_total[5m])) by (category) + +sum(rate(mo_arrow_load_payload_bytes_total{kind="borrowed"}[5m])) +/ +clamp_min(sum(rate(mo_arrow_load_payload_bytes_total{kind="eligible"}[5m])), 1) + +sum(rate(mo_arrow_load_copy_bytes_total{layer="arrow_to_mo"}[5m])) +/ +clamp_min(sum(rate(mo_arrow_load_payload_bytes_total{kind="eligible"}[5m])), 1) + +histogram_quantile( + 0.99, + sum(rate(mo_arrow_load_phase_duration_seconds_bucket[5m])) by (le, phase) +) +``` + +## Alert candidates + +- page on a sustained `resource_exhausted`, `internal`, or `object_changed` + error rate above the environment's LOAD baseline; +- warn when pinned bytes remain high while published rows do not increase; +- warn when pin-amplification fallback or Arrow-to-MO copy ratio changes + materially after a rollout; +- compare open/convert/publish p99 with ordinary LOAD latency before enabling + the next rollout stage. + +Thresholds belong to deployment owners and must be derived from representative +workloads. This repository does not hard-code cluster-specific alert values. + +For a local canary/release rehearsal, use these reference gates until deployment +owners replace them with workload-specific values: + +- zero partial commits, successful statements with internal errors, leaked + allocation-account bytes, or pinned bytes that fail to return to baseline; +- `internal` errors page on the first occurrence; `resource_exhausted` and + `object_changed` warn on any new rate and page when sustained for 5 minutes; +- Arrow p99 may not regress more than 20% and throughput may not fall below 90% + of the accepted control for the same data, topology, cache state, and binary; +- after the last Arrow statement terminates, pinned bytes must return to the + pre-run baseline within 60 seconds. + +These are rollout acceptance gates, not universal production SLOs. Compare +against ordinary LOAD/INSERT and the forced-materialize control before advancing +local file, S3/stage, and distributed stages. + +## Triage + +1. Check the error category and the failing phase; retain the query/trace ID + from normal logs, not a source path metric label. +2. For `object_changed`, verify object-store versioning and whether a producer + overwrote or deleted a key, or deleted the planned version, between planning + and execution. Conditional `404`/`NoSuchVersion` and precondition failures + are both classified here. The statement must retry as a whole against a + newly planned object set. +3. For `resource_exhausted`, compare retained capacity with borrowed payload, + copy ratio, statement memory, and FileService cache pressure. +4. For stalled cancellation, verify no new rows are published and pinned bytes + return to the pre-query baseline after all retained consumers release. +5. If correctness, lease accounting, or mixed-version behavior is uncertain, + disable Arrow admission and use the existing Parquet/INSERT path while the + incident is investigated. diff --git a/go.mod b/go.mod index e67949f536755..a5e8a2f55253f 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/aliyun/alibaba-cloud-sdk-go v1.63.34 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/aliyun/credentials-go v1.3.10 + github.com/apache/arrow-go/v18 v18.7.0 github.com/aws/aws-sdk-go v1.55.5 github.com/aws/aws-sdk-go-v2 v1.32.5 github.com/aws/aws-sdk-go-v2/config v1.28.5 @@ -50,6 +51,7 @@ require ( github.com/golang/mock v1.6.0 github.com/golang/snappy v1.0.0 github.com/google/btree v1.1.2 + github.com/google/flatbuffers v25.12.19+incompatible github.com/google/gofuzz v1.2.0 github.com/google/gops v0.3.25 github.com/google/pprof v0.0.0-20260302011040-a15ffb7f9dcc @@ -63,7 +65,7 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/jonboulle/clockwork v0.4.0 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.18.7 + github.com/klauspost/compress v1.19.0 github.com/lni/dragonboat/v4 v4.0.0-20220815145555-6f622e8bcbef github.com/lni/goutils v1.3.1-0.20220604063047-388d67b4dbc4 github.com/lni/vfs v0.2.1-0.20220616104132-8852fd867376 @@ -76,7 +78,7 @@ require ( github.com/panjf2000/ants/v2 v2.12.0 github.com/parquet-go/parquet-go v0.25.1 github.com/petermattis/goid v0.0.0-20241025130422-66cb2e6d7274 - github.com/pierrec/lz4/v4 v4.1.26 + github.com/pierrec/lz4/v4 v4.1.27 github.com/pkg/errors v0.9.1 github.com/plar/go-adaptive-radix-tree v1.0.5 github.com/prashantv/gostub v1.1.0 @@ -90,7 +92,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spkg/bom v1.0.0 github.com/stretchr/testify v1.11.1 - github.com/substrait-io/substrait-protobuf/go v0.78.0 + github.com/substrait-io/substrait-protobuf/go v0.85.0 github.com/syncthing/notify v0.0.0-20250528144937-c7027d4f7465 github.com/tencentyun/cos-go-sdk-v5 v0.7.55 github.com/ti-mo/conntrack v0.5.1 @@ -98,8 +100,13 @@ require ( github.com/tidwall/btree v1.7.0 github.com/tidwall/pretty v1.2.1 github.com/tmc/langchaingo v0.1.13 + github.com/twmb/franz-go v1.21.6 + github.com/twmb/franz-go/pkg/kadm v1.18.0 + github.com/twmb/franz-go/pkg/kfake v0.0.0-20260820024614-9b174ed31afe + github.com/twmb/franz-go/pkg/kmsg v1.13.1 github.com/uber/h3-go/v4 v4.5.0 github.com/unum-cloud/usearch/golang v0.0.0-20260524141737-9fd6b0115dcd + github.com/xdg-go/scram v1.2.0 github.com/xeipuuv/gojsonschema v1.2.0 github.com/yalue/onnxruntime_go v1.31.0 github.com/yanyiwu/gojieba v1.4.7 @@ -109,11 +116,11 @@ require ( go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 - golang.org/x/sync v0.20.0 - golang.org/x/sys v0.44.0 + golang.org/x/sync v0.22.0 + golang.org/x/sys v0.47.0 golang.org/x/time v0.15.0 - gonum.org/v1/gonum v0.14.0 - google.golang.org/grpc v1.65.0 + gonum.org/v1/gonum v0.17.0 + google.golang.org/grpc v1.82.0 google.golang.org/protobuf v1.36.11 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -126,7 +133,7 @@ require ( github.com/alibabacloud-go/debug v1.0.1 // indirect github.com/alibabacloud-go/tea v1.2.2 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect - github.com/andybalholm/brotli v1.1.0 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect @@ -144,7 +151,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.24.2 // indirect github.com/blevesearch/mmap-go v1.2.0 // indirect - github.com/bufbuild/protocompile v0.6.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect @@ -154,12 +160,13 @@ require ( github.com/charmbracelet/x/term v0.2.1 // indirect github.com/cilium/ebpf v0.9.1 // indirect github.com/clbanning/mxj v1.8.4 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/pebble v0.0.0-20220407171941-2120d145e292 // indirect github.com/cockroachdb/redact v1.1.3 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dlclark/regexp2 v1.10.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/dylibso/observe-sdk/go v0.0.0-20240819160327-2d926c5d788a // indirect @@ -167,11 +174,12 @@ require ( github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/getsentry/sentry-go v0.12.0 // indirect github.com/go-ini/ini v1.67.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/gobwas/glob v0.2.3 // indirect + github.com/goccy/go-json v0.10.6 // indirect github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -202,7 +210,7 @@ require ( github.com/josharian/native v1.1.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kamstrup/intmap v0.5.2 // indirect - github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -210,7 +218,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mdlayher/netlink v1.7.2 // indirect github.com/mdlayher/socket v0.5.1 // indirect @@ -231,12 +239,12 @@ require ( github.com/philhofer/fwd v1.2.0 // indirect github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7 // indirect github.com/pkoukk/tiktoken-go v0.1.6 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/segmentio/asm v1.1.3 // indirect @@ -250,34 +258,31 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/twmb/franz-go v1.21.6 // indirect - github.com/twmb/franz-go/pkg/kadm v1.18.0 // indirect - github.com/twmb/franz-go/pkg/kfake v0.0.0-20260820024614-9b174ed31afe // indirect - github.com/twmb/franz-go/pkg/kmsg v1.13.1 // indirect github.com/valyala/fastrand v1.1.0 // indirect github.com/valyala/histogram v1.2.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect - go.opentelemetry.io/otel v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/crypto v0.51.0 // indirect - golang.org/x/mod v0.35.0 // indirect - golang.org/x/net v0.53.0 // indirect + golang.org/x/mod v0.36.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/text v0.37.0 // indirect - golang.org/x/tools v0.44.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect + golang.org/x/tools v0.45.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8bc04bb36e0ea..8e759305a866f 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= -github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -37,10 +29,6 @@ github.com/KimMachineGun/automemlimit v0.6.0 h1:p/BXkH+K40Hax+PuWWPQ478hPjsp9h1C github.com/KimMachineGun/automemlimit v0.6.0/go.mod h1:T7xYht7B8r6AG/AqFcUdc7fzd2bIdBKmepfP2S1svPY= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= -github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/RoaringBitmap/roaring/v2 v2.16.0 h1:Kys1UNf49d5W8Tq3bpuAhIr/Z8/yPB+59CO8A6c/BbE= github.com/RoaringBitmap/roaring/v2 v2.16.0/go.mod h1:eq4wdNXxtJIS/oikeCzdX1rBzek7ANzbth041hrU8Q4= @@ -62,8 +50,12 @@ github.com/aliyun/credentials-go v1.3.10 h1:45Xxrae/evfzQL9V10zL3xX31eqgLWEaIdCo github.com/aliyun/credentials-go v1.3.10/go.mod h1:Jm6d+xIgwJVLVWT561vy67ZRP4lPTQxMbEYRuT2Ti1U= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= +github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= +github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= +github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -121,10 +113,6 @@ github.com/blevesearch/mmap-go v1.2.0 h1:l33nNKPFcBjJUMwem6sAYJPUzhUCABoK9FxZDGi github.com/blevesearch/mmap-go v1.2.0/go.mod h1:Vd6+20GBhEdwJnU1Xohgt88XCD/CTWcqbCNxkZpyBo0= github.com/blevesearch/vellum v1.2.0 h1:xkDiOEsHc2t3Cp0NsNZZ36pvc130sCzcGKOPMzXe+e0= github.com/blevesearch/vellum v1.2.0/go.mod h1:uEcfBJz7mAOf0Kvq6qoEKQQkLODBF46SINYNkZNae4k= -github.com/bufbuild/protocompile v0.6.0 h1:Uu7WiSQ6Yj9DbkdnOe7U4mNKp58y9WDMKDn28/ZlunY= -github.com/bufbuild/protocompile v0.6.0/go.mod h1:YNP35qEYoYGme7QMtz5SBCoN4kL4g12jTtjuzRNdjpE= -github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= -github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= @@ -135,9 +123,6 @@ github.com/bytedance/sonic/loader v0.5.1 h1:Ygpfa9zwRCCKSlrp5bBP/b/Xzc3VxsAW+5NI github.com/bytedance/sonic/loader v0.5.1/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -162,6 +147,8 @@ github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnx github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -186,20 +173,8 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/colinmarc/hdfs/v2 v2.4.0 h1:v6R8oBx/Wu9fHpdPoJJjpGSUxo8NhHIwrwsfhFvU9W0= github.com/colinmarc/hdfs/v2 v2.4.0/go.mod h1:0NAO+/3knbMx6+5pCv+Hcbaz4xn/Zzbn9+WIib2rKVI= -github.com/compose-spec/compose-go/v2 v2.0.0-rc.2 h1:eJ01FpliL/02KvsaPyH1bSLbM1S70yWQUojHVRbyvy4= -github.com/compose-spec/compose-go/v2 v2.0.0-rc.2/go.mod h1:IVsvFyGVhw4FASzUtlWNVaAOhYmakXAFY9IlZ7LAuD8= github.com/containerd/cgroups/v3 v3.0.1 h1:4hfGvu8rfGIwVIDd+nLzn/B9ZXx4BcCjzt5ToenJRaE= github.com/containerd/cgroups/v3 v3.0.1/go.mod h1:/vtwk1VXrtoa5AaZLkypuOJgA/6DyPMZHJPGQNtlHnw= -github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.7.15 h1:afEHXdil9iAm03BmhjzKyXnnEBtjaLJefdU7DV0IFes= -github.com/containerd/containerd v1.7.15/go.mod h1:ISzRRTMF8EXNpJlTzyr2XMhN+j9K302C21/+cr3kUnY= -github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= -github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= -github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -207,14 +182,13 @@ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzA github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpegeric/pdftotext-go v0.0.0-20241112123704-49cb86a3790e h1:tQSCiEjYPRU+AuuVR+zd+xYVOsEqX1clPhmIAM6FCHU= github.com/cpegeric/pdftotext-go v0.0.0-20241112123704-49cb86a3790e/go.mod h1:zt7uTOYu0EEeKatGaTi9JiP0I9ePHpDvjAwpfPXh/N0= -github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= -github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= @@ -222,28 +196,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-metro v0.0.0-20250106013310-edb8663e5e33 h1:ucRHb6/lvW/+mTEIGbvhcYU3S8+uSNkuMjx/qZFfhtM= github.com/dgryski/go-metro v0.0.0-20250106013310-edb8663e5e33/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= -github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= -github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/buildx v0.12.0-rc2.0.20231219140829-617f538cb315 h1:UZxx9xBADdf/9UmSdEUi+pdJoPKpgcf9QUAY5gEIYmY= -github.com/docker/buildx v0.12.0-rc2.0.20231219140829-617f538cb315/go.mod h1:X8ZHhuW6ncwtoJ36TlU+gyaROTcBkTE01VHYmTStQCE= -github.com/docker/cli v25.0.1+incompatible h1:mFpqnrS6Hsm3v1k7Wa/BO23oz0k121MTbTO1lpcGSkU= -github.com/docker/cli v25.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/compose/v2 v2.24.3 h1:BVc1oDV7aQgksH64pDKTvcI95G36uJ+Mz9DGGBBoZeQ= -github.com/docker/compose/v2 v2.24.3/go.mod h1:D8Nv9+juzD7xiMyyHJ7G2J/MOYiGBmb9SvdIW5+2zKo= -github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= -github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v25.0.5+incompatible h1:UmQydMduGkrD5nQde1mecF/YnSbTOaPeFIeP5C4W+DE= -github.com/docker/docker v25.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8= -github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40= -github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0= -github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c/go.mod h1:CADgU4DSXK5QUlFslkQu2yW2TKzFZcXq/leZfM0UH5Q= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ= @@ -260,8 +214,6 @@ github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHo github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= github.com/elastic/go-elasticsearch/v8 v8.15.0 h1:IZyJhe7t7WI3NEFdcHnf6IJXqpRf+8S8QWLtZYYyBYk= github.com/elastic/go-elasticsearch/v8 v8.15.0/go.mod h1:HCON3zj4btpqs2N1jjsAy4a/fiAul+YBP00mBH4xik8= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -278,18 +230,12 @@ github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/fgprof v0.9.6-0.20240831122612-49987e680f04 h1:6exK5FZYMJwsXo4xGqax5rdCHCYeQz+zgTO+Sk+tNHE= github.com/felixge/fgprof v0.9.6-0.20240831122612-49987e680f04/go.mod h1:YXcxFS6VIHoyXkktueL71Xqrp0tXKJzxODNlZDBt9nw= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/fsnotify/fsevents v0.1.1 h1:/125uxJvvoSDDBPen6yUZbil8J9ydKZnnl3TWWmvnkw= -github.com/fsnotify/fsevents v0.1.1/go.mod h1:+d+hS27T6k5J8CRaPLKFgwKYcpS7GwW3Ule9+SC2ZRc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo= -github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk= github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= @@ -304,19 +250,13 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= @@ -326,12 +266,13 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -374,8 +315,8 @@ github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85q github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -410,8 +351,6 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v1.12.80 h1:aC68NT6VK715WeUapxcPSFq/a3gZdS32HdtghdOIgAo= github.com/gopherjs/gopherjs v1.12.80/go.mod h1:d55Q4EjGQHeJVms+9LGtXul6ykz5Xzx1E1gaXQXdimY= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= @@ -422,17 +361,11 @@ github.com/gosimple/slug v1.13.1 h1:bQ+kpX9Qa6tHRaK+fZR0A0M2Kd7Pa5eHPPsb1JpHD+Q= github.com/gosimple/slug v1.13.1/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 h1:CWyXh/jylQWp2dtiV33mY4iSSp6yf4lmn+c7/tN+ObI= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0/go.mod h1:nCLIt0w3Ept2NwF8ThLmrppXsfT07oC8k0XNDxd8sVU= github.com/hamba/avro/v2 v2.31.0 h1:wv3nmua7lCEIwWsb6vqsTS3pXktTxcKg5eoyNu0VhrU= github.com/hamba/avro/v2 v2.31.0/go.mod h1:t6lJYAGE5Mswfn17zjtyQsssRQgnqO6TXLBCHHWRqrw= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= @@ -447,8 +380,6 @@ github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -463,11 +394,7 @@ github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:q github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/ianlancetaylor/demangle v0.0.0-20250417193237-f615e6bd150b h1:ogbOPx86mIhFy764gGkqnkFC8m5PJA7sPzlk9ppLVQA= github.com/ianlancetaylor/demangle v0.0.0-20250417193237-f615e6bd150b/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= -github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= -github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/in-toto/in-toto-golang v0.5.0 h1:hb8bgwr0M2hGdDsLjkJ3ZqJ8JFLL/tgYdAxF/XEFBbY= -github.com/in-toto/in-toto-golang v0.5.0/go.mod h1:/Rq0IZHLV7Ku5gielPT4wPHJfH1GdHMCq8+WPxw8/BE= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -507,8 +434,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -533,8 +458,6 @@ github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2 github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -543,14 +466,12 @@ github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= -github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= -github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= -github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= -github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -569,14 +490,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matrixorigin/dragonboat/v4 v4.0.0-20260706084232-ea0aa61b062b h1:IlemEj5LthYSgF15ROKOwI59sjXT39HDmAQQUW19Kxw= github.com/matrixorigin/dragonboat/v4 v4.0.0-20260706084232-ea0aa61b062b/go.mod h1:B1VfuDzGm7Uk0xoRkRJynA6n4i31m/1ZJIPfIyJfYQg= -github.com/matrixorigin/goetty/v2 v2.0.0-20240611082008-a4de209fff3d h1:wSlkJlWXZ3if1sH8Bc/lUUOWhTw91lgYGSFOHqy1tcw= -github.com/matrixorigin/goetty/v2 v2.0.0-20240611082008-a4de209fff3d/go.mod h1:OwIBpVwRW1HjF/Jhc2Av3UvG2NygMg+bdqGxZaqwhU0= github.com/matrixorigin/goetty/v2 v2.0.0-20260811155919-8edc32ac370a h1:D8kXjySDJHA8nyZzzi/4cx3KB6qrSPQeugUCLkgoEho= github.com/matrixorigin/goetty/v2 v2.0.0-20260811155919-8edc32ac370a/go.mod h1:OwIBpVwRW1HjF/Jhc2Av3UvG2NygMg+bdqGxZaqwhU0= github.com/matrixorigin/gosigar v0.14.3-0.20241204071856-40aab500bfac h1:KRPOwOcSZRfWT7w8mr7BmFLoFB75ljgqjkg47xkN/Pc= @@ -594,8 +509,6 @@ github.com/matrixorigin/vfs v0.2.1-0.20220616104132-8852fd867376/go.mod h1:LOatf github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= @@ -605,10 +518,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= -github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= -github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= -github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= +github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -619,49 +530,21 @@ github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTa github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.53 h1:ZBkuHr5dxHtB1caEOlZTLPo7D3L3TWckgUUs/RHfDxw= github.com/miekg/dns v1.1.53/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= -github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= -github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= -github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= -github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= -github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/buildkit v0.13.0-beta1.0.20231219135447-957cb50df991 h1:r80LLQ91uOLxU1ElAvrB1o8oBsph51lPzVnr7t2b200= -github.com/moby/buildkit v0.13.0-beta1.0.20231219135447-957cb50df991/go.mod h1:6MddWPSL5jxy+W8eMMHWDOfZzzRRKWXPZqajw72YHBc= -github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= -github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= -github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= -github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= -github.com/moby/sys/symlink v0.2.0 h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZc= -github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= -github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= -github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -671,8 +554,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= @@ -684,8 +565,6 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= @@ -710,10 +589,6 @@ github.com/openacid/must v0.1.3/go.mod h1:luPiXCuJlEo3UUFQngVQokV0MPGryeYvtCbQPs github.com/openacid/slimarray v0.1.3 h1:+/+G8k+Nz4p8QUj4J2kd7IzFC5DiJzk5H2QPp/BpHHk= github.com/openacid/slimarray v0.1.3/go.mod h1:9PM3kQPSUP02hll5jerjjT1dvtjSOGdHjFqEeZkPL1U= github.com/openacid/testutil v0.1.1/go.mod h1:qgfN+myXuX8gc+JveuP+sts//cpvCGRM5BIqwpYnzIs= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= @@ -727,16 +602,12 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/petermattis/goid v0.0.0-20241025130422-66cb2e6d7274 h1:qli3BGQK0tYDkSEvZ/FzZTi9ZrOX86Q6CIhKLGc489A= github.com/petermattis/goid v0.0.0-20241025130422-66cb2e6d7274/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= -github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= +github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -751,8 +622,9 @@ github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAc github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= github.com/plar/go-adaptive-radix-tree v1.0.5 h1:rHR89qy/6c24TBAHullFMrJsU9hGlKmPibdBGU6/gbM= github.com/plar/go-adaptive-radix-tree v1.0.5/go.mod h1:15VOUO7R9MhJL8HOJdpydR0rvanrtRE6fA6XSa/tqWE= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= @@ -766,7 +638,6 @@ github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdO github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -776,8 +647,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -789,18 +660,12 @@ github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtm github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/FulN9fTtqYUdS5+Oxzt+DUE= -github.com/secure-systems-lab/go-securesystemslib v0.4.0/go.mod h1:FGBZgq2tXWICsxWQW1msNf49F0Pf2Op5Htayx335Qbs= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.4.0 h1:MEBYvRqiUB2nfR2criEXWqwdY6HJOUrCn5hboVOVmy8= github.com/segmentio/encoding v0.4.0/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrWPgtJHYZSnI= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002 h1:ka9QPuQg2u4LGipiZGsgkg3rJCo4iIUCy75FddM0GRQ= -github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= -github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= -github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/shirou/gopsutil/v3 v3.22.4/go.mod h1:D01hZJ4pVHPpCTZ3m3T2+wDF2YAGfd+H4ifUguaQzHM= github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= @@ -853,24 +718,18 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/substrait-io/substrait-protobuf/go v0.78.0 h1:AMhcYhFIT14dznECF6kVDZHJcnm8kWI3FuHt4Jh8/9s= -github.com/substrait-io/substrait-protobuf/go v0.78.0/go.mod h1:hn+Szm1NmZZc91FwWK9EXD/lmuGBSRTJ5IvHhlG1YnQ= +github.com/substrait-io/substrait-protobuf/go v0.85.0 h1:zk6MtNWLtDSl8a7qCZRFH0+EIIXVrrd/hsgYK/SQTgM= +github.com/substrait-io/substrait-protobuf/go v0.85.0/go.mod h1:hn+Szm1NmZZc91FwWK9EXD/lmuGBSRTJ5IvHhlG1YnQ= github.com/syncthing/notify v0.0.0-20250528144937-c7027d4f7465 h1:yhxdTGmFkAM2TFA65c3NgGwpnIkUM8oVqPX2e9S7IVg= github.com/syncthing/notify v0.0.0-20250528144937-c7027d4f7465/go.mod h1:J0q59IWjLtpRIJulohwqEZvjzwOfTEPp8SVhDJl+y0Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= github.com/tencentyun/cos-go-sdk-v5 v0.7.55 h1:9DfH3umWUd0I2jdqcUxrU1kLfUPOydULNy4T9qN5PF8= github.com/tencentyun/cos-go-sdk-v5 v0.7.55/go.mod h1:8+hG+mQMuRP/OIS9d83syAvXvrMj9HhkND6Q1fLghw0= -github.com/testcontainers/testcontainers-go v0.31.0 h1:W0VwIhcEVhRflwL9as3dhY6jXjVCA27AkmbnZ+UTh3U= -github.com/testcontainers/testcontainers-go v0.31.0/go.mod h1:D2lAoA0zUFiSY+eAflqK5mcUx/A5hrrORaEQrd0SefI= -github.com/testcontainers/testcontainers-go/modules/compose v0.29.1 h1:47ipPM+s+ltCDOP3Sa1j95AkNb+z+WGiHLDbLU8ixuc= -github.com/testcontainers/testcontainers-go/modules/compose v0.29.1/go.mod h1:Sqh+Ef2ESdbJQjTJl57UOkEHkOc7gXvQLg1b5xh6f1Y= github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 h1:ZF+QBjOI+tILZjBaFj3HgFonKXUcwgJ4djLb6i42S3Q= github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834/go.mod h1:m9ymHTgNSEjuxvw8E7WWe4Pl4hZQHXONY8wE6dMLaRk= github.com/tetratelabs/wazero v1.8.1-0.20240916092830-1353ca24fef0 h1:NCRnJ+X6eZt3awiReoHCcDuC6Wf+CgWk6p4IDkIuxTo= github.com/tetratelabs/wazero v1.8.1-0.20240916092830-1353ca24fef0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= -github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c= -github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw= github.com/ti-mo/conntrack v0.5.1 h1:opEwkFICnDbQc0BUXl73PHBK0h23jEIFVjXsqvF4GY0= github.com/ti-mo/conntrack v0.5.1/go.mod h1:T6NCbkMdVU4qEIgwL0njA6lw/iCAbzchlnwm1Sa314o= github.com/ti-mo/netfilter v0.5.2 h1:CTjOwFuNNeZ9QPdRXt1MZFLFUf84cKtiQutNauHWd40= @@ -879,8 +738,6 @@ github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 h1:QB54BJwA6x8QU9nHY3xJSZR2kX9bgpZekRKGkLTmEXA= -github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375/go.mod h1:xRroudyp5iVtxKqZCrA6n2TLFRBf8bmnjr1UD4x+z7g= github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= @@ -891,12 +748,6 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/langchaingo v0.1.13 h1:rcpMWBIi2y3B90XxfE4Ao8dhCQPVDMaNPnN5cGB1CaA= github.com/tmc/langchaingo v0.1.13/go.mod h1:vpQ5NOIhpzxDfTZK9B6tf2GM/MoaHewPWM5KXXGh7hg= -github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302 h1:ZT8ibgassurSISJ1Pj26NsM3vY2jxFZn63Nd/TpHmRw= -github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302/go.mod h1:9kMVqMyQ/Sx2df5LtnGG+nbrmiZzCS7V6gjW3oGHsvI= -github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0= -github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk= -github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531 h1:Y/M5lygoNPKwVNLMPXgVfsRT40CSFKXCxuU8LoHySjs= -github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/twmb/franz-go v1.21.6 h1:+v0dQJVIIuw9uPmPWmPrkoUHs1pPeV8MSwA4eU/Y2kY= @@ -948,6 +799,8 @@ github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yalue/onnxruntime_go v1.31.0 h1:1ln4YW1SFOFfGJZXe3jNOb2JUSt+l2pEneZfV8HdtFA= github.com/yalue/onnxruntime_go v1.31.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4= @@ -965,38 +818,24 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mongodb.org/mongo-driver/v2 v2.8.0 h1:CxWDGQYY8QQwNjAl/aq2sfWakdnWZynnqJ9F4DhHbP8= go.mongodb.org/mongo-driver/v2 v2.8.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 h1:A3SayB3rNyt+1S6qpI9mHPkeHTZbD7XILEqWnYZb2l0= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0 h1:2ea0IkZBsWH+HA2GkD+7+hRw2u97jzdFyRtXuO14a1s= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0/go.mod h1:4m3RnBBb+7dB9d21y510oO1pdB1V4J6smNf14WXcBFQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 h1:NmnYCiR0qNufkldjVvyQfZTHSdzeHoZ41zggMsdMcLM= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0/go.mod h1:UVAO61+umUsHLtYb8KXXRoHtxUkdOPkYidzW3gipRLQ= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 h1:wNMDy/LVGLj2h3p6zg4d0gypKfWKSWI14E1C4smOgl8= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0/go.mod h1:YfbDdXAAkemWJK3H/DshvlrxqFB2rtW4rY6ky/3x/H0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= -go.opentelemetry.io/otel/exporters/prometheus v0.42.0 h1:jwV9iQdvp38fxXi8ZC+lNpxjK16MRcZlpDYvbuO1FiA= -go.opentelemetry.io/otel/exporters/prometheus v0.42.0/go.mod h1:f3bYiqNqhoPxkvI2LrXqQVC546K7BuRDL/kKuxkujhA= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8Ni+hx+8i1k= -go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.starlark.net v0.0.0-20250701195324-d457b4515e0e h1:/WX+ZvcgVJxdIxVR9J3u45ds+Bl4IWPIHRSSICp0t3Q= @@ -1011,8 +850,6 @@ go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= -go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -1041,8 +878,6 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1072,10 +907,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1101,13 +934,9 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1118,8 +947,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180807162357-acbc56fc7007/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1169,18 +998,13 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= -golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= -golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -1191,8 +1015,6 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1224,10 +1046,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= -golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1235,8 +1055,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.1/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0= -gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -1246,12 +1066,8 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20240528184218-531527333157 h1:u7WMYrIrVvs0TF5yaKwKNbcJyySYf+HAIFXxWltJOXE= -google.golang.org/genproto v0.0.0-20240528184218-531527333157/go.mod h1:ubQlAQnzejB8uZzszhrTCU2Fyp6Vi7ZE5nn0c3W8+qQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -1259,8 +1075,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1284,8 +1100,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -1309,27 +1123,7 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/api v0.26.7 h1:Lf4iEBEJb5OFNmawtBfSZV/UNi9riSJ0t1qdhyZqI40= -k8s.io/api v0.26.7/go.mod h1:Vk9bMadzA49UHPmHB//lX7VRCQSXGoVwfLd3Sc1SSXI= -k8s.io/apimachinery v0.26.7 h1:590jSBwaSHCAFCqltaEogY/zybFlhGsnLteLpuF2wig= -k8s.io/apimachinery v0.26.7/go.mod h1:qYzLkrQ9lhrZRh0jNKo2cfvf/R1/kQONnSiyB7NUJU0= -k8s.io/apiserver v0.26.7 h1:NX/zBZZn4R+Cq6shwyn8Pn8REd0yJJ16dbtv9WkEVEU= -k8s.io/apiserver v0.26.7/go.mod h1:r0wDRWHI7VL/KlQLTkJJBVGZ3KeNfv+VetlyRtr86xs= -k8s.io/client-go v0.26.7 h1:hyU9aKHlwVOykgyxzGYkrDSLCc4+mimZVyUJjPyUn1E= -k8s.io/client-go v0.26.7/go.mod h1:okYjy0jtq6sdeztALDvCh24tg4opOQS1XNvsJlERDAo= -k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= -k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= -k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -tags.cncf.io/container-device-interface v0.6.2 h1:dThE6dtp/93ZDGhqaED2Pu374SOeUkBfuvkLuiTdwzg= -tags.cncf.io/container-device-interface v0.6.2/go.mod h1:Shusyhjs1A5Na/kqPVLL0KqnHQHuunol9LFeUNkuGVE= diff --git a/optools/compose_bvt/compose_bvt.sh b/optools/compose_bvt/compose_bvt.sh index 91c3d203f9d5e..dce8d7ca4ccb8 100755 --- a/optools/compose_bvt/compose_bvt.sh +++ b/optools/compose_bvt/compose_bvt.sh @@ -26,4 +26,4 @@ function compose_bvt() { #create the dir for export logs rm -rf ${MO_WORKSPACE}/docker-compose-log && mkdir -p ${MO_WORKSPACE}/docker-compose-log -compose_bvt \ No newline at end of file +compose_bvt diff --git a/pkg/common/mpool/allocation_owner.go b/pkg/common/mpool/allocation_owner.go index 5054a657b8537..88d83da5303e0 100644 --- a/pkg/common/mpool/allocation_owner.go +++ b/pkg/common/mpool/allocation_owner.go @@ -32,12 +32,13 @@ const ( AllocationOwnerFulltext AllocationOwnerDML AllocationOwnerSample + AllocationOwnerExternal ) const ( // AllocationOwnerCatalogMax is the largest owner implemented by this // binary. Append new catalog entries immediately before it. - AllocationOwnerCatalogMax = AllocationOwnerSample + AllocationOwnerCatalogMax = AllocationOwnerExternal // AllocationOwnerMax preserves the existing public bound and reserves IDs // for rolling-version terminal summaries. Unknown owners remain observable // on the wire but cannot allocate locally until catalogued by this binary. @@ -72,6 +73,8 @@ func (o AllocationOwner) String() string { return "dml" case AllocationOwnerSample: return "sample" + case AllocationOwnerExternal: + return "external" default: return "owner-" + strconv.FormatUint(uint64(o), 10) } diff --git a/pkg/common/mpool/allocation_owner_test.go b/pkg/common/mpool/allocation_owner_test.go index b4c14f87b007b..ffd7937bde42c 100644 --- a/pkg/common/mpool/allocation_owner_test.go +++ b/pkg/common/mpool/allocation_owner_test.go @@ -34,6 +34,7 @@ func TestAllocationOwnerCatalog(t *testing.T) { {AllocationOwnerFulltext, 10, "fulltext"}, {AllocationOwnerDML, 11, "dml"}, {AllocationOwnerSample, 12, "sample"}, + {AllocationOwnerExternal, 13, "external"}, } seen := make(map[AllocationOwner]struct{}, len(owners)) seenNames := make(map[string]struct{}, len(owners)) @@ -56,8 +57,8 @@ func TestAllocationOwnerCatalog(t *testing.T) { } seenNames[entry.name] = struct{}{} } - if AllocationOwnerCatalogMax != AllocationOwnerSample { - t.Fatalf("catalog max = %d, want %d", AllocationOwnerCatalogMax, AllocationOwnerSample) + if AllocationOwnerCatalogMax != AllocationOwnerExternal { + t.Fatalf("catalog max = %d, want %d", AllocationOwnerCatalogMax, AllocationOwnerExternal) } if AllocationOwnerCatalogMax > AllocationOwnerMax { t.Fatalf("catalog max %d exceeds reserved max %d", AllocationOwnerCatalogMax, AllocationOwnerMax) diff --git a/pkg/common/mpool/capacity_lease.go b/pkg/common/mpool/capacity_lease.go new file mode 100644 index 0000000000000..6c66921079041 --- /dev/null +++ b/pkg/common/mpool/capacity_lease.go @@ -0,0 +1,155 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mpool + +import "sync/atomic" + +const ( + capacityReservationOpen uint32 = iota + capacityReservationCommitted + capacityReservationAborted +) + +// CapacityReservation pre-admits an upper bound before a non-MPool backing is +// allocated or pinned. Commit transfers the actual charge to one CapacityLease; +// Abort returns the whole reservation. Both terminal operations are idempotent. +type CapacityReservation struct { + account *AllocationAccount + owner AllocationOwner + site AllocationSite + capacityClass AllocationCapacityClass + upperBound uint64 + state atomic.Uint32 +} + +// CapacityLease is the sole release owner for committed non-MPool capacity. +type CapacityLease struct { + account *AllocationAccount + owner AllocationOwner + site AllocationSite + capacityClass AllocationCapacityClass + capacity uint64 + released atomic.Bool +} + +// ReserveCapacity reserves non-MPool capacity against the statement's default +// capacity controller. +func (a *AllocationAccount) ReserveCapacity( + upperBound uint64, + owner AllocationOwner, + site AllocationSite, +) (*CapacityReservation, error) { + return a.ReserveCapacityWithClass( + upperBound, + AllocationCapacityClassDefault, + owner, + site, + ) +} + +// ReserveCapacityWithClass is the execution-local capacity-class counterpart +// of ReserveCapacity. +func (a *AllocationAccount) ReserveCapacityWithClass( + upperBound uint64, + capacityClass AllocationCapacityClass, + owner AllocationOwner, + site AllocationSite, +) (*CapacityReservation, error) { + if a == nil || site < AllocationSiteMin || site > AllocationSiteMax { + return nil, ErrAllocationAccountInvalid + } + if err := a.acquireWithCapacityClass(upperBound, capacityClass, owner); err != nil { + return nil, err + } + return &CapacityReservation{ + account: a, + owner: owner, + site: site, + capacityClass: capacityClass, + upperBound: upperBound, + }, nil +} + +func (r *CapacityReservation) UpperBound() uint64 { + if r == nil { + return 0 + } + return r.upperBound +} + +// Commit transfers actualCapacity to the returned lease. A failed oversized +// commit leaves the reservation open so its caller can release acquired +// backing and Abort without losing the original admission charge. +func (r *CapacityReservation) Commit(actualCapacity uint64) (*CapacityLease, error) { + if r == nil || r.account == nil { + return nil, ErrAllocationAccountInvalid + } + if actualCapacity > r.upperBound { + return nil, newAllocationAccountCapacityError( + r.upperBound, + actualCapacity-r.upperBound, + r.upperBound, + ) + } + if !r.state.CompareAndSwap(capacityReservationOpen, capacityReservationCommitted) { + return nil, ErrAllocationAccountMismatch + } + if unused := r.upperBound - actualCapacity; unused > 0 { + r.account.releaseWithCapacityClass(unused, r.capacityClass, r.owner) + } + return &CapacityLease{ + account: r.account, + owner: r.owner, + site: r.site, + capacityClass: r.capacityClass, + capacity: actualCapacity, + }, nil +} + +func (r *CapacityReservation) Abort() { + if r == nil || r.account == nil || + !r.state.CompareAndSwap(capacityReservationOpen, capacityReservationAborted) { + return + } + r.account.releaseWithCapacityClass(r.upperBound, r.capacityClass, r.owner) +} + +func (l *CapacityLease) Capacity() uint64 { + if l == nil { + return 0 + } + return l.capacity +} + +func (l *CapacityLease) Owner() AllocationOwner { + if l == nil { + return 0 + } + return l.owner +} + +func (l *CapacityLease) Site() AllocationSite { + if l == nil { + return 0 + } + return l.site +} + +func (l *CapacityLease) Release() { + if l == nil || l.account == nil || !l.released.CompareAndSwap(false, true) { + return + } + l.account.releaseWithCapacityClass(l.capacity, l.capacityClass, l.owner) +} diff --git a/pkg/common/mpool/capacity_lease_test.go b/pkg/common/mpool/capacity_lease_test.go new file mode 100644 index 0000000000000..82dafcff0fcd4 --- /dev/null +++ b/pkg/common/mpool/capacity_lease_test.go @@ -0,0 +1,86 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mpool + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCapacityReservationCommitAndRelease(t *testing.T) { + registry, account := newTestAllocationAccount(t, 1024, 1) + reservation, err := account.ReserveCapacity(800, AllocationOwnerExternal, 1) + require.NoError(t, err) + require.Equal(t, uint64(800), account.Snapshot().Used) + + lease, err := reservation.Commit(512) + require.NoError(t, err) + require.Equal(t, uint64(512), lease.Capacity()) + require.Equal(t, uint64(512), account.Snapshot().Used) + lease.Release() + lease.Release() + require.Zero(t, account.Snapshot().Used) + finalizeTestAllocationAccount(t, registry, account) +} + +func TestCapacityReservationAbortAndOversizedCommit(t *testing.T) { + registry, account := newTestAllocationAccount(t, 1024, 1) + reservation, err := account.ReserveCapacity(128, AllocationOwnerExternal, 2) + require.NoError(t, err) + _, err = reservation.Commit(129) + require.ErrorIs(t, err, ErrAllocationAccountCapacity) + require.Equal(t, uint64(128), account.Snapshot().Used) + reservation.Abort() + reservation.Abort() + require.Zero(t, account.Snapshot().Used) + finalizeTestAllocationAccount(t, registry, account) +} + +func TestCapacityReservationCommitAbortRace(t *testing.T) { + for range 100 { + registry, account := newTestAllocationAccount(t, 16, 1) + reservation, err := account.ReserveCapacity(16, AllocationOwnerExternal, 3) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(2) + var lease *CapacityLease + go func() { + defer wg.Done() + lease, _ = reservation.Commit(8) + }() + go func() { + defer wg.Done() + reservation.Abort() + }() + wg.Wait() + if lease != nil { + lease.Release() + } + require.Zero(t, account.Snapshot().Used) + finalizeTestAllocationAccount(t, registry, account) + } +} + +func TestCapacityReservationRejectsAfterSeal(t *testing.T) { + registry, account := newTestAllocationAccount(t, 16, 1) + account.Seal() + _, err := account.ReserveCapacity(1, AllocationOwnerExternal, 1) + require.ErrorIs(t, err, ErrAllocationAccountSealed) + _, err = registry.Finalize(account) + require.NoError(t, err) +} diff --git a/pkg/config/arrow_load_test.go b/pkg/config/arrow_load_test.go new file mode 100644 index 0000000000000..053b8d8741fef --- /dev/null +++ b/pkg/config/arrow_load_test.go @@ -0,0 +1,122 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "os" + "path/filepath" + "testing" + + "github.com/BurntSushi/toml" + "github.com/stretchr/testify/require" +) + +func TestArrowLoadDefaultsAndProgrammaticOptIn(t *testing.T) { + var frontend FrontendParameters + frontend.SetDefaultValues() + require.False(t, frontend.ArrowLoad.Enabled) + require.False(t, frontend.ArrowLoad.S3Enabled) + require.False(t, frontend.ArrowLoad.DistributedEnabled) + require.False(t, frontend.ArrowLoad.ForceMaterialize) + + parameters := NewArrowLoadParameters() + parameters.Enabled = true + parameters.S3Enabled = true + parameters.DistributedEnabled = true + parameters.SetDefaultValues() + require.True(t, parameters.Enabled) + require.True(t, parameters.S3Enabled) + require.True(t, parameters.DistributedEnabled) +} + +func TestLaunchTAEComposeProfileKeepsArrowLoadFailClosed(t *testing.T) { + for _, name := range []string{"cn-0.toml", "cn-1.toml"} { + t.Run(name, func(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", "etc", "launch-tae-compose", "config", name)) + require.NoError(t, err) + + var decoded struct { + CN struct { + Frontend FrontendParameters `toml:"frontend"` + } `toml:"cn"` + } + _, err = toml.Decode(string(data), &decoded) + require.NoError(t, err) + decoded.CN.Frontend.SetDefaultValues() + + require.False(t, decoded.CN.Frontend.ArrowLoad.Enabled) + require.False(t, decoded.CN.Frontend.ArrowLoad.S3Enabled) + require.False(t, decoded.CN.Frontend.ArrowLoad.DistributedEnabled) + }) + } +} + +func TestArrowLoadTOMLDefaultsAndExplicitOptOut(t *testing.T) { + for _, test := range []struct { + name string + input string + enabled bool + s3Enabled bool + distributedEnabled bool + forceMaterialize bool + }{ + {name: "section omitted"}, + { + name: "enable fields omitted", input: "[arrow-load]\nforce-materialize = true\n", + forceMaterialize: true, + }, + { + name: "explicit opt in", input: `[arrow-load] + enabled = true +s3-enabled = true +distributed-enabled = true +`, + enabled: true, s3Enabled: true, distributedEnabled: true, + }, + { + name: "one case-insensitive opt in", input: "[arrow-load]\nENABLED = true\nS3-ENABLED = true\n", + enabled: true, s3Enabled: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + decoded := struct { + ArrowLoad ArrowLoadParameters `toml:"arrow-load"` + }{ArrowLoad: *NewArrowLoadParameters()} + _, err := toml.Decode(test.input, &decoded) + require.NoError(t, err) + decoded.ArrowLoad.SetDefaultValues() + + require.Equal(t, test.enabled, decoded.ArrowLoad.Enabled) + require.Equal(t, test.s3Enabled, decoded.ArrowLoad.S3Enabled) + require.Equal(t, test.distributedEnabled, decoded.ArrowLoad.DistributedEnabled) + require.Equal(t, test.forceMaterialize, decoded.ArrowLoad.ForceMaterialize) + + // Service validation may apply defaults more than once. An explicit + // false must remain an opt-out on every later pass. + decoded.ArrowLoad.SetDefaultValues() + require.Equal(t, test.enabled, decoded.ArrowLoad.Enabled) + require.Equal(t, test.s3Enabled, decoded.ArrowLoad.S3Enabled) + require.Equal(t, test.distributedEnabled, decoded.ArrowLoad.DistributedEnabled) + }) + } +} + +func TestArrowLoadRejectsConflictingGateKeys(t *testing.T) { + var decoded struct { + ArrowLoad ArrowLoadParameters `toml:"arrow-load"` + } + _, err := toml.Decode("[arrow-load]\nenabled = false\nENABLED = true\n", &decoded) + require.ErrorContains(t, err, "conflicting enabled keys") +} diff --git a/pkg/config/configuration.go b/pkg/config/configuration.go index b69bc6a2770dc..6b696498a5369 100644 --- a/pkg/config/configuration.go +++ b/pkg/config/configuration.go @@ -287,6 +287,97 @@ type MongoDBParameters struct { enableDefaulted bool } +const ( + arrowLoadEnabledConfigured uint8 = 1 << iota + arrowLoadS3EnabledConfigured + arrowLoadDistributedEnabledConfigured +) + +// ArrowLoadParameters controls the LOAD-only Arrow IPC surface. Local files are +// disabled by default. Each source and execution surface requires an explicit +// deployment opt-in until its release-readiness gates are accepted. +// +// configuredFields distinguishes an omitted TOML key from an explicit false. +// defaultsApplied makes repeated service validation idempotent, so a later +// programmatic false is not silently changed back to true. +type ArrowLoadParameters struct { + Enabled bool `toml:"enabled" user_setting:"advanced"` + S3Enabled bool `toml:"s3-enabled" user_setting:"advanced"` + DistributedEnabled bool `toml:"distributed-enabled" user_setting:"advanced"` + // ForceMaterialize disables the Arrow-to-MO borrow path without disabling + // Arrow LOAD itself. It is a rollout diagnostic and emergency fallback, not + // the normal execution policy, so its zero value keeps borrowing enabled. + ForceMaterialize bool `toml:"force-materialize" user_setting:"advanced"` + + configuredFields uint8 + defaultsApplied bool +} + +// NewArrowLoadParameters returns the default fail-closed Arrow LOAD settings. +// Callers that adjust a programmatic service configuration should start from +// this value so an explicit setting survives later validation and defaulting +// passes. +func NewArrowLoadParameters() *ArrowLoadParameters { + parameters := &ArrowLoadParameters{} + parameters.SetDefaultValues() + return parameters +} + +// UnmarshalTOML records explicit opt-outs while preserving defaults for omitted +// keys. BurntSushi TOML matches field names case-insensitively, so conflicting +// case variants are rejected instead of making the selected value ambiguous. +func (parameters *ArrowLoadParameters) UnmarshalTOML(value interface{}) error { + table, ok := value.(map[string]interface{}) + if !ok { + return moerr.NewBadConfigNoCtx("arrow-load configuration must be a TOML table") + } + + var configured uint8 + for key := range table { + var field uint8 + var name string + switch { + case strings.EqualFold(key, "enabled"): + field, name = arrowLoadEnabledConfigured, "enabled" + case strings.EqualFold(key, "s3-enabled"): + field, name = arrowLoadS3EnabledConfigured, "s3-enabled" + case strings.EqualFold(key, "distributed-enabled"): + field, name = arrowLoadDistributedEnabledConfigured, "distributed-enabled" + default: + continue + } + if configured&field != 0 { + return moerr.NewBadConfigNoCtxf( + "arrow-load configuration contains conflicting %s keys", name, + ) + } + configured |= field + } + + var encoded bytes.Buffer + if err := btoml.NewEncoder(&encoded).Encode(table); err != nil { + return err + } + type plainArrowLoadParameters ArrowLoadParameters + decoded := plainArrowLoadParameters(*parameters) + if _, err := btoml.Decode(encoded.String(), &decoded); err != nil { + return err + } + *parameters = ArrowLoadParameters(decoded) + parameters.configuredFields |= configured + return nil +} + +// SetDefaultValues preserves the zero-value fail-closed policy. Deployment +// configuration must explicitly opt in to Arrow LOAD; S3-backed sources and +// distributed execution require their corresponding opt-ins as well. +func (parameters *ArrowLoadParameters) SetDefaultValues() { + if parameters.defaultsApplied { + return + } + parameters.defaultsApplied = true +} + // NewMongoDBParameters returns MongoDB parameters with defaults that must be // established before TOML decoding. Initializing Enable here lets an explicit // false from either TOML or programmatic configuration remain meaningful when @@ -660,8 +751,9 @@ type FrontendParameters struct { // globally for new sessions with SET GLOBAL sidecar_url = '...'. SidecarURL string `toml:"sidecarUrl" user_setting:"advanced"` - Iceberg IcebergParameters `toml:"iceberg" user_setting:"advanced"` - MongoDB MongoDBParameters `toml:"mongodb" user_setting:"advanced"` + Iceberg IcebergParameters `toml:"iceberg" user_setting:"advanced"` + MongoDB MongoDBParameters `toml:"mongodb" user_setting:"advanced"` + ArrowLoad ArrowLoadParameters `toml:"arrow-load" user_setting:"advanced"` } func (fp *FrontendParameters) SetDefaultValues() { @@ -821,6 +913,7 @@ func (fp *FrontendParameters) SetDefaultValues() { fp.Iceberg.SetDefaultValues() fp.MongoDB.SetDefaultValues() + fp.ArrowLoad.SetDefaultValues() } func (fp *FrontendParameters) SetMaxMessageSize(size uint64) { diff --git a/pkg/container/arrowbridge/bind.go b/pkg/container/arrowbridge/bind.go new file mode 100644 index 0000000000000..5a2b826b665d7 --- /dev/null +++ b/pkg/container/arrowbridge/bind.go @@ -0,0 +1,539 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowbridge + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "hash" + "sort" + "strings" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/decimal128" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +const ( + // MaxFields and MaxNestingDepth bound the already-decoded Arrow schema. + // Wire-level FlatBuffers vectors have separate, stricter validation in + // container/arrowipc before Arrow-Go is allowed to construct this schema. + MaxFields = 4096 + MaxNestingDepth = 32 + // ConversionPlanVersion fences distributed LOAD plans from binaries that + // implement a different binding/conversion contract. + ConversionPlanVersion = uint32(1) +) + +// MatchMode controls how source fields are bound to target columns. +type MatchMode uint8 + +const ( + // MatchByName binds case-insensitively and rejects missing or ambiguous + // source names. + MatchByName MatchMode = iota + // MatchByPosition ignores source names but still fingerprints both schemas + // so execution cannot accept a drifted record. + MatchByPosition +) + +// TargetColumn is the stable table-side contract used by the bridge. MOIndex +// and AttrName let a caller keep output ordering independent of source order. +type TargetColumn struct { + Name string + Type types.Type + NotNull bool + MOIndex int + AttrName string +} + +type conversionKind uint8 + +const ( + conversionBorrowFixed conversionKind = iota + conversionBorrowVarlen + conversionMaterializeBool + conversionMaterializeDate32 + conversionMaterializeDate64 + conversionMaterializeTimestamp + conversionMaterializeDictionary + conversionMaterializeWiden + conversionMaterializeTime + conversionMaterializeNull +) + +type columnPlan struct { + source int + target TargetColumn + kind conversionKind +} + +// Plan is immutable after BindLoad and may be reused for records carrying the +// exact same Arrow schema. +type Plan struct { + schemaFingerprint [sha256.Size]byte + conversionFingerprint [sha256.Size]byte + columns []columnPlan + attrs []string +} + +// Bind is the compatibility spelling for BindLoad. +// +// Deprecated: new callers must use BindLoad so the LOAD conversion policy is +// visible at the call site. Other Arrow consumers, especially Python UDF, must +// not treat this policy as their wire ABI. +func Bind( + ctx context.Context, + schema *arrow.Schema, + targets []TargetColumn, + mode MatchMode, +) (*Plan, error) { + return BindLoad(ctx, schema, targets, mode) +} + +// BindLoad validates cardinality, mapping ambiguity, and the supported LOAD +// conversion matrix before any record data is acquired. The matrix permits a +// small set of checked widening and temporal conversions that are useful for +// ingestion but are intentionally forbidden by exact result protocols. +func BindLoad( + ctx context.Context, + schema *arrow.Schema, + targets []TargetColumn, + mode MatchMode, +) (*Plan, error) { + if schema == nil { + return nil, moerr.NewInvalidInput(ctx, "Arrow schema is nil") + } + if err := validateSchemaShape(ctx, schema); err != nil { + return nil, err + } + if len(targets) != schema.NumFields() { + return nil, moerr.NewInvalidInputf(ctx, "Arrow source has %d fields but target has %d columns", schema.NumFields(), len(targets)) + } + if mode != MatchByName && mode != MatchByPosition { + return nil, moerr.NewInvalidInputf(ctx, "unknown Arrow column match mode %d", mode) + } + + plan := &Plan{ + schemaFingerprint: schemaFingerprint(schema), + columns: make([]columnPlan, len(targets)), + attrs: make([]string, len(targets)), + } + explicitOutputOrder := false + for _, target := range targets { + explicitOutputOrder = explicitOutputOrder || target.MOIndex != 0 + } + used := make([]bool, schema.NumFields()) + outputUsed := make([]bool, len(targets)) + for targetIndex, target := range targets { + if !explicitOutputOrder { + target.MOIndex = targetIndex + } + if target.MOIndex < 0 || target.MOIndex >= len(targets) { + return nil, moerr.NewInvalidInputf(ctx, "invalid MatrixOne output column index %d", target.MOIndex) + } + // AttrName is allowed to be empty at this transport-neutral boundary, + // so it cannot double as an occupancy sentinel for output ordering. + if outputUsed[target.MOIndex] { + return nil, moerr.NewInvalidInputf(ctx, "duplicate MatrixOne output column index %d", target.MOIndex) + } + outputUsed[target.MOIndex] = true + if target.AttrName == "" { + target.AttrName = target.Name + } + + sourceIndex := targetIndex + if mode == MatchByName { + matches := fieldIndicesFold(schema, target.Name) + if len(matches) == 0 { + return nil, moerr.NewInvalidInputf(ctx, "Arrow field %q is missing", target.Name) + } + if len(matches) != 1 { + return nil, moerr.NewInvalidInputf(ctx, "Arrow field %q is ambiguous", target.Name) + } + sourceIndex = matches[0] + } + if used[sourceIndex] { + return nil, moerr.NewInvalidInputf(ctx, "Arrow source field %q is bound more than once", schema.Field(sourceIndex).Name) + } + used[sourceIndex] = true + + kind, err := selectLoadConversion(schema.Field(sourceIndex).Type, target.Type) + if err != nil { + return nil, moerr.NewNotSupportedf(ctx, "Arrow field %q (%s) to MatrixOne column %q (%s): %v", + schema.Field(sourceIndex).Name, schema.Field(sourceIndex).Type, target.Name, target.Type, err) + } + if err := validateLoadTargetType(ctx, target.Type); err != nil { + return nil, err + } + plan.columns[target.MOIndex] = columnPlan{source: sourceIndex, target: target, kind: kind} + plan.attrs[target.MOIndex] = target.AttrName + } + plan.conversionFingerprint = planFingerprint(plan, mode) + return plan, nil +} + +func validateSchemaShape(ctx context.Context, schema *arrow.Schema) error { + if schema.NumFields() == 0 { + return moerr.NewInvalidInput(ctx, "Arrow schema does not contain fields") + } + type pendingField struct { + field arrow.Field + depth int + } + pending := make([]pendingField, 0, schema.NumFields()) + for _, field := range schema.Fields() { + pending = append(pending, pendingField{field: field, depth: 1}) + } + total := 0 + for len(pending) > 0 { + last := len(pending) - 1 + current := pending[last] + pending = pending[:last] + total++ + if total > MaxFields { + return moerr.NewInvalidInputf(ctx, "Arrow total field count exceeds %d", MaxFields) + } + // BindLoad is also a public in-process boundary; callers may construct an + // Arrow schema directly rather than through the validated IPC decoder. + if current.field.Type == nil { + return moerr.NewInvalidInputf(ctx, "Arrow field %q type is nil", current.field.Name) + } + if err := validateArrowTypeContract(ctx, current.field.Name, current.field.Type); err != nil { + return err + } + if current.depth > MaxNestingDepth { + return moerr.NewInvalidInputf(ctx, + "Arrow field %q nesting depth exceeds %d", current.field.Name, MaxNestingDepth) + } + if nested, ok := current.field.Type.(arrow.NestedType); ok { + children := nested.Fields() + for _, child := range children { + pending = append(pending, pendingField{field: child, depth: current.depth + 1}) + } + } + } + return nil +} + +// validateArrowTypeContract checks type metadata that later conversion code +// relies on for bounded arithmetic. Dictionary value types are not exposed as +// NestedType children by Arrow, so they need an explicit recursive check here. +func validateArrowTypeContract(ctx context.Context, fieldName string, typ arrow.DataType) error { + switch typed := typ.(type) { + case *arrow.Decimal128Type: + if typed.Precision < 1 || typed.Precision > decimal128.MaxPrecision { + return moerr.NewInvalidInputf(ctx, + "Arrow field %q has invalid Decimal128 precision %d", fieldName, typed.Precision) + } + case *arrow.TimestampType: + if !validArrowTimeUnit(typed.Unit) { + return moerr.NewInvalidInputf(ctx, + "Arrow field %q has invalid timestamp time unit %d", fieldName, typed.Unit) + } + // GetZone validates both IANA names and fixed offsets. Do this while + // binding the schema so an invalid timezone cannot survive until the + // first record happens to exercise the timestamp conversion. + if _, err := typed.GetZone(); err != nil { + return moerr.NewInvalidInputf(ctx, + "Arrow field %q has invalid timestamp timezone %q: %v", fieldName, typed.TimeZone, err) + } + case *arrow.Time32Type: + if typed.Unit != arrow.Second && typed.Unit != arrow.Millisecond { + return moerr.NewInvalidInputf(ctx, + "Arrow field %q has invalid time32 time unit %d", fieldName, typed.Unit) + } + case *arrow.Time64Type: + if typed.Unit != arrow.Microsecond && typed.Unit != arrow.Nanosecond { + return moerr.NewInvalidInputf(ctx, + "Arrow field %q has invalid time64 time unit %d", fieldName, typed.Unit) + } + case *arrow.DictionaryType: + if typed.ValueType == nil { + return moerr.NewInvalidInputf(ctx, "Arrow field %q has a dictionary with nil value type", fieldName) + } + return validateArrowTypeContract(ctx, fieldName, typed.ValueType) + } + return nil +} + +func validArrowTimeUnit(unit arrow.TimeUnit) bool { + return unit == arrow.Second || unit == arrow.Millisecond || + unit == arrow.Microsecond || unit == arrow.Nanosecond +} + +func validateLoadTargetType(ctx context.Context, typ types.Type) error { + var expected int32 + switch typ.Oid { + case types.T_bool, types.T_int8, types.T_uint8: + expected = 1 + case types.T_int16, types.T_uint16, types.T_year: + expected = 2 + case types.T_int32, types.T_uint32, types.T_date, types.T_float32: + expected = 4 + case types.T_int64, types.T_uint64, types.T_datetime, types.T_time, types.T_timestamp, + types.T_float64, types.T_decimal64: + expected = 8 + case types.T_decimal128: + expected = 16 + case types.T_char, types.T_varchar, types.T_text, types.T_binary, types.T_varbinary, types.T_blob: + expected = int32(types.VarlenaSize) + default: + return nil + } + if typ.Size != expected { + return moerr.NewInvalidInputf(ctx, + "invalid MatrixOne target type size %d for %s, expected %d", + typ.Size, typ.Oid, expected) + } + return nil +} + +// Fingerprint returns the immutable logical source-schema and conversion +// contract identity. It is safe to serialize into a distributed scan plan. +func (p *Plan) Fingerprint() [sha256.Size]byte { + if p == nil { + return [sha256.Size]byte{} + } + return p.conversionFingerprint +} + +func schemaFingerprint(schema *arrow.Schema) [sha256.Size]byte { + h := sha256.New() + writeFingerprintString(h, "matrixone-arrow-schema-v1") + if schema == nil { + return [sha256.Size]byte{} + } + writeFingerprintUint64(h, uint64(schema.Endianness())) + writeFingerprintMetadata(h, schema.Metadata()) + for _, field := range schema.Fields() { + writeFingerprintField(h, field) + } + var fingerprint [sha256.Size]byte + copy(fingerprint[:], h.Sum(nil)) + return fingerprint +} + +func writeFingerprintField(h hash.Hash, field arrow.Field) { + writeFingerprintString(h, field.Name) + writeFingerprintBool(h, field.Nullable) + if field.Type == nil { + writeFingerprintString(h, "") + writeFingerprintMetadata(h, field.Metadata) + return + } + writeFingerprintString(h, field.Type.Fingerprint()) + writeFingerprintMetadata(h, field.Metadata) + if nested, ok := field.Type.(arrow.NestedType); ok { + for _, child := range nested.Fields() { + writeFingerprintField(h, child) + } + } +} + +func writeFingerprintMetadata(h hash.Hash, metadata arrow.Metadata) { + keys := metadata.Keys() + values := metadata.Values() + indices := make([]int, len(keys)) + for i := range indices { + indices[i] = i + } + sort.Slice(indices, func(i, j int) bool { + if keys[indices[i]] == keys[indices[j]] { + return values[indices[i]] < values[indices[j]] + } + return keys[indices[i]] < keys[indices[j]] + }) + writeFingerprintUint64(h, uint64(len(indices))) + for _, index := range indices { + writeFingerprintString(h, keys[index]) + writeFingerprintString(h, values[index]) + } +} + +func planFingerprint(plan *Plan, mode MatchMode) [sha256.Size]byte { + h := sha256.New() + writeFingerprintString(h, "matrixone-arrow-conversion-plan") + writeFingerprintUint64(h, uint64(ConversionPlanVersion)) + _, _ = h.Write(plan.schemaFingerprint[:]) + writeFingerprintUint64(h, uint64(mode)) + for _, column := range plan.columns { + writeFingerprintUint64(h, uint64(column.source)) + writeFingerprintString(h, column.target.Name) + writeFingerprintString(h, column.target.AttrName) + writeFingerprintUint64(h, uint64(column.target.MOIndex)) + writeFingerprintBool(h, column.target.NotNull) + writeFingerprintUint64(h, uint64(column.target.Type.Oid)) + writeFingerprintUint64(h, uint64(column.target.Type.Charset)) + writeFingerprintUint64(h, uint64(uint32(column.target.Type.Size))) + writeFingerprintUint64(h, uint64(uint32(column.target.Type.Width))) + writeFingerprintUint64(h, uint64(uint32(column.target.Type.Scale))) + writeFingerprintUint64(h, uint64(column.kind)) + } + var fingerprint [sha256.Size]byte + copy(fingerprint[:], h.Sum(nil)) + return fingerprint +} + +func writeFingerprintString(h hash.Hash, value string) { + writeFingerprintUint64(h, uint64(len(value))) + _, _ = h.Write([]byte(value)) +} + +func writeFingerprintUint64(h hash.Hash, value uint64) { + var buffer [8]byte + binary.LittleEndian.PutUint64(buffer[:], value) + _, _ = h.Write(buffer[:]) +} + +func writeFingerprintBool(h hash.Hash, value bool) { + if value { + _, _ = h.Write([]byte{1}) + } else { + _, _ = h.Write([]byte{0}) + } +} + +func fieldIndicesFold(schema *arrow.Schema, name string) []int { + indices := make([]int, 0, 1) + for i, field := range schema.Fields() { + if strings.EqualFold(field.Name, name) { + indices = append(indices, i) + } + } + return indices +} + +// selectLoadConversion is intentionally private: its result is an execution +// kernel choice, not a reusable Arrow ABI declaration. Consumers with an +// exact protocol must first validate their own versioned logical descriptor. +func selectLoadConversion(source arrow.DataType, target types.Type) (conversionKind, error) { + if source.ID() == arrow.DICTIONARY { + dictionary, ok := source.(*arrow.DictionaryType) + if !ok || dictionary.IndexType == nil || dictionary.ValueType == nil { + return 0, moerr.NewInvalidInputNoCtx("invalid Arrow dictionary type") + } + switch dictionary.IndexType.ID() { + case arrow.INT8, arrow.INT16, arrow.INT32, arrow.INT64, + arrow.UINT8, arrow.UINT16, arrow.UINT32, arrow.UINT64: + default: + return 0, moerr.NewInvalidInputNoCtxf("dictionary index type %s is not an integer", dictionary.IndexType) + } + if dictionary.ValueType.ID() == arrow.DICTIONARY { + return 0, moerr.NewInvalidInputNoCtx("nested Arrow dictionaries are not supported") + } + if _, err := selectLoadConversion(dictionary.ValueType, target); err != nil { + return 0, err + } + return conversionMaterializeDictionary, nil + } + if exactFixedLayout(source, target) { + return conversionBorrowFixed, nil + } + switch source.ID() { + case arrow.STRING, arrow.LARGE_STRING: + if target.Oid == types.T_char || target.Oid == types.T_varchar || target.Oid == types.T_text { + return conversionBorrowVarlen, nil + } + case arrow.BINARY, arrow.LARGE_BINARY, arrow.FIXED_SIZE_BINARY: + if target.Oid == types.T_binary || target.Oid == types.T_varbinary || target.Oid == types.T_blob { + return conversionBorrowVarlen, nil + } + case arrow.BOOL: + if target.Oid == types.T_bool { + return conversionMaterializeBool, nil + } + case arrow.DATE32: + if target.Oid == types.T_date || target.Oid == types.T_datetime { + return conversionMaterializeDate32, nil + } + case arrow.DATE64: + if target.Oid == types.T_date || target.Oid == types.T_datetime { + return conversionMaterializeDate64, nil + } + case arrow.TIMESTAMP: + if (target.Oid == types.T_timestamp || target.Oid == types.T_datetime) && + target.Scale >= 0 && target.Scale <= 6 { + return conversionMaterializeTimestamp, nil + } + case arrow.TIME32, arrow.TIME64: + if target.Oid == types.T_time && target.Scale >= 0 && target.Scale <= 6 { + return conversionMaterializeTime, nil + } + case arrow.NULL: + return conversionMaterializeNull, nil + } + if isCheckedWidening(source.ID(), target.Oid) { + return conversionMaterializeWiden, nil + } + return 0, moerr.NewInvalidInputNoCtx("no exact long-term conversion") +} + +func isCheckedWidening(source arrow.Type, target types.T) bool { + switch source { + case arrow.INT8: + return target == types.T_int16 || target == types.T_int32 || target == types.T_int64 + case arrow.INT16: + return target == types.T_int32 || target == types.T_int64 + case arrow.INT32: + return target == types.T_int64 + case arrow.UINT8: + return target == types.T_uint16 || target == types.T_uint32 || target == types.T_uint64 + case arrow.UINT16: + return target == types.T_uint32 || target == types.T_uint64 + case arrow.UINT32: + return target == types.T_uint64 + case arrow.FLOAT32: + return target == types.T_float64 + default: + return false + } +} + +func exactFixedLayout(source arrow.DataType, target types.Type) bool { + matched := false + switch source.ID() { + case arrow.INT8: + matched = target.Oid == types.T_int8 + case arrow.INT16: + matched = target.Oid == types.T_int16 + case arrow.INT32: + matched = target.Oid == types.T_int32 + case arrow.INT64: + matched = target.Oid == types.T_int64 + case arrow.UINT8: + matched = target.Oid == types.T_uint8 + case arrow.UINT16: + matched = target.Oid == types.T_uint16 + case arrow.UINT32: + matched = target.Oid == types.T_uint32 + case arrow.UINT64: + matched = target.Oid == types.T_uint64 + case arrow.FLOAT32: + matched = target.Oid == types.T_float32 + case arrow.FLOAT64: + matched = target.Oid == types.T_float64 + case arrow.DECIMAL128: + decimal, ok := source.(*arrow.Decimal128Type) + matched = ok && target.Oid == types.T_decimal128 && + target.Width == decimal.Precision && target.Scale == decimal.Scale + case arrow.TIME64: + timeType, ok := source.(*arrow.Time64Type) + matched = ok && timeType.Unit == arrow.Microsecond && target.Oid == types.T_time && target.Scale == 6 + } + return matched && source.ID() != arrow.BOOL && target.TypeSize() > 0 +} diff --git a/pkg/container/arrowbridge/bridge_test.go b/pkg/container/arrowbridge/bridge_test.go new file mode 100644 index 0000000000000..90a5d2c30f9a9 --- /dev/null +++ b/pkg/container/arrowbridge/bridge_test.go @@ -0,0 +1,2528 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowbridge + +import ( + "context" + "math/big" + "reflect" + "testing" + "time" + "unsafe" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/decimal128" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/stretchr/testify/require" +) + +// contextCheckBudget turns every cancellation checkpoint into an observable +// unit of work. It lets a regression test distinguish the selected one-row +// window from an accidental rescan of a large immutable dictionary. +type contextCheckBudget struct { + context.Context + limit int + checks int +} + +func (c *contextCheckBudget) Err() error { + c.checks++ + if c.checks > c.limit { + return context.Canceled + } + return nil +} + +func TestBindByNameAndPosition(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{ + {Name: "B", Type: arrow.PrimitiveTypes.Int64}, + {Name: "a", Type: arrow.BinaryTypes.String}, + }, nil) + targets := []TargetColumn{ + {Name: "A", Type: types.T_varchar.ToType(), MOIndex: 1, AttrName: "a_out"}, + {Name: "b", Type: types.T_int64.ToType(), MOIndex: 0, AttrName: "b_out"}, + } + plan, err := Bind(context.Background(), schema, targets, MatchByName) + require.NoError(t, err) + require.Equal(t, []string{"b_out", "a_out"}, plan.attrs) + require.Equal(t, 0, plan.columns[0].source) + require.Equal(t, 1, plan.columns[1].source) + + positionTargets := []TargetColumn{ + {Name: "first", Type: types.T_int64.ToType()}, + {Name: "second", Type: types.T_varchar.ToType()}, + } + positionPlan, err := Bind(context.Background(), schema, positionTargets, MatchByPosition) + require.NoError(t, err) + require.Equal(t, []string{"first", "second"}, positionPlan.attrs) +} + +func TestAppendDictionaryNullSupportsEveryTargetStorageClass(t *testing.T) { + // Dictionary NULLs bypass their physical Arrow value type. Exercise every + // MatrixOne storage representation so a nullable dictionary remains a + // logical NULL regardless of the target column type. + for _, oid := range []types.T{ + types.T_varchar, + types.T_bool, types.T_bit, + types.T_int8, types.T_int16, types.T_int32, types.T_int64, + types.T_uint8, types.T_uint16, types.T_uint32, types.T_uint64, + types.T_float32, types.T_float64, + types.T_year, types.T_enum, + types.T_decimal64, types.T_decimal128, types.T_decimal256, + types.T_uuid, types.T_TS, types.T_Rowid, types.T_Blockid, + types.T_date, types.T_time, types.T_datetime, types.T_timestamp, + } { + t.Run(oid.String(), func(t *testing.T) { + mp := mpool.MustNewZero() + vec := vector.NewVec(oid.ToType()) + require.NoError(t, appendDictionaryNull(vec, oid.ToType(), mp)) + require.Equal(t, 1, vec.Length()) + require.True(t, vec.IsNull(0)) + vec.Free(mp) + require.Equal(t, int64(0), mp.CurrNB()) + }) + } +} + +func TestCheckedDictionaryIndexAcceptsEveryArrowIndexWidth(t *testing.T) { + tests := []struct { + name string + build func(memory.Allocator) arrow.Array + }{ + {"int8", func(alloc memory.Allocator) arrow.Array { + b := array.NewInt8Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + {"int16", func(alloc memory.Allocator) arrow.Array { + b := array.NewInt16Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + {"int32", func(alloc memory.Allocator) arrow.Array { + b := array.NewInt32Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + {"int64", func(alloc memory.Allocator) arrow.Array { + b := array.NewInt64Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + {"uint8", func(alloc memory.Allocator) arrow.Array { + b := array.NewUint8Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + {"uint16", func(alloc memory.Allocator) arrow.Array { + b := array.NewUint16Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + {"uint32", func(alloc memory.Allocator) arrow.Array { + b := array.NewUint32Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + {"uint64", func(alloc memory.Allocator) arrow.Array { + b := array.NewUint64Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indices := test.build(alloc) + valuesBuilder := array.NewInt64Builder(alloc) + valuesBuilder.Append(7) + values := valuesBuilder.NewArray() + valuesBuilder.Release() + dictionaryType := &arrow.DictionaryType{IndexType: indices.DataType(), ValueType: values.DataType()} + dictionary := array.NewDictionaryArray(dictionaryType, indices, values) + + index, err := checkedDictionaryIndex(context.Background(), dictionary, 0, values.Len()) + require.NoError(t, err) + require.Equal(t, 0, index) + + dictionary.Release() + indices.Release() + values.Release() + alloc.AssertSize(t, 0) + }) + } +} + +func TestCheckedDictionaryIndexRejectsNegativeAndOverflowedValues(t *testing.T) { + tests := []struct { + name string + build func(memory.Allocator) arrow.Array + err string + }{ + {"negative", func(alloc memory.Allocator) arrow.Array { + b := array.NewInt8Builder(alloc) + defer b.Release() + b.Append(-1) + return b.NewArray() + }, "outside"}, + {"overflow", func(alloc memory.Allocator) arrow.Array { + b := array.NewUint64Builder(alloc) + defer b.Release() + b.Append(^uint64(0)) + return b.NewArray() + }, "overflows"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indices := test.build(alloc) + valuesBuilder := array.NewInt64Builder(alloc) + valuesBuilder.Append(7) + values := valuesBuilder.NewArray() + valuesBuilder.Release() + dictionaryType := &arrow.DictionaryType{IndexType: indices.DataType(), ValueType: values.DataType()} + dictionary := array.NewDictionaryArray(dictionaryType, indices, values) + + _, err := checkedDictionaryIndex(context.Background(), dictionary, 0, values.Len()) + require.ErrorContains(t, err, test.err) + + dictionary.Release() + indices.Release() + values.Release() + alloc.AssertSize(t, 0) + }) + } +} + +func TestAppendDictionaryNullRejectsUnknownTarget(t *testing.T) { + vec := new(vector.Vector) + require.ErrorContains(t, appendDictionaryNull(vec, types.T_any.ToType(), mpool.MustNewZero()), "unknown") +} + +func TestAppendDictionaryValuePreservesPrimitiveArrowValues(t *testing.T) { + tests := []struct { + name string + target types.T + build func(memory.Allocator) arrow.Array + }{ + {"int8", types.T_int8, func(alloc memory.Allocator) arrow.Array { + b := array.NewInt8Builder(alloc) + defer b.Release() + b.Append(7) + return b.NewArray() + }}, + {"int16", types.T_int16, func(alloc memory.Allocator) arrow.Array { + b := array.NewInt16Builder(alloc) + defer b.Release() + b.Append(7) + return b.NewArray() + }}, + {"int32", types.T_int32, func(alloc memory.Allocator) arrow.Array { + b := array.NewInt32Builder(alloc) + defer b.Release() + b.Append(7) + return b.NewArray() + }}, + {"int64", types.T_int64, func(alloc memory.Allocator) arrow.Array { + b := array.NewInt64Builder(alloc) + defer b.Release() + b.Append(7) + return b.NewArray() + }}, + {"uint8", types.T_uint8, func(alloc memory.Allocator) arrow.Array { + b := array.NewUint8Builder(alloc) + defer b.Release() + b.Append(7) + return b.NewArray() + }}, + {"uint16", types.T_uint16, func(alloc memory.Allocator) arrow.Array { + b := array.NewUint16Builder(alloc) + defer b.Release() + b.Append(7) + return b.NewArray() + }}, + {"uint32", types.T_uint32, func(alloc memory.Allocator) arrow.Array { + b := array.NewUint32Builder(alloc) + defer b.Release() + b.Append(7) + return b.NewArray() + }}, + {"uint64", types.T_uint64, func(alloc memory.Allocator) arrow.Array { + b := array.NewUint64Builder(alloc) + defer b.Release() + b.Append(7) + return b.NewArray() + }}, + {"float32", types.T_float32, func(alloc memory.Allocator) arrow.Array { + b := array.NewFloat32Builder(alloc) + defer b.Release() + b.Append(7) + return b.NewArray() + }}, + {"float64", types.T_float64, func(alloc memory.Allocator) arrow.Array { + b := array.NewFloat64Builder(alloc) + defer b.Release() + b.Append(7) + return b.NewArray() + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + values := test.build(alloc) + mp := mpool.MustNewZero() + vec := vector.NewVec(test.target.ToType()) + + copied, err := appendDictionaryValue(context.Background(), vec, values, conversionMaterializeBool, 0, 0, false, test.target.ToType(), mp, time.UTC) + require.NoError(t, err) + require.Zero(t, copied) + require.Equal(t, 1, vec.Length()) + + vec.Free(mp) + values.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) + }) + } +} + +func TestAppendDictionaryValuePreservesVariableWidthArrowValues(t *testing.T) { + tests := []struct { + name string + build func(memory.Allocator) arrow.Array + }{ + {"string", func(alloc memory.Allocator) arrow.Array { + b := array.NewStringBuilder(alloc) + defer b.Release() + b.Append("value") + return b.NewArray() + }}, + {"large_string", func(alloc memory.Allocator) arrow.Array { + b := array.NewLargeStringBuilder(alloc) + defer b.Release() + b.Append("value") + return b.NewArray() + }}, + {"binary", func(alloc memory.Allocator) arrow.Array { + b := array.NewBinaryBuilder(alloc, arrow.BinaryTypes.Binary) + defer b.Release() + b.Append([]byte("value")) + return b.NewArray() + }}, + {"large_binary", func(alloc memory.Allocator) arrow.Array { + b := array.NewBinaryBuilder(alloc, arrow.BinaryTypes.LargeBinary) + defer b.Release() + b.Append([]byte("value")) + return b.NewArray() + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + values := test.build(alloc) + mp := mpool.MustNewZero() + target := types.New(types.T_varchar, 16, 0) + vec := vector.NewVec(target) + + _, err := appendDictionaryValue(context.Background(), vec, values, conversionBorrowVarlen, 0, 0, false, target, mp, time.UTC) + require.NoError(t, err) + require.Equal(t, 1, vec.Length()) + + vec.Free(mp) + values.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) + }) + } +} + +func TestArrowTemporalHelpersRejectOutOfRangeValues(t *testing.T) { + maxInt64 := int64(^uint64(0) >> 1) + _, err := timestampToMicros(maxInt64, arrow.Second) + require.Error(t, err) + _, err = timestampToMicros(maxInt64, arrow.Millisecond) + require.Error(t, err) + _, err = arrowMicrosToTimestamp(maxSupportedUnixMicros+1, true, time.UTC) + require.Error(t, err) + _, err = arrowMicrosToDatetime(maxSupportedUnixMicros+1, true, time.UTC) + require.Error(t, err) +} + +func TestAppendDictionaryValueConvertsDateAndTimeValues(t *testing.T) { + tests := []struct { + name string + target types.T + build func(memory.Allocator) arrow.Array + }{ + {"date32_to_date", types.T_date, func(alloc memory.Allocator) arrow.Array { + b := array.NewDate32Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + {"date32_to_datetime", types.T_datetime, func(alloc memory.Allocator) arrow.Array { + b := array.NewDate32Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + {"date64_to_date", types.T_date, func(alloc memory.Allocator) arrow.Array { + b := array.NewDate64Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + {"date64_to_datetime", types.T_datetime, func(alloc memory.Allocator) arrow.Array { + b := array.NewDate64Builder(alloc) + defer b.Release() + b.Append(0) + return b.NewArray() + }}, + {"time64", types.T_time, func(alloc memory.Allocator) arrow.Array { + b := array.NewTime64Builder(alloc, &arrow.Time64Type{Unit: arrow.Microsecond}) + defer b.Release() + b.Append(1) + return b.NewArray() + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + values := test.build(alloc) + mp := mpool.MustNewZero() + vec := vector.NewVec(test.target.ToType()) + + copied, err := appendDictionaryValue(context.Background(), vec, values, conversionMaterializeBool, 0, 0, false, test.target.ToType(), mp, time.UTC) + require.NoError(t, err) + require.Zero(t, copied) + require.Equal(t, 1, vec.Length()) + + vec.Free(mp) + values.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) + }) + } +} + +func TestAppendDictionaryValueHandlesDecimalAndRejectsInvalidTemporalValues(t *testing.T) { + t.Run("decimal", func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + decimalType := &arrow.Decimal128Type{Precision: 18, Scale: 2} + builder := array.NewDecimal128Builder(alloc, decimalType) + builder.Append(decimal128.FromBigInt(big.NewInt(7))) + values := builder.NewArray() + builder.Release() + mp := mpool.MustNewZero() + target := types.New(types.T_decimal128, 18, 2) + vec := vector.NewVec(target) + + copied, err := appendDictionaryValue(context.Background(), vec, values, conversionMaterializeBool, 0, 0, false, target, mp, time.UTC) + require.NoError(t, err) + require.Zero(t, copied) + require.Equal(t, 1, vec.Length()) + + vec.Free(mp) + values.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) + }) + + for _, test := range []struct { + name string + target types.T + build func(memory.Allocator) arrow.Array + err string + }{ + {"fractional_date64", types.T_date, func(alloc memory.Allocator) arrow.Array { + b := array.NewDate64Builder(alloc) + defer b.Release() + b.Append(1) + return b.NewArray() + }, "integral"}, + {"out_of_range_time64", types.T_time, func(alloc memory.Allocator) arrow.Array { + b := array.NewTime64Builder(alloc, &arrow.Time64Type{Unit: arrow.Microsecond}) + defer b.Release() + b.Append(-1) + return b.NewArray() + }, "outside"}, + } { + t.Run(test.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + values := test.build(alloc) + mp := mpool.MustNewZero() + vec := vector.NewVec(test.target.ToType()) + + _, err := appendDictionaryValue(context.Background(), vec, values, conversionMaterializeBool, 0, 0, false, test.target.ToType(), mp, time.UTC) + require.ErrorContains(t, err, test.err) + + vec.Free(mp) + values.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) + }) + } + + t.Run("unsupported_value_type", func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewDurationBuilder(alloc, &arrow.DurationType{Unit: arrow.Microsecond}) + builder.Append(1) + values := builder.NewArray() + builder.Release() + mp := mpool.MustNewZero() + vec := vector.NewVec(types.T_int64.ToType()) + + _, err := appendDictionaryValue(context.Background(), vec, values, conversionMaterializeBool, 0, 0, false, types.T_int64.ToType(), mp, time.UTC) + require.ErrorContains(t, err, "invalid Arrow dictionary values") + + vec.Free(mp) + values.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) + }) +} + +func TestPlanFingerprintCoversSchemaMetadataAndConversionContract(t *testing.T) { + ctx := context.Background() + fieldMetadata := arrow.NewMetadata([]string{"field-key"}, []string{"field-value"}) + schemaMetadata := arrow.NewMetadata([]string{"schema-key"}, []string{"schema-value"}) + baseSchema := arrow.NewSchema([]arrow.Field{{ + Name: "v", Type: arrow.BinaryTypes.String, Nullable: true, Metadata: fieldMetadata, + }}, &schemaMetadata) + baseTarget := []TargetColumn{{ + Name: "v", AttrName: "v", Type: types.New(types.T_varchar, 100, 0), NotNull: false, + }} + base, err := Bind(ctx, baseSchema, baseTarget, MatchByName) + require.NoError(t, err) + require.NotEqual(t, [32]byte{}, base.Fingerprint()) + + // Metadata ordering is not a semantic schema change. + reorderedMetadata := arrow.NewMetadata( + []string{"second", "schema-key"}, []string{"two", "schema-value"}, + ) + reorderedSchema := arrow.NewSchema([]arrow.Field{{ + Name: "v", Type: arrow.BinaryTypes.String, Nullable: true, Metadata: fieldMetadata, + }}, &reorderedMetadata) + reorderedBaseMetadata := arrow.NewMetadata( + []string{"schema-key", "second"}, []string{"schema-value", "two"}, + ) + reorderedBaseSchema := arrow.NewSchema([]arrow.Field{{ + Name: "v", Type: arrow.BinaryTypes.String, Nullable: true, Metadata: fieldMetadata, + }}, &reorderedBaseMetadata) + first, err := Bind(ctx, reorderedSchema, baseTarget, MatchByName) + require.NoError(t, err) + second, err := Bind(ctx, reorderedBaseSchema, baseTarget, MatchByName) + require.NoError(t, err) + require.Equal(t, first.Fingerprint(), second.Fingerprint()) + + changedFieldMetadata := arrow.NewMetadata([]string{"field-key"}, []string{"changed"}) + changedSchema := arrow.NewSchema([]arrow.Field{{ + Name: "v", Type: arrow.BinaryTypes.String, Nullable: true, Metadata: changedFieldMetadata, + }}, &schemaMetadata) + changed, err := Bind(ctx, changedSchema, baseTarget, MatchByName) + require.NoError(t, err) + require.NotEqual(t, base.Fingerprint(), changed.Fingerprint()) + + widerTarget := append([]TargetColumn(nil), baseTarget...) + widerTarget[0].Type = types.New(types.T_varchar, 101, 0) + wider, err := Bind(ctx, baseSchema, widerTarget, MatchByName) + require.NoError(t, err) + require.NotEqual(t, base.Fingerprint(), wider.Fingerprint()) + + position, err := Bind(ctx, baseSchema, baseTarget, MatchByPosition) + require.NoError(t, err) + require.NotEqual(t, base.Fingerprint(), position.Fingerprint()) +} + +func TestBindRejectsAmbiguousMissingAndUnsupported(t *testing.T) { + ctx := context.Background() + _, err := Bind(ctx, arrow.NewSchema([]arrow.Field{ + {Name: "A", Type: arrow.PrimitiveTypes.Int64}, + {Name: "a", Type: arrow.PrimitiveTypes.Int64}, + }, nil), []TargetColumn{ + {Name: "a", Type: types.T_int64.ToType()}, + {Name: "x", Type: types.T_int64.ToType()}, + }, MatchByName) + require.ErrorContains(t, err, "ambiguous") + + _, err = Bind(ctx, arrow.NewSchema([]arrow.Field{{Name: "a", Type: arrow.PrimitiveTypes.Int64}}, nil), + []TargetColumn{{Name: "missing", Type: types.T_int64.ToType()}}, MatchByName) + require.ErrorContains(t, err, "missing") + + _, err = Bind(ctx, arrow.NewSchema([]arrow.Field{{Name: "a", Type: arrow.PrimitiveTypes.Int64}}, nil), + []TargetColumn{{Name: "a", Type: types.T_int32.ToType()}}, MatchByName) + require.ErrorContains(t, err, "no exact long-term conversion") + + _, err = Bind(ctx, arrow.NewSchema([]arrow.Field{{Name: "a", Type: arrow.PrimitiveTypes.Int64}}, nil), + []TargetColumn{{Name: "a", Type: types.T_int64.ToType()}, {Name: "b", Type: types.T_int64.ToType()}}, MatchByName) + require.ErrorContains(t, err, "target has 2") +} + +func TestBindRejectsDuplicateOutputIndexWhenAttributeNameIsEmpty(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{ + {Name: "", Type: arrow.PrimitiveTypes.Int64}, + {Name: "", Type: arrow.PrimitiveTypes.Int64}, + {Name: "third", Type: arrow.PrimitiveTypes.Int64}, + }, nil) + _, err := BindLoad(context.Background(), schema, []TargetColumn{ + {Name: "", Type: types.T_int64.ToType(), MOIndex: 0}, + {Name: "", Type: types.T_int64.ToType(), MOIndex: 0}, + {Name: "third", Type: types.T_int64.ToType(), MOIndex: 1}, + }, MatchByPosition) + require.ErrorContains(t, err, "duplicate MatrixOne output column index 0") +} + +func TestBindRejectsMalformedFixedTargetSize(t *testing.T) { + target := types.T_int64.ToType() + target.Size = 1 + _, err := BindLoad(context.Background(), arrow.NewSchema([]arrow.Field{{ + Name: "value", Type: arrow.PrimitiveTypes.Int64, + }}, nil), []TargetColumn{{Name: "value", Type: target}}, MatchByName) + require.ErrorContains(t, err, "invalid MatrixOne target type size") +} + +func TestBindBoundsTotalFieldsAndNestingBeforeFingerprint(t *testing.T) { + allowed := arrow.DataType(arrow.PrimitiveTypes.Int64) + for range MaxNestingDepth - 1 { + allowed = arrow.ListOf(allowed) + } + _, err := Bind(context.Background(), arrow.NewSchema([]arrow.Field{{Name: "v", Type: allowed}}, nil), + []TargetColumn{{Name: "v", Type: types.T_json.ToType()}}, MatchByName) + require.ErrorContains(t, err, "no exact long-term conversion", + "a schema at the limit must reach ordinary type validation") + + tooDeep := arrow.ListOf(allowed) + _, err = Bind(context.Background(), arrow.NewSchema([]arrow.Field{{Name: "v", Type: tooDeep}}, nil), + []TargetColumn{{Name: "v", Type: types.T_json.ToType()}}, MatchByName) + require.ErrorContains(t, err, "nesting depth exceeds") + + children := make([]arrow.Field, MaxFields) + for index := range children { + children[index] = arrow.Field{Name: "f", Type: arrow.PrimitiveTypes.Int8} + } + tooMany := arrow.StructOf(children...) + _, err = Bind(context.Background(), arrow.NewSchema([]arrow.Field{{Name: "v", Type: tooMany}}, nil), + []TargetColumn{{Name: "v", Type: types.T_json.ToType()}}, MatchByName) + require.ErrorContains(t, err, "total field count exceeds") +} + +func TestFixedBorrowValidityLifetimeAndWindow(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewInt64Builder(alloc) + builder.AppendValues([]int64{10, 20, 30, 40}, []bool{true, false, true, true}) + base := builder.NewArray() + builder.Release() + sliced := array.NewSlice(base, 1, 4) + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: arrow.PrimitiveTypes.Int64, Nullable: true}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{sliced}, 3) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "v", Type: types.T_int64.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + bat, stats, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.Equal(t, int64(24), stats.BorrowedPayloadBytes) + require.Equal(t, int64(24), stats.EligiblePayloadBytes) + require.Equal(t, int64(1), stats.BorrowedColumns) + require.Zero(t, stats.MaterializedColumns) + require.Zero(t, stats.PinAmplificationFallbacks) + require.Zero(t, stats.UnalignedFallbacks) + require.True(t, bat.Vecs[0].HasBorrowedBacking()) + require.True(t, bat.Vecs[0].GetNulls().HasBorrowedValidity()) + expectedData := sliced.Data().Buffers()[1].Bytes()[sliced.Data().Offset()*8:] + require.Equal(t, + uintptr(unsafe.Pointer(unsafe.SliceData(expectedData))), + uintptr(unsafe.Pointer(unsafe.SliceData(bat.Vecs[0].GetData()))), + ) + require.Equal(t, []int64{20, 30, 40}, vector.MustFixedColNoTypeCheck[int64](bat.Vecs[0])) + require.True(t, bat.Vecs[0].IsNull(0)) + require.True(t, bat.Vecs[0].GetNulls().HasBorrowedValidity(), "Contains must preserve the validity view") + + window, err := bat.Vecs[0].Window(1, 3) + require.NoError(t, err) + record.Release() + sliced.Release() + base.Release() + bat.Clean(mp) + require.Equal(t, []int64{30, 40}, vector.MustFixedColNoTypeCheck[int64](window)) + require.False(t, window.IsNull(0)) + window.Free(mp) + require.Equal(t, int64(0), mp.CurrNB()) + alloc.AssertSize(t, 0) +} + +func TestExactFixedWidthTypeMatrixBorrowsPayload(t *testing.T) { + tests := []struct { + name string + source arrow.DataType + target types.Type + append func(array.Builder) + }{ + {name: "int8", source: arrow.PrimitiveTypes.Int8, target: types.T_int8.ToType(), append: func(b array.Builder) { b.(*array.Int8Builder).Append(-8) }}, + {name: "int16", source: arrow.PrimitiveTypes.Int16, target: types.T_int16.ToType(), append: func(b array.Builder) { b.(*array.Int16Builder).Append(-16) }}, + {name: "int32", source: arrow.PrimitiveTypes.Int32, target: types.T_int32.ToType(), append: func(b array.Builder) { b.(*array.Int32Builder).Append(-32) }}, + {name: "int64", source: arrow.PrimitiveTypes.Int64, target: types.T_int64.ToType(), append: func(b array.Builder) { b.(*array.Int64Builder).Append(-64) }}, + {name: "uint8", source: arrow.PrimitiveTypes.Uint8, target: types.T_uint8.ToType(), append: func(b array.Builder) { b.(*array.Uint8Builder).Append(8) }}, + {name: "uint16", source: arrow.PrimitiveTypes.Uint16, target: types.T_uint16.ToType(), append: func(b array.Builder) { b.(*array.Uint16Builder).Append(16) }}, + {name: "uint32", source: arrow.PrimitiveTypes.Uint32, target: types.T_uint32.ToType(), append: func(b array.Builder) { b.(*array.Uint32Builder).Append(32) }}, + {name: "uint64", source: arrow.PrimitiveTypes.Uint64, target: types.T_uint64.ToType(), append: func(b array.Builder) { b.(*array.Uint64Builder).Append(64) }}, + {name: "float32", source: arrow.PrimitiveTypes.Float32, target: types.T_float32.ToType(), append: func(b array.Builder) { b.(*array.Float32Builder).Append(3.25) }}, + {name: "float64", source: arrow.PrimitiveTypes.Float64, target: types.T_float64.ToType(), append: func(b array.Builder) { b.(*array.Float64Builder).Append(6.5) }}, + { + name: "decimal128", + source: &arrow.Decimal128Type{Precision: 18, Scale: 2}, + target: types.New(types.T_decimal128, 18, 2), + append: func(b array.Builder) { b.(*array.Decimal128Builder).Append(decimal128.FromI64(-12345)) }, + }, + { + name: "time64 microsecond", + source: &arrow.Time64Type{Unit: arrow.Microsecond}, + target: types.New(types.T_time, 0, 6), + append: func(b array.Builder) { b.(*array.Time64Builder).Append(12_345_678) }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewBuilder(alloc, test.source) + test.append(builder) + values := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: test.source}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{values}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "v", Type: test.target}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + bat, stats, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.True(t, bat.Vecs[0].HasBorrowedBacking()) + require.Equal(t, int64(test.target.TypeSize()), stats.BorrowedPayloadBytes) + require.Equal(t, int64(test.target.TypeSize()), stats.EligiblePayloadBytes) + require.Equal(t, int64(1), stats.BorrowedColumns) + require.Zero(t, stats.MaterializedColumns) + require.Zero(t, stats.MaterializedPayloadBytes) + sourceData := values.Data().Buffers()[1].Bytes() + require.Equal(t, + uintptr(unsafe.Pointer(unsafe.SliceData(sourceData))), + uintptr(unsafe.Pointer(unsafe.SliceData(bat.Vecs[0].GetData()))), + ) + + record.Release() + values.Release() + bat.Clean(mp) + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) + }) + } +} + +func TestDecimalExactLayoutContractRejectsRescale(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{ + Name: "d", Type: &arrow.Decimal128Type{Precision: 18, Scale: 2}, + }}, nil) + _, err := Bind(context.Background(), schema, []TargetColumn{{ + Name: "d", Type: types.New(types.T_decimal128, 18, 3), + }}, MatchByName) + require.ErrorContains(t, err, "no exact long-term conversion") +} + +func TestBindRejectsInvalidDecimal128Precision(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{ + Name: "d", Type: &arrow.Decimal128Type{Precision: decimal128.MaxPrecision + 1, Scale: 0}, + }}, nil) + _, err := Bind(context.Background(), schema, []TargetColumn{{ + Name: "d", Type: types.New(types.T_decimal128, decimal128.MaxPrecision+1, 0), + }}, MatchByName) + require.ErrorContains(t, err, "precision") +} + +func TestDecimalExactLayoutRejectsValuePrecisionOverflow(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + decimalType := &arrow.Decimal128Type{Precision: 18, Scale: 2} + builder := array.NewDecimal128Builder(alloc, decimalType) + // The raw scaled value has 19 digits and cannot fit DECIMAL(18,2). + builder.Append(decimal128.FromBigInt(big.NewInt(1_000_000_000_000_000_000))) + values := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "d", Type: decimalType}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{values}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "d", Type: types.New(types.T_decimal128, 18, 2)}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + _, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.ErrorContains(t, err, "precision") + require.Zero(t, mp.CurrNB()) + record.Release() + values.Release() + alloc.AssertSize(t, 0) +} + +func TestDictionaryDecimalPrecisionOverflow(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + decimalType := &arrow.Decimal128Type{Precision: 18, Scale: 2} + valueBuilder := array.NewDecimal128Builder(alloc, decimalType) + valueBuilder.Append(decimal128.FromBigInt(big.NewInt(1_000_000_000_000_000_000))) + values := valueBuilder.NewArray() + valueBuilder.Release() + indexBuilder := array.NewInt8Builder(alloc) + indexBuilder.Append(0) + indices := indexBuilder.NewArray() + indexBuilder.Release() + dictType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: decimalType} + dictionary := array.NewDictionaryArray(dictType, indices, values) + indices.Release() + values.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "d", Type: dictType}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "d", Type: types.New(types.T_decimal128, 18, 2)}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + _, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.ErrorContains(t, err, "precision") + require.Zero(t, mp.CurrNB()) + record.Release() + dictionary.Release() + alloc.AssertSize(t, 0) +} + +func TestFixedExplicitCOW(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewInt32Builder(alloc) + builder.AppendValues([]int32{1, 2}, nil) + arr := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: arrow.PrimitiveTypes.Int32}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{arr}, 2) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "v", Type: types.T_int32.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.NoError(t, bat.Vecs[0].MaterializeOwned(mp)) + require.False(t, bat.Vecs[0].HasBorrowedBacking()) + vector.SetFixedAtWithTypeCheck(bat.Vecs[0], 0, int32(9)) + require.Equal(t, []int32{9, 2}, vector.MustFixedColNoTypeCheck[int32](bat.Vecs[0])) + record.Release() + arr.Release() + bat.Clean(mp) + require.Equal(t, int64(0), mp.CurrNB()) + alloc.AssertSize(t, 0) +} + +func TestMaterializedConversionUsesStatementAllocationSelection(t *testing.T) { + registry, err := mpool.NewAllocationAccountRegistry(1, 32) + require.NoError(t, err) + account, err := registry.Open(1 << 20) + require.NoError(t, err) + selection, err := vector.NewAllocationAccountSelection( + account, mpool.AllocationOwnerExternal, 20, 21, 22, 23, + ) + require.NoError(t, err) + allocator := memory.NewGoAllocator() + builder := array.NewBooleanBuilder(allocator) + builder.AppendValues([]bool{true, false, true}, []bool{true, false, true}) + values := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: arrow.FixedWidthTypes.Boolean, Nullable: true}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{values}, 3) + plan, err := Bind(context.Background(), schema, []TargetColumn{{ + Name: "v", Type: types.T_bool.ToType(), + }}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{Allocation: selection}) + require.NoError(t, err) + require.Same(t, selection, bat.Vecs[0].AllocationAccountSelection()) + require.Positive(t, account.Snapshot().Used) + record.Release() + values.Release() + bat.Clean(mp) + require.Zero(t, account.Snapshot().Used) + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) +} + +func TestConvertReleasesUnpublishedVectorWhenPreExtendIsRejected(t *testing.T) { + registry, err := mpool.NewAllocationAccountRegistry(1, 16) + require.NoError(t, err) + // One byte cannot admit either a fixed-width value buffer or a varlena + // descriptor. This deterministically fails while the new vector is still + // owned by the conversion helper rather than by the output batch. + account, err := registry.Open(1) + require.NoError(t, err) + selection, err := vector.NewAllocationAccountSelection( + account, mpool.AllocationOwnerExternal, 20, 21, 22, 23, + ) + require.NoError(t, err) + + allocator := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewInt64Builder(allocator) + builder.Append(42) + values := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: arrow.PrimitiveTypes.Int64}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{values}, 1) + plan, err := BindLoad(context.Background(), schema, []TargetColumn{{ + Name: "v", Type: types.T_int64.ToType(), + }}, MatchByName) + require.NoError(t, err) + + mp := mpool.MustNewZero() + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{ + Allocation: selection, + ForceMaterialize: true, + }) + require.Error(t, err) + require.Nil(t, bat) + require.Zero(t, account.Snapshot().Used) + require.Zero(t, mp.CurrNB()) + + record.Release() + values.Release() + allocator.AssertSize(t, 0) + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) +} + +func TestConvertReleasesBorrowedVarlenVectorWhenDescriptorAdmissionIsRejected(t *testing.T) { + registry, err := mpool.NewAllocationAccountRegistry(1, 16) + require.NoError(t, err) + account, err := registry.Open(1) + require.NoError(t, err) + selection, err := vector.NewAllocationAccountSelection( + account, mpool.AllocationOwnerExternal, 20, 21, 22, 23, + ) + require.NoError(t, err) + + allocator := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewStringBuilder(allocator) + builder.Append("a borrowed Arrow value that is longer than the inline threshold") + values := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: arrow.BinaryTypes.String}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{values}, 1) + plan, err := BindLoad(context.Background(), schema, []TargetColumn{{ + Name: "v", Type: types.T_varchar.ToType(), + }}, MatchByName) + require.NoError(t, err) + + mp := mpool.MustNewZero() + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{ + Allocation: selection, + MaxPinAmplification: 100, + }) + require.Error(t, err) + require.Nil(t, bat) + require.Zero(t, account.Snapshot().Used) + require.Zero(t, mp.CurrNB()) + + record.Release() + values.Release() + allocator.AssertSize(t, 0) + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) +} + +func TestVarlenBorrowLongInlineShortAndLifetime(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewStringBuilder(alloc) + long := "this payload is definitely longer than twenty three bytes" + builder.AppendValues([]string{"tiny", long, "also tiny"}, []bool{true, true, false}) + arr := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "s", Type: arrow.BinaryTypes.String, Nullable: true}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{arr}, 3) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "s", Type: types.T_varchar.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + bat, stats, err := plan.Convert(context.Background(), record, mp, ConvertOptions{MaxPinAmplification: 100}) + require.NoError(t, err) + require.Equal(t, int64(len(long)), stats.BorrowedPayloadBytes) + require.Equal(t, int64(len(long)), stats.EligiblePayloadBytes) + require.Equal(t, int64(len("tiny")), stats.MaterializedPayloadBytes) + require.Equal(t, int64(1), stats.BorrowedColumns) + require.Zero(t, stats.MaterializedColumns) + require.True(t, bat.Vecs[0].HasBorrowedBacking()) + descriptors, area := vector.MustVarlenaRawData(bat.Vecs[0]) + require.True(t, descriptors[0].IsSmall()) + require.False(t, descriptors[1].IsSmall()) + require.Equal(t, uintptr(unsafe.Pointer(unsafe.SliceData(arr.(*array.String).ValueBytes()))), + uintptr(unsafe.Pointer(unsafe.SliceData(area)))) + require.Equal(t, "tiny", string(bat.Vecs[0].GetBytesAt(0))) + require.Equal(t, long, string(bat.Vecs[0].GetBytesAt(1))) + require.True(t, bat.Vecs[0].IsNull(2)) + + record.Release() + arr.Release() + require.Equal(t, long, string(bat.Vecs[0].GetBytesAt(1)), "vector lease must outlive Arrow record") + bat.Clean(mp) + require.Equal(t, int64(0), mp.CurrNB()) + alloc.AssertSize(t, 0) +} + +func TestVarlenPinAmplificationMaterializes(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewStringBuilder(alloc) + long := "a single long payload longer than twenty three bytes" + builder.Append(long) + arr := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "s", Type: arrow.BinaryTypes.String}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{arr}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "s", Type: types.T_varchar.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + bat, stats, err := plan.Convert(context.Background(), record, mp, ConvertOptions{MaxPinAmplification: 0.0001}) + require.NoError(t, err) + require.False(t, bat.Vecs[0].HasBorrowedBacking()) + require.Zero(t, stats.BorrowedPayloadBytes) + require.Equal(t, int64(len(long)), stats.EligiblePayloadBytes) + require.Equal(t, int64(len(long)), stats.MaterializedPayloadBytes) + require.Equal(t, int64(1), stats.MaterializedColumns) + require.Equal(t, int64(1), stats.PinAmplificationFallbacks) + require.Equal(t, long, string(bat.Vecs[0].GetBytesAt(0))) + record.Release() + arr.Release() + bat.Clean(mp) + require.Equal(t, int64(0), mp.CurrNB()) + alloc.AssertSize(t, 0) +} + +func TestFixedBinaryConversionPadsToDeclaredWidth(t *testing.T) { + for _, forceMaterialize := range []bool{false, true} { + t.Run(map[bool]string{false: "normal", true: "forced-materialize"}[forceMaterialize], func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewBinaryBuilder(alloc, arrow.BinaryTypes.Binary) + value := []byte("payload-longer-than-inline") + builder.Append(value) + arr := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "b", Type: arrow.BinaryTypes.Binary}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{arr}, 1) + target := types.New(types.T_binary, int32(len(value)+3), 0) + plan, err := BindLoad(context.Background(), schema, + []TargetColumn{{Name: "b", Type: target}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{ + ForceMaterialize: forceMaterialize, + MaxPinAmplification: 100, + }) + require.NoError(t, err) + expected := append(append([]byte(nil), value...), 0, 0, 0) + require.Equal(t, expected, bat.Vecs[0].GetBytesAt(0)) + require.False(t, bat.Vecs[0].HasBorrowedBacking(), + "padding requires an owned MatrixOne value") + + bat.Clean(mp) + record.Release() + arr.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) + }) + } + + t.Run("dictionary", func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indicesBuilder := array.NewInt8Builder(alloc) + indicesBuilder.Append(0) + indices := indicesBuilder.NewArray() + indicesBuilder.Release() + valuesBuilder := array.NewBinaryBuilder(alloc, arrow.BinaryTypes.Binary) + valuesBuilder.Append([]byte("ab")) + values := valuesBuilder.NewArray() + valuesBuilder.Release() + dictionaryType := &arrow.DictionaryType{ + IndexType: arrow.PrimitiveTypes.Int8, + ValueType: arrow.BinaryTypes.Binary, + } + dictionary := array.NewDictionaryArray(dictionaryType, indices, values) + schema := arrow.NewSchema([]arrow.Field{{Name: "b", Type: dictionaryType}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, 1) + plan, err := BindLoad(context.Background(), schema, []TargetColumn{{ + Name: "b", Type: types.New(types.T_binary, 4, 0), + }}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.Equal(t, []byte{'a', 'b', 0, 0}, bat.Vecs[0].GetBytesAt(0)) + + bat.Clean(mp) + record.Release() + dictionary.Release() + indices.Release() + values.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) + }) +} + +func TestForceMaterializeBorrowEligibleColumns(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + ints := array.NewInt64Builder(alloc) + ints.AppendValues([]int64{10, 20}, nil) + intValues := ints.NewArray() + ints.Release() + strings := array.NewStringBuilder(alloc) + long := "a payload deliberately longer than twenty three bytes" + strings.AppendValues([]string{long, long}, nil) + stringValues := strings.NewArray() + strings.Release() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "i", Type: arrow.PrimitiveTypes.Int64}, + {Name: "s", Type: arrow.BinaryTypes.String}, + }, nil) + record := array.NewRecordBatch(schema, []arrow.Array{intValues, stringValues}, 2) + plan, err := Bind(context.Background(), schema, []TargetColumn{ + {Name: "i", Type: types.T_int64.ToType()}, + {Name: "s", Type: types.T_varchar.ToType()}, + }, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + borrowed, borrowedStats, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.Equal(t, int64(2), borrowedStats.BorrowedColumns) + require.Zero(t, borrowedStats.MaterializedColumns) + require.Equal(t, int64(16+2*len(long)), borrowedStats.EligiblePayloadBytes) + require.Equal(t, borrowedStats.EligiblePayloadBytes, borrowedStats.BorrowedPayloadBytes) + + materialized, materializedStats, err := plan.Convert( + context.Background(), record, mp, ConvertOptions{ForceMaterialize: true}, + ) + require.NoError(t, err) + require.Zero(t, materializedStats.BorrowedColumns) + require.Equal(t, int64(2), materializedStats.MaterializedColumns) + require.Equal(t, borrowedStats.EligiblePayloadBytes, materializedStats.EligiblePayloadBytes) + require.Equal(t, borrowedStats.EligiblePayloadBytes, materializedStats.MaterializedPayloadBytes) + require.Zero(t, materializedStats.PinAmplificationFallbacks) + require.False(t, materialized.Vecs[0].HasBorrowedBacking()) + require.False(t, materialized.Vecs[1].HasBorrowedBacking()) + + borrowed.Clean(mp) + materialized.Clean(mp) + record.Release() + intValues.Release() + stringValues.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) +} + +func TestVarlenSliceOffsetsAreNormalized(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewBinaryBuilder(alloc, arrow.BinaryTypes.Binary) + one := []byte("discarded-prefix-that-is-long") + two := []byte("the-kept-value-is-longer-than-inline") + three := []byte("another-kept-value-longer-than-inline") + builder.AppendValues([][]byte{one, two, three}, nil) + base := builder.NewArray() + builder.Release() + sliced := array.NewSlice(base, 1, 3) + schema := arrow.NewSchema([]arrow.Field{{Name: "b", Type: arrow.BinaryTypes.Binary}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{sliced}, 2) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "b", Type: types.T_varbinary.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{MaxPinAmplification: 100}) + require.NoError(t, err) + require.Equal(t, two, bat.Vecs[0].GetBytesAt(0)) + require.Equal(t, three, bat.Vecs[0].GetBytesAt(1)) + desc, _ := vector.MustVarlenaRawData(bat.Vecs[0]) + offset, _ := desc[0].OffsetLen() + require.Zero(t, offset) + record.Release() + sliced.Release() + base.Release() + bat.Clean(mp) + alloc.AssertSize(t, 0) +} + +func TestVarlenRejectsInvalidUTF8AndLength(t *testing.T) { + for _, tc := range []struct { + name string + value string + target types.Type + }{ + {name: "utf8", value: string([]byte{0xff}), target: types.T_varchar.ToType()}, + {name: "length", value: "abcd", target: types.New(types.T_varchar, 3, 0)}, + } { + t.Run(tc.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewStringBuilder(alloc) + builder.Append(tc.value) + arr := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "s", Type: arrow.BinaryTypes.String}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{arr}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "s", Type: tc.target}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + _, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.Error(t, err) + require.Equal(t, int64(0), mp.CurrNB()) + record.Release() + arr.Release() + alloc.AssertSize(t, 0) + }) + } +} + +func TestVarlenRejectsNegativeOffsetsWithoutPanicking(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + offsets := memory.NewBufferBytes(arrow.Int32Traits.CastToBytes([]int32{-1, 0})) + values := memory.NewBufferBytes([]byte{'x'}) + data := array.NewData(arrow.BinaryTypes.String, 1, []*memory.Buffer{nil, offsets, values}, nil, 0, 0) + valuesArray := array.NewStringData(data) + schema := arrow.NewSchema([]arrow.Field{{Name: "s", Type: arrow.BinaryTypes.String}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{valuesArray}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "s", Type: types.T_varchar.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + var rows int + require.NotPanics(t, func() { + rows, err = plan.MaxOutputRows(context.Background(), record, 0, 1, 1<<20) + }) + require.Error(t, err) + require.Zero(t, rows) + + var converted *batch.Batch + require.NotPanics(t, func() { + converted, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + }) + require.Error(t, err) + require.Nil(t, converted) + require.Zero(t, mp.CurrNB()) + + record.Release() + valuesArray.Release() + offsets.Release() + values.Release() + data.Release() + alloc.AssertSize(t, 0) +} + +func TestRejectsMismatchedArrowNullCount(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + validity := memory.NewBufferBytes([]byte{0xff}) + values := memory.NewBufferBytes(arrow.Int64Traits.CastToBytes([]int64{7})) + data := array.NewData(arrow.PrimitiveTypes.Int64, 1, []*memory.Buffer{validity, values}, nil, 1, 0) + valuesArray := array.NewInt64Data(data) + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: arrow.PrimitiveTypes.Int64, Nullable: true}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{valuesArray}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "v", Type: types.T_int64.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + // Output budgeting deliberately performs only O(columns) structural checks. + // ArrowReader validates this immutable record once before splitting it, so + // repeated windows cannot rescan the complete validity bitmap. + rows, err := plan.MaxOutputRows(context.Background(), record, 0, 1, 1<<20) + require.NoError(t, err) + require.Equal(t, 1, rows) + err = plan.ValidateRecord(context.Background(), record) + require.ErrorContains(t, err, "validity bitmap") + + converted, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.ErrorContains(t, err, "validity bitmap") + require.Nil(t, converted) + require.Zero(t, mp.CurrNB()) + + record.Release() + valuesArray.Release() + validity.Release() + values.Release() + data.Release() + alloc.AssertSize(t, 0) +} + +func TestNullArrowColumnConvertsWithoutValidityBuffer(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + values := array.NewNull(2) + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: arrow.Null}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{values}, 2) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "v", Type: types.T_varchar.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + converted, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.Equal(t, 2, converted.RowCount()) + require.True(t, converted.Vecs[0].IsNull(0)) + require.True(t, converted.Vecs[0].IsNull(1)) + + converted.Clean(mp) + record.Release() + values.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) +} + +func TestDictionaryRejectsMalformedIndicesWithoutPanicking(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + valueBuilder := array.NewInt64Builder(alloc) + valueBuilder.Append(7) + values := valueBuilder.NewArray() + valueBuilder.Release() + indicesData := array.NewData(arrow.PrimitiveTypes.Int8, 1, []*memory.Buffer{nil, nil}, nil, 0, 0) + indices := array.NewInt8Data(indicesData) + dictType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.PrimitiveTypes.Int64} + dictionary := array.NewDictionaryArray(dictType, indices, values) + indices.Release() + values.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: dictType}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "v", Type: types.T_int64.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + var converted *batch.Batch + require.NotPanics(t, func() { + converted, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + }) + require.Error(t, err) + require.Nil(t, converted) + require.Zero(t, mp.CurrNB()) + + record.Release() + dictionary.Release() + indicesData.Release() + alloc.AssertSize(t, 0) +} + +func TestVarlenIgnoresUnobservableNullPayload(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewStringBuilder(alloc) + builder.AppendValues( + []string{string([]byte{0xff}), "visible"}, + []bool{false, true}, + ) + values := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{ + Name: "s", Type: arrow.BinaryTypes.String, Nullable: true, + }}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{values}, 2) + plan, err := Bind(context.Background(), schema, []TargetColumn{{ + Name: "s", Type: types.New(types.T_varchar, 7, 0), + }}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + converted, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.True(t, converted.Vecs[0].IsNull(0)) + require.Equal(t, "visible", string(converted.Vecs[0].GetBytesAt(1))) + + converted.Clean(mp) + record.Release() + values.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) +} + +func TestMaterializedBoolAndDates(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + boolBuilder := array.NewBooleanBuilder(alloc) + boolBuilder.AppendValues([]bool{true, false}, []bool{true, false}) + bools := boolBuilder.NewArray() + boolBuilder.Release() + date32Builder := array.NewDate32Builder(alloc) + date32Builder.AppendValues([]arrow.Date32{0, 1}, nil) + date32s := date32Builder.NewArray() + date32Builder.Release() + date64Builder := array.NewDate64Builder(alloc) + date64Builder.AppendValues([]arrow.Date64{0, 86_400_000}, nil) + date64s := date64Builder.NewArray() + date64Builder.Release() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "b", Type: arrow.FixedWidthTypes.Boolean, Nullable: true}, + {Name: "d32", Type: arrow.FixedWidthTypes.Date32}, + {Name: "d64", Type: arrow.FixedWidthTypes.Date64}, + }, nil) + record := array.NewRecordBatch(schema, []arrow.Array{bools, date32s, date64s}, 2) + plan, err := Bind(context.Background(), schema, []TargetColumn{ + {Name: "b", Type: types.T_bool.ToType()}, + {Name: "d32", Type: types.T_date.ToType()}, + {Name: "d64", Type: types.T_date.ToType()}, + }, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + bat, stats, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.Zero(t, stats.BorrowedPayloadBytes) + require.Equal(t, []bool{true, false}, vector.MustFixedColNoTypeCheck[bool](bat.Vecs[0])) + require.True(t, bat.Vecs[0].IsNull(1)) + expectedDates := []types.Date{ + types.DaysFromUnixEpochToDate(0), + types.DaysFromUnixEpochToDate(1), + } + require.Equal(t, expectedDates, vector.MustFixedColNoTypeCheck[types.Date](bat.Vecs[1])) + require.Equal(t, expectedDates, vector.MustFixedColNoTypeCheck[types.Date](bat.Vecs[2])) + bat.Clean(mp) + record.Release() + bools.Release() + date32s.Release() + date64s.Release() + require.Equal(t, int64(0), mp.CurrNB()) + alloc.AssertSize(t, 0) +} + +func TestTemporalValidationAndTime64Borrow(t *testing.T) { + t.Run("date32 target date range", func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewDate32Builder(alloc) + // 10000-01-01 is a representable Arrow Date32 value but is outside + // MatrixOne's storable DATE range. + builder.Append(arrow.Date32(time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix() / (24 * 60 * 60))) + arr := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "d", Type: arrow.FixedWidthTypes.Date32}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{arr}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "d", Type: types.T_date.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + _, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.ErrorContains(t, err, "out of MatrixOne range") + require.Zero(t, mp.CurrNB()) + record.Release() + arr.Release() + alloc.AssertSize(t, 0) + }) + + t.Run("date64 target date range", func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewDate64Builder(alloc) + builder.Append(arrow.Date64(time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli())) + arr := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "d", Type: arrow.FixedWidthTypes.Date64}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{arr}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "d", Type: types.T_date.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + _, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.ErrorContains(t, err, "out of MatrixOne range") + require.Zero(t, mp.CurrNB()) + record.Release() + arr.Release() + alloc.AssertSize(t, 0) + }) + + t.Run("date64 fractional day", func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewDate64Builder(alloc) + builder.Append(1) + arr := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "d", Type: arrow.FixedWidthTypes.Date64}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{arr}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "d", Type: types.T_date.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + _, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.ErrorContains(t, err, "integral representable day") + require.Equal(t, int64(0), mp.CurrNB()) + record.Release() + arr.Release() + alloc.AssertSize(t, 0) + }) + + t.Run("time64 range", func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + timeType := &arrow.Time64Type{Unit: arrow.Microsecond} + builder := array.NewTime64Builder(alloc, timeType) + builder.Append(arrow.Time64(24 * 60 * 60 * 1_000_000)) + arr := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "t", Type: timeType}}, nil) + target := types.T_time.ToType() + target.Scale = 6 + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "t", Type: target}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + record := array.NewRecordBatch(schema, []arrow.Array{arr}, 1) + _, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.ErrorContains(t, err, "outside [0,24h)") + require.Equal(t, int64(0), mp.CurrNB()) + record.Release() + arr.Release() + alloc.AssertSize(t, 0) + }) +} + +func TestDictionaryDateRangeAndNullPayload(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + valueBuilder := array.NewDate32Builder(alloc) + valueBuilder.AppendValues( + []arrow.Date32{arrow.Date32(time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix() / (24 * 60 * 60))}, + []bool{false}, + ) + values := valueBuilder.NewArray() + valueBuilder.Release() + indexBuilder := array.NewInt8Builder(alloc) + indexBuilder.Append(0) + indices := indexBuilder.NewArray() + indexBuilder.Release() + dictType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.FixedWidthTypes.Date32} + dictionary := array.NewDictionaryArray(dictType, indices, values) + indices.Release() + values.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "d", Type: dictType, Nullable: true}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "d", Type: types.T_date.ToType()}}, MatchMode(MatchByName)) + require.NoError(t, err) + mp := mpool.MustNewZero() + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err, "a logical NULL must ignore the dictionary payload") + require.True(t, bat.Vecs[0].IsNull(0)) + bat.Clean(mp) + record.Release() + dictionary.Release() + alloc.AssertSize(t, 0) +} + +func TestDictionaryNullValuesConvertToLogicalNulls(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indexBuilder := array.NewInt8Builder(alloc) + indexBuilder.AppendValues([]int8{0, 0}, nil) + indices := indexBuilder.NewArray() + indexBuilder.Release() + values := array.NewNull(1) + dictType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.Null} + dictionary := array.NewDictionaryArray(dictType, indices, values) + indices.Release() + values.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: dictType, Nullable: true}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, 2) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "v", Type: types.T_int64.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.True(t, bat.Vecs[0].IsNull(0)) + require.True(t, bat.Vecs[0].IsNull(1)) + bat.Clean(mp) + record.Release() + dictionary.Release() + alloc.AssertSize(t, 0) +} + +func TestDictionaryNullValuesConvertToVarlenLogicalNulls(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indexBuilder := array.NewInt8Builder(alloc) + indexBuilder.Append(0) + indices := indexBuilder.NewArray() + indexBuilder.Release() + values := array.NewNull(1) + dictType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.Null} + dictionary := array.NewDictionaryArray(dictType, indices, values) + indices.Release() + values.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: dictType, Nullable: true}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "v", Type: types.T_varchar.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.True(t, bat.Vecs[0].IsNull(0)) + bat.Clean(mp) + record.Release() + dictionary.Release() + alloc.AssertSize(t, 0) +} + +func TestDictionaryNullValuesSupportAllFixedTargets(t *testing.T) { + targets := []struct { + name string + typ types.Type + }{ + {name: "bit", typ: types.T_bit.ToType()}, + {name: "year", typ: types.T_year.ToType()}, + {name: "enum", typ: types.T_enum.ToType()}, + {name: "decimal64", typ: types.T_decimal64.ToType()}, + {name: "decimal256", typ: types.T_decimal256.ToType()}, + {name: "uuid", typ: types.T_uuid.ToType()}, + {name: "transaction timestamp", typ: types.T_TS.ToType()}, + {name: "rowid", typ: types.T_Rowid.ToType()}, + {name: "blockid", typ: types.T_Blockid.ToType()}, + } + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indexBuilder := array.NewInt8Builder(alloc) + indexBuilder.Append(0) + indices := indexBuilder.NewArray() + indexBuilder.Release() + values := array.NewNull(1) + dictType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.Null} + dictionary := array.NewDictionaryArray(dictType, indices, values) + indices.Release() + values.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: dictType, Nullable: true}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "v", Type: target.typ}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.True(t, bat.Vecs[0].IsNull(0)) + bat.Clean(mp) + record.Release() + dictionary.Release() + alloc.AssertSize(t, 0) + }) + } +} + +func TestDictionaryNullIndexWithEmptyValues(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indexBuilder := array.NewInt8Builder(alloc) + indexBuilder.AppendValues([]int8{0}, []bool{false}) + indices := indexBuilder.NewArray() + indexBuilder.Release() + valueBuilder := array.NewInt64Builder(alloc) + values := valueBuilder.NewArray() + valueBuilder.Release() + dictType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.PrimitiveTypes.Int64} + dictionary := array.NewDictionaryArray(dictType, indices, values) + indices.Release() + values.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: dictType, Nullable: true}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "v", Type: types.T_int64.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.True(t, bat.Vecs[0].IsNull(0)) + bat.Clean(mp) + record.Release() + dictionary.Release() + alloc.AssertSize(t, 0) +} + +func TestMaterializedWidenTimeDateTimeAndNull(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + intBuilder := array.NewInt8Builder(alloc) + intBuilder.AppendValues([]int8{-1, 2}, nil) + ints := intBuilder.NewArray() + intBuilder.Release() + timeType := &arrow.Time32Type{Unit: arrow.Millisecond} + timeBuilder := array.NewTime32Builder(alloc, timeType) + timeBuilder.AppendValues([]arrow.Time32{1_000, 1_234}, nil) + times := timeBuilder.NewArray() + timeBuilder.Release() + dateBuilder := array.NewDate32Builder(alloc) + dateBuilder.AppendValues([]arrow.Date32{0, 1}, nil) + dates := dateBuilder.NewArray() + dateBuilder.Release() + nulls := array.NewNull(2) + schema := arrow.NewSchema([]arrow.Field{ + {Name: "i", Type: arrow.PrimitiveTypes.Int8}, + {Name: "t", Type: timeType}, + {Name: "d", Type: arrow.FixedWidthTypes.Date32}, + {Name: "n", Type: arrow.Null, Nullable: true}, + }, nil) + timeTarget := types.T_time.ToType() + timeTarget.Scale = 3 + plan, err := Bind(context.Background(), schema, []TargetColumn{ + {Name: "i", Type: types.T_int64.ToType()}, + {Name: "t", Type: timeTarget}, + {Name: "d", Type: types.T_datetime.ToType()}, + {Name: "n", Type: types.T_varchar.ToType()}, + }, MatchByName) + require.NoError(t, err) + record := array.NewRecordBatch(schema, []arrow.Array{ints, times, dates, nulls}, 2) + mp := mpool.MustNewZero() + + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.Equal(t, []int64{-1, 2}, vector.MustFixedColNoTypeCheck[int64](bat.Vecs[0])) + require.Equal(t, []types.Time{types.Time(1_000_000), types.Time(1_234_000)}, + vector.MustFixedColNoTypeCheck[types.Time](bat.Vecs[1])) + require.Equal(t, []string{"1970-01-01 00:00:00", "1970-01-02 00:00:00"}, []string{ + vector.MustFixedColNoTypeCheck[types.Datetime](bat.Vecs[2])[0].String(), + vector.MustFixedColNoTypeCheck[types.Datetime](bat.Vecs[2])[1].String(), + }) + require.True(t, bat.Vecs[3].IsNull(0)) + require.True(t, bat.Vecs[3].IsNull(1)) + + bat.Clean(mp) + record.Release() + ints.Release() + times.Release() + dates.Release() + nulls.Release() + alloc.AssertSize(t, 0) +} + +func TestCheckedWideningTypeMatrix(t *testing.T) { + tests := []struct { + source arrow.DataType + target types.Type + }{ + {source: arrow.PrimitiveTypes.Int8, target: types.T_int16.ToType()}, + {source: arrow.PrimitiveTypes.Int8, target: types.T_int32.ToType()}, + {source: arrow.PrimitiveTypes.Int8, target: types.T_int64.ToType()}, + {source: arrow.PrimitiveTypes.Int16, target: types.T_int32.ToType()}, + {source: arrow.PrimitiveTypes.Int16, target: types.T_int64.ToType()}, + {source: arrow.PrimitiveTypes.Int32, target: types.T_int64.ToType()}, + {source: arrow.PrimitiveTypes.Uint8, target: types.T_uint16.ToType()}, + {source: arrow.PrimitiveTypes.Uint8, target: types.T_uint32.ToType()}, + {source: arrow.PrimitiveTypes.Uint8, target: types.T_uint64.ToType()}, + {source: arrow.PrimitiveTypes.Uint16, target: types.T_uint32.ToType()}, + {source: arrow.PrimitiveTypes.Uint16, target: types.T_uint64.ToType()}, + {source: arrow.PrimitiveTypes.Uint32, target: types.T_uint64.ToType()}, + {source: arrow.PrimitiveTypes.Float32, target: types.T_float64.ToType()}, + } + + for _, test := range tests { + name := test.source.String() + " to " + test.target.String() + t.Run(name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewBuilder(alloc, test.source) + switch typed := builder.(type) { + case *array.Int8Builder: + typed.Append(-7) + case *array.Int16Builder: + typed.Append(-7) + case *array.Int32Builder: + typed.Append(-7) + case *array.Uint8Builder: + typed.Append(7) + case *array.Uint16Builder: + typed.Append(7) + case *array.Uint32Builder: + typed.Append(7) + case *array.Float32Builder: + typed.Append(1.25) + default: + t.Fatalf("missing source builder %T", builder) + } + values := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "v", Type: test.source}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{values}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "v", Type: test.target}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + bat, stats, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.False(t, bat.Vecs[0].HasBorrowedBacking()) + require.Equal(t, int64(test.target.TypeSize()), stats.MaterializedPayloadBytes) + switch test.target.Oid { + case types.T_int16: + require.Equal(t, []int16{-7}, vector.MustFixedColNoTypeCheck[int16](bat.Vecs[0])) + case types.T_int32: + require.Equal(t, []int32{-7}, vector.MustFixedColNoTypeCheck[int32](bat.Vecs[0])) + case types.T_int64: + require.Equal(t, []int64{-7}, vector.MustFixedColNoTypeCheck[int64](bat.Vecs[0])) + case types.T_uint16: + require.Equal(t, []uint16{7}, vector.MustFixedColNoTypeCheck[uint16](bat.Vecs[0])) + case types.T_uint32: + require.Equal(t, []uint32{7}, vector.MustFixedColNoTypeCheck[uint32](bat.Vecs[0])) + case types.T_uint64: + require.Equal(t, []uint64{7}, vector.MustFixedColNoTypeCheck[uint64](bat.Vecs[0])) + case types.T_float64: + require.Equal(t, []float64{1.25}, vector.MustFixedColNoTypeCheck[float64](bat.Vecs[0])) + } + + bat.Clean(mp) + record.Release() + values.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) + }) + } +} + +func TestMaterializedTimeRejectsPrecisionLoss(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + timeType := &arrow.Time32Type{Unit: arrow.Millisecond} + builder := array.NewTime32Builder(alloc, timeType) + builder.Append(1) + values := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "t", Type: timeType}}, nil) + target := types.T_time.ToType() + target.Scale = 2 + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "t", Type: target}}, MatchByName) + require.NoError(t, err) + record := array.NewRecordBatch(schema, []arrow.Array{values}, 1) + mp := mpool.MustNewZero() + _, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.ErrorContains(t, err, "precision") + require.Zero(t, mp.CurrNB()) + record.Release() + values.Release() + alloc.AssertSize(t, 0) +} + +func TestTimestampTimezoneSemantics(t *testing.T) { + location, err := time.LoadLocation("Asia/Shanghai") + require.NoError(t, err) + for _, tc := range []struct { + name string + timezone string + target types.Type + input arrow.Timestamp + expected string + timestampS int64 + }{ + {name: "wall clock to timestamp", timezone: "", target: types.T_timestamp.ToType(), input: arrow.Timestamp((8*60*60 + 1) * 1_000_000), expected: "1970-01-01 08:00:01.000000", timestampS: 1}, + {name: "wall clock to datetime", timezone: "", target: types.T_datetime.ToType(), expected: "1970-01-01 00:00:00"}, + {name: "instant to datetime", timezone: "UTC", target: types.T_datetime.ToType(), expected: "1970-01-01 08:00:00"}, + } { + t.Run(tc.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + timestampType := &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: tc.timezone} + builder := array.NewTimestampBuilder(alloc, timestampType) + builder.Append(tc.input) + arr := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "ts", Type: timestampType}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{arr}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "ts", Type: tc.target}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{Location: location}) + require.NoError(t, err) + if tc.target.Oid == types.T_timestamp { + value := vector.MustFixedColNoTypeCheck[types.Timestamp](bat.Vecs[0])[0] + require.Equal(t, tc.timestampS, value.Unix()) + require.Contains(t, value.String2(location, 6), tc.expected) + } else { + value := vector.MustFixedColNoTypeCheck[types.Datetime](bat.Vecs[0])[0] + require.Equal(t, tc.expected, value.String()) + } + bat.Clean(mp) + record.Release() + arr.Release() + require.Equal(t, int64(0), mp.CurrNB()) + alloc.AssertSize(t, 0) + }) + } +} + +func TestBindRejectsInvalidTimestampTimezone(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{ + Name: "ts", + Type: &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: "not/a-real-timezone"}, + }}, nil) + + _, err := Bind(context.Background(), schema, []TargetColumn{{ + Name: "ts", + Type: types.T_datetime.ToType(), + }}, MatchByName) + require.ErrorContains(t, err, "timezone") +} + +func TestBindRejectsInvalidTimeUnit(t *testing.T) { + for _, test := range []struct { + name string + typ arrow.DataType + }{ + {name: "time32", typ: &arrow.Time32Type{Unit: arrow.TimeUnit(99)}}, + {name: "time64", typ: &arrow.Time64Type{Unit: arrow.TimeUnit(99)}}, + {name: "timestamp", typ: &arrow.TimestampType{Unit: arrow.TimeUnit(99)}}, + } { + t.Run(test.name, func(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{Name: "t", Type: test.typ}}, nil) + var err error + require.NotPanics(t, func() { + _, err = Bind(context.Background(), schema, []TargetColumn{{ + Name: "t", Type: types.T_time.ToType(), + }}, MatchByName) + }) + require.ErrorContains(t, err, "time unit") + }) + } +} + +func TestTimestampUnitAndPrecisionMatrix(t *testing.T) { + for _, test := range []struct { + name string + unit arrow.TimeUnit + value arrow.Timestamp + }{ + {name: "second", unit: arrow.Second, value: 2}, + {name: "millisecond", unit: arrow.Millisecond, value: 2_000}, + {name: "microsecond", unit: arrow.Microsecond, value: 2_000_000}, + {name: "nanosecond exact", unit: arrow.Nanosecond, value: 2_000_000_000}, + } { + t.Run(test.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + timestampType := &arrow.TimestampType{Unit: test.unit} + builder := array.NewTimestampBuilder(alloc, timestampType) + builder.Append(test.value) + values := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "ts", Type: timestampType}}, nil) + target := types.New(types.T_datetime, 0, 6) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "ts", Type: target}}, MatchByName) + require.NoError(t, err) + record := array.NewRecordBatch(schema, []arrow.Array{values}, 1) + mp := mpool.MustNewZero() + + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.Equal(t, "1970-01-01 00:00:02", vector.MustFixedColNoTypeCheck[types.Datetime](bat.Vecs[0])[0].String()) + + bat.Clean(mp) + record.Release() + values.Release() + alloc.AssertSize(t, 0) + }) + } + + for _, test := range []struct { + name string + unit arrow.TimeUnit + value arrow.Timestamp + scale int32 + valid bool + errText string + }{ + {name: "nanosecond precision loss", unit: arrow.Nanosecond, value: 1, scale: 6, valid: true, errText: "out of range"}, + {name: "target precision loss", unit: arrow.Millisecond, value: 1, scale: 2, valid: true, errText: "precision"}, + {name: "overflow", unit: arrow.Second, value: arrow.Timestamp(^uint64(0) >> 1), scale: 6, valid: true, errText: "out of range"}, + {name: "null payload ignored", unit: arrow.Second, value: arrow.Timestamp(^uint64(0) >> 1), scale: 6, valid: false}, + } { + t.Run(test.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + timestampType := &arrow.TimestampType{Unit: test.unit} + builder := array.NewTimestampBuilder(alloc, timestampType) + builder.AppendValues([]arrow.Timestamp{test.value}, []bool{test.valid}) + values := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "ts", Type: timestampType, Nullable: true}}, nil) + target := types.New(types.T_timestamp, 0, test.scale) + plan, err := Bind(context.Background(), schema, []TargetColumn{{Name: "ts", Type: target}}, MatchByName) + require.NoError(t, err) + record := array.NewRecordBatch(schema, []arrow.Array{values}, 1) + mp := mpool.MustNewZero() + + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + if test.errText != "" { + require.ErrorContains(t, err, test.errText) + require.Nil(t, bat) + } else { + require.NoError(t, err) + require.True(t, bat.Vecs[0].IsNull(0)) + bat.Clean(mp) + } + require.Zero(t, mp.CurrNB()) + record.Release() + values.Release() + alloc.AssertSize(t, 0) + }) + } + + invalidScale := types.New(types.T_timestamp, 0, 7) + _, err := Bind(context.Background(), arrow.NewSchema([]arrow.Field{{ + Name: "ts", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, + }}, nil), []TargetColumn{{Name: "ts", Type: invalidScale}}, MatchByName) + require.ErrorContains(t, err, "no exact long-term conversion") +} + +func TestDictionaryGatherMaterializesAndPropagatesLogicalNulls(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indicesBuilder := array.NewInt8Builder(alloc) + indicesBuilder.AppendValues([]int8{0, 1, 0, 2}, []bool{true, true, false, true}) + indices := indicesBuilder.NewArray() + indicesBuilder.Release() + valuesBuilder := array.NewStringBuilder(alloc) + long := "dictionary payload longer than MatrixOne inline varlena" + valuesBuilder.AppendValues([]string{long, "ignored", "tiny"}, []bool{true, false, true}) + values := valuesBuilder.NewArray() + valuesBuilder.Release() + dictionaryType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.BinaryTypes.String} + dictionary := array.NewDictionaryArray(dictionaryType, indices, values) + schema := arrow.NewSchema([]arrow.Field{{Name: "s", Type: dictionaryType, Nullable: true}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, 4) + plan, err := Bind(context.Background(), schema, + []TargetColumn{{Name: "s", Type: types.T_varchar.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + bat, stats, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.False(t, bat.Vecs[0].HasBorrowedBacking(), "dictionary indices must be gathered") + require.Equal(t, int64(len(long)+len("tiny")), stats.MaterializedPayloadBytes) + require.Equal(t, long, string(bat.Vecs[0].GetBytesAt(0))) + require.True(t, bat.Vecs[0].IsNull(1), "a null dictionary value is a logical null") + require.True(t, bat.Vecs[0].IsNull(2), "a null dictionary index is a logical null") + require.Equal(t, "tiny", string(bat.Vecs[0].GetBytesAt(3))) + + notNullPlan, err := Bind(context.Background(), schema, + []TargetColumn{{Name: "s", Type: types.T_varchar.ToType(), NotNull: true}}, MatchByName) + require.NoError(t, err) + _, _, err = notNullPlan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.ErrorContains(t, err, "NOT NULL") + + bat.Clean(mp) + record.Release() + dictionary.Release() + indices.Release() + values.Release() + require.Equal(t, int64(0), mp.CurrNB()) + alloc.AssertSize(t, 0) +} + +func TestValidatedDictionaryWindowsDoNotRescanImmutableValues(t *testing.T) { + const dictionaryRows = 1024 + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indicesBuilder := array.NewInt32Builder(alloc) + valuesBuilder := array.NewStringBuilder(alloc) + for row := 0; row < dictionaryRows; row++ { + indicesBuilder.Append(int32(row)) + valuesBuilder.Append("dictionary-value") + } + indices := indicesBuilder.NewArray() + indicesBuilder.Release() + values := valuesBuilder.NewArray() + valuesBuilder.Release() + dictionaryType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int32, ValueType: arrow.BinaryTypes.String} + dictionary := array.NewDictionaryArray(dictionaryType, indices, values) + schema := arrow.NewSchema([]arrow.Field{{Name: "s", Type: dictionaryType}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, dictionaryRows) + plan, err := Bind(context.Background(), schema, + []TargetColumn{{Name: "s", Type: types.T_varchar.ToType()}}, MatchByName) + require.NoError(t, err) + require.NoError(t, plan.ValidateRecord(context.Background(), record)) + + // Arrow slices retain the full dictionary values array. Convert every row as + // its own output window, as ReadBatch does under a small wire budget. The + // context checkpoints are a deterministic work counter: rescanning all + // dictionary values for each window would exceed this linear bound. + budget := &contextCheckBudget{Context: context.Background(), limit: dictionaryRows * 8} + mp := mpool.MustNewZero() + for row := int64(0); row < dictionaryRows; row++ { + view := record.NewSlice(row, row+1) + converted, _, err := plan.ConvertValidatedRecordWindow(budget, view, mp, ConvertOptions{}) + require.NoError(t, err) + require.Equal(t, "dictionary-value", converted.Vecs[0].GetStringAt(0)) + converted.Clean(mp) + view.Release() + } + require.LessOrEqual(t, budget.checks, dictionaryRows*8, + "many output windows must not revalidate all immutable dictionary values") + record.Release() + dictionary.Release() + indices.Release() + values.Release() + require.Zero(t, mp.CurrNB()) + alloc.AssertSize(t, 0) +} + +func TestDictionaryFixedWidthAndTemporalGather(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indicesBuilder := array.NewUint16Builder(alloc) + indicesBuilder.AppendValues([]uint16{1, 0}, nil) + indices := indicesBuilder.NewArray() + indicesBuilder.Release() + intBuilder := array.NewInt64Builder(alloc) + intBuilder.AppendValues([]int64{11, 22}, nil) + intValues := intBuilder.NewArray() + intBuilder.Release() + dateBuilder := array.NewDate64Builder(alloc) + dateBuilder.AppendValues([]arrow.Date64{0, 86_400_000}, nil) + dateValues := dateBuilder.NewArray() + dateBuilder.Release() + intType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Uint16, ValueType: arrow.PrimitiveTypes.Int64} + dateType := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Uint16, ValueType: arrow.FixedWidthTypes.Date64} + intDictionary := array.NewDictionaryArray(intType, indices, intValues) + dateDictionary := array.NewDictionaryArray(dateType, indices, dateValues) + schema := arrow.NewSchema([]arrow.Field{ + {Name: "i", Type: intType}, + {Name: "d", Type: dateType}, + }, nil) + record := array.NewRecordBatch(schema, []arrow.Array{intDictionary, dateDictionary}, 2) + plan, err := Bind(context.Background(), schema, []TargetColumn{ + {Name: "i", Type: types.T_int64.ToType()}, + {Name: "d", Type: types.T_date.ToType()}, + }, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + bat, stats, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.NoError(t, err) + require.Equal(t, []int64{22, 11}, vector.MustFixedColNoTypeCheck[int64](bat.Vecs[0])) + require.Equal(t, []types.Date{ + types.DaysFromUnixEpochToDate(1), + types.DaysFromUnixEpochToDate(0), + }, vector.MustFixedColNoTypeCheck[types.Date](bat.Vecs[1])) + require.Equal(t, int64(2*(types.T_int64.ToType().TypeSize()+types.T_date.ToType().TypeSize())), stats.MaterializedPayloadBytes) + + bat.Clean(mp) + record.Release() + intDictionary.Release() + dateDictionary.Release() + indices.Release() + intValues.Release() + dateValues.Release() + alloc.AssertSize(t, 0) +} + +func TestDictionaryTimestampUsesCanonicalRangePrecisionAndNullSemantics(t *testing.T) { + tests := []struct { + name string + unit arrow.TimeUnit + value arrow.Timestamp + valid bool + scale int32 + errText string + }{ + {name: "valid millisecond", unit: arrow.Millisecond, value: 1_000, valid: true, scale: 3}, + {name: "target precision loss", unit: arrow.Millisecond, value: 1_001, valid: true, scale: 2, errText: "precision"}, + {name: "nanosecond precision loss", unit: arrow.Nanosecond, value: 1, valid: true, scale: 6, errText: "out of range"}, + {name: "overflow", unit: arrow.Second, value: arrow.Timestamp(^uint64(0) >> 1), valid: true, scale: 6, errText: "out of range"}, + {name: "null payload ignored", unit: arrow.Second, value: arrow.Timestamp(^uint64(0) >> 1), valid: false, scale: 6}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indicesBuilder := array.NewInt8Builder(alloc) + indicesBuilder.Append(0) + indices := indicesBuilder.NewArray() + indicesBuilder.Release() + + timestampType := &arrow.TimestampType{Unit: test.unit} + valuesBuilder := array.NewTimestampBuilder(alloc, timestampType) + valuesBuilder.AppendValues([]arrow.Timestamp{test.value}, []bool{test.valid}) + values := valuesBuilder.NewArray() + valuesBuilder.Release() + dictionaryType := &arrow.DictionaryType{ + IndexType: arrow.PrimitiveTypes.Int8, + ValueType: timestampType, + } + dictionary := array.NewDictionaryArray(dictionaryType, indices, values) + schema := arrow.NewSchema([]arrow.Field{{Name: "ts", Type: dictionaryType, Nullable: true}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, 1) + plan, err := Bind(context.Background(), schema, []TargetColumn{{ + Name: "ts", + Type: types.New(types.T_timestamp, 0, test.scale), + }}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + bat, _, err := plan.Convert(context.Background(), record, mp, ConvertOptions{}) + if test.errText != "" { + require.ErrorContains(t, err, test.errText) + require.Nil(t, bat) + } else { + require.NoError(t, err) + if test.valid { + require.False(t, bat.Vecs[0].IsNull(0)) + } else { + require.True(t, bat.Vecs[0].IsNull(0)) + } + bat.Clean(mp) + } + require.Zero(t, mp.CurrNB()) + + record.Release() + dictionary.Release() + indices.Release() + values.Release() + alloc.AssertSize(t, 0) + }) + } +} + +func TestDictionaryRejectsMalformedIndicesAndNestedValues(t *testing.T) { + for _, test := range []struct { + name string + indexType arrow.DataType + build func(memory.Allocator) arrow.Array + }{ + { + name: "negative signed index", + indexType: arrow.PrimitiveTypes.Int16, + build: func(alloc memory.Allocator) arrow.Array { + builder := array.NewInt16Builder(alloc) + defer builder.Release() + builder.Append(-1) + return builder.NewArray() + }, + }, + { + name: "too large unsigned index", + indexType: arrow.PrimitiveTypes.Uint64, + build: func(alloc memory.Allocator) arrow.Array { + builder := array.NewUint64Builder(alloc) + defer builder.Release() + builder.Append(9) + return builder.NewArray() + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + indices := test.build(alloc) + valuesBuilder := array.NewInt32Builder(alloc) + valuesBuilder.Append(7) + values := valuesBuilder.NewArray() + valuesBuilder.Release() + dictionaryType := &arrow.DictionaryType{IndexType: test.indexType, ValueType: arrow.PrimitiveTypes.Int32} + dictionary := array.NewDictionaryArray(dictionaryType, indices, values) + schema := arrow.NewSchema([]arrow.Field{{Name: "i", Type: dictionaryType}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{dictionary}, 1) + plan, err := Bind(context.Background(), schema, + []TargetColumn{{Name: "i", Type: types.T_int32.ToType()}}, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + + require.NotPanics(t, func() { + _, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + }) + require.ErrorContains(t, err, "outside") + require.Equal(t, int64(0), mp.CurrNB()) + + record.Release() + dictionary.Release() + indices.Release() + values.Release() + alloc.AssertSize(t, 0) + }) + } + + nested := &arrow.DictionaryType{ + IndexType: arrow.PrimitiveTypes.Int8, + ValueType: &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.BinaryTypes.String}, + } + _, err := Bind(context.Background(), arrow.NewSchema([]arrow.Field{{Name: "s", Type: nested}}, nil), + []TargetColumn{{Name: "s", Type: types.T_varchar.ToType()}}, MatchByName) + require.ErrorContains(t, err, "nested") +} + +func TestMaxOutputRowsHonorsByteBudgetAndProgress(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + intBuilder := array.NewInt64Builder(alloc) + intBuilder.AppendValues([]int64{1, 2, 3, 4}, nil) + ints := intBuilder.NewArray() + intBuilder.Release() + stringBuilder := array.NewStringBuilder(alloc) + value := "0123456789012345678901234567890123456789" + require.Len(t, value, 40) + stringBuilder.AppendValues([]string{value, value, value, value}, nil) + strings := stringBuilder.NewArray() + stringBuilder.Release() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "i", Type: arrow.PrimitiveTypes.Int64}, + {Name: "s", Type: arrow.BinaryTypes.String}, + }, nil) + record := array.NewRecordBatch(schema, []arrow.Array{ints, strings}, 4) + plan, err := Bind(context.Background(), schema, []TargetColumn{ + {Name: "i", Type: types.T_int64.ToType()}, + {Name: "s", Type: types.T_varchar.ToType()}, + }, MatchByName) + require.NoError(t, err) + + const rowBytes = uint64(8 + types.VarlenaSize + 40) + rows, err := plan.MaxOutputRows(context.Background(), record, 0, 4, 2*rowBytes) + require.NoError(t, err) + require.Equal(t, 2, rows) + rows, err = plan.MaxOutputRows(context.Background(), record, 1, 3, rowBytes) + require.NoError(t, err) + require.Equal(t, 1, rows) + rows, err = plan.MaxOutputRows(context.Background(), record, 0, 4, 1) + require.NoError(t, err) + require.Equal(t, 1, rows, "one oversized row must still make progress") + + record.Release() + ints.Release() + strings.Release() + alloc.AssertSize(t, 0) +} + +func TestMaxOutputRowsRejectsMismatchedColumnRows(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + builder := array.NewStringBuilder(alloc) + builder.Append("one") + values := builder.NewArray() + builder.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "value", Type: arrow.BinaryTypes.String}}, nil) + record := &mismatchedRowsRecordBatch{ + RecordBatch: array.NewRecordBatch(schema, []arrow.Array{values}, 1), + rows: 2, + } + plan, err := BindLoad(context.Background(), schema, []TargetColumn{{ + Name: "value", Type: types.T_varchar.ToType(), + }}, MatchByName) + require.NoError(t, err) + + var rows int + require.NotPanics(t, func() { + rows, err = plan.MaxOutputRows(context.Background(), record, 0, 2, 1024) + }) + require.Zero(t, rows) + require.ErrorContains(t, err, "rows") + + record.Release() + values.Release() + alloc.AssertSize(t, 0) +} + +type mismatchedRowsRecordBatch struct { + arrow.RecordBatch + rows int64 +} + +func (r *mismatchedRowsRecordBatch) NumRows() int64 { + return r.rows +} + +func TestMaxOutputRowsRejectsNilRecordSchema(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{Name: "value", Type: arrow.PrimitiveTypes.Int64}}, nil) + values := array.NewInt64Builder(memory.NewGoAllocator()) + values.Append(1) + record := &nilSchemaRecordBatch{ + RecordBatch: array.NewRecordBatch(schema, []arrow.Array{values.NewArray()}, 1), + } + values.Release() + plan, err := BindLoad(context.Background(), schema, []TargetColumn{{ + Name: "value", Type: types.T_int64.ToType(), + }}, MatchByName) + require.NoError(t, err) + + var rows int + require.NotPanics(t, func() { + rows, err = plan.MaxOutputRows(context.Background(), record, 0, 1, 1024) + }) + require.Zero(t, rows) + require.ErrorContains(t, err, "schema") + record.Release() +} + +func TestConvertRejectsNilRecordSchema(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{Name: "value", Type: arrow.PrimitiveTypes.Int64}}, nil) + values := array.NewInt64Builder(memory.NewGoAllocator()) + values.Append(1) + record := &nilSchemaRecordBatch{ + RecordBatch: array.NewRecordBatch(schema, []arrow.Array{values.NewArray()}, 1), + } + values.Release() + plan, err := BindLoad(context.Background(), schema, []TargetColumn{{ + Name: "value", Type: types.T_int64.ToType(), + }}, MatchByName) + require.NoError(t, err) + + mp := mpool.MustNewZero() + var converted *batch.Batch + require.NotPanics(t, func() { + converted, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + }) + require.Nil(t, converted) + require.ErrorContains(t, err, "schema") + record.Release() +} + +func TestConvertRejectsNilRecordColumn(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{Name: "value", Type: arrow.PrimitiveTypes.Int64}}, nil) + values := array.NewInt64Builder(memory.NewGoAllocator()) + values.Append(1) + record := &nilColumnRecordBatch{ + RecordBatch: array.NewRecordBatch(schema, []arrow.Array{values.NewArray()}, 1), + } + values.Release() + plan, err := BindLoad(context.Background(), schema, []TargetColumn{{ + Name: "value", Type: types.T_int64.ToType(), + }}, MatchByName) + require.NoError(t, err) + + rows, budgetErr := plan.MaxOutputRows(context.Background(), record, 0, 1, 1024) + require.Zero(t, rows) + require.ErrorContains(t, budgetErr, "column") + + mp := mpool.MustNewZero() + var converted *batch.Batch + require.NotPanics(t, func() { + converted, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + }) + require.Nil(t, converted) + require.ErrorContains(t, err, "column") + record.Release() +} + +func TestConvertRejectsRecordColumnTypeDrift(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{Name: "value", Type: arrow.PrimitiveTypes.Int64}}, nil) + intValues := array.NewInt64Builder(memory.NewGoAllocator()) + intValues.Append(1) + record := &driftedTypeRecordBatch{ + RecordBatch: array.NewRecordBatch(schema, []arrow.Array{intValues.NewArray()}, 1), + } + intValues.Release() + floatValues := array.NewFloat64Builder(memory.NewGoAllocator()) + floatValues.Append(1.5) + record.column = floatValues.NewArray() + floatValues.Release() + plan, err := BindLoad(context.Background(), schema, []TargetColumn{{ + Name: "value", Type: types.T_int64.ToType(), + }}, MatchByName) + require.NoError(t, err) + + mp := mpool.MustNewZero() + var converted *batch.Batch + require.NotPanics(t, func() { + converted, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + }) + require.Nil(t, converted) + require.ErrorContains(t, err, "data type") + record.Release() + record.column.Release() +} + +func TestConvertRejectsMalformedRecordSchema(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{Name: "value", Type: arrow.PrimitiveTypes.Int64}}, nil) + values := array.NewInt64Builder(memory.NewGoAllocator()) + values.Append(1) + record := &customSchemaRecordBatch{ + RecordBatch: array.NewRecordBatch(schema, []arrow.Array{values.NewArray()}, 1), + schema: schemaWithNilFieldType(schema), + } + values.Release() + plan, err := BindLoad(context.Background(), schema, []TargetColumn{{ + Name: "value", Type: types.T_int64.ToType(), + }}, MatchByName) + require.NoError(t, err) + + rows, budgetErr := plan.MaxOutputRows(context.Background(), record, 0, 1, 1024) + require.Zero(t, rows) + require.ErrorContains(t, budgetErr, "schema") + + mp := mpool.MustNewZero() + var converted *batch.Batch + require.NotPanics(t, func() { + converted, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + }) + require.Nil(t, converted) + require.ErrorContains(t, err, "schema") + record.Release() +} + +func schemaWithNilFieldType(schema *arrow.Schema) *arrow.Schema { + malformed := *schema + fields := malformed.Fields() + fields[0].Type = nil + privateFields := reflect.ValueOf(&malformed).Elem().FieldByName("fields") + reflect.NewAt(privateFields.Type(), unsafe.Pointer(privateFields.UnsafeAddr())).Elem().Set(reflect.ValueOf(fields)) + return &malformed +} + +type nilSchemaRecordBatch struct { + arrow.RecordBatch +} + +func (r *nilSchemaRecordBatch) Schema() *arrow.Schema { + return nil +} + +type nilColumnRecordBatch struct { + arrow.RecordBatch +} + +func (r *nilColumnRecordBatch) Column(int) arrow.Array { + return nil +} + +type driftedTypeRecordBatch struct { + arrow.RecordBatch + column arrow.Array +} + +func (r *driftedTypeRecordBatch) Column(int) arrow.Array { + return r.column +} + +type customSchemaRecordBatch struct { + arrow.RecordBatch + schema *arrow.Schema +} + +func (r *customSchemaRecordBatch) Schema() *arrow.Schema { + return r.schema +} + +func TestConvertRollbackSchemaDriftNotNullAndCancel(t *testing.T) { + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + intsBuilder := array.NewInt64Builder(alloc) + intsBuilder.AppendValues([]int64{1, 2}, nil) + ints := intsBuilder.NewArray() + intsBuilder.Release() + stringsBuilder := array.NewStringBuilder(alloc) + stringsBuilder.AppendValues([]string{"ok", "bad"}, []bool{true, false}) + strings := stringsBuilder.NewArray() + stringsBuilder.Release() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "i", Type: arrow.PrimitiveTypes.Int64}, + {Name: "s", Type: arrow.BinaryTypes.String, Nullable: true}, + }, nil) + record := array.NewRecordBatch(schema, []arrow.Array{ints, strings}, 2) + plan, err := Bind(context.Background(), schema, []TargetColumn{ + {Name: "i", Type: types.T_int64.ToType()}, + {Name: "s", Type: types.T_varchar.ToType(), NotNull: true}, + }, MatchByName) + require.NoError(t, err) + mp := mpool.MustNewZero() + _, _, err = plan.Convert(context.Background(), record, mp, ConvertOptions{}) + require.ErrorContains(t, err, "NOT NULL") + require.Equal(t, int64(0), mp.CurrNB()) + + driftSchema := arrow.NewSchema([]arrow.Field{ + {Name: "i", Type: arrow.PrimitiveTypes.Int64}, + {Name: "renamed", Type: arrow.BinaryTypes.String, Nullable: true}, + }, nil) + driftRecord := array.NewRecordBatch(driftSchema, []arrow.Array{ints, strings}, 2) + _, _, err = plan.Convert(context.Background(), driftRecord, mp, ConvertOptions{}) + require.ErrorContains(t, err, "does not match") + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + _, _, err = plan.Convert(canceled, record, mp, ConvertOptions{}) + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, int64(0), mp.CurrNB()) + + driftRecord.Release() + record.Release() + ints.Release() + strings.Release() + alloc.AssertSize(t, 0) +} diff --git a/pkg/container/arrowbridge/budget.go b/pkg/container/arrowbridge/budget.go new file mode 100644 index 0000000000000..cd3725e89fa8d --- /dev/null +++ b/pkg/container/arrowbridge/budget.go @@ -0,0 +1,239 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowbridge + +import ( + "context" + "math" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// MaxOutputRows selects the largest non-empty prefix that fits the logical MO +// batch budget. A single oversized row is selected to guarantee planning +// progress; the canonical wire-size admission step still rejects it before +// execution if that row exceeds the hard statement budget. The estimate is +// deliberately conservative for materialized varlen dictionaries and exact +// for borrowed Arrow varlen areas. +func (p *Plan) MaxOutputRows( + ctx context.Context, + record arrow.RecordBatch, + start int64, + maxRows int, + maxBytes uint64, +) (int, error) { + if p == nil || record == nil || start < 0 || start >= record.NumRows() || maxRows <= 0 { + return 0, moerr.NewInvalidInput(ctx, "invalid Arrow output budget input") + } + recordSchema := record.Schema() + if recordSchema == nil { + return 0, moerr.NewInvalidInput(ctx, "Arrow record schema is nil") + } + if schemaFingerprint(recordSchema) != p.schemaFingerprint { + return 0, moerr.NewInvalidInput(ctx, "Arrow record schema does not match the bound schema") + } + // ReadBatch validates a newly received immutable record once before it is + // split into output windows. Keep this hot path to structural checks: a + // full validity-bitmap scan here for every window turns one large record + // into quadratic work. Convert validates each selected window instead. + if err := validateRecordShape(ctx, record, recordSchema, p.columns); err != nil { + return 0, err + } + available := record.NumRows() - start + if int64(maxRows) > available { + maxRows = int(available) + } + if maxBytes == 0 { + return maxRows, nil + } + + var used uint64 + for relativeRow := 0; relativeRow < maxRows; relativeRow++ { + if err := checkConvertContext(ctx, relativeRow); err != nil { + return 0, err + } + row := int(start) + relativeRow + var rowBytes uint64 + for _, binding := range p.columns { + column := record.Column(binding.source) + bytes, err := estimateColumnRowBytes(ctx, column, binding, row) + if err != nil { + return 0, err + } + if rowBytes > math.MaxUint64-bytes { + return 0, moerr.NewInvalidInput(ctx, "Arrow output row size overflows") + } + rowBytes += bytes + } + if used > maxBytes || rowBytes > maxBytes-used { + if relativeRow == 0 { + return 1, nil + } + return relativeRow, nil + } + used += rowBytes + } + return maxRows, nil +} + +func validateRecordColumns( + ctx context.Context, + record arrow.RecordBatch, + schema *arrow.Schema, + columns []columnPlan, + validateDictionaryValues bool, +) error { + if err := validateRecordShape(ctx, record, schema, columns); err != nil { + return err + } + for _, binding := range columns { + column := record.Column(binding.source) + if _, err := validateArrowArrayValidity(ctx, column, validateDictionaryValues); err != nil { + return err + } + } + return nil +} + +// ValidateRecord validates an immutable record once before a caller splits it +// into output windows. It includes validity metadata; window conversion still +// validates its own sliced arrays so a standalone Convert remains safe. +func (p *Plan) ValidateRecord(ctx context.Context, record arrow.RecordBatch) error { + if p == nil || record == nil { + return moerr.NewInvalidInput(ctx, "invalid Arrow record") + } + recordSchema := record.Schema() + if recordSchema == nil { + return moerr.NewInvalidInput(ctx, "Arrow record schema is nil") + } + if schemaFingerprint(recordSchema) != p.schemaFingerprint { + return moerr.NewInvalidInput(ctx, "Arrow record schema does not match the bound schema") + } + return validateRecordColumns(ctx, record, recordSchema, p.columns, true) +} + +func validateRecordShape( + ctx context.Context, + record arrow.RecordBatch, + schema *arrow.Schema, + columns []columnPlan, +) error { + if record.NumCols() != int64(len(columns)) { + return moerr.NewInvalidInputf(ctx, "Arrow record has %d columns, expected %d", record.NumCols(), len(columns)) + } + for _, binding := range columns { + column := record.Column(binding.source) + if column == nil { + return moerr.NewInvalidInputf(ctx, "Arrow column %q is nil", binding.target.Name) + } + if int64(column.Len()) != record.NumRows() { + return moerr.NewInvalidInputf(ctx, + "Arrow column %q has %d rows, expected %d", + binding.target.Name, column.Len(), record.NumRows()) + } + if binding.source < 0 || binding.source >= schema.NumFields() || schema.Field(binding.source).Type == nil || + column.DataType() == nil || column.DataType().Fingerprint() != schema.Field(binding.source).Type.Fingerprint() { + return moerr.NewInvalidInputf(ctx, + "Arrow column %q data type does not match the bound schema", binding.target.Name) + } + } + return nil +} + +func estimateColumnRowBytes( + ctx context.Context, + column arrow.Array, + binding columnPlan, + row int, +) (uint64, error) { + fixed := binding.target.Type.TypeSize() + if binding.kind != conversionBorrowVarlen && binding.kind != conversionMaterializeDictionary { + if fixed < 0 { + return 0, moerr.NewInvalidInput(ctx, "invalid MatrixOne target width") + } + return uint64(fixed), nil + } + if binding.kind == conversionMaterializeDictionary { + dictionary, ok := column.(*array.Dictionary) + if !ok { + return 0, moerr.NewInvalidInput(ctx, "invalid Arrow Dictionary array") + } + valueKind, err := selectLoadConversion(dictionary.Dictionary().DataType(), binding.target.Type) + if err != nil { + return 0, err + } + if valueKind != conversionBorrowVarlen { + if fixed < 0 { + return 0, moerr.NewInvalidInput(ctx, "invalid MatrixOne target width") + } + return uint64(fixed), nil + } + if dictionary.IsNull(row) { + return uint64(fixed), nil + } + index, err := checkedDictionaryIndex(ctx, dictionary, row, dictionary.Dictionary().Len()) + if err != nil { + return 0, err + } + if dictionary.Dictionary().IsNull(index) { + return uint64(fixed), nil + } + length, err := varlenValueLength(ctx, dictionary.Dictionary(), index) + if err != nil { + return 0, err + } + return checkedRowBytes(ctx, fixed, length) + } + + length, err := varlenValueLength(ctx, column, row) + if err != nil { + return 0, err + } + // Borrowed varlen keeps the whole Arrow values window, including physical + // bytes associated with a logical null, until canonical serialization. + return checkedRowBytes(ctx, fixed, length) +} + +func checkedRowBytes(ctx context.Context, fixed int, variable int) (uint64, error) { + if fixed < 0 || variable < 0 || uint64(fixed) > math.MaxUint64-uint64(variable) { + return 0, moerr.NewInvalidInput(ctx, "Arrow output row size overflows") + } + return uint64(fixed) + uint64(variable), nil +} + +func varlenValueLength(ctx context.Context, values arrow.Array, row int) (length int, err error) { + defer func() { + if recovered := recover(); recovered != nil { + length = 0 + err = moerr.NewInvalidInputf(ctx, "invalid Arrow varlen value at row %d: %v", row, recovered) + } + }() + switch typed := values.(type) { + case *array.String: + return len(typed.Value(row)), nil + case *array.LargeString: + return len(typed.Value(row)), nil + case *array.Binary: + return len(typed.Value(row)), nil + case *array.LargeBinary: + return len(typed.Value(row)), nil + case *array.FixedSizeBinary: + return len(typed.Value(row)), nil + default: + return 0, moerr.NewInvalidInputNoCtxf("invalid Arrow varlen array %T", values) + } +} diff --git a/pkg/container/arrowbridge/convert.go b/pkg/container/arrowbridge/convert.go new file mode 100644 index 0000000000000..f86d62734cab5 --- /dev/null +++ b/pkg/container/arrowbridge/convert.go @@ -0,0 +1,1550 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowbridge + +import ( + "context" + "math" + "time" + "unicode/utf8" + "unsafe" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/decimal128" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/bufferlease" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" +) + +// DefaultMaxPinAmplification is the largest retained-capacity/payload ratio +// for which LOAD keeps a borrowed varlen value area. Above it, materializing +// avoids pinning a large source allocation for a small logical slice. +const DefaultMaxPinAmplification = 4.0 + +// ConvertOptions contains execution semantics that are intentionally outside +// the Arrow schema. Location is the MatrixOne session timezone. +type ConvertOptions struct { + // Location defines LOAD's session-timezone conversion for timestamp values. + // Exact-ABI consumers must not use this option to reinterpret wire types. + Location *time.Location + // MaxPinAmplification overrides the LOAD varlen retention threshold. + MaxPinAmplification float64 + // Allocation charges every materialized MO backing to the caller's + // statement account. Borrowed Arrow capacity is charged by its source lease. + Allocation *vector.AllocationAccountSelection + // ForceMaterialize is a verification/rollback switch used to compare the + // borrowed path with identical conversion semantics. + ForceMaterialize bool +} + +// ConvertStats separates avoided payload copies from the capacity retained to +// achieve them. Descriptor and mandatory layout conversions are materialized. +type ConvertStats struct { + // BorrowedPayloadBytes is logical source payload whose copy was avoided. + BorrowedPayloadBytes int64 + // MaterializedPayloadBytes is payload copied into MO-owned vectors. + MaterializedPayloadBytes int64 + // RetainedCapacityBytes is physical Arrow capacity pinned by borrowed views. + RetainedCapacityBytes int64 + // EligiblePayloadBytes is payload that could use the borrowed layout before + // forced-materialize and pin-amplification policy are applied. + EligiblePayloadBytes int64 + BorrowedColumns int64 + MaterializedColumns int64 + PinAmplificationFallbacks int64 + UnalignedFallbacks int64 +} + +func newOutputVector( + typ types.Type, + selection *vector.AllocationAccountSelection, +) (*vector.Vector, error) { + return vector.NewOffHeapVecWithTypeAndAllocation(typ, selection) +} + +// Convert builds a complete batch transactionally. On any error every vector +// and lease already installed in the partial batch is released. +func (p *Plan) Convert( + ctx context.Context, + record arrow.RecordBatch, + mp *mpool.MPool, + options ConvertOptions, +) (_ *batch.Batch, stats ConvertStats, err error) { + return p.convert(ctx, record, mp, options, true) +} + +// ConvertValidatedRecordWindow converts a slice of a record that was already +// accepted by ValidateRecord. It revalidates the selected rows and dictionary +// indices, but it deliberately does not rescan immutable dictionary values. +// Callers must use only slices derived from that exact immutable record. +func (p *Plan) ConvertValidatedRecordWindow( + ctx context.Context, + record arrow.RecordBatch, + mp *mpool.MPool, + options ConvertOptions, +) (_ *batch.Batch, stats ConvertStats, err error) { + return p.convert(ctx, record, mp, options, false) +} + +func (p *Plan) convert( + ctx context.Context, + record arrow.RecordBatch, + mp *mpool.MPool, + options ConvertOptions, + validateDictionaryValues bool, +) (_ *batch.Batch, stats ConvertStats, err error) { + if p == nil || record == nil || mp == nil { + return nil, stats, moerr.NewInvalidInput(ctx, "invalid Arrow conversion input") + } + if record.NumRows() < 0 || record.NumRows() > int64(math.MaxInt) { + return nil, stats, moerr.NewInvalidInputf(ctx, "Arrow record row count %d is invalid", record.NumRows()) + } + recordSchema := record.Schema() + if recordSchema == nil { + return nil, stats, moerr.NewInvalidInput(ctx, "Arrow record schema is nil") + } + if schemaFingerprint(recordSchema) != p.schemaFingerprint { + return nil, stats, moerr.NewInvalidInput(ctx, "Arrow record schema does not match the bound schema") + } + if err := validateRecordColumns(ctx, record, recordSchema, p.columns, validateDictionaryValues); err != nil { + return nil, stats, err + } + if options.Location == nil { + options.Location = time.UTC + } + if options.MaxPinAmplification <= 0 { + options.MaxPinAmplification = DefaultMaxPinAmplification + } + + rows := int(record.NumRows()) + bat := batch.NewOffHeap(p.attrs) + defer func() { + if err != nil { + bat.Clean(mp) + } + }() + for outputIndex, binding := range p.columns { + if err = ctx.Err(); err != nil { + return nil, stats, err + } + column := record.Column(binding.source) + nullCount, validityErr := validateArrowArrayValidity(ctx, column, validateDictionaryValues) + if validityErr != nil { + return nil, stats, validityErr + } + if binding.target.NotNull && nullCount != 0 { + return nil, stats, moerr.NewConstraintViolationf(ctx, "Arrow column %q contains NULL for a NOT NULL target", binding.target.Name) + } + + var vec *vector.Vector + var columnStats ConvertStats + switch binding.kind { + case conversionBorrowFixed: + vec, columnStats, err = convertFixed( + ctx, column, binding.target.Type, mp, options.Allocation, options.ForceMaterialize, nullCount, + ) + case conversionBorrowVarlen: + vec, columnStats, err = convertVarlen( + ctx, column, binding.target.Type, mp, options.MaxPinAmplification, + options.Allocation, options.ForceMaterialize, nullCount, + ) + default: + vec, columnStats, err = materializeConverted(ctx, column, binding, mp, options.Location, options.Allocation) + } + if err != nil { + return nil, stats, err + } + bat.Vecs[outputIndex] = vec + stats.BorrowedPayloadBytes += columnStats.BorrowedPayloadBytes + stats.MaterializedPayloadBytes += columnStats.MaterializedPayloadBytes + stats.RetainedCapacityBytes += columnStats.RetainedCapacityBytes + stats.EligiblePayloadBytes += columnStats.EligiblePayloadBytes + stats.BorrowedColumns += columnStats.BorrowedColumns + stats.MaterializedColumns += columnStats.MaterializedColumns + stats.PinAmplificationFallbacks += columnStats.PinAmplificationFallbacks + stats.UnalignedFallbacks += columnStats.UnalignedFallbacks + } + bat.SetRowCount(rows) + return bat, stats, nil +} + +func validateArrowArrayValidity( + ctx context.Context, + column arrow.Array, + validateDictionaryValues bool, +) (int, error) { + if column == nil || column.Data() == nil { + return 0, moerr.NewInvalidInput(ctx, "Arrow column has no array data") + } + data := column.Data() + rows := data.Len() + nullCount := data.NullN() + if rows < 0 || nullCount < array.UnknownNullCount || nullCount > rows { + return 0, moerr.NewInvalidInput(ctx, "Arrow column has invalid null metadata") + } + if data.DataType() != nil && data.DataType().ID() == arrow.NULL { + if nullCount >= 0 && nullCount != rows { + return 0, moerr.NewInvalidInput(ctx, "Arrow NULL array has invalid null metadata") + } + return rows, nil + } + actualNulls := 0 + buffers := data.Buffers() + if len(buffers) == 0 || buffers[0] == nil { + if nullCount > 0 { + return 0, moerr.NewInvalidInput(ctx, "Arrow column declares NULLs without a validity buffer") + } + } else if rows > 0 { + bitOffset := data.Offset() + if bitOffset < 0 || int64(bitOffset) > math.MaxInt64-int64(rows) { + return 0, moerr.NewInvalidInput(ctx, "Arrow validity bit offset overflows") + } + lastBit := int64(bitOffset) + int64(rows) + if lastBit > math.MaxInt64-7 { + return 0, moerr.NewInvalidInput(ctx, "Arrow validity bitmap length overflows") + } + requiredBytes := (lastBit + 7) / 8 + validity := buffers[0].Bytes() + if requiredBytes > int64(len(validity)) { + return 0, moerr.NewInvalidInput(ctx, "Arrow validity buffer is too short") + } + for row := 0; row < rows; row++ { + if err := checkConvertContext(ctx, row); err != nil { + return 0, err + } + bit := int64(bitOffset) + int64(row) + if validity[bit>>3]&(1<= 0 && actualNulls != nullCount { + return 0, moerr.NewInvalidInputf(ctx, + "Arrow validity bitmap has %d NULLs; metadata declares %d", actualNulls, nullCount) + } + if dictionary, ok := column.(*array.Dictionary); ok { + values := dictionary.Dictionary() + if values == nil { + return 0, moerr.NewInvalidInput(ctx, "Arrow dictionary has no values array") + } + if err := validateDictionaryIndices(ctx, dictionary, values.Len()); err != nil { + return 0, err + } + if validateDictionaryValues { + if _, err := validateArrowArrayValidity(ctx, values, true); err != nil { + return 0, err + } + switch values.(type) { + case *array.String, *array.LargeString, *array.Binary, *array.LargeBinary, *array.FixedSizeBinary: + if _, err := inspectVarlen(ctx, values); err != nil { + return 0, err + } + } + } + } + return actualNulls, nil +} + +func validateDictionaryIndices( + ctx context.Context, + dictionary *array.Dictionary, + dictionaryLength int, +) (err error) { + defer func() { + if recovered := recover(); recovered != nil { + err = moerr.NewInvalidInputf(ctx, "invalid Arrow dictionary index array: %v", recovered) + } + }() + indices := dictionary.Indices() + if indices == nil || indices.Data() == nil { + return moerr.NewInvalidInput(ctx, "Arrow dictionary indices are missing") + } + if indices.Len() != dictionary.Len() { + return moerr.NewInvalidInput(ctx, "Arrow dictionary indices length does not match the dictionary array") + } + for row := 0; row < dictionary.Len(); row++ { + if dictionary.IsNull(row) { + continue + } + if _, err := checkedDictionaryIndex(ctx, dictionary, row, dictionaryLength); err != nil { + return err + } + } + return nil +} + +func convertFixed( + ctx context.Context, + column arrow.Array, + target types.Type, + mp *mpool.MPool, + selection *vector.AllocationAccountSelection, + forceMaterialize bool, + nullCount int, +) (*vector.Vector, ConvertStats, error) { + var stats ConvertStats + if column.DataType().ID() == arrow.TIME64 { + values, ok := column.(*array.Time64) + if !ok { + return nil, stats, moerr.NewInvalidInput(ctx, "invalid Arrow Time64 array") + } + const microsPerDay = arrow.Time64(24 * 60 * 60 * 1_000_000) + for row, value := range values.Time64Values() { + if err := checkConvertContext(ctx, row); err != nil { + return nil, stats, err + } + if !column.IsNull(row) && (value < 0 || value >= microsPerDay) { + return nil, stats, moerr.NewConstraintViolationf(ctx, "Arrow Time64 value at row %d is outside [0,24h)", row) + } + } + } + if column.DataType().ID() == arrow.DECIMAL128 { + decimalType, ok := column.DataType().(*arrow.Decimal128Type) + values, valuesOK := column.(*array.Decimal128) + if !ok || !valuesOK { + return nil, stats, moerr.NewInvalidInput(ctx, "invalid Arrow Decimal128 array") + } + for row := 0; row < values.Len(); row++ { + if err := checkConvertContext(ctx, row); err != nil { + return nil, stats, err + } + if !column.IsNull(row) { + if err := validateDecimal128Value(ctx, values.Value(row), decimalType.Precision, row); err != nil { + return nil, stats, err + } + } + } + } + + data := column.Data() + buffers := data.Buffers() + if len(buffers) < 2 || buffers[1] == nil { + if column.Len() == 0 { + vec, err := newOutputVector(target, selection) + return vec, stats, err + } + return nil, stats, moerr.NewInvalidInput(ctx, "Arrow fixed-width value buffer is missing") + } + width := target.TypeSize() + if width <= 0 || data.Offset() > math.MaxInt/width || column.Len() > math.MaxInt/width { + return nil, stats, moerr.NewInvalidInput(ctx, "Arrow fixed-width buffer size overflows") + } + start := data.Offset() * width + length := column.Len() * width + values := buffers[1].Bytes() + if start < 0 || length < 0 || start > len(values) || length > len(values)-start { + return nil, stats, moerr.NewInvalidInput(ctx, "Arrow fixed-width value buffer is out of bounds") + } + view := values[start : start+length] + if forceMaterialize { + vec, stats, err := materializeFixedLayout(ctx, column, target, view, mp, selection) + stats.EligiblePayloadBytes = int64(length) + return vec, stats, err + } + if len(view) > 0 && uintptr(unsafe.Pointer(unsafe.SliceData(view)))%uintptr(min(width, 8)) != 0 { + vec, stats, err := materializeFixedLayout(ctx, column, target, view, mp, selection) + stats.EligiblePayloadBytes = int64(length) + stats.UnalignedFallbacks = 1 + return vec, stats, err + } + + lease, err := newArrayDataLease(data, view, int64(buffers[1].Cap())) + if err != nil { + return nil, stats, err + } + vec, err := vector.NewBorrowedFixedVectorWithAllocation(target, column.Len(), view, lease, selection) + lease.Release() + if err != nil { + return nil, stats, err + } + if err = installBorrowedValidity(column, vec, mp, nullCount); err != nil { + vec.Free(mp) + return nil, stats, err + } + stats.BorrowedPayloadBytes = int64(length) + stats.RetainedCapacityBytes = int64(buffers[1].Cap()) + stats.EligiblePayloadBytes = int64(length) + stats.BorrowedColumns = 1 + return vec, stats, nil +} + +func materializeFixedLayout( + ctx context.Context, + column arrow.Array, + target types.Type, + view []byte, + mp *mpool.MPool, + selection *vector.AllocationAccountSelection, +) (*vector.Vector, ConvertStats, error) { + stats := ConvertStats{MaterializedPayloadBytes: int64(len(view)), MaterializedColumns: 1} + vec, err := newOutputVector(target, selection) + if err != nil { + return nil, stats, err + } + if err := vec.PreExtend(column.Len(), mp); err != nil { + // PreExtend may have grown one or more vector buffers before a later + // allocation or account reservation fails. The vector has not been + // published into the output batch yet, so this helper is its sole owner. + vec.Free(mp) + return nil, stats, err + } + vec.SetLength(column.Len()) + copy(vec.GetData(), view) + for row := 0; row < column.Len(); row++ { + if err := checkConvertContext(ctx, row); err != nil { + vec.Free(mp) + return nil, stats, err + } + if column.IsNull(row) { + vec.SetNull(uint64(row)) + } + } + return vec, stats, nil +} + +func newArrayDataLease( + data arrow.ArrayData, + view []byte, + accounted int64, +) (*bufferlease.RefCounted, error) { + // Arrow ArrayData is the physical lifetime root visible to this package. A + // File reader may in turn have attached a RangeLease to that object graph; + // retaining ArrayData therefore preserves both layers without teaching the + // container bridge about FileService. + data.Retain() + lease, err := bufferlease.NewRefCounted(view, accounted, data.Release) + if err != nil { + data.Release() + return nil, err + } + return lease, nil +} + +func installBorrowedValidity(column arrow.Array, vec *vector.Vector, mp *mpool.MPool, nullCount int) error { + if nullCount == 0 { + return nil + } + // MO nulls use an inverted bitmap. Reserve the possible legacy-materialized + // bitmap before publishing the readonly Arrow validity view, so a later + // compatibility consumer cannot allocate outside statement admission. + if err := vec.PrepareBorrowedValidity(column.Len(), mp); err != nil { + return err + } + data := column.Data() + buffers := data.Buffers() + if len(buffers) == 0 || buffers[0] == nil { + return moerr.NewInvalidInputNoCtx("Arrow validity buffer is missing") + } + validity := buffers[0].Bytes() + lease, err := newArrayDataLease(data, validity, int64(buffers[0].Cap())) + if err != nil { + return err + } + err = vec.GetNulls().InstallBorrowedValidity(validity, data.Offset(), column.Len(), nullCount, lease) + lease.Release() + return err +} + +type varlenView struct { + values []byte + offsets32 []int32 + offsets64 []int64 + baseOffset int64 + fixedWidth int64 + text bool + retainedCapacity int64 +} + +func convertVarlen( + ctx context.Context, + column arrow.Array, + target types.Type, + mp *mpool.MPool, + maxPinAmplification float64, + selection *vector.AllocationAccountSelection, + forceMaterialize bool, + nullCount int, +) (*vector.Vector, ConvertStats, error) { + view, err := inspectVarlen(ctx, column) + if err != nil { + return nil, ConvertStats{}, err + } + var ( + avoided int64 + inlineCopied int64 + fixedBinaryPadRow bool + ) + for row := 0; row < column.Len(); row++ { + if err = checkConvertContext(ctx, row); err != nil { + return nil, ConvertStats{}, err + } + // Arrow permits arbitrary, semantically invisible bytes behind a NULL + // slot. Validate only values that can reach SQL; requiring UTF-8 or a + // target width for a NULL payload would reject a valid IPC array. + if column.IsNull(row) { + continue + } + value := view.value(row) + if err = validateVarlenValue(ctx, row, value, view.text, target); err != nil { + return nil, ConvertStats{}, err + } + fixedBinaryPadRow = fixedBinaryPadRow || requiresFixedBinaryPadding(target, value) + if len(value) > types.VarlenaInlineSize { + avoided += int64(len(value)) + } else { + inlineCopied += int64(len(value)) + } + } + // Short MO varlena values must remain canonical inline descriptors. Borrowing + // only the long values keeps that invariant while still avoiding their copy. + // BINARY(N) assignment stores exactly N bytes, so any short value forces an + // owned, zero-padded image for the entire column. + if forceMaterialize || avoided == 0 || fixedBinaryPadRow { + vec, stats, err := materializeVarlen(ctx, column, target, view, mp, selection) + if !fixedBinaryPadRow { + stats.EligiblePayloadBytes = avoided + } + return vec, stats, err + } + // Admission follows physical retained capacity, while this policy compares + // it with the useful bytes. A tiny slice of a large Arrow allocation should + // not remain pinned merely because its descriptor can be borrowed. + if float64(view.retainedCapacity)/float64(avoided) > maxPinAmplification { + vec, stats, err := materializeVarlen(ctx, column, target, view, mp, selection) + stats.EligiblePayloadBytes = avoided + stats.PinAmplificationFallbacks = 1 + return vec, stats, err + } + + vec, err := newOutputVector(target, selection) + if err != nil { + return nil, ConvertStats{}, err + } + if err = vec.PreExtend(column.Len(), mp); err != nil { + // Keep failure transactional even when PreExtend performed partial work; + // Convert cannot clean this vector until it is installed in the batch. + vec.Free(mp) + return nil, ConvertStats{}, err + } + vec.SetLength(column.Len()) + descriptors := vector.MustFixedColNoTypeCheck[types.Varlena](vec) + for row := 0; row < column.Len(); row++ { + if column.IsNull(row) { + continue + } + value := view.value(row) + if len(value) <= types.VarlenaInlineSize { + descriptors[row][0] = byte(len(value)) + copy(descriptors[row][1:], value) + continue + } + offset := view.offset(row) + if offset < 0 || offset > math.MaxUint32 || len(value) > math.MaxUint32 { + vec.Free(mp) + return nil, ConvertStats{}, moerr.NewInvalidInputf(ctx, "Arrow value at row %d exceeds MatrixOne varlen limits", row) + } + descriptors[row].SetOffsetLen(uint32(offset), uint32(len(value))) + } + + data := column.Data() + lease, err := newArrayDataLease(data, view.values, view.retainedCapacity) + if err != nil { + vec.Free(mp) + return nil, ConvertStats{}, err + } + err = vec.InstallBorrowedArea(view.values, lease) + lease.Release() + if err != nil { + vec.Free(mp) + return nil, ConvertStats{}, err + } + if err = installBorrowedValidity(column, vec, mp, nullCount); err != nil { + vec.Free(mp) + return nil, ConvertStats{}, err + } + return vec, ConvertStats{ + BorrowedPayloadBytes: avoided, + MaterializedPayloadBytes: inlineCopied, + RetainedCapacityBytes: view.retainedCapacity, + EligiblePayloadBytes: avoided, + BorrowedColumns: 1, + }, nil +} + +func inspectVarlen(ctx context.Context, column arrow.Array) (view varlenView, err error) { + // Arrow-Go constructors validate only part of the offset contract. Keep + // their slice-based accessors behind this trust boundary so a malformed + // in-process array is reported as invalid input instead of panicking. + defer func() { + if recovered := recover(); recovered != nil { + view = varlenView{} + err = moerr.NewInvalidInputf(ctx, "invalid Arrow varlen array: %v", recovered) + } + }() + data := column.Data() + buffers := data.Buffers() + if len(buffers) < 2 { + return view, moerr.NewInvalidInput(ctx, "Arrow varlen buffers are missing") + } + + switch values := column.(type) { + case *array.String: + view.text = true + view.values = values.ValueBytes() + view.offsets32 = values.ValueOffsets() + view.baseOffset = int64(view.offsets32[0]) + case *array.LargeString: + view.text = true + view.values = values.ValueBytes() + view.offsets64 = values.ValueOffsets() + view.baseOffset = view.offsets64[0] + case *array.Binary: + view.values = values.ValueBytes() + view.offsets32 = values.ValueOffsets() + view.baseOffset = int64(view.offsets32[0]) + case *array.LargeBinary: + view.values = values.ValueBytes() + view.offsets64 = values.ValueOffsets() + view.baseOffset = view.offsets64[0] + case *array.FixedSizeBinary: + widthType, ok := column.DataType().(*arrow.FixedSizeBinaryType) + if !ok || widthType.ByteWidth <= 0 || data.Offset() > math.MaxInt/int(widthType.ByteWidth) || + column.Len() > math.MaxInt/int(widthType.ByteWidth) { + return view, moerr.NewInvalidInput(ctx, "invalid Arrow FixedSizeBinary width") + } + if buffers[1] == nil { + if column.Len() == 0 { + return view, nil + } + return view, moerr.NewInvalidInput(ctx, "Arrow FixedSizeBinary value buffer is missing") + } + start := data.Offset() * int(widthType.ByteWidth) + length := column.Len() * int(widthType.ByteWidth) + if start > len(buffers[1].Bytes()) || length > len(buffers[1].Bytes())-start { + return view, moerr.NewInvalidInput(ctx, "Arrow FixedSizeBinary value buffer is out of bounds") + } + view.values = buffers[1].Bytes()[start : start+length] + view.fixedWidth = int64(widthType.ByteWidth) + default: + return view, moerr.NewInvalidInputf(ctx, "invalid Arrow varlen array %T", column) + } + if len(buffers) > 2 && buffers[2] != nil { + view.retainedCapacity = int64(buffers[2].Cap()) + } else if buffers[1] != nil { + view.retainedCapacity = int64(buffers[1].Cap()) + } + if len(view.offsets32) != 0 && len(view.offsets32) != column.Len()+1 || + len(view.offsets64) != 0 && len(view.offsets64) != column.Len()+1 || + len(view.offsets32) == 0 && len(view.offsets64) == 0 && view.fixedWidth == 0 { + return view, moerr.NewInvalidInput(ctx, "invalid Arrow varlen offsets") + } + for row := 0; row < column.Len(); row++ { + if err := checkConvertContext(ctx, row); err != nil { + return view, err + } + start, end := view.offset(row), view.offset(row+1) + if start < 0 || start > end || end > int64(len(view.values)) { + return view, moerr.NewInvalidInputf(ctx, "invalid Arrow varlen offsets at row %d", row) + } + } + return view, nil +} + +func (v varlenView) offset(row int) int64 { + if len(v.offsets32) != 0 { + return int64(v.offsets32[row]) - v.baseOffset + } + if len(v.offsets64) != 0 { + return v.offsets64[row] - v.baseOffset + } + return int64(row) * v.fixedWidth +} + +func (v varlenView) value(row int) []byte { + return v.values[v.offset(row):v.offset(row+1)] +} + +func validateVarlenValue( + ctx context.Context, + row int, + value []byte, + text bool, + target types.Type, +) error { + if len(value) > math.MaxUint32 { + return moerr.NewConstraintViolationf(ctx, "Arrow value at row %d exceeds MatrixOne varlen capacity", row) + } + logicalLength := len(value) + if text { + if !utf8.Valid(value) { + return moerr.NewInvalidInputf(ctx, "Arrow UTF-8 value at row %d is invalid", row) + } + logicalLength = utf8.RuneCount(value) + } + if target.Width > 0 && int64(logicalLength) > int64(target.Width) { + return moerr.NewConstraintViolationf(ctx, "Arrow value at row %d has length %d, target limit is %d", row, logicalLength, target.Width) + } + return nil +} + +func requiresFixedBinaryPadding(target types.Type, value []byte) bool { + return target.Oid == types.T_binary && target.Width > 0 && len(value) < int(target.Width) +} + +// appendLoadVarlenValue applies storage semantics normally supplied by the +// DML assignment cast. Arrow LOAD emits target-typed vectors directly, so +// fixed BINARY padding has to happen at this bridge boundary. +func appendLoadVarlenValue( + vec *vector.Vector, + value []byte, + isNull bool, + target types.Type, + mp *mpool.MPool, +) (int64, error) { + if isNull || !requiresFixedBinaryPadding(target, value) { + if err := vector.AppendBytes(vec, value, isNull, mp); err != nil { + return 0, err + } + if isNull { + return 0, nil + } + return int64(len(value)), nil + } + stored := int(target.Width) + err := vector.AppendBytesWithWriter(vec, stored, mp, func(dst []byte) error { + copy(dst, value) + clear(dst[len(value):]) + return nil + }) + return int64(stored), err +} + +func materializeVarlen( + ctx context.Context, + column arrow.Array, + target types.Type, + view varlenView, + mp *mpool.MPool, + selection *vector.AllocationAccountSelection, +) (*vector.Vector, ConvertStats, error) { + vec, err := newOutputVector(target, selection) + if err != nil { + return nil, ConvertStats{}, err + } + var copied int64 + for row := 0; row < column.Len(); row++ { + if row&1023 == 0 { + if err := ctx.Err(); err != nil { + vec.Free(mp) + return nil, ConvertStats{}, err + } + } + value := view.value(row) + stored, err := appendLoadVarlenValue( + vec, value, column.IsNull(row), target, mp, + ) + if err != nil { + vec.Free(mp) + return nil, ConvertStats{}, err + } + copied += stored + } + return vec, ConvertStats{MaterializedPayloadBytes: copied, MaterializedColumns: 1}, nil +} + +func materializeConverted( + ctx context.Context, + column arrow.Array, + binding columnPlan, + mp *mpool.MPool, + location *time.Location, + selection *vector.AllocationAccountSelection, +) (*vector.Vector, ConvertStats, error) { + if binding.kind == conversionMaterializeDictionary { + return materializeDictionary(ctx, column, binding, mp, location, selection) + } + vec, err := newOutputVector(binding.target.Type, selection) + if err != nil { + return nil, ConvertStats{}, err + } + stats := ConvertStats{ + MaterializedPayloadBytes: int64(column.Len() * binding.target.Type.TypeSize()), + MaterializedColumns: 1, + } + fail := func(err error) (*vector.Vector, ConvertStats, error) { + vec.Free(mp) + return nil, stats, err + } + + switch binding.kind { + case conversionMaterializeBool: + values, ok := column.(*array.Boolean) + if !ok { + return fail(moerr.NewInvalidInput(ctx, "invalid Arrow Boolean array")) + } + for row := 0; row < values.Len(); row++ { + if err := checkConvertContext(ctx, row); err != nil { + return fail(err) + } + if err := vector.AppendFixed(vec, values.Value(row), values.IsNull(row), mp); err != nil { + return fail(err) + } + } + case conversionMaterializeDate32: + values, ok := column.(*array.Date32) + if !ok { + return fail(moerr.NewInvalidInput(ctx, "invalid Arrow Date32 array")) + } + for row, value := range values.Date32Values() { + if err := checkConvertContext(ctx, row); err != nil { + return fail(err) + } + if binding.target.Type.Oid == types.T_date { + date := types.Date(0) + if !values.IsNull(row) { + date, err = arrowDaysToDate(int64(value)) + if err != nil { + return fail(moerr.NewConstraintViolationf(ctx, "Arrow Date32 value at row %d is out of MatrixOne range", row)) + } + } + if err := vector.AppendFixed(vec, date, values.IsNull(row), mp); err != nil { + return fail(err) + } + } else { + datetime, err := arrowDaysToDatetime(int64(value)) + if err != nil && !values.IsNull(row) { + return fail(moerr.NewConstraintViolationf(ctx, "Arrow Date32 value at row %d is out of MatrixOne range", row)) + } + if err := vector.AppendFixed(vec, datetime, values.IsNull(row), mp); err != nil { + return fail(err) + } + } + } + case conversionMaterializeDate64: + values, ok := column.(*array.Date64) + if !ok { + return fail(moerr.NewInvalidInput(ctx, "invalid Arrow Date64 array")) + } + const millisPerDay = int64(24 * 60 * 60 * 1000) + for row, value := range values.Date64Values() { + if err := checkConvertContext(ctx, row); err != nil { + return fail(err) + } + days := int64(value) / millisPerDay + if !values.IsNull(row) && (int64(value)%millisPerDay != 0 || days < math.MinInt32 || days > math.MaxInt32) { + return fail(moerr.NewConstraintViolationf(ctx, "Arrow Date64 value at row %d is not an integral representable day", row)) + } + if binding.target.Type.Oid == types.T_date { + date := types.Date(0) + if !values.IsNull(row) { + date, err = arrowDaysToDate(days) + if err != nil { + return fail(moerr.NewConstraintViolationf(ctx, "Arrow Date64 value at row %d is out of MatrixOne range", row)) + } + } + if err := vector.AppendFixed(vec, date, values.IsNull(row), mp); err != nil { + return fail(err) + } + } else { + datetime, err := arrowDaysToDatetime(days) + if err != nil && !values.IsNull(row) { + return fail(moerr.NewConstraintViolationf(ctx, "Arrow Date64 value at row %d is out of MatrixOne range", row)) + } + if err := vector.AppendFixed(vec, datetime, values.IsNull(row), mp); err != nil { + return fail(err) + } + } + } + case conversionMaterializeTimestamp: + values, ok := column.(*array.Timestamp) + if !ok { + return fail(moerr.NewInvalidInput(ctx, "invalid Arrow Timestamp array")) + } + timestampType, ok := column.DataType().(*arrow.TimestampType) + if !ok { + return fail(moerr.NewInvalidInput(ctx, "invalid Arrow Timestamp type")) + } + if _, err := timestampType.GetZone(); err != nil { + return fail(moerr.NewInvalidInputf(ctx, "invalid Arrow timezone %q: %v", timestampType.TimeZone, err)) + } + for row, value := range values.TimestampValues() { + if err := checkConvertContext(ctx, row); err != nil { + return fail(err) + } + isNull := values.IsNull(row) + micros, err := timestampToMicros(int64(value), timestampType.Unit) + if err != nil && !isNull { + return fail(moerr.NewConstraintViolationf(ctx, "Arrow Timestamp at row %d is out of range", row)) + } + if !isNull { + if err := validateTimestampPrecision(ctx, row, micros, binding.target.Type); err != nil { + return fail(err) + } + } + if binding.target.Type.Oid == types.T_timestamp { + converted, err := arrowMicrosToTimestamp(micros, timestampType.TimeZone != "", location) + if err != nil && !isNull { + return fail(moerr.NewConstraintViolationf(ctx, "Arrow Timestamp at row %d is out of MatrixOne range", row)) + } + if err := vector.AppendFixed(vec, converted, isNull, mp); err != nil { + return fail(err) + } + } else { + converted, err := arrowMicrosToDatetime(micros, timestampType.TimeZone != "", location) + if err != nil && !isNull { + return fail(moerr.NewConstraintViolationf(ctx, "Arrow Timestamp at row %d is out of MatrixOne range", row)) + } + if err := vector.AppendFixed(vec, converted, isNull, mp); err != nil { + return fail(err) + } + } + } + case conversionMaterializeWiden: + for row := 0; row < column.Len(); row++ { + if err := checkConvertContext(ctx, row); err != nil { + return fail(err) + } + if err := appendWidenedValue(ctx, vec, column, row, column.IsNull(row), binding.target.Type, mp); err != nil { + return fail(err) + } + } + case conversionMaterializeTime: + for row := 0; row < column.Len(); row++ { + if err := checkConvertContext(ctx, row); err != nil { + return fail(err) + } + if err := appendTimeValue(ctx, vec, column, row, column.IsNull(row), binding.target.Type, mp); err != nil { + return fail(err) + } + } + case conversionMaterializeNull: + if err := vec.PreExtend(column.Len(), mp); err != nil { + return fail(err) + } + vec.SetLength(column.Len()) + for row := 0; row < column.Len(); row++ { + if err := checkConvertContext(ctx, row); err != nil { + return fail(err) + } + vec.SetNull(uint64(row)) + } + default: + return fail(moerr.NewInternalErrorNoCtx("unknown Arrow conversion plan")) + } + return vec, stats, nil +} + +func appendWidenedValue( + ctx context.Context, + vec *vector.Vector, + values arrow.Array, + row int, + isNull bool, + target types.Type, + mp *mpool.MPool, +) error { + switch typed := values.(type) { + case *array.Int8: + value := typed.Value(row) + switch target.Oid { + case types.T_int16: + return appendDictionaryFixed(vec, int16(value), isNull, mp) + case types.T_int32: + return appendDictionaryFixed(vec, int32(value), isNull, mp) + case types.T_int64: + return appendDictionaryFixed(vec, int64(value), isNull, mp) + } + case *array.Int16: + value := typed.Value(row) + switch target.Oid { + case types.T_int32: + return appendDictionaryFixed(vec, int32(value), isNull, mp) + case types.T_int64: + return appendDictionaryFixed(vec, int64(value), isNull, mp) + } + case *array.Int32: + if target.Oid == types.T_int64 { + return appendDictionaryFixed(vec, int64(typed.Value(row)), isNull, mp) + } + case *array.Uint8: + value := typed.Value(row) + switch target.Oid { + case types.T_uint16: + return appendDictionaryFixed(vec, uint16(value), isNull, mp) + case types.T_uint32: + return appendDictionaryFixed(vec, uint32(value), isNull, mp) + case types.T_uint64: + return appendDictionaryFixed(vec, uint64(value), isNull, mp) + } + case *array.Uint16: + value := typed.Value(row) + switch target.Oid { + case types.T_uint32: + return appendDictionaryFixed(vec, uint32(value), isNull, mp) + case types.T_uint64: + return appendDictionaryFixed(vec, uint64(value), isNull, mp) + } + case *array.Uint32: + if target.Oid == types.T_uint64 { + return appendDictionaryFixed(vec, uint64(typed.Value(row)), isNull, mp) + } + case *array.Float32: + if target.Oid == types.T_float64 { + return appendDictionaryFixed(vec, float64(typed.Value(row)), isNull, mp) + } + } + return moerr.NewInvalidInputf(ctx, "invalid Arrow widening from %s to %s", values.DataType(), target) +} + +func appendTimeValue( + ctx context.Context, + vec *vector.Vector, + values arrow.Array, + row int, + isNull bool, + target types.Type, + mp *mpool.MPool, +) error { + var value int64 + var unit arrow.TimeUnit + switch typed := values.(type) { + case *array.Time32: + value = int64(typed.Value(row)) + timeType, ok := values.DataType().(*arrow.Time32Type) + if !ok { + return moerr.NewInvalidInput(ctx, "invalid Arrow Time32 type") + } + unit = timeType.Unit + case *array.Time64: + value = int64(typed.Value(row)) + timeType, ok := values.DataType().(*arrow.Time64Type) + if !ok { + return moerr.NewInvalidInput(ctx, "invalid Arrow Time64 type") + } + unit = timeType.Unit + default: + return moerr.NewInvalidInputf(ctx, "invalid Arrow time array %T", values) + } + micros, err := timeToMicros(value, unit) + if err != nil && !isNull { + return moerr.NewConstraintViolationf(ctx, "Arrow time value at row %d is not representable in microseconds", row) + } + const microsPerDay = int64(24 * 60 * 60 * 1_000_000) + if !isNull && (micros < 0 || micros >= microsPerDay) { + return moerr.NewConstraintViolationf(ctx, "Arrow time value at row %d is outside [0,24h)", row) + } + precisionFactor := int64(1) + for scale := target.Scale; scale < 6; scale++ { + precisionFactor *= 10 + } + if !isNull && micros%precisionFactor != 0 { + return moerr.NewConstraintViolationf(ctx, + "Arrow time value at row %d exceeds MatrixOne TIME(%d) precision", row, target.Scale) + } + return appendDictionaryFixed(vec, types.Time(micros), isNull, mp) +} + +func timeToMicros(value int64, unit arrow.TimeUnit) (int64, error) { + switch unit { + case arrow.Second: + if value > math.MaxInt64/1_000_000 || value < math.MinInt64/1_000_000 { + return 0, moerr.NewOutOfRangeNoCtx("time", "microsecond") + } + return value * 1_000_000, nil + case arrow.Millisecond: + if value > math.MaxInt64/1_000 || value < math.MinInt64/1_000 { + return 0, moerr.NewOutOfRangeNoCtx("time", "microsecond") + } + return value * 1_000, nil + case arrow.Microsecond: + return value, nil + case arrow.Nanosecond: + if value%1_000 != 0 { + return 0, moerr.NewOutOfRangeNoCtx("time", "microsecond") + } + return value / 1_000, nil + default: + return 0, moerr.NewInvalidInputNoCtx("invalid Arrow time unit") + } +} + +// validateDecimal128Value closes the gap between Arrow's schema precision and +// the actual raw scaled integer stored in a Decimal128 array. Exact-layout +// conversion borrows that integer, so accepting an oversized value would let +// a target DECIMAL column observe a value outside its declared range. +func validateDecimal128Value(ctx context.Context, value decimal128.Num, precision int32, row int) error { + if !value.FitsInPrecision(precision) { + return moerr.NewConstraintViolationf(ctx, + "Arrow Decimal128 value at row %d exceeds precision %d", row, precision) + } + return nil +} + +func arrowDaysToDate(days int64) (types.Date, error) { + if days < math.MinInt32 || days > math.MaxInt32 { + return 0, moerr.NewOutOfRangeNoCtx("date", "MatrixOne") + } + value := types.DaysFromUnixEpochToDate(int32(days)) + year, month, day, _ := value.Calendar(true) + if !types.ValidDate(year, month, day) { + return 0, moerr.NewOutOfRangeNoCtx("date", "MatrixOne") + } + return value, nil +} + +func arrowDaysToDatetime(days int64) (types.Datetime, error) { + if days > math.MaxInt64/(24*60*60) || days < math.MinInt64/(24*60*60) { + return 0, moerr.NewOutOfRangeNoCtx("date", "MatrixOne datetime") + } + value := types.DatetimeFromUnixWithNsec(time.UTC, days*24*60*60, 0) + year, _, _, _ := value.ToDate().Calendar(true) + if year < types.MinDatetimeYear || year > types.MaxDatetimeYear { + return 0, moerr.NewOutOfRangeNoCtx("date", "MatrixOne datetime") + } + return value, nil +} + +func materializeDictionary( + ctx context.Context, + column arrow.Array, + binding columnPlan, + mp *mpool.MPool, + location *time.Location, + selection *vector.AllocationAccountSelection, +) (*vector.Vector, ConvertStats, error) { + dictionary, ok := column.(*array.Dictionary) + if !ok { + return nil, ConvertStats{}, moerr.NewInvalidInput(ctx, "invalid Arrow Dictionary array") + } + values := dictionary.Dictionary() + valueKind, err := selectLoadConversion(values.DataType(), binding.target.Type) + if err != nil || valueKind == conversionMaterializeDictionary { + return nil, ConvertStats{}, moerr.NewInvalidInputf(ctx, "invalid Arrow Dictionary value type %s", values.DataType()) + } + + vec, err := newOutputVector(binding.target.Type, selection) + if err != nil { + return nil, ConvertStats{}, err + } + stats := ConvertStats{MaterializedColumns: 1} + if valueKind != conversionBorrowVarlen { + stats.MaterializedPayloadBytes = int64(column.Len() * binding.target.Type.TypeSize()) + } + fail := func(err error) (*vector.Vector, ConvertStats, error) { + vec.Free(mp) + return nil, stats, err + } + + for row := 0; row < dictionary.Len(); row++ { + if err := checkConvertContext(ctx, row); err != nil { + return fail(err) + } + logicalNull := dictionary.IsNull(row) + index := 0 + if !logicalNull { + index, err = checkedDictionaryIndex(ctx, dictionary, row, values.Len()) + if err != nil { + return fail(err) + } + logicalNull = values.IsNull(index) + } + if binding.target.NotNull && logicalNull { + return fail(moerr.NewConstraintViolationf(ctx, + "Arrow column %q contains NULL for a NOT NULL target", binding.target.Name)) + } + copied, err := appendDictionaryValue( + ctx, vec, values, valueKind, index, row, logicalNull, binding.target.Type, mp, location, + ) + if err != nil { + return fail(err) + } + stats.MaterializedPayloadBytes += copied + } + return vec, stats, nil +} + +func checkedDictionaryIndex( + ctx context.Context, + dictionary *array.Dictionary, + row int, + dictionaryLength int, +) (int, error) { + var index int64 + switch indices := dictionary.Indices().(type) { + case *array.Int8: + index = int64(indices.Value(row)) + case *array.Int16: + index = int64(indices.Value(row)) + case *array.Int32: + index = int64(indices.Value(row)) + case *array.Int64: + index = indices.Value(row) + case *array.Uint8: + index = int64(indices.Value(row)) + case *array.Uint16: + index = int64(indices.Value(row)) + case *array.Uint32: + index = int64(indices.Value(row)) + case *array.Uint64: + value := indices.Value(row) + if value > uint64(math.MaxInt) { + return 0, moerr.NewInvalidInputf(ctx, "Arrow dictionary index at row %d overflows", row) + } + index = int64(value) + default: + return 0, moerr.NewInvalidInputf(ctx, "invalid Arrow dictionary index array %T", dictionary.Indices()) + } + if index < 0 || index >= int64(dictionaryLength) { + return 0, moerr.NewInvalidInputf(ctx, + "Arrow dictionary index %d at row %d is outside [0,%d)", index, row, dictionaryLength) + } + return int(index), nil +} + +func appendDictionaryValue( + ctx context.Context, + vec *vector.Vector, + values arrow.Array, + valueKind conversionKind, + index int, + row int, + isNull bool, + target types.Type, + mp *mpool.MPool, + location *time.Location, +) (int64, error) { + if valueKind == conversionBorrowVarlen { + var value []byte + text := false + if !isNull { + switch typed := values.(type) { + case *array.String: + value, text = []byte(typed.Value(index)), true + case *array.LargeString: + value, text = []byte(typed.Value(index)), true + case *array.Binary: + value = typed.Value(index) + case *array.LargeBinary: + value = typed.Value(index) + case *array.FixedSizeBinary: + value = typed.Value(index) + default: + return 0, moerr.NewInvalidInputf(ctx, "invalid Arrow dictionary varlen values %T", values) + } + if err := validateVarlenValue(ctx, row, value, text, target); err != nil { + return 0, err + } + } + return appendLoadVarlenValue(vec, value, isNull, target, mp) + } + if isNull { + return 0, appendDictionaryNull(vec, target, mp) + } + if valueKind == conversionMaterializeWiden { + return 0, appendWidenedValue(ctx, vec, values, index, false, target, mp) + } + if valueKind == conversionMaterializeTime { + return 0, appendTimeValue(ctx, vec, values, index, false, target, mp) + } + + switch typed := values.(type) { + case *array.Int8: + return 0, appendDictionaryFixed(vec, typed.Value(index), isNull, mp) + case *array.Int16: + return 0, appendDictionaryFixed(vec, typed.Value(index), isNull, mp) + case *array.Int32: + return 0, appendDictionaryFixed(vec, typed.Value(index), isNull, mp) + case *array.Int64: + return 0, appendDictionaryFixed(vec, typed.Value(index), isNull, mp) + case *array.Uint8: + return 0, appendDictionaryFixed(vec, typed.Value(index), isNull, mp) + case *array.Uint16: + return 0, appendDictionaryFixed(vec, typed.Value(index), isNull, mp) + case *array.Uint32: + return 0, appendDictionaryFixed(vec, typed.Value(index), isNull, mp) + case *array.Uint64: + return 0, appendDictionaryFixed(vec, typed.Value(index), isNull, mp) + case *array.Float32: + return 0, appendDictionaryFixed(vec, typed.Value(index), isNull, mp) + case *array.Float64: + return 0, appendDictionaryFixed(vec, typed.Value(index), isNull, mp) + case *array.Decimal128: + value := typed.Value(index) + if !isNull { + decimalType, ok := values.DataType().(*arrow.Decimal128Type) + if !ok { + return 0, moerr.NewInvalidInputf(ctx, "invalid Arrow Decimal128 dictionary type %T", values.DataType()) + } + if err := validateDecimal128Value(ctx, value, decimalType.Precision, row); err != nil { + return 0, err + } + } + converted := types.Decimal128{B0_63: value.LowBits(), B64_127: uint64(value.HighBits())} + return 0, appendDictionaryFixed(vec, converted, isNull, mp) + case *array.Time64: + value := typed.Value(index) + const microsPerDay = arrow.Time64(24 * 60 * 60 * 1_000_000) + if !isNull && (value < 0 || value >= microsPerDay) { + return 0, moerr.NewConstraintViolationf(ctx, + "Arrow Time64 value at row %d is outside [0,24h)", row) + } + return 0, appendDictionaryFixed(vec, types.Time(value), isNull, mp) + case *array.Boolean: + return 0, appendDictionaryFixed(vec, typed.Value(index), isNull, mp) + case *array.Null: + return 0, appendDictionaryNull(vec, target, mp) + case *array.Date32: + if target.Oid == types.T_date { + value := types.Date(0) + if !isNull { + var err error + value, err = arrowDaysToDate(int64(typed.Value(index))) + if err != nil { + return 0, moerr.NewConstraintViolationf(ctx, "Arrow Date32 value at row %d is out of MatrixOne range", row) + } + } + return 0, appendDictionaryFixed(vec, value, isNull, mp) + } + value := types.Datetime(0) + if !isNull { + var err error + value, err = arrowDaysToDatetime(int64(typed.Value(index))) + if err != nil { + return 0, moerr.NewConstraintViolationf(ctx, "Arrow Date32 value at row %d is out of MatrixOne range", row) + } + } + return 0, appendDictionaryFixed(vec, value, isNull, mp) + case *array.Date64: + value := int64(typed.Value(index)) + const millisPerDay = int64(24 * 60 * 60 * 1000) + days := value / millisPerDay + if !isNull && (value%millisPerDay != 0 || days < math.MinInt32 || days > math.MaxInt32) { + return 0, moerr.NewConstraintViolationf(ctx, + "Arrow Date64 value at row %d is not an integral representable day", row) + } + if target.Oid == types.T_date { + date := types.Date(0) + if !isNull { + var err error + date, err = arrowDaysToDate(days) + if err != nil { + return 0, moerr.NewConstraintViolationf(ctx, "Arrow Date64 value at row %d is out of MatrixOne range", row) + } + } + return 0, appendDictionaryFixed(vec, date, isNull, mp) + } + datetime := types.Datetime(0) + if !isNull { + var err error + datetime, err = arrowDaysToDatetime(days) + if err != nil { + return 0, moerr.NewConstraintViolationf(ctx, "Arrow Date64 value at row %d is out of MatrixOne range", row) + } + } + return 0, appendDictionaryFixed(vec, datetime, isNull, mp) + case *array.Timestamp: + timestampType, ok := values.DataType().(*arrow.TimestampType) + if !ok { + return 0, moerr.NewInvalidInput(ctx, "invalid Arrow Timestamp dictionary type") + } + if _, err := timestampType.GetZone(); err != nil { + return 0, moerr.NewInvalidInputf(ctx, "invalid Arrow timezone %q: %v", timestampType.TimeZone, err) + } + micros, err := timestampToMicros(int64(typed.Value(index)), timestampType.Unit) + if err != nil && !isNull { + return 0, moerr.NewConstraintViolationf(ctx, "Arrow Timestamp at row %d is out of range", row) + } + if !isNull { + if err := validateTimestampPrecision(ctx, row, micros, target); err != nil { + return 0, err + } + } + if target.Oid == types.T_timestamp { + converted, err := arrowMicrosToTimestamp(micros, timestampType.TimeZone != "", location) + if err != nil && !isNull { + return 0, moerr.NewConstraintViolationf(ctx, "Arrow Timestamp at row %d is out of MatrixOne range", row) + } + return 0, appendDictionaryFixed(vec, converted, isNull, mp) + } + converted, err := arrowMicrosToDatetime(micros, timestampType.TimeZone != "", location) + if err != nil && !isNull { + return 0, moerr.NewConstraintViolationf(ctx, "Arrow Timestamp at row %d is out of MatrixOne range", row) + } + return 0, appendDictionaryFixed(vec, converted, isNull, mp) + default: + return 0, moerr.NewInvalidInputf(ctx, "invalid Arrow dictionary values %T", values) + } +} + +func appendDictionaryFixed[T any](vec *vector.Vector, value T, isNull bool, mp *mpool.MPool) error { + var zero T + if isNull { + value = zero + } + return vector.AppendFixed(vec, value, isNull, mp) +} + +func appendDictionaryNull(vec *vector.Vector, target types.Type, mp *mpool.MPool) error { + if target.IsVarlen() { + return vector.AppendBytes(vec, nil, true, mp) + } + switch target.Oid { + case types.T_bool: + return vector.AppendFixed(vec, false, true, mp) + case types.T_bit: + return vector.AppendFixed(vec, uint64(0), true, mp) + case types.T_int8: + return vector.AppendFixed(vec, int8(0), true, mp) + case types.T_int16: + return vector.AppendFixed(vec, int16(0), true, mp) + case types.T_int32: + return vector.AppendFixed(vec, int32(0), true, mp) + case types.T_int64: + return vector.AppendFixed(vec, int64(0), true, mp) + case types.T_uint8: + return vector.AppendFixed(vec, uint8(0), true, mp) + case types.T_uint16: + return vector.AppendFixed(vec, uint16(0), true, mp) + case types.T_uint32: + return vector.AppendFixed(vec, uint32(0), true, mp) + case types.T_uint64: + return vector.AppendFixed(vec, uint64(0), true, mp) + case types.T_float32: + return vector.AppendFixed(vec, float32(0), true, mp) + case types.T_float64: + return vector.AppendFixed(vec, float64(0), true, mp) + case types.T_year: + return vector.AppendFixed(vec, types.MoYear(0), true, mp) + case types.T_enum: + return vector.AppendFixed(vec, types.Enum(0), true, mp) + case types.T_decimal64: + return vector.AppendFixed(vec, types.Decimal64(0), true, mp) + case types.T_decimal128: + return vector.AppendFixed(vec, types.Decimal128{}, true, mp) + case types.T_decimal256: + return vector.AppendFixed(vec, types.Decimal256{}, true, mp) + case types.T_uuid: + return vector.AppendFixed(vec, types.Uuid{}, true, mp) + case types.T_TS: + return vector.AppendFixed(vec, types.TS{}, true, mp) + case types.T_Rowid: + return vector.AppendFixed(vec, types.Rowid{}, true, mp) + case types.T_Blockid: + return vector.AppendFixed(vec, types.Blockid{}, true, mp) + case types.T_date: + return vector.AppendFixed(vec, types.Date(0), true, mp) + case types.T_time: + return vector.AppendFixed(vec, types.Time(0), true, mp) + case types.T_datetime: + return vector.AppendFixed(vec, types.Datetime(0), true, mp) + case types.T_timestamp: + return vector.AppendFixed(vec, types.Timestamp(0), true, mp) + default: + return moerr.NewInternalErrorNoCtx("unknown MatrixOne dictionary target type") + } +} + +func checkConvertContext(ctx context.Context, row int) error { + if row&1023 == 0 { + return ctx.Err() + } + return nil +} + +func timestampToMicros(value int64, unit arrow.TimeUnit) (int64, error) { + switch unit { + case arrow.Second: + if value > math.MaxInt64/1_000_000 || value < math.MinInt64/1_000_000 { + return 0, moerr.NewOutOfRangeNoCtx("timestamp", "microsecond") + } + return value * 1_000_000, nil + case arrow.Millisecond: + if value > math.MaxInt64/1_000 || value < math.MinInt64/1_000 { + return 0, moerr.NewOutOfRangeNoCtx("timestamp", "microsecond") + } + return value * 1_000, nil + case arrow.Microsecond: + return value, nil + case arrow.Nanosecond: + if value%1_000 != 0 { + return 0, moerr.NewOutOfRangeNoCtx("timestamp", "microsecond") + } + return value / 1_000, nil + default: + return 0, moerr.NewInvalidInputNoCtx("invalid Arrow timestamp unit") + } +} + +func validateTimestampPrecision( + ctx context.Context, + row int, + micros int64, + target types.Type, +) error { + precisionFactor := int64(1) + for scale := target.Scale; scale < 6; scale++ { + precisionFactor *= 10 + } + if micros%precisionFactor != 0 { + return moerr.NewConstraintViolationf(ctx, + "Arrow Timestamp at row %d exceeds MatrixOne %s(%d) precision", + row, target.Oid, target.Scale) + } + return nil +} + +var ( + minSupportedUnixMicros = time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).UnixMicro() + maxSupportedUnixMicros = time.Date(9999, 12, 31, 23, 59, 59, 999999000, time.UTC).UnixMicro() +) + +func arrowMicrosToTimestamp(micros int64, zoned bool, location *time.Location) (types.Timestamp, error) { + if !zoned { + wall := time.UnixMicro(micros).UTC() + micros = time.Date( + wall.Year(), wall.Month(), wall.Day(), wall.Hour(), wall.Minute(), wall.Second(), wall.Nanosecond(), location, + ).UnixMicro() + } + if micros < minSupportedUnixMicros || micros > maxSupportedUnixMicros { + return 0, moerr.NewOutOfRangeNoCtx("timestamp", "MatrixOne") + } + value := types.UnixMicroToTimestamp(micros) + if value < types.TimestampMinValue || value > types.TimestampMaxValue { + return 0, moerr.NewOutOfRangeNoCtx("timestamp", "MatrixOne") + } + return value, nil +} + +func arrowMicrosToDatetime(micros int64, zoned bool, location *time.Location) (types.Datetime, error) { + conversionLocation := time.UTC + if zoned { + conversionLocation = location + } + if micros < minSupportedUnixMicros || micros > maxSupportedUnixMicros { + return 0, moerr.NewOutOfRangeNoCtx("datetime", "MatrixOne") + } + seconds := micros / 1_000_000 + nanos := micros % 1_000_000 * 1_000 + value := types.DatetimeFromUnixWithNsec(conversionLocation, seconds, nanos) + year, _, _, _ := value.ToDate().Calendar(true) + if year < types.MinDatetimeYear || year > types.MaxDatetimeYear { + return 0, moerr.NewOutOfRangeNoCtx("datetime", "MatrixOne") + } + return value, nil +} diff --git a/pkg/container/arrowbridge/convert_benchmark_test.go b/pkg/container/arrowbridge/convert_benchmark_test.go new file mode 100644 index 0000000000000..cdffae0332513 --- /dev/null +++ b/pkg/container/arrowbridge/convert_benchmark_test.go @@ -0,0 +1,195 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowbridge + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/decimal128" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +type arrowBridgeBenchmarkCase struct { + name string + record arrow.RecordBatch + targets []TargetColumn + logicalBytes int64 +} + +func BenchmarkArrowBridgeMaterializeAB(b *testing.B) { + const rows = 4096 + alloc := memory.NewCheckedAllocator(memory.NewGoAllocator()) + cases := []arrowBridgeBenchmarkCase{ + makeNumericDecimalBenchmarkCase(b, alloc, rows), + makeTimestampShortStringBenchmarkCase(b, alloc, rows), + makeLongBinaryBenchmarkCase(b, alloc, rows), + } + defer func() { + for _, test := range cases { + test.record.Release() + } + alloc.AssertSize(b, 0) + }() + + for _, test := range cases { + plan, err := BindLoad(context.Background(), test.record.Schema(), test.targets, MatchByName) + if err != nil { + b.Fatal(err) + } + for _, forceMaterialize := range []bool{false, true} { + policy := "borrow" + if forceMaterialize { + policy = "materialize" + } + b.Run(test.name+"/"+policy, func(b *testing.B) { + mp := mpool.MustNewZero() + var stats ConvertStats + b.ReportAllocs() + b.SetBytes(test.logicalBytes) + b.ResetTimer() + for index := 0; index < b.N; index++ { + converted, current, err := plan.Convert(context.Background(), test.record, mp, ConvertOptions{ + ForceMaterialize: forceMaterialize, + }) + if err != nil { + b.Fatal(err) + } + stats = current + converted.Clean(mp) + } + b.StopTimer() + if mp.CurrNB() != 0 { + b.Fatalf("bridge retained %d MPool bytes", mp.CurrNB()) + } + b.ReportMetric(float64(stats.EligiblePayloadBytes), "eligible_B/op") + b.ReportMetric(float64(stats.BorrowedPayloadBytes), "borrowed_B/op") + b.ReportMetric(float64(stats.MaterializedPayloadBytes), "copied_B/op") + b.ReportMetric(float64(stats.RetainedCapacityBytes), "retained_B/op") + }) + } + } +} + +func makeNumericDecimalBenchmarkCase( + b *testing.B, + alloc memory.Allocator, + rows int, +) arrowBridgeBenchmarkCase { + b.Helper() + ints := array.NewInt64Builder(alloc) + intValues := make([]int64, rows) + valid := make([]bool, rows) + for index := range intValues { + intValues[index] = int64(index) + valid[index] = index%17 != 0 + } + ints.AppendValues(intValues, valid) + intArray := ints.NewArray() + ints.Release() + + decimalType := &arrow.Decimal128Type{Precision: 18, Scale: 2} + decimals := array.NewDecimal128Builder(alloc, decimalType) + for index := 0; index < rows; index++ { + decimals.Append(decimal128.FromI64(int64(index * 100))) + } + decimalArray := decimals.NewArray() + decimals.Release() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "i", Type: arrow.PrimitiveTypes.Int64, Nullable: true}, + {Name: "d", Type: decimalType}, + }, nil) + record := array.NewRecordBatch(schema, []arrow.Array{intArray, decimalArray}, int64(rows)) + intArray.Release() + decimalArray.Release() + return arrowBridgeBenchmarkCase{ + name: "numeric_decimal", record: record, + targets: []TargetColumn{ + {Name: "i", Type: types.T_int64.ToType()}, + {Name: "d", Type: types.New(types.T_decimal128, 18, 2)}, + }, + logicalBytes: int64(rows * (8 + 16)), + } +} + +func makeTimestampShortStringBenchmarkCase( + b *testing.B, + alloc memory.Allocator, + rows int, +) arrowBridgeBenchmarkCase { + b.Helper() + timestampType := &arrow.TimestampType{Unit: arrow.Microsecond} + timestamps := array.NewTimestampBuilder(alloc, timestampType) + strings := array.NewStringBuilder(alloc) + baseMicros := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).UnixMicro() + var logicalBytes int64 + for index := 0; index < rows; index++ { + timestamps.Append(arrow.Timestamp(baseMicros + int64(index)*1_000_000)) + value := fmt.Sprintf("event-%04d", index) + strings.Append(value) + logicalBytes += 8 + int64(len(value)) + } + timestampArray := timestamps.NewArray() + stringArray := strings.NewArray() + timestamps.Release() + strings.Release() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "ts", Type: timestampType}, + {Name: "message", Type: arrow.BinaryTypes.String}, + }, nil) + record := array.NewRecordBatch(schema, []arrow.Array{timestampArray, stringArray}, int64(rows)) + timestampArray.Release() + stringArray.Release() + return arrowBridgeBenchmarkCase{ + name: "timestamp_short_string", record: record, + targets: []TargetColumn{ + {Name: "ts", Type: types.New(types.T_timestamp, 0, 6)}, + {Name: "message", Type: types.T_varchar.ToType()}, + }, + logicalBytes: logicalBytes, + } +} + +func makeLongBinaryBenchmarkCase( + b *testing.B, + alloc memory.Allocator, + rows int, +) arrowBridgeBenchmarkCase { + b.Helper() + binaries := array.NewBinaryBuilder(alloc, arrow.BinaryTypes.Binary) + payload := make([]byte, 256) + for index := range payload { + payload[index] = byte(index) + } + for index := 0; index < rows; index++ { + binaries.Append(payload) + } + binaryArray := binaries.NewArray() + binaries.Release() + schema := arrow.NewSchema([]arrow.Field{{Name: "payload", Type: arrow.BinaryTypes.Binary}}, nil) + record := array.NewRecordBatch(schema, []arrow.Array{binaryArray}, int64(rows)) + binaryArray.Release() + return arrowBridgeBenchmarkCase{ + name: "long_binary", record: record, + targets: []TargetColumn{{Name: "payload", Type: types.T_blob.ToType()}}, + logicalBytes: int64(rows * len(payload)), + } +} diff --git a/pkg/container/arrowbridge/doc.go b/pkg/container/arrowbridge/doc.go new file mode 100644 index 0000000000000..65e9d9e07d6a6 --- /dev/null +++ b/pkg/container/arrowbridge/doc.go @@ -0,0 +1,30 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package arrowbridge owns the in-process Arrow-to-MatrixOne container +// boundary. BindLoad freezes a LOAD schema and conversion policy before data +// is acquired; Convert then either retains immutable Arrow backing or +// materializes an owned vector transactionally under the caller's allocation +// account. +// +// The package intentionally knows nothing about files, object stores, Flight, +// SQL transactions, or runtime credentials. LOAD and external runtimes may +// share its physical conversion and lease rules while retaining independent +// protocol, authorization, error, and exact-type policies. In particular, +// BindLoad implements the LOAD conversion matrix. It is deliberately named +// for that consumer because Python UDF results have a stricter, versioned ABI: +// a future UDF binder must validate its logical metadata and exact type matrix +// before constructing a Plan, rather than silently accepting LOAD widening or +// temporal conversions. +package arrowbridge diff --git a/pkg/container/arrowipc/ipcflatbuf/metadata_generated.go b/pkg/container/arrowipc/ipcflatbuf/metadata_generated.go new file mode 100644 index 0000000000000..c2a99cb2beb15 --- /dev/null +++ b/pkg/container/arrowipc/ipcflatbuf/metadata_generated.go @@ -0,0 +1,589 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file incorporates generated metadata derived from Apache Arrow. +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file distributed +// with this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0. + +// Code generated from Apache Arrow format/Message.fbs by flatc and reduced to +// the read-only metadata surface used by MatrixOne. DO NOT EDIT. + +package ipcflatbuf + +import flatbuffers "github.com/google/flatbuffers/go" + +const ( + blockSize = flatbuffers.UOffsetT(24) + fieldNodeSize = flatbuffers.UOffsetT(16) + bufferSize = flatbuffers.UOffsetT(16) +) + +type MessageHeader byte + +const ( + MessageHeaderDictionaryBatch MessageHeader = 2 + MessageHeaderRecordBatch MessageHeader = 3 + MessageHeaderSchema MessageHeader = 1 +) + +type Type byte + +const ( + TypeNone Type = 0 + TypeTimestamp Type = 10 + TypeUnion Type = 14 +) + +type CompressionType int8 + +const ( + CompressionTypeLZ4Frame CompressionType = 0 + CompressionTypeZSTD CompressionType = 1 +) + +type BodyCompressionMethod int8 + +const ( + BodyCompressionMethodBuffer BodyCompressionMethod = 0 +) + +type Block struct { + tab flatbuffers.Struct +} + +func (b *Block) Init(buf []byte, position flatbuffers.UOffsetT) { + b.tab.Bytes = buf + b.tab.Pos = position +} + +func (b *Block) Offset() int64 { + return b.tab.GetInt64(b.tab.Pos) +} + +func (b *Block) MetadataLength() int32 { + return b.tab.GetInt32(b.tab.Pos + 8) +} + +func (b *Block) BodyLength() int64 { + return b.tab.GetInt64(b.tab.Pos + 16) +} + +type Footer struct { + tab flatbuffers.Table +} + +func GetRootAsFooter(buf []byte) *Footer { + root := flatbuffers.GetUOffsetT(buf) + footer := new(Footer) + footer.Init(buf, root) + return footer +} + +func (f *Footer) Init(buf []byte, position flatbuffers.UOffsetT) { + f.tab.Bytes = buf + f.tab.Pos = position +} + +func (f *Footer) Schema(schema *Schema) *Schema { + offset := flatbuffers.UOffsetT(f.tab.Offset(6)) + if offset == 0 { + return nil + } + position := f.tab.Indirect(offset + f.tab.Pos) + if schema == nil { + schema = new(Schema) + } + schema.Init(f.tab.Bytes, position) + return schema +} + +func (f *Footer) Dictionaries(block *Block, index int) bool { + offset := flatbuffers.UOffsetT(f.tab.Offset(8)) + if offset == 0 { + return false + } + position := f.tab.Vector(offset) + flatbuffers.UOffsetT(index)*blockSize + block.Init(f.tab.Bytes, position) + return true +} + +func (f *Footer) DictionariesLength() int { + offset := flatbuffers.UOffsetT(f.tab.Offset(8)) + if offset == 0 { + return 0 + } + return f.tab.VectorLen(offset) +} + +func (f *Footer) RecordBatches(block *Block, index int) bool { + offset := flatbuffers.UOffsetT(f.tab.Offset(10)) + if offset == 0 { + return false + } + position := f.tab.Vector(offset) + flatbuffers.UOffsetT(index)*blockSize + block.Init(f.tab.Bytes, position) + return true +} + +func (f *Footer) RecordBatchesLength() int { + offset := flatbuffers.UOffsetT(f.tab.Offset(10)) + if offset == 0 { + return 0 + } + return f.tab.VectorLen(offset) +} + +type Message struct { + tab flatbuffers.Table +} + +func GetRootAsMessage(buf []byte) *Message { + root := flatbuffers.GetUOffsetT(buf) + message := new(Message) + message.Init(buf, root) + return message +} + +func (m *Message) Init(buf []byte, position flatbuffers.UOffsetT) { + m.tab.Bytes = buf + m.tab.Pos = position +} + +func (m *Message) HeaderType() MessageHeader { + offset := flatbuffers.UOffsetT(m.tab.Offset(6)) + if offset == 0 { + return 0 + } + return MessageHeader(m.tab.GetByte(offset + m.tab.Pos)) +} + +func (m *Message) BodyLength() int64 { + offset := flatbuffers.UOffsetT(m.tab.Offset(10)) + if offset == 0 { + return 0 + } + return m.tab.GetInt64(offset + m.tab.Pos) +} + +func (m *Message) RecordBatch(batch *RecordBatch) bool { + if m.HeaderType() != MessageHeaderRecordBatch { + return false + } + return m.header(func(table flatbuffers.Table) { batch.Init(table.Bytes, table.Pos) }) +} + +func (m *Message) Schema(schema *Schema) bool { + if m.HeaderType() != MessageHeaderSchema { + return false + } + return m.header(func(table flatbuffers.Table) { schema.Init(table.Bytes, table.Pos) }) +} + +func (m *Message) DictionaryBatch(batch *DictionaryBatch) bool { + if m.HeaderType() != MessageHeaderDictionaryBatch { + return false + } + return m.header(func(table flatbuffers.Table) { batch.Init(table.Bytes, table.Pos) }) +} + +func (m *Message) header(install func(flatbuffers.Table)) bool { + offset := flatbuffers.UOffsetT(m.tab.Offset(8)) + if offset == 0 { + return false + } + var table flatbuffers.Table + m.tab.Union(&table, offset) + install(table) + return true +} + +type Schema struct { + tab flatbuffers.Table +} + +func (s *Schema) Init(buf []byte, position flatbuffers.UOffsetT) { + s.tab.Bytes = buf + s.tab.Pos = position +} + +func (s *Schema) Fields(field *Field, index int) bool { + offset := flatbuffers.UOffsetT(s.tab.Offset(6)) + if offset == 0 { + return false + } + position := s.tab.Vector(offset) + flatbuffers.UOffsetT(index)*4 + position = s.tab.Indirect(position) + field.Init(s.tab.Bytes, position) + return true +} + +func (s *Schema) FieldsLength() int { + offset := flatbuffers.UOffsetT(s.tab.Offset(6)) + if offset == 0 { + return 0 + } + return s.tab.VectorLen(offset) +} + +func (s *Schema) CustomMetadata(metadata *KeyValue, index int) bool { + offset := flatbuffers.UOffsetT(s.tab.Offset(8)) + if offset == 0 { + return false + } + position := s.tab.Vector(offset) + flatbuffers.UOffsetT(index)*4 + position = s.tab.Indirect(position) + metadata.Init(s.tab.Bytes, position) + return true +} + +func (s *Schema) CustomMetadataLength() int { + offset := flatbuffers.UOffsetT(s.tab.Offset(8)) + if offset == 0 { + return 0 + } + return s.tab.VectorLen(offset) +} + +func (s *Schema) Features(index int) int64 { + offset := flatbuffers.UOffsetT(s.tab.Offset(10)) + if offset == 0 { + return 0 + } + position := s.tab.Vector(offset) + flatbuffers.UOffsetT(index)*8 + return s.tab.GetInt64(position) +} + +func (s *Schema) FeaturesLength() int { + offset := flatbuffers.UOffsetT(s.tab.Offset(10)) + if offset == 0 { + return 0 + } + return s.tab.VectorLen(offset) +} + +type Field struct { + tab flatbuffers.Table +} + +func (f *Field) Init(buf []byte, position flatbuffers.UOffsetT) { + f.tab.Bytes = buf + f.tab.Pos = position +} + +func (f *Field) Name() []byte { + offset := flatbuffers.UOffsetT(f.tab.Offset(4)) + if offset == 0 { + return nil + } + return f.tab.ByteVector(offset + f.tab.Pos) +} + +func (f *Field) TypeType() Type { + offset := flatbuffers.UOffsetT(f.tab.Offset(8)) + if offset == 0 { + return TypeNone + } + return Type(f.tab.GetByte(offset + f.tab.Pos)) +} + +func (f *Field) Type(table *flatbuffers.Table) bool { + offset := flatbuffers.UOffsetT(f.tab.Offset(10)) + if offset == 0 { + return false + } + f.tab.Union(table, offset) + return true +} + +func (f *Field) Children(child *Field, index int) bool { + offset := flatbuffers.UOffsetT(f.tab.Offset(14)) + if offset == 0 { + return false + } + position := f.tab.Vector(offset) + flatbuffers.UOffsetT(index)*4 + position = f.tab.Indirect(position) + child.Init(f.tab.Bytes, position) + return true +} + +func (f *Field) ChildrenLength() int { + offset := flatbuffers.UOffsetT(f.tab.Offset(14)) + if offset == 0 { + return 0 + } + return f.tab.VectorLen(offset) +} + +func (f *Field) CustomMetadata(metadata *KeyValue, index int) bool { + offset := flatbuffers.UOffsetT(f.tab.Offset(16)) + if offset == 0 { + return false + } + position := f.tab.Vector(offset) + flatbuffers.UOffsetT(index)*4 + position = f.tab.Indirect(position) + metadata.Init(f.tab.Bytes, position) + return true +} + +func (f *Field) CustomMetadataLength() int { + offset := flatbuffers.UOffsetT(f.tab.Offset(16)) + if offset == 0 { + return 0 + } + return f.tab.VectorLen(offset) +} + +type KeyValue struct { + tab flatbuffers.Table +} + +func (k *KeyValue) Init(buf []byte, position flatbuffers.UOffsetT) { + k.tab.Bytes = buf + k.tab.Pos = position +} + +func (k *KeyValue) Key() []byte { + offset := flatbuffers.UOffsetT(k.tab.Offset(4)) + if offset == 0 { + return nil + } + return k.tab.ByteVector(offset + k.tab.Pos) +} + +func (k *KeyValue) Value() []byte { + offset := flatbuffers.UOffsetT(k.tab.Offset(6)) + if offset == 0 { + return nil + } + return k.tab.ByteVector(offset + k.tab.Pos) +} + +type Union struct { + tab flatbuffers.Table +} + +type Timestamp struct { + tab flatbuffers.Table +} + +func (t *Timestamp) Init(buf []byte, position flatbuffers.UOffsetT) { + t.tab.Bytes = buf + t.tab.Pos = position +} + +func (t *Timestamp) Timezone() []byte { + offset := flatbuffers.UOffsetT(t.tab.Offset(6)) + if offset == 0 { + return nil + } + return t.tab.ByteVector(offset + t.tab.Pos) +} + +func (u *Union) Init(buf []byte, position flatbuffers.UOffsetT) { + u.tab.Bytes = buf + u.tab.Pos = position +} + +func (u *Union) TypeIDs(index int) int32 { + offset := flatbuffers.UOffsetT(u.tab.Offset(6)) + if offset == 0 { + return 0 + } + position := u.tab.Vector(offset) + flatbuffers.UOffsetT(index)*4 + return u.tab.GetInt32(position) +} + +func (u *Union) TypeIDsLength() int { + offset := flatbuffers.UOffsetT(u.tab.Offset(6)) + if offset == 0 { + return 0 + } + return u.tab.VectorLen(offset) +} + +type DictionaryBatch struct { + tab flatbuffers.Table +} + +func (d *DictionaryBatch) Init(buf []byte, position flatbuffers.UOffsetT) { + d.tab.Bytes = buf + d.tab.Pos = position +} + +func (d *DictionaryBatch) ID() int64 { + offset := flatbuffers.UOffsetT(d.tab.Offset(4)) + if offset == 0 { + return 0 + } + return d.tab.GetInt64(offset + d.tab.Pos) +} + +func (d *DictionaryBatch) Data(batch *RecordBatch) bool { + offset := flatbuffers.UOffsetT(d.tab.Offset(6)) + if offset == 0 { + return false + } + position := d.tab.Indirect(offset + d.tab.Pos) + batch.Init(d.tab.Bytes, position) + return true +} + +func (d *DictionaryBatch) IsDelta() bool { + offset := flatbuffers.UOffsetT(d.tab.Offset(8)) + return offset != 0 && d.tab.GetBool(offset+d.tab.Pos) +} + +type RecordBatch struct { + tab flatbuffers.Table +} + +func (r *RecordBatch) Init(buf []byte, position flatbuffers.UOffsetT) { + r.tab.Bytes = buf + r.tab.Pos = position +} + +func (r *RecordBatch) Length() int64 { + offset := flatbuffers.UOffsetT(r.tab.Offset(4)) + if offset == 0 { + return 0 + } + return r.tab.GetInt64(offset + r.tab.Pos) +} + +func (r *RecordBatch) Nodes(node *FieldNode, index int) bool { + offset := flatbuffers.UOffsetT(r.tab.Offset(6)) + if offset == 0 { + return false + } + position := r.tab.Vector(offset) + flatbuffers.UOffsetT(index)*fieldNodeSize + node.Init(r.tab.Bytes, position) + return true +} + +func (r *RecordBatch) NodesLength() int { + offset := flatbuffers.UOffsetT(r.tab.Offset(6)) + if offset == 0 { + return 0 + } + return r.tab.VectorLen(offset) +} + +func (r *RecordBatch) Buffers(buffer *Buffer, index int) bool { + offset := flatbuffers.UOffsetT(r.tab.Offset(8)) + if offset == 0 { + return false + } + position := r.tab.Vector(offset) + flatbuffers.UOffsetT(index)*bufferSize + buffer.Init(r.tab.Bytes, position) + return true +} + +func (r *RecordBatch) BuffersLength() int { + offset := flatbuffers.UOffsetT(r.tab.Offset(8)) + if offset == 0 { + return 0 + } + return r.tab.VectorLen(offset) +} + +func (r *RecordBatch) Compression(compression *BodyCompression) *BodyCompression { + offset := flatbuffers.UOffsetT(r.tab.Offset(10)) + if offset == 0 { + return nil + } + position := r.tab.Indirect(offset + r.tab.Pos) + if compression == nil { + compression = new(BodyCompression) + } + compression.Init(r.tab.Bytes, position) + return compression +} + +func (r *RecordBatch) VariadicBufferCounts(index int) int64 { + offset := flatbuffers.UOffsetT(r.tab.Offset(12)) + if offset == 0 { + return 0 + } + position := r.tab.Vector(offset) + flatbuffers.UOffsetT(index)*8 + return r.tab.GetInt64(position) +} + +func (r *RecordBatch) VariadicBufferCountsLength() int { + offset := flatbuffers.UOffsetT(r.tab.Offset(12)) + if offset == 0 { + return 0 + } + return r.tab.VectorLen(offset) +} + +type BodyCompression struct { + tab flatbuffers.Table +} + +func (c *BodyCompression) Init(buf []byte, position flatbuffers.UOffsetT) { + c.tab.Bytes = buf + c.tab.Pos = position +} + +func (c *BodyCompression) Codec() CompressionType { + offset := flatbuffers.UOffsetT(c.tab.Offset(4)) + if offset == 0 { + return CompressionTypeLZ4Frame + } + return CompressionType(c.tab.GetInt8(offset + c.tab.Pos)) +} + +func (c *BodyCompression) Method() BodyCompressionMethod { + offset := flatbuffers.UOffsetT(c.tab.Offset(6)) + if offset == 0 { + return BodyCompressionMethodBuffer + } + return BodyCompressionMethod(c.tab.GetInt8(offset + c.tab.Pos)) +} + +type FieldNode struct { + tab flatbuffers.Struct +} + +func (n *FieldNode) Init(buf []byte, position flatbuffers.UOffsetT) { + n.tab.Bytes = buf + n.tab.Pos = position +} + +func (n *FieldNode) Length() int64 { + return n.tab.GetInt64(n.tab.Pos) +} + +func (n *FieldNode) NullCount() int64 { + return n.tab.GetInt64(n.tab.Pos + 8) +} + +type Buffer struct { + tab flatbuffers.Struct +} + +func (b *Buffer) Init(buf []byte, position flatbuffers.UOffsetT) { + b.tab.Bytes = buf + b.tab.Pos = position +} + +func (b *Buffer) Offset() int64 { + return b.tab.GetInt64(b.tab.Pos) +} + +func (b *Buffer) Length() int64 { + return b.tab.GetInt64(b.tab.Pos + 8) +} diff --git a/pkg/container/arrowipc/message.go b/pkg/container/arrowipc/message.go new file mode 100644 index 0000000000000..feac0b31ff135 --- /dev/null +++ b/pkg/container/arrowipc/message.go @@ -0,0 +1,349 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package arrowipc validates Arrow IPC framing and FlatBuffers metadata at a +// shared trust boundary. Object/range transport and consumer-specific type +// policy deliberately remain outside this package. +package arrowipc + +import ( + "context" + "encoding/binary" + "math" + "sort" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/arrowipc/ipcflatbuf" +) + +const ( + // DefaultMaxMetadataBytes is a structural safety ceiling, not a transport + // frame limit. A consumer may negotiate a lower value but must not raise it. + DefaultMaxMetadataBytes int64 = 1 << 20 + // ContinuationToken prefixes modern stream-framed Arrow IPC metadata. + ContinuationToken = uint32(math.MaxUint32) + + MessageHeaderSchema = byte(ipcflatbuf.MessageHeaderSchema) + MessageHeaderDictionaryBatch = byte(ipcflatbuf.MessageHeaderDictionaryBatch) + MessageHeaderRecordBatch = byte(ipcflatbuf.MessageHeaderRecordBatch) +) + +// ValidationOptions supplies caller-owned limits. BodyEnvelopeBytes is -1 +// when the exact framed body size is not known yet; otherwise it may include +// at most seven bytes of IPC alignment padding. +type ValidationOptions struct { + // MaxMetadataBytes bounds the complete framed metadata input. + MaxMetadataBytes int64 + // MaxBodyBytes bounds Message.bodyLength before any decoder allocation. + MaxBodyBytes int64 + // BodyEnvelopeBytes is the body span supplied by the outer transport. Use + // -1 only while inspecting metadata before that envelope is available. + BodyEnvelopeBytes int64 + // Body is read only for compressed buffers' decoded-size prefixes. + Body []byte + // ValidateBody enables body-range and decoded-size inspection. + ValidateBody bool + // MaxDecodedRecordBytes bounds the sum of all decoded Arrow buffers. + MaxDecodedRecordBytes int64 +} + +// MessageInfo is transport-neutral metadata needed by File, Stream, and +// Flight consumers before a decoder may allocate or retain the message body. +// It contains no object identity or invocation sequence because those are +// owned by the transport that supplied the message. +type MessageInfo struct { + // HeaderType identifies Schema, DictionaryBatch, or RecordBatch metadata. + HeaderType byte + // Rows is populated for record and dictionary batches. + Rows int64 + // DictionaryID and IsDelta are populated only for dictionary batches. + DictionaryID int64 + IsDelta bool + // BodyBytes is the validated Message.bodyLength, excluding IPC padding. + BodyBytes int64 +} + +// Metadata accepts a raw Message flatbuffer, continuation framing, or the +// legacy four-byte length prefix and returns a bounded raw metadata view. The +// caller may lower maxBytes, but values above DefaultMaxMetadataBytes are +// clamped so this trust-boundary ceiling cannot be bypassed accidentally. +func Metadata(ctx context.Context, wire []byte, maxBytes int64) ([]byte, error) { + if ctx == nil { + ctx = context.Background() + } + if maxBytes > DefaultMaxMetadataBytes { + maxBytes = DefaultMaxMetadataBytes + } + if maxBytes < 4 || int64(len(wire)) > maxBytes { + return nil, moerr.NewInvalidInputf(ctx, + "Arrow IPC metadata length %d exceeds limit %d", len(wire), maxBytes) + } + if len(wire) < 4 { + return nil, moerr.NewInvalidInput(ctx, "Arrow IPC metadata is truncated") + } + if binary.LittleEndian.Uint32(wire[:4]) == ContinuationToken { + if len(wire) < 8 { + return nil, moerr.NewInvalidInput(ctx, "Arrow IPC continuation header is truncated") + } + length := uint64(binary.LittleEndian.Uint32(wire[4:8])) + if length == 0 || length > uint64(len(wire)-8) { + return nil, moerr.NewInvalidInput(ctx, "Arrow IPC metadata length is invalid") + } + return wire[8 : 8+length], nil + } + length := uint64(binary.LittleEndian.Uint32(wire[:4])) + if length != 0 && length == uint64(len(wire)-4) { + return wire[4:], nil + } + return wire, nil +} + +// InspectMessage validates the complete untrusted IPC metadata graph and, when +// requested, compressed-buffer decoded sizes before a downstream decoder can +// allocate from values controlled by the message. Success is structural only: +// callers still enforce metadata version, expected header kind, schema/type +// identity, row cardinality, and protocol sequence as their own contracts. +func InspectMessage( + ctx context.Context, + wire []byte, + options ValidationOptions, +) (_ MessageInfo, retErr error) { + if ctx == nil { + ctx = context.Background() + } + if options.MaxMetadataBytes == 0 { + options.MaxMetadataBytes = DefaultMaxMetadataBytes + } + if options.BodyEnvelopeBytes < -1 { + return MessageInfo{}, moerr.NewInvalidInput(ctx, "invalid Arrow IPC body envelope length") + } + if options.MaxBodyBytes < 0 || options.MaxDecodedRecordBytes <= 0 { + return MessageInfo{}, moerr.NewInvalidInput(ctx, "invalid Arrow IPC validation limits") + } + metadata, err := Metadata(ctx, wire, options.MaxMetadataBytes) + if err != nil { + return MessageInfo{}, err + } + // FlatBuffers' generated accessors panic on some malformed offset graphs. + // Recovery belongs at this shared trust boundary, before consumer-specific + // code can allocate or retain attacker-described buffers. + defer func() { + if recovered := recover(); recovered != nil { + retErr = moerr.NewInvalidInputf(ctx, "invalid Arrow IPC message metadata: %v", recovered) + } + }() + root := binary.LittleEndian.Uint32(metadata) + if uint64(root) >= uint64(len(metadata)) { + return MessageInfo{}, moerr.NewInvalidInput(ctx, "Arrow IPC message root is out of bounds") + } + message := ipcflatbuf.GetRootAsMessage(metadata) + headerType := message.HeaderType() + if headerType == 0 { + return MessageInfo{}, moerr.NewInvalidInput(ctx, "Arrow IPC message header is missing") + } + if headerType != ipcflatbuf.MessageHeaderSchema && + headerType != ipcflatbuf.MessageHeaderDictionaryBatch && + headerType != ipcflatbuf.MessageHeaderRecordBatch { + return MessageInfo{}, moerr.NewInvalidInputf(ctx, + "unsupported Arrow IPC message header %d", headerType) + } + bodyLength := message.BodyLength() + if bodyLength < 0 || bodyLength > options.MaxBodyBytes || bodyLength > int64(math.MaxInt) { + return MessageInfo{}, moerr.NewInvalidInputf(ctx, + "Arrow IPC message body length %d exceeds limit %d", bodyLength, options.MaxBodyBytes) + } + if options.BodyEnvelopeBytes >= 0 && + (bodyLength > options.BodyEnvelopeBytes || options.BodyEnvelopeBytes-bodyLength >= 8) { + return MessageInfo{}, moerr.NewInvalidInputf(ctx, + "Arrow IPC message body length %d does not match envelope body length %d", + bodyLength, options.BodyEnvelopeBytes) + } + if options.ValidateBody && + (int64(len(options.Body)) < bodyLength || int64(len(options.Body))-bodyLength >= 8) { + return MessageInfo{}, moerr.NewInvalidInputf(ctx, + "Arrow IPC message body length %d does not match available body length %d", + bodyLength, len(options.Body)) + } + if headerType == ipcflatbuf.MessageHeaderSchema && bodyLength != 0 { + return MessageInfo{}, moerr.NewInvalidInputf(ctx, + "invalid Arrow IPC schema message body length %d", bodyLength) + } + + result := MessageInfo{HeaderType: byte(headerType), BodyBytes: bodyLength} + switch headerType { + case ipcflatbuf.MessageHeaderSchema: + var schema ipcflatbuf.Schema + if !message.Schema(&schema) { + return MessageInfo{}, moerr.NewInvalidInput(ctx, "Arrow schema header is missing") + } + if err := ValidateSchemaMetadata(ctx, &schema, len(metadata)); err != nil { + return MessageInfo{}, err + } + case ipcflatbuf.MessageHeaderRecordBatch: + var record ipcflatbuf.RecordBatch + if !message.RecordBatch(&record) { + return MessageInfo{}, moerr.NewInvalidInput(ctx, "Arrow record data header is missing") + } + if err := validateRecordBatchMetadata(ctx, &record, len(metadata), bodyLength, options); err != nil { + return MessageInfo{}, err + } + result.Rows = record.Length() + case ipcflatbuf.MessageHeaderDictionaryBatch: + var dictionary ipcflatbuf.DictionaryBatch + if !message.DictionaryBatch(&dictionary) { + return MessageInfo{}, moerr.NewInvalidInput(ctx, "Arrow dictionary header is missing") + } + var record ipcflatbuf.RecordBatch + if !dictionary.Data(&record) { + return MessageInfo{}, moerr.NewInvalidInput(ctx, "Arrow dictionary data header is missing") + } + if err := validateRecordBatchMetadata(ctx, &record, len(metadata), bodyLength, options); err != nil { + return MessageInfo{}, err + } + result.DictionaryID = dictionary.ID() + result.IsDelta = dictionary.IsDelta() + result.Rows = record.Length() + } + return result, nil +} + +func validateRecordBatchMetadata( + ctx context.Context, + record *ipcflatbuf.RecordBatch, + metadataBytes int, + bodyBytes int64, + options ValidationOptions, +) error { + rows := record.Length() + if rows < 0 || rows > int64(math.MaxInt) { + return moerr.NewInvalidInputf(ctx, "Arrow IPC message has invalid row count %d", rows) + } + + nodeCount := record.NodesLength() + if nodeCount < 0 || nodeCount > metadataBytes/16 { + return moerr.NewInvalidInputf(ctx, "Arrow IPC field-node count %d exceeds metadata", nodeCount) + } + // FieldNodes drive downstream array lengths and null allocations. Merely + // checking the vector byte range is insufficient; every tuple is validated. + var node ipcflatbuf.FieldNode + for index := 0; index < nodeCount; index++ { + if !record.Nodes(&node, index) { + return moerr.NewInvalidInputf(ctx, "Arrow IPC field node %d is missing", index) + } + length, nullCount := node.Length(), node.NullCount() + if length < 0 || nullCount < 0 || nullCount > length { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC field node %d has invalid length %d and null count %d", + index, length, nullCount) + } + } + + compression := record.Compression(nil) + if compression != nil { + codec := compression.Codec() + if codec != ipcflatbuf.CompressionTypeLZ4Frame && codec != ipcflatbuf.CompressionTypeZSTD { + return moerr.NewInvalidInputf(ctx, "Arrow IPC compression codec %d is unsupported", codec) + } + if method := compression.Method(); method != ipcflatbuf.BodyCompressionMethodBuffer { + return moerr.NewInvalidInputf(ctx, "Arrow IPC compression method %d is unsupported", method) + } + } + + bufferCount := record.BuffersLength() + if bufferCount < 0 || bufferCount > metadataBytes/16 { + return moerr.NewInvalidInputf(ctx, "Arrow IPC buffer count %d exceeds metadata", bufferCount) + } + // Buffer ranges are checked with subtraction to avoid offset+length overflow. + // When compressed, the first eight bytes are an Arrow decoded-size prefix; + // the aggregate decoded budget is enforced before decompression. + ranges := make([]arrowBufferRange, 0, bufferCount) + var buffer ipcflatbuf.Buffer + var decodedBytes int64 + for index := 0; index < bufferCount; index++ { + if !record.Buffers(&buffer, index) { + return moerr.NewInvalidInputf(ctx, "Arrow IPC buffer %d is missing", index) + } + offset, length := buffer.Offset(), buffer.Length() + if offset < 0 || length < 0 || offset > bodyBytes || length > bodyBytes-offset { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC buffer %d range [%d,%d) exceeds message body %d", + index, offset, offset+length, bodyBytes) + } + if offset%8 != 0 { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC buffer %d has unaligned buffer offset %d", index, offset) + } + if length > 0 { + ranges = append(ranges, arrowBufferRange{index: index, offset: offset, length: length}) + } + if !options.ValidateBody { + continue + } + decodedLength := length + if compression != nil && length != 0 { + if length < 8 { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC compressed buffer %d is shorter than its decoded-size prefix", index) + } + declared := int64(binary.LittleEndian.Uint64(options.Body[int(offset) : int(offset)+8])) + switch { + case declared == -1: + decodedLength = length - 8 + case declared < 0 || declared > int64(math.MaxInt): + return moerr.NewInvalidInputf(ctx, + "Arrow IPC compressed buffer %d has invalid decoded size %d", index, declared) + default: + decodedLength = declared + } + } + if decodedLength > options.MaxDecodedRecordBytes-decodedBytes { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC decoded record body exceeds limit %d", options.MaxDecodedRecordBytes) + } + decodedBytes += decodedLength + } + sort.Slice(ranges, func(i, j int) bool { + return ranges[i].offset < ranges[j].offset + }) + for index := 1; index < len(ranges); index++ { + previous, current := ranges[index-1], ranges[index] + if current.offset < previous.offset+previous.length { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC buffer %d overlaps buffer %d", current.index, previous.index) + } + } + + variadicCount := record.VariadicBufferCountsLength() + if variadicCount < 0 || variadicCount > metadataBytes/8 { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC variadic-buffer count %d exceeds metadata", variadicCount) + } + var variadicBuffers int64 + for index := 0; index < variadicCount; index++ { + count := record.VariadicBufferCounts(index) + if count < 0 || count > int64(bufferCount)-variadicBuffers { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC variadic-buffer count %d at index %d exceeds buffer count %d", + count, index, bufferCount) + } + variadicBuffers += count + } + return nil +} + +type arrowBufferRange struct { + index int + offset int64 + length int64 +} diff --git a/pkg/container/arrowipc/message_test.go b/pkg/container/arrowipc/message_test.go new file mode 100644 index 0000000000000..6ad2734b6dea8 --- /dev/null +++ b/pkg/container/arrowipc/message_test.go @@ -0,0 +1,287 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowipc + +import ( + "bytes" + "context" + "encoding/binary" + "math" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + flatbuffers "github.com/google/flatbuffers/go" + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/container/arrowipc/ipcflatbuf" +) + +func TestMetadataAcceptsRawAndFramedMessages(t *testing.T) { + raw := []byte{0, 0, 0, 0, 9} + metadata, err := Metadata(context.Background(), raw, DefaultMaxMetadataBytes) + require.NoError(t, err) + require.Equal(t, raw, metadata) + + legacy := []byte{1, 0, 0, 0, 7} + metadata, err = Metadata(context.Background(), legacy, DefaultMaxMetadataBytes) + require.NoError(t, err) + require.Equal(t, []byte{7}, metadata) + + continuation := make([]byte, 9) + binary.LittleEndian.PutUint32(continuation, ContinuationToken) + binary.LittleEndian.PutUint32(continuation[4:], 1) + metadata, err = Metadata(context.Background(), continuation, DefaultMaxMetadataBytes) + require.NoError(t, err) + require.Equal(t, []byte{0}, metadata) +} + +func TestMetadataRejectsMalformedOrOversizedFraming(t *testing.T) { + _, err := Metadata(context.Background(), nil, DefaultMaxMetadataBytes) + require.ErrorContains(t, err, "truncated") + _, err = Metadata(context.Background(), make([]byte, 5), 4) + require.ErrorContains(t, err, "exceeds limit") + _, err = Metadata(context.Background(), []byte{0xff, 0xff, 0xff, 0xff}, DefaultMaxMetadataBytes) + require.ErrorContains(t, err, "continuation header") + + continuation := make([]byte, 9) + binary.LittleEndian.PutUint32(continuation, math.MaxUint32) + binary.LittleEndian.PutUint32(continuation[4:], 2) + _, err = Metadata(context.Background(), continuation, DefaultMaxMetadataBytes) + require.ErrorContains(t, err, "length is invalid") + + // A consumer cannot turn the shared structural limit into an allocation + // escape hatch merely by supplying a larger local option. + _, err = Metadata(context.Background(), make([]byte, DefaultMaxMetadataBytes+1), 2*DefaultMaxMetadataBytes) + require.ErrorContains(t, err, "exceeds limit 1048576") +} + +func TestInspectMessageValidatesGeneratedSchemaBeforeConsumerPolicy(t *testing.T) { + // Generate through Arrow-Go so this package test covers the public IPC wire + // shape without depending on a hand-authored FlatBuffers fixture. + schema := arrow.NewSchema([]arrow.Field{{ + Name: "value", Type: arrow.PrimitiveTypes.Int64, + }}, nil) + var stream bytes.Buffer + writer := ipc.NewWriter(&stream, ipc.WithSchema(schema)) + require.NoError(t, writer.Close()) + + wire := firstStreamMetadata(t, stream.Bytes()) + info, err := InspectMessage(context.Background(), wire, ValidationOptions{ + MaxBodyBytes: 0, + BodyEnvelopeBytes: 0, + MaxDecodedRecordBytes: 1, + }) + require.NoError(t, err) + require.Equal(t, MessageHeaderSchema, info.HeaderType) + require.Zero(t, info.BodyBytes) + + malformed := append([]byte(nil), wire...) + metadata, err := Metadata(context.Background(), malformed, DefaultMaxMetadataBytes) + require.NoError(t, err) + // Keep the malformed root distinct from the continuation token so the + // framing parser reaches the FlatBuffers root bounds check. + binary.LittleEndian.PutUint32(metadata, math.MaxUint32-1) + _, err = InspectMessage(context.Background(), malformed, ValidationOptions{ + MaxBodyBytes: 0, + BodyEnvelopeBytes: 0, + MaxDecodedRecordBytes: 1, + }) + require.ErrorContains(t, err, "root is out of bounds") +} + +func TestInspectMessageRejectsInvalidBodyEnvelopeSentinel(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{ + Name: "value", Type: arrow.PrimitiveTypes.Int64, + }}, nil) + var stream bytes.Buffer + writer := ipc.NewWriter(&stream, ipc.WithSchema(schema)) + require.NoError(t, writer.Close()) + wire := firstStreamMetadata(t, stream.Bytes()) + + _, err := InspectMessage(context.Background(), wire, ValidationOptions{ + MaxBodyBytes: 0, + BodyEnvelopeBytes: -2, + MaxDecodedRecordBytes: 1, + }) + require.ErrorContains(t, err, "body envelope") +} + +func TestInspectMessageRejectsUnsupportedHeaderType(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{ + Name: "value", Type: arrow.PrimitiveTypes.Int64, + }}, nil) + var stream bytes.Buffer + writer := ipc.NewWriter(&stream, ipc.WithSchema(schema)) + require.NoError(t, writer.Close()) + wire := firstStreamMetadata(t, stream.Bytes()) + metadata, err := Metadata(context.Background(), wire, DefaultMaxMetadataBytes) + require.NoError(t, err) + + root := binary.LittleEndian.Uint32(metadata) + message := flatbuffers.Table{Bytes: metadata, Pos: flatbuffers.UOffsetT(root)} + headerOffset := flatbuffers.UOffsetT(message.Offset(6)) + require.NotZero(t, headerOffset) + metadata[headerOffset+message.Pos] = 4 // Tensor, not an IPC scan message. + + _, err = InspectMessage(context.Background(), metadata, ValidationOptions{ + MaxBodyBytes: 0, + BodyEnvelopeBytes: 0, + MaxDecodedRecordBytes: 1, + }) + require.ErrorContains(t, err, "unsupported Arrow IPC message header") +} + +func TestInspectMessageRejectsSchemaBody(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{ + Name: "value", Type: arrow.PrimitiveTypes.Int64, + }}, nil) + var stream bytes.Buffer + writer := ipc.NewWriter(&stream, ipc.WithSchema(schema)) + alloc := memory.NewGoAllocator() + values := array.NewInt64Builder(alloc) + values.Append(1) + record := array.NewRecordBatch(schema, []arrow.Array{values.NewArray()}, 1) + values.Release() + require.NoError(t, writer.Write(record)) + record.Release() + require.NoError(t, writer.Close()) + metadata := streamMetadataAt(t, stream.Bytes(), 1) + + root := binary.LittleEndian.Uint32(metadata) + message := flatbuffers.Table{Bytes: metadata, Pos: flatbuffers.UOffsetT(root)} + headerOffset := flatbuffers.UOffsetT(message.Offset(6)) + require.NotZero(t, headerOffset) + metadata[headerOffset+message.Pos] = byte(ipcflatbuf.MessageHeaderSchema) + + _, err := InspectMessage(context.Background(), metadata, ValidationOptions{ + MaxBodyBytes: 8, + BodyEnvelopeBytes: 8, + MaxDecodedRecordBytes: 1, + }) + require.ErrorContains(t, err, "schema message body") +} + +func TestInspectMessageRejectsUnalignedBufferOffset(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{ + Name: "value", Type: arrow.PrimitiveTypes.Int64, + }}, nil) + var stream bytes.Buffer + writer := ipc.NewWriter(&stream, ipc.WithSchema(schema)) + alloc := memory.NewGoAllocator() + values := array.NewInt64Builder(alloc) + values.Append(1) + record := array.NewRecordBatch(schema, []arrow.Array{values.NewArray()}, 1) + values.Release() + require.NoError(t, writer.Write(record)) + record.Release() + require.NoError(t, writer.Close()) + metadata := streamMetadataAt(t, stream.Bytes(), 1) + + root := binary.LittleEndian.Uint32(metadata) + messageTable := flatbuffers.Table{Bytes: metadata, Pos: flatbuffers.UOffsetT(root)} + headerOffset := flatbuffers.UOffsetT(messageTable.Offset(8)) + require.NotZero(t, headerOffset) + var recordTable flatbuffers.Table + messageTable.Union(&recordTable, headerOffset) + buffersOffset := flatbuffers.UOffsetT(recordTable.Offset(8)) + require.NotZero(t, buffersOffset) + bufferPos := recordTable.Vector(buffersOffset) + binary.LittleEndian.PutUint64(recordTable.Bytes[bufferPos:], 1) + + bodyLength := ipcflatbuf.GetRootAsMessage(metadata).BodyLength() + _, err := InspectMessage(context.Background(), metadata, ValidationOptions{ + MaxBodyBytes: bodyLength, + BodyEnvelopeBytes: bodyLength, + MaxDecodedRecordBytes: 1, + }) + require.ErrorContains(t, err, "unaligned buffer offset") +} + +func TestInspectMessageRejectsOverlappingBufferRanges(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{ + {Name: "left", Type: arrow.PrimitiveTypes.Int64}, + {Name: "right", Type: arrow.PrimitiveTypes.Int64}, + }, nil) + var stream bytes.Buffer + writer := ipc.NewWriter(&stream, ipc.WithSchema(schema)) + alloc := memory.NewGoAllocator() + left := array.NewInt64Builder(alloc) + left.Append(1) + right := array.NewInt64Builder(alloc) + right.Append(2) + record := array.NewRecordBatch(schema, []arrow.Array{left.NewArray(), right.NewArray()}, 1) + left.Release() + right.Release() + require.NoError(t, writer.Write(record)) + record.Release() + require.NoError(t, writer.Close()) + metadata := streamMetadataAt(t, stream.Bytes(), 1) + + root := binary.LittleEndian.Uint32(metadata) + messageTable := flatbuffers.Table{Bytes: metadata, Pos: flatbuffers.UOffsetT(root)} + headerOffset := flatbuffers.UOffsetT(messageTable.Offset(8)) + require.NotZero(t, headerOffset) + var recordTable flatbuffers.Table + messageTable.Union(&recordTable, headerOffset) + buffersOffset := flatbuffers.UOffsetT(recordTable.Offset(8)) + require.NotZero(t, buffersOffset) + bufferPos := recordTable.Vector(buffersOffset) + firstOffset := binary.LittleEndian.Uint64(recordTable.Bytes[bufferPos+16:]) + firstLength := binary.LittleEndian.Uint64(recordTable.Bytes[bufferPos+24:]) + require.Greater(t, firstLength, uint64(0)) + // Point the second non-empty logical buffer at the first buffer. Both + // ranges remain aligned and in bounds, but they no longer describe the + // serialized RecordBatch layout. + binary.LittleEndian.PutUint64(recordTable.Bytes[bufferPos+48:], firstOffset) + + bodyLength := ipcflatbuf.GetRootAsMessage(metadata).BodyLength() + _, err := InspectMessage(context.Background(), metadata, ValidationOptions{ + MaxBodyBytes: bodyLength, + BodyEnvelopeBytes: bodyLength, + MaxDecodedRecordBytes: 1, + }) + require.ErrorContains(t, err, "overlaps buffer") +} + +func firstStreamMetadata(t *testing.T, stream []byte) []byte { + return streamMetadataAt(t, stream, 0) +} + +func streamMetadataAt(t *testing.T, stream []byte, target int) []byte { + t.Helper() + position := 0 + for index := 0; index <= target; index++ { + require.LessOrEqual(t, position+8, len(stream)) + require.Equal(t, ContinuationToken, binary.LittleEndian.Uint32(stream[position:])) + length := int(binary.LittleEndian.Uint32(stream[position+4:])) + require.Positive(t, length) + metadataStart := position + 8 + metadataEnd := metadataStart + length + require.LessOrEqual(t, metadataEnd, len(stream)) + if index == target { + return stream[metadataStart:metadataEnd] + } + metadata := stream[metadataStart:metadataEnd] + message := ipcflatbuf.GetRootAsMessage(metadata) + bodyLength := message.BodyLength() + require.GreaterOrEqual(t, bodyLength, int64(0)) + position = metadataStart + (length+7)/8*8 + (int(bodyLength)+7)/8*8 + } + t.Fatalf("stream message %d is missing", target) + return nil +} diff --git a/pkg/container/arrowipc/schema_metadata.go b/pkg/container/arrowipc/schema_metadata.go new file mode 100644 index 0000000000000..d93c76fc4ab7c --- /dev/null +++ b/pkg/container/arrowipc/schema_metadata.go @@ -0,0 +1,266 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowipc + +import ( + "context" + + flatbuffers "github.com/google/flatbuffers/go" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/arrowipc/ipcflatbuf" +) + +const ( + // Keep the total field limit aligned with plan.TableColumnCountLimit. It + // applies to top-level and nested fields together so a deeply nested schema + // cannot multiply decoder allocations behind the table limit. These bounds + // are shared by file and Flight trust boundaries; a consumer may impose a + // lower negotiated limit but must not raise them locally. + MaxSchemaFields = 4096 + MaxSchemaDepth = 64 + MaxSchemaMetadataEntries = 4096 + MaxSchemaFeatures = 64 + MaxUnionTypeIDsPerField = 128 + MaxSchemaUnionTypeIDs = 4096 +) + +type schemaMetadataBudget struct { + metadataBytes int + fields int + metadata int + unionTypeIDs int + decodedStrings int +} + +// ValidateSchemaMetadata bounds every schema vector and recursively walks it +// before a decoder can allocate slices from untrusted vector lengths. Walking +// every element also proves that declared vector and string ranges fit in the +// metadata buffer. decodedStrings prevents aliased FlatBuffers offsets from +// amplifying a bounded wire message into unbounded Go string allocations. This +// validates structure, not a consumer's SQL type or ABI policy. +func ValidateSchemaMetadata( + ctx context.Context, + schema *ipcflatbuf.Schema, + metadataBytes int, +) (retErr error) { + defer func() { + if recovered := recover(); recovered != nil { + retErr = moerr.NewInvalidInputf(ctx, "invalid Arrow IPC schema metadata: %v", recovered) + } + }() + if schema == nil || metadataBytes < 4 { + return moerr.NewInvalidInput(ctx, "Arrow IPC schema metadata is missing") + } + budget := schemaMetadataBudget{metadataBytes: metadataBytes} + if err := budget.validateSchema(ctx, schema); err != nil { + return err + } + return nil +} + +func (b *schemaMetadataBudget) validateSchema(ctx context.Context, schema *ipcflatbuf.Schema) error { + fieldCount := schema.FieldsLength() + if err := b.consumeVector(ctx, "field", fieldCount, 4); err != nil { + return err + } + if err := b.consumeFields(ctx, fieldCount); err != nil { + return err + } + if err := b.validateCustomMetadata( + ctx, "schema", schema.CustomMetadataLength(), schema.CustomMetadata, + ); err != nil { + return err + } + + featureCount := schema.FeaturesLength() + if err := b.consumeVector(ctx, "feature", featureCount, 8); err != nil { + return err + } + if featureCount > MaxSchemaFeatures { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC schema feature count %d exceeds limit %d", + featureCount, MaxSchemaFeatures) + } + for index := 0; index < featureCount; index++ { + _ = schema.Features(index) + } + + for index := 0; index < fieldCount; index++ { + var field ipcflatbuf.Field + if !schema.Fields(&field, index) { + return moerr.NewInvalidInputf(ctx, "Arrow IPC schema field %d is missing", index) + } + if err := b.validateField(ctx, &field, 1); err != nil { + return moerr.NewInvalidInputf(ctx, "invalid Arrow IPC schema field %d: %v", index, err) + } + } + return nil +} + +func (b *schemaMetadataBudget) validateField( + ctx context.Context, + field *ipcflatbuf.Field, + depth int, +) error { + if depth > MaxSchemaDepth { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC schema nesting depth %d exceeds limit %d", depth, MaxSchemaDepth) + } + if err := b.consumeStringBytes(ctx, len(field.Name())); err != nil { + return err + } + if err := b.validateCustomMetadata( + ctx, "field", field.CustomMetadataLength(), field.CustomMetadata, + ); err != nil { + return err + } + + childCount := field.ChildrenLength() + if err := b.consumeVector(ctx, "child field", childCount, 4); err != nil { + return err + } + if err := b.consumeFields(ctx, childCount); err != nil { + return err + } + for index := 0; index < childCount; index++ { + var child ipcflatbuf.Field + if !field.Children(&child, index) { + return moerr.NewInvalidInputf(ctx, "Arrow IPC schema child field %d is missing", index) + } + if err := b.validateField(ctx, &child, depth+1); err != nil { + return moerr.NewInvalidInputf(ctx, "invalid Arrow IPC schema child field %d: %v", index, err) + } + } + + typeID := field.TypeType() + var typeTable flatbuffers.Table + if typeID == ipcflatbuf.TypeNone || !field.Type(&typeTable) { + return moerr.NewInvalidInput(ctx, "Arrow IPC schema field type is missing") + } + if typeID == ipcflatbuf.TypeTimestamp { + var timestamp ipcflatbuf.Timestamp + timestamp.Init(typeTable.Bytes, typeTable.Pos) + if err := b.consumeStringBytes(ctx, len(timestamp.Timezone())); err != nil { + return err + } + } + if typeID != ipcflatbuf.TypeUnion { + return nil + } + if childCount > MaxUnionTypeIDsPerField { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC union child count %d exceeds limit %d", + childCount, MaxUnionTypeIDsPerField) + } + var union ipcflatbuf.Union + union.Init(typeTable.Bytes, typeTable.Pos) + typeIDCount := union.TypeIDsLength() + if err := b.consumeVector(ctx, "union type ID", typeIDCount, 4); err != nil { + return err + } + if typeIDCount > MaxUnionTypeIDsPerField { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC union type ID count %d exceeds per-field limit %d", + typeIDCount, MaxUnionTypeIDsPerField) + } + if typeIDCount > MaxSchemaUnionTypeIDs-b.unionTypeIDs { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC union type ID count exceeds limit %d", MaxSchemaUnionTypeIDs) + } + if typeIDCount != 0 && typeIDCount != childCount { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC union type ID count %d does not match child count %d", + typeIDCount, childCount) + } + b.unionTypeIDs += typeIDCount + seenTypeIDs := make(map[int32]struct{}, typeIDCount) + for index := 0; index < typeIDCount; index++ { + value := union.TypeIDs(index) + if value < 0 || value >= MaxUnionTypeIDsPerField { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC union type ID %d at index %d is out of bounds", value, index) + } + if _, exists := seenTypeIDs[value]; exists { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC duplicate union type ID %d at index %d", value, index) + } + seenTypeIDs[value] = struct{}{} + } + return nil +} + +func (b *schemaMetadataBudget) validateCustomMetadata( + ctx context.Context, + owner string, + count int, + read func(*ipcflatbuf.KeyValue, int) bool, +) error { + if err := b.consumeVector(ctx, owner+" custom-metadata", count, 4); err != nil { + return err + } + if count > MaxSchemaMetadataEntries-b.metadata { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC schema custom metadata entry count exceeds limit %d", + MaxSchemaMetadataEntries) + } + b.metadata += count + for index := 0; index < count; index++ { + var metadata ipcflatbuf.KeyValue + if !read(&metadata, index) { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC %s custom metadata entry %d is missing", owner, index) + } + if err := b.consumeStringBytes(ctx, len(metadata.Key())); err != nil { + return err + } + if err := b.consumeStringBytes(ctx, len(metadata.Value())); err != nil { + return err + } + } + return nil +} + +func (b *schemaMetadataBudget) consumeFields(ctx context.Context, count int) error { + if count < 0 || count > MaxSchemaFields-b.fields { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC schema field count exceeds limit %d", MaxSchemaFields) + } + b.fields += count + return nil +} + +func (b *schemaMetadataBudget) consumeStringBytes(ctx context.Context, count int) error { + if count < 0 || count > b.metadataBytes-b.decodedStrings { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC schema decoded string bytes exceed metadata size %d", b.metadataBytes) + } + b.decodedStrings += count + return nil +} + +func (b *schemaMetadataBudget) consumeVector( + ctx context.Context, + name string, + count int, + elementBytes int, +) error { + if count < 0 || elementBytes <= 0 || + uint64(count) > uint64(b.metadataBytes)/uint64(elementBytes) { + return moerr.NewInvalidInputf(ctx, + "Arrow IPC schema %s vector count %d exceeds metadata size %d", + name, count, b.metadataBytes) + } + return nil +} diff --git a/pkg/container/bufferlease/lease.go b/pkg/container/bufferlease/lease.go new file mode 100644 index 0000000000000..90512927c19e1 --- /dev/null +++ b/pkg/container/bufferlease/lease.go @@ -0,0 +1,115 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bufferlease owns immutable external byte-buffer lifetimes shared by +// containers, decoders, and FileService adapters. +package bufferlease + +import ( + "sync" + "sync/atomic" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// BufferLease is a ref-counted immutable byte backing. Retain must fail after +// the refcount reaches zero; every successful Retain requires one Release. +// Bytes is valid only while the caller owns a live reference. +type BufferLease interface { + Retain() bool + Release() + Bytes() []byte + AccountedBytes() int64 +} + +// RefCounted is the default BufferLease implementation. NewRefCounted returns +// one initial owner reference. +type RefCounted struct { + refs atomic.Int64 + accounted int64 + mu sync.RWMutex + data []byte + releaseOne func() +} + +func NewRefCounted( + data []byte, + accountedBytes int64, + releaseOne func(), +) (*RefCounted, error) { + if accountedBytes < 0 { + return nil, moerr.NewInvalidInputNoCtx("negative buffer lease accounting") + } + lease := &RefCounted{ + data: data, + accounted: accountedBytes, + releaseOne: releaseOne, + } + lease.refs.Store(1) + return lease, nil +} + +func (l *RefCounted) Retain() bool { + if l == nil { + return false + } + for { + refs := l.refs.Load() + if refs <= 0 { + return false + } + if l.refs.CompareAndSwap(refs, refs+1) { + return true + } + } +} + +func (l *RefCounted) Release() { + if l == nil { + panic("release nil buffer lease") + } + refs := l.refs.Add(-1) + if refs < 0 { + panic("buffer lease release underflow") + } + if refs != 0 { + return + } + + // The successful 1 -> 0 transition is the only backing cleanup owner. + l.mu.Lock() + l.data = nil + releaseOne := l.releaseOne + l.releaseOne = nil + l.mu.Unlock() + if releaseOne != nil { + releaseOne() + } +} + +func (l *RefCounted) Bytes() []byte { + if l == nil || l.refs.Load() <= 0 { + return nil + } + l.mu.RLock() + defer l.mu.RUnlock() + return l.data +} + +func (l *RefCounted) AccountedBytes() int64 { + if l == nil { + return 0 + } + return l.accounted +} diff --git a/pkg/container/bufferlease/lease_test.go b/pkg/container/bufferlease/lease_test.go new file mode 100644 index 0000000000000..bef98fbad7e17 --- /dev/null +++ b/pkg/container/bufferlease/lease_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufferlease + +import ( + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRefCountedTerminalLifecycle(t *testing.T) { + var cleanup atomic.Int32 + lease, err := NewRefCounted([]byte("payload"), 64, func() { cleanup.Add(1) }) + require.NoError(t, err) + require.Equal(t, []byte("payload"), lease.Bytes()) + require.Equal(t, int64(64), lease.AccountedBytes()) + require.True(t, lease.Retain()) + lease.Release() + require.Equal(t, int32(0), cleanup.Load()) + lease.Release() + require.Equal(t, int32(1), cleanup.Load()) + require.Nil(t, lease.Bytes()) + require.False(t, lease.Retain(), "a terminal lease must not be resurrected") + require.Panics(t, lease.Release, "release underflow is an ownership violation") +} + +func TestRefCountedConcurrentLastRelease(t *testing.T) { + const holders = 128 + var cleanup atomic.Int32 + lease, err := NewRefCounted(make([]byte, 8), 8, func() { cleanup.Add(1) }) + require.NoError(t, err) + for range holders { + require.True(t, lease.Retain()) + } + + var wait sync.WaitGroup + var missing atomic.Int32 + wait.Add(holders) + for range holders { + go func() { + defer wait.Done() + if lease.Bytes() == nil { + missing.Add(1) + } + lease.Release() + }() + } + wait.Wait() + require.Zero(t, missing.Load()) + require.Equal(t, int32(0), cleanup.Load()) + lease.Release() + require.Equal(t, int32(1), cleanup.Load()) +} + +func TestRefCountedRejectsNegativeAccounting(t *testing.T) { + lease, err := NewRefCounted(nil, -1, nil) + require.Error(t, err) + require.Nil(t, lease) +} diff --git a/pkg/container/nulls/borrowed_validity.go b/pkg/container/nulls/borrowed_validity.go new file mode 100644 index 0000000000000..916a053e19053 --- /dev/null +++ b/pkg/container/nulls/borrowed_validity.go @@ -0,0 +1,128 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nulls + +import ( + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/bufferlease" +) + +// InstallBorrowedValidity installs an immutable Arrow validity bitmap. Arrow +// bit 1 means valid, so Contains exposes its inverse as an MO NULL marker. +// The lease is retained on success; the caller keeps its incoming reference. +func (nsp *Nulls) InstallBorrowedValidity( + validity []byte, + bitOffset int, + length int, + nullCount int, + lease bufferlease.BufferLease, +) error { + if nsp == nil || lease == nil || bitOffset < 0 || length < 0 || + nullCount <= 0 || nullCount > length || nsp.validityLease != nil || + nsp.np.Len() != 0 || bitOffset > int(^uint(0)>>1)-length || + (bitOffset+length+7)/8 > len(validity) { + return moerr.NewInvalidInputNoCtx("invalid borrowed Arrow validity view") + } + if !lease.Retain() { + return moerr.NewInternalErrorNoCtx("buffer lease is already released") + } + nsp.validity = validity + nsp.validityOffset = bitOffset + nsp.validityLength = length + nsp.validityNulls = nullCount + nsp.validityLease = lease + return nil +} + +// HasBorrowedValidity reports whether reads still use the Arrow bitmap. +func (nsp *Nulls) HasBorrowedValidity() bool { + return nsp != nil && nsp.validityLease != nil +} + +// BorrowedAccountedBytes returns the charge attached to the shared validity +// owner. Lease-group accounting may de-duplicate it across retained views. +func (nsp *Nulls) BorrowedAccountedBytes() int64 { + if nsp == nil || nsp.validityLease == nil { + return 0 + } + return nsp.validityLease.AccountedBytes() +} + +// InitBorrowedWindow creates a retained logical sub-view. It returns false +// when the requested range contains no NULL, in which case dst remains empty. +func (nsp *Nulls) InitBorrowedWindow(dst *Nulls, start, end int) (bool, error) { + if nsp == nil || dst == nil || nsp.validityLease == nil || + start < 0 || end < start || end > nsp.validityLength { + return false, moerr.NewInvalidInputNoCtx("invalid borrowed validity window") + } + nulls := 0 + for row := start; row < end; row++ { + if nsp.validityContainsNull(uint64(row)) { + nulls++ + } + } + dst.Reset() + if nulls == 0 { + return false, nil + } + err := dst.InstallBorrowedValidity( + nsp.validity, + nsp.validityOffset+start, + end-start, + nulls, + nsp.validityLease, + ) + return err == nil, err +} + +func (nsp *Nulls) validityContainsNull(row uint64) bool { + if nsp.validityLease == nil || row >= uint64(nsp.validityLength) { + return false + } + bit := nsp.validityOffset + int(row) + return nsp.validity[bit>>3]&(byte(1)< 0 || !nsp.np.EmptyByFlag()) } func (nsp *Nulls) IsEmpty() bool { - return nsp == nil || nsp.np.IsEmpty() + return nsp == nil || (nsp.validityNulls == 0 && nsp.np.IsEmpty()) } func (nsp *Nulls) EmptyByFlag() bool { - return nsp == nil || nsp.np.EmptyByFlag() + return nsp == nil || (nsp.validityNulls == 0 && nsp.np.EmptyByFlag()) } func (nsp *Nulls) Set(row uint64) { @@ -410,6 +457,7 @@ func (nsp *Nulls) Set(row uint64) { // Call it unset to match set. Clear or reset are taken. func (nsp *Nulls) Unset(row uint64) { if nsp != nil { + nsp.materializeValidity() nsp.np.Remove(row) } } @@ -419,39 +467,131 @@ func (nsp *Nulls) Count() int { if nsp == nil { return 0 } + if nsp.validityLease != nil { + return nsp.validityNulls + } return nsp.np.Count() } +// CountRange returns the number of NULL rows in [start, end) without forcing +// a borrowed Arrow validity bitmap to materialize. +func (nsp *Nulls) CountRange(start, end uint64) int { + if nsp == nil || start >= end { + return 0 + } + if nsp.validityLease != nil { + if start >= uint64(nsp.validityLength) { + return 0 + } + if end > uint64(nsp.validityLength) { + end = uint64(nsp.validityLength) + } + count := 0 + for row := start; row < end; row++ { + if nsp.validityContainsNull(row) { + count++ + } + } + return count + } + return nsp.np.CountRange(start, end) +} + +// Len returns the logical bitmap domain without forcing a borrowed Arrow +// validity view to materialize. +func (nsp *Nulls) Len() int64 { + if nsp == nil { + return 0 + } + if nsp.validityLease != nil { + return int64(nsp.validityLength) + } + return nsp.np.Len() +} + func (nsp *Nulls) Show() ([]byte, error) { - if nsp.np.EmptyByFlag() { + if nsp.EmptyByFlag() { return nil, nil } + nsp.materializeValidity() return nsp.np.Marshal(), nil } func (nsp *Nulls) MarshalSize() int { - if nsp == nil || nsp.np.EmptyByFlag() { + if nsp == nil || nsp.EmptyByFlag() { return 0 } + if nsp.validityLease != nil { + return bitmap.MarshalHeaderSize + (nsp.validityLength+63)/64*8 + } + nsp.materializeValidity() return nsp.np.MarshalSize() } func (nsp *Nulls) MarshalTo(w io.Writer) error { - if nsp == nil || nsp.np.EmptyByFlag() { + if nsp == nil || nsp.EmptyByFlag() { return nil } + if nsp.validityLease != nil { + return nsp.marshalBorrowedValidityTo(w) + } + nsp.materializeValidity() return nsp.np.MarshalTo(w) } +func (nsp *Nulls) marshalBorrowedValidityTo(w io.Writer) error { + if w == nil { + return io.ErrClosedPipe + } + words := (nsp.validityLength + 63) / 64 + var value [8]byte + writeUint64 := func(v uint64) error { + binary.LittleEndian.PutUint64(value[:], v) + written, err := w.Write(value[:]) + if err != nil { + return err + } + if written != len(value) { + return io.ErrShortWrite + } + return nil + } + if err := writeUint64(uint64(nsp.validityNulls)); err != nil { + return err + } + if err := writeUint64(uint64(nsp.validityLength)); err != nil { + return err + } + if err := writeUint64(uint64(words * 8)); err != nil { + return err + } + for wordIndex := 0; wordIndex < words; wordIndex++ { + var word uint64 + start := wordIndex * 64 + end := min(start+64, nsp.validityLength) + for row := start; row < end; row++ { + if nsp.validityContainsNull(uint64(row)) { + word |= uint64(1) << uint(row-start) + } + } + if err := writeUint64(word); err != nil { + return err + } + } + return nil +} + // ShowV1 in version 1, bitmap is v1 func (nsp *Nulls) ShowV1() ([]byte, error) { - if nsp.np.EmptyByFlag() { + if nsp.EmptyByFlag() { return nil, nil } + nsp.materializeValidity() return nsp.np.MarshalV1(), nil } func (nsp *Nulls) Read(data []byte) error { + nsp.releaseValidity() if len(data) == 0 { // don't we need to reset? Or we always, Read into a blank Nulls? // nsp.np.Reset() @@ -462,6 +602,7 @@ func (nsp *Nulls) Read(data []byte) error { } func (nsp *Nulls) ReadNoCopy(data []byte) error { + nsp.releaseValidity() if len(data) == 0 { return nil } @@ -470,6 +611,7 @@ func (nsp *Nulls) ReadNoCopy(data []byte) error { } func (nsp *Nulls) ReadNoCopyV1(data []byte) error { + nsp.releaseValidity() if len(data) == 0 { return nil } @@ -478,13 +620,15 @@ func (nsp *Nulls) ReadNoCopyV1(data []byte) error { } func (nsp *Nulls) OrBitmap(m *bitmap.Bitmap) { + nsp.materializeValidity() orBitmapInto(nsp, m) } // Or the m Nulls into nsp. func (nsp *Nulls) Or(m *Nulls) { if m != nil { - orBitmapInto(nsp, &m.np) + nsp.materializeValidity() + orBitmapInto(nsp, m.GetBitmap()) } } @@ -496,20 +640,38 @@ func (nsp *Nulls) IsSame(m *Nulls) bool { return false } - return nsp.np.IsSame(&m.np) + return nsp.GetBitmap().IsSame(m.GetBitmap()) } func (nsp *Nulls) ToArray() []uint64 { - if nsp == nil || nsp.np.EmptyByFlag() { + if nsp == nil || nsp.EmptyByFlag() { return []uint64{} } + if nsp.validityLease != nil { + rows := make([]uint64, 0, nsp.validityNulls) + for row := 0; row < nsp.validityLength; row++ { + if nsp.validityContainsNull(uint64(row)) { + rows = append(rows, uint64(row)) + } + } + return rows + } return nsp.np.ToArray() } func (nsp *Nulls) ToI64Array() []int64 { - if nsp == nil || nsp.np.EmptyByFlag() { + if nsp == nil || nsp.EmptyByFlag() { return []int64{} } + if nsp.validityLease != nil { + rows := make([]int64, 0, nsp.validityNulls) + for row := 0; row < nsp.validityLength; row++ { + if nsp.validityContainsNull(uint64(row)) { + rows = append(rows, int64(row)) + } + } + return rows + } return nsp.np.ToI64Array(nil) } @@ -521,6 +683,14 @@ func (nsp *Nulls) Foreach(fn func(uint64) bool) { if nsp.IsEmpty() { return } + if nsp.validityLease != nil { + for row := 0; row < nsp.validityLength; row++ { + if nsp.validityContainsNull(uint64(row)) && !fn(uint64(row)) { + break + } + } + return + } itr := nsp.np.Iterator() for itr.HasNext() { row := itr.Next() @@ -532,7 +702,8 @@ func (nsp *Nulls) Foreach(fn func(uint64) bool) { func (nsp *Nulls) Merge(other *Nulls) { if other != nil { - orBitmapInto(nsp, &other.np) + nsp.materializeValidity() + orBitmapInto(nsp, other.GetBitmap()) } } @@ -540,6 +711,9 @@ func (nsp *Nulls) String() string { if nsp.IsEmpty() { return fmt.Sprintf("%v", []uint64{}) } + if nsp.validityLease != nil { + return fmt.Sprintf("%v", nsp.ToArray()) + } return nsp.np.String() } @@ -548,10 +722,9 @@ func ToArray[T constraints.Integer](nsp *Nulls) []T { return []T{} } ret := make([]T, 0, nsp.Count()) - it := nsp.np.Iterator() - for it.HasNext() { - r := it.Next() - ret = append(ret, T(r)) - } + nsp.Foreach(func(row uint64) bool { + ret = append(ret, T(row)) + return true + }) return ret } diff --git a/pkg/container/pSpool/buffer.go b/pkg/container/pSpool/buffer.go index c79ff95f258c5..a7209d8de8f3f 100644 --- a/pkg/container/pSpool/buffer.go +++ b/pkg/container/pSpool/buffer.go @@ -59,7 +59,8 @@ func (b *spoolBuffer) putCacheID(mp *mpool.MPool, id uint32, bat *batch.Batch) { // 1. const vector size was too small, // 2. vector doesn't own its data and area, // we don't need to cache it. - if !vec.IsConst() && !vec.NeedDup() { + if !vec.IsConst() && !vec.NeedDup() && + vec.CanDetach(vector.BackingData) && vec.CanDetach(vector.BackingArea) { data := vector.DetachVectorData(vec) area := vector.DetachVectorArea(vec) if data.Capacity() != 0 { diff --git a/pkg/container/pSpool/copy.go b/pkg/container/pSpool/copy.go index 7a1ba91d20c80..07af26ce65d38 100644 --- a/pkg/container/pSpool/copy.go +++ b/pkg/container/pSpool/copy.go @@ -107,6 +107,19 @@ func (cb *cachedBatch) GetCopiedBatch( if vec == nil || dst.Vecs[i] != nil { continue } + if vec.HasBorrowedBacking() { + dst.Vecs[i], err = vec.RetainedReadonlyViewWithMP(cb.mp) + if err != nil { + cb.CacheBatch(true, cacheID, dst) + return nil, false, 0, err + } + for j := i + 1; j < len(src.Vecs); j++ { + if dst.Vecs[j] == nil && src.Vecs[j] == vec { + dst.Vecs[j] = dst.Vecs[i] + } + } + continue + } typ := *vec.GetType() selection := vec.AllocationAccountSelection() diff --git a/pkg/container/pSpool/sender_test.go b/pkg/container/pSpool/sender_test.go index eac48ce3ecfba..724534bc71c98 100644 --- a/pkg/container/pSpool/sender_test.go +++ b/pkg/container/pSpool/sender_test.go @@ -18,6 +18,7 @@ import ( "context" "testing" "time" + "unsafe" "github.com/stretchr/testify/require" @@ -61,6 +62,99 @@ func TestCachedBatchPreservesPrepareParamKind(t *testing.T) { constantCache.free() } +func TestSpoolCacheNeverDetachesBorrowedVectorBacking(t *testing.T) { + data := types.EncodeSlice([]int64{11}) + lease, err := vector.NewRefCountedBufferLease(data, int64(cap(data)), nil) + require.NoError(t, err) + vec, err := vector.NewBorrowedFixedVector(types.T_int64.ToType(), 1, data, lease) + require.NoError(t, err) + lease.Release() + bat := batch.NewOffHeap([]string{"v"}) + bat.Vecs[0] = vec + bat.SetRowCount(1) + + buffer := initSpoolBuffer(1) + cacheID, _ := buffer.getCacheID() + buffer.putCacheID(nil, cacheID, bat) + require.Empty(t, buffer.bytesCache[0].buffers) + require.Nil(t, lease.Bytes()) +} + +func TestSpoolRetainsBorrowedPayloadWithoutCopy(t *testing.T) { + mp := mpool.MustNewZero() + data := types.EncodeSlice([]int64{11, 22}) + lease, err := vector.NewRefCountedBufferLease(data, int64(cap(data)), nil) + require.NoError(t, err) + vec, err := vector.NewBorrowedFixedVector(types.T_int64.ToType(), 2, data, lease) + require.NoError(t, err) + lease.Release() + source := batch.NewOffHeap([]string{"v"}) + source.Vecs[0] = vec + source.SetRowCount(2) + + cache := initCachedBatch(mp, 1) + copied, useCache, cacheID, err := cache.GetCopiedBatch(source) + require.NoError(t, err) + require.True(t, useCache) + require.Equal(t, + uintptr(unsafe.Pointer(unsafe.SliceData(source.Vecs[0].GetData()))), + uintptr(unsafe.Pointer(unsafe.SliceData(copied.Vecs[0].GetData()))), + ) + source.Clean(mp) + require.Equal(t, []int64{11, 22}, vector.MustFixedColNoTypeCheck[int64](copied.Vecs[0])) + require.NotNil(t, lease.Bytes()) + + cache.CacheBatch(useCache, cacheID, copied) + require.Nil(t, lease.Bytes()) + cache.free() + require.Zero(t, mp.CurrNB()) +} + +func TestSpoolCopiesVarlenDescriptorsAndRetainsBorrowedPayload(t *testing.T) { + mp := mpool.MustNewZero() + first := []byte("first payload longer than twenty three bytes") + second := []byte("second payload longer than twenty three bytes") + area := append(append([]byte(nil), first...), second...) + lease, err := vector.NewRefCountedBufferLease(area, int64(cap(area)), nil) + require.NoError(t, err) + + vec := vector.NewOffHeapVecWithType(types.T_varchar.ToType()) + require.NoError(t, vec.PreExtend(2, mp)) + vec.SetLength(2) + descriptors := vector.MustFixedColNoTypeCheck[types.Varlena](vec) + descriptors[0].SetOffsetLen(0, uint32(len(first))) + descriptors[1].SetOffsetLen(uint32(len(first)), uint32(len(second))) + require.NoError(t, vec.InstallBorrowedArea(area, lease)) + lease.Release() + + source := batch.NewOffHeap([]string{"v"}) + source.Vecs[0] = vec + source.SetRowCount(2) + sourceDataPointer := uintptr(unsafe.Pointer(unsafe.SliceData(vec.GetData()))) + sourceAreaPointer := uintptr(unsafe.Pointer(unsafe.SliceData(vec.GetArea()))) + + cache := initCachedBatch(mp, 1) + copied, useCache, cacheID, err := cache.GetCopiedBatch(source) + require.NoError(t, err) + require.True(t, useCache) + require.NotEqual(t, sourceDataPointer, + uintptr(unsafe.Pointer(unsafe.SliceData(copied.Vecs[0].GetData()))), + "mutable varlena descriptors must not be shared") + require.Equal(t, sourceAreaPointer, + uintptr(unsafe.Pointer(unsafe.SliceData(copied.Vecs[0].GetArea()))), + "long-value payload should stay zero-copy") + + source.Clean(mp) + require.Equal(t, string(first), copied.Vecs[0].GetStringAt(0)) + require.Equal(t, string(second), copied.Vecs[0].GetStringAt(1)) + require.NotNil(t, lease.Bytes()) + + cache.CacheBatch(useCache, cacheID, copied) + require.Nil(t, lease.Bytes()) + cache.free() + require.Zero(t, mp.CurrNB()) +} + func TestCachedBatchPreservesMixedBinaryStringRows(t *testing.T) { mp := mpool.MustNewZeroNoFixed() defer mpool.DeleteMPool(mp) diff --git a/pkg/container/vector/allocation_account.go b/pkg/container/vector/allocation_account.go index 9aa1a9ea447ca..3c7bd865c3e6a 100644 --- a/pkg/container/vector/allocation_account.go +++ b/pkg/container/vector/allocation_account.go @@ -312,7 +312,8 @@ func (v *Vector) hasBackingStorage() bool { // borrowed aliases; ordinary bitmap backing is Go-owned and remains GC-visible // after replacement. Accounted bitmap storage is explicit external storage. func (v *Vector) hasOwnedBackingStorage() bool { - return cap(v.data) != 0 && !v.cantFreeData || + return v.dataLease != nil || v.areaLease != nil || + cap(v.data) != 0 && !v.cantFreeData || cap(v.area) != 0 && !v.cantFreeArea || cap(v.prepareParamKinds) != 0 || cap(v.stringSources) != 0 || @@ -541,6 +542,11 @@ func (v *Vector) allocateBitmapGrowth( } func (v *Vector) freeBitmapStorage(mp *mpool.MPool) { + if v.nsp.HasBorrowedValidity() { + // Release the source view without materializing it. The bitmap may still + // carry the admitted MPool COW destination reserved before publication. + v.nsp.Reset() + } for _, value := range []*bitmap.Bitmap{ v.nsp.GetBitmap(), v.gsp.GetBitmap(), @@ -611,6 +617,16 @@ func (v *Vector) growOwned( size int, data bool, ) ([]byte, error) { + if v.HasBorrowedBacking() { + if err := v.MaterializeOwned(mp); err != nil { + return nil, err + } + if data { + old = v.data + } else { + old = v.area + } + } if size <= cap(old) { return old[:size], nil } diff --git a/pkg/container/vector/buffer_lease.go b/pkg/container/vector/buffer_lease.go new file mode 100644 index 0000000000000..3b04cdc4dc4f3 --- /dev/null +++ b/pkg/container/vector/buffer_lease.go @@ -0,0 +1,373 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vector + +import ( + "math" + "reflect" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/bufferlease" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// BackingKind describes the release authority for one physical Vector backing. +// The zero value deliberately preserves the existing MPool-owned behavior. +type BackingKind uint8 + +const ( + OwnedMPoolUnique BackingKind = iota + BorrowedLease + LegacyAlias +) + +// BackingPart selects one independently-owned Vector backing. +type BackingPart uint8 + +const ( + BackingData BackingPart = iota + BackingArea +) + +// BufferLease is a ref-counted immutable byte backing. Retain must fail after +// the refcount reaches zero; every successful Retain requires one Release. +// Bytes is valid only while the caller owns a live reference. +type BufferLease = bufferlease.BufferLease + +// RefCountedBufferLease is the default lease implementation used by external +// decoders and FileService adapters. NewRefCountedBufferLease returns one +// initial owner reference. +type RefCountedBufferLease = bufferlease.RefCounted + +func NewRefCountedBufferLease( + data []byte, + accountedBytes int64, + releaseOne func(), +) (*RefCountedBufferLease, error) { + return bufferlease.NewRefCounted(data, accountedBytes, releaseOne) +} + +func (v *Vector) DataBackingKind() BackingKind { + if v == nil { + return OwnedMPoolUnique + } + if v.dataLease != nil { + return BorrowedLease + } + if v.cantFreeData { + return LegacyAlias + } + return OwnedMPoolUnique +} + +func (v *Vector) AreaBackingKind() BackingKind { + if v == nil { + return OwnedMPoolUnique + } + if v.areaLease != nil { + return BorrowedLease + } + if v.cantFreeArea { + return LegacyAlias + } + return OwnedMPoolUnique +} + +func (v *Vector) HasBorrowedBacking() bool { + return v != nil && (v.dataLease != nil || v.areaLease != nil || v.nsp.HasBorrowedValidity()) +} + +func (v *Vector) BorrowedAccountedBytes() int64 { + if v == nil { + return 0 + } + var bytes int64 + if v.dataLease != nil { + bytes += v.dataLease.AccountedBytes() + } + if v.areaLease != nil && !sameBufferLease(v.areaLease, v.dataLease) { + bytes += v.areaLease.AccountedBytes() + } + bytes += v.nsp.BorrowedAccountedBytes() + return bytes +} + +// sameBufferLease only de-duplicates implementations whose dynamic values are +// comparable. BufferLease is a public interface and can legally be +// implemented by slice-backed value types; comparing those interfaces +// directly would panic. When identity cannot be proven, accounting both +// references is the safe upper bound. +func sameBufferLease(left, right BufferLease) bool { + if left == nil || right == nil { + return false + } + leftValue := reflect.ValueOf(left) + rightValue := reflect.ValueOf(right) + return leftValue.Type() == rightValue.Type() && + leftValue.Comparable() && leftValue.Interface() == rightValue.Interface() +} + +func (v *Vector) CanDetach(part BackingPart) bool { + if v == nil { + return false + } + switch part { + case BackingData: + return v.dataLease == nil && !v.cantFreeData + case BackingArea: + return v.areaLease == nil && !v.cantFreeArea + default: + return false + } +} + +// InstallBorrowedData atomically installs a retained read-only data view. The +// Vector must not already own data capacity; callers must Free or materialize +// the previous generation first. +func (v *Vector) InstallBorrowedData(data []byte, lease BufferLease) error { + if v == nil || lease == nil || cap(v.data) != 0 || v.dataLease != nil { + return moerr.NewInternalErrorNoCtx("cannot install borrowed vector data") + } + if !lease.Retain() { + return moerr.NewInternalErrorNoCtx("buffer lease is already released") + } + v.data = data + v.dataLease = lease + v.cantFreeData = true + return nil +} + +// InstallBorrowedArea is the independently-owned area counterpart of +// InstallBorrowedData. +func (v *Vector) InstallBorrowedArea(area []byte, lease BufferLease) error { + if v == nil || lease == nil || cap(v.area) != 0 || v.areaLease != nil { + return moerr.NewInternalErrorNoCtx("cannot install borrowed vector area") + } + if !lease.Retain() { + return moerr.NewInternalErrorNoCtx("buffer lease is already released") + } + v.area = area + v.areaLease = lease + v.cantFreeArea = true + return nil +} + +// PrepareBorrowedValidity reserves the owned COW destination before a +// borrowed Arrow validity view is published. Nulls' legacy mutation APIs +// cannot return allocation errors, so admission must happen here while the +// bridge can still fail transactionally. The bitmap remains logically empty +// until the validity view is materialized. +func (v *Vector) PrepareBorrowedValidity(rows int, mp *mpool.MPool) error { + if v == nil || mp == nil || rows < 0 || rows > math.MaxInt-63 || + v.nsp.HasBorrowedValidity() || v.nsp.Len() != 0 { + return moerr.NewInvalidInputNoCtx("invalid borrowed validity reservation") + } + requiredWords := (rows + 63) / 64 + bitmap := v.nsp.GetBitmap() + if requiredWords > bitmap.ExternalStorageCapacity() { + var ( + storage []uint64 + err error + ) + if v.allocationAccount == nil { + storage, err = mpool.MakeSlice[uint64](requiredWords, mp, v.offHeap) + } else { + storage, err = v.allocateBitmapGrowth( + bitmap, rows, mp, v.allocationAccount.nullsSite, + ) + } + if err != nil { + return err + } + if cap(storage) > 0 { + previous := bitmap.InstallExternalStorage(storage) + mpool.FreeSlice(mp, previous) + } + } + bitmap.Reset() + return nil +} + +// NewBorrowedFixedVector constructs a fixed-width immutable Vector. The +// constructor retains lease; the caller continues to own its incoming ref. +func NewBorrowedFixedVector( + typ types.Type, + rows int, + data []byte, + lease BufferLease, +) (*Vector, error) { + return NewBorrowedFixedVectorWithAllocation(typ, rows, data, lease, nil) +} + +// NewBorrowedFixedVectorWithAllocation installs selection before borrowed +// backing, so any later COW or bitmap materialization remains in the same +// statement account. +func NewBorrowedFixedVectorWithAllocation( + typ types.Type, + rows int, + data []byte, + lease BufferLease, + selection *AllocationAccountSelection, +) (*Vector, error) { + if rows < 0 || typ.IsVarlen() || typ.TypeSize() <= 0 || + uint64(rows) > uint64(^uint(0)>>1)/uint64(typ.TypeSize()) || + len(data) != rows*typ.TypeSize() { + return nil, moerr.NewInvalidInputNoCtx("invalid borrowed fixed vector layout") + } + vec, err := NewOffHeapVecWithTypeAndAllocation(typ, selection) + if err != nil { + return nil, err + } + if err := vec.InstallBorrowedData(data, lease); err != nil { + vec.Free(nil) + return nil, err + } + vec.length = rows + return vec, nil +} + +func (v *Vector) releaseBorrowedData() { + if v == nil || v.dataLease == nil { + return + } + lease := v.dataLease + v.dataLease = nil + v.data = nil + v.cantFreeData = false + lease.Release() +} + +func (v *Vector) releaseBorrowedArea() { + if v == nil || v.areaLease == nil { + return + } + lease := v.areaLease + v.areaLease = nil + v.area = nil + v.cantFreeArea = false + lease.Release() +} + +func (v *Vector) releaseBorrowedBacking() { + if v == nil { + return + } + v.releaseBorrowedArea() + v.releaseBorrowedData() +} + +// MaterializeOwned performs copy-on-write transactionally. Allocation failure +// leaves the borrowed Vector unchanged and readable. +func (v *Vector) MaterializeOwned(mp *mpool.MPool) error { + if v == nil || !v.HasBorrowedBacking() { + return nil + } + if mp == nil { + return moerr.NewInternalErrorNoCtx("borrowed vector materialization does not have a mpool") + } + owned, err := v.dup(mp, true, true, v.allocationAccount) + if err != nil { + return err + } + old := *v + *v = *owned + old.Free(mp) + return nil +} + +// RetainedReadonlyView returns an explicitly retained full-row view. Legacy +// aliases are excluded because they do not carry a lifetime owner. +func (v *Vector) RetainedReadonlyView() (*Vector, error) { + if v == nil || v.DataBackingKind() == LegacyAlias || v.AreaBackingKind() == LegacyAlias { + return nil, moerr.NewInternalErrorNoCtx("vector backing has no retainable owner") + } + if !v.HasBorrowedBacking() { + return nil, moerr.NewInternalErrorNoCtx("owned vector has no shared backing lease") + } + return v.WindowByLogicalRows(0, v.length) +} + +// RetainedReadonlyViewWithMP creates an asynchronously safe full-row snapshot. +// Borrowed backings are retained; unique-owned backings are copied so the +// source may be recycled immediately. The result remains immutable while any +// retained backing is present and must be materialized before mutation. +func (v *Vector) RetainedReadonlyViewWithMP(mp *mpool.MPool) (*Vector, error) { + if v == nil { + return nil, moerr.NewInvalidInputNoCtx("retained vector snapshot requires a vector and mpool") + } + return v.RetainedReadonlyWindowWithMP(0, v.length, mp) +} + +// RetainedReadonlyWindowWithMP is the row-range form of +// RetainedReadonlyViewWithMP. It preserves leased payload backings while +// copying source-owned descriptors and other unique-owned slices. +func (v *Vector) RetainedReadonlyWindowWithMP(start, end int, mp *mpool.MPool) (*Vector, error) { + if v == nil || mp == nil { + return nil, moerr.NewInvalidInputNoCtx("retained vector snapshot requires a vector and mpool") + } + if v.DataBackingKind() == LegacyAlias || v.AreaBackingKind() == LegacyAlias { + return nil, moerr.NewInternalErrorNoCtx("vector backing has no retainable owner") + } + if !v.HasBorrowedBacking() { + return nil, moerr.NewInternalErrorNoCtx("owned vector has no shared backing lease") + } + + var ( + view *Vector + err error + ) + if v.allocationAccount != nil { + view, err = v.WindowByLogicalRowsWithAllocation( + start, end, mp, v.allocationAccount, + ) + } else { + view, err = v.window(start, end, mp, nil, true) + } + if err != nil { + return nil, err + } + view.offHeap = true + + // Window can safely alias only a leased source backing. Copy every + // source-owned part before the source becomes reusable. + if v.dataLease == nil && len(view.data) > 0 { + owned, allocErr := view.allocOwned(mp, len(view.data), true, true) + if allocErr != nil { + view.Free(mp) + return nil, allocErr + } + copy(owned, view.data) + view.data = owned + view.cantFreeData = false + } + if v.areaLease == nil && len(view.area) > 0 { + owned, allocErr := view.allocOwned(mp, len(view.area), true, false) + if allocErr != nil { + view.Free(mp) + return nil, allocErr + } + copy(owned, view.area) + view.area = owned + view.cantFreeArea = false + } + if view.dataLease == nil && len(view.data) == 0 { + view.cantFreeData = false + } + if view.areaLease == nil && len(view.area) == 0 { + view.cantFreeArea = false + } + return view, nil +} diff --git a/pkg/container/vector/buffer_lease_test.go b/pkg/container/vector/buffer_lease_test.go new file mode 100644 index 0000000000000..32acd8baeb969 --- /dev/null +++ b/pkg/container/vector/buffer_lease_test.go @@ -0,0 +1,335 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vector + +import ( + "bytes" + "sync" + "sync/atomic" + "testing" + "unsafe" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +type nonComparableBufferLease []byte + +func (l nonComparableBufferLease) Retain() bool { return true } +func (l nonComparableBufferLease) Release() {} +func (l nonComparableBufferLease) Bytes() []byte { return l } +func (l nonComparableBufferLease) AccountedBytes() int64 { return int64(cap(l)) } + +func TestRefCountedBufferLeaseTerminalState(t *testing.T) { + var cleanup atomic.Int32 + lease, err := NewRefCountedBufferLease([]byte{1, 2, 3}, 64, func() { + cleanup.Add(1) + }) + require.NoError(t, err) + require.Equal(t, int64(64), lease.AccountedBytes()) + require.True(t, lease.Retain()) + + lease.Release() + require.Equal(t, int32(0), cleanup.Load()) + require.Equal(t, []byte{1, 2, 3}, lease.Bytes()) + lease.Release() + require.Equal(t, int32(1), cleanup.Load()) + require.Nil(t, lease.Bytes()) + require.False(t, lease.Retain()) + require.Panics(t, lease.Release) +} + +func TestRefCountedBufferLeaseConcurrentRetainRelease(t *testing.T) { + var cleanup atomic.Int32 + lease, err := NewRefCountedBufferLease(make([]byte, 32), 32, func() { + cleanup.Add(1) + }) + require.NoError(t, err) + + const workers = 64 + var wg sync.WaitGroup + wg.Add(workers) + for range workers { + go func() { + defer wg.Done() + require.True(t, lease.Retain()) + lease.Release() + }() + } + wg.Wait() + lease.Release() + require.Equal(t, int32(1), cleanup.Load()) +} + +func TestBorrowedFixedVectorWindowAndCleanRelease(t *testing.T) { + data := types.EncodeSlice([]int64{11, 22, 33, 44}) + var cleanup atomic.Int32 + lease, err := NewRefCountedBufferLease(data, int64(cap(data)), func() { + cleanup.Add(1) + }) + require.NoError(t, err) + + vec, err := NewBorrowedFixedVector(types.T_int64.ToType(), 4, data, lease) + require.NoError(t, err) + lease.Release() // transfer the source reference to vec + require.Equal(t, BorrowedLease, vec.DataBackingKind()) + // Borrowed storage has a stable retained lifetime, but it is still + // immutable and non-owning. Legacy ownership/COW boundaries must duplicate + // it before mutation or receiver reuse. + require.True(t, vec.NeedDup()) + require.False(t, vec.CanDetach(BackingData)) + detached := DetachVectorData(vec) + require.Zero(t, detached.Capacity()) + require.Equal(t, []int64{11, 22, 33, 44}, MustFixedColNoTypeCheck[int64](vec)) + + window, err := vec.Window(1, 3) + require.NoError(t, err) + require.Equal(t, []int64{22, 33}, MustFixedColNoTypeCheck[int64](window)) + vec.CleanOnlyData() + require.Zero(t, vec.Length()) + require.Equal(t, int32(0), cleanup.Load()) + require.Equal(t, []int64{22, 33}, MustFixedColNoTypeCheck[int64](window)) + window.Free(nil) + require.Equal(t, int32(1), cleanup.Load()) +} + +func TestBorrowedVectorMaterializeOwnedIsTransactional(t *testing.T) { + values := make([]int64, mpool.MB/8+1) + copy(values, []int64{7, 8, 9}) + data := types.EncodeSlice(values) + var cleanup atomic.Int32 + lease, err := NewRefCountedBufferLease(data, int64(cap(data)), func() { + cleanup.Add(1) + }) + require.NoError(t, err) + vec, err := NewBorrowedFixedVector(types.T_int64.ToType(), len(values), data, lease) + require.NoError(t, err) + lease.Release() + + tooSmall, err := mpool.NewMPool("borrowed-cow-failure", mpool.MB, mpool.NoFixed) + require.NoError(t, err) + require.Error(t, vec.MaterializeOwned(tooSmall)) + require.Equal(t, BorrowedLease, vec.DataBackingKind()) + require.Equal(t, []int64{7, 8, 9}, MustFixedColNoTypeCheck[int64](vec)[:3]) + require.Equal(t, int32(0), cleanup.Load()) + + mp := mpool.MustNewZero() + require.NoError(t, vec.MaterializeOwned(mp)) + require.Equal(t, OwnedMPoolUnique, vec.DataBackingKind()) + require.Equal(t, []int64{7, 8, 9}, MustFixedColNoTypeCheck[int64](vec)[:3]) + require.Equal(t, int32(1), cleanup.Load()) + require.True(t, vec.CanDetach(BackingData)) + vec.Free(mp) + require.Zero(t, mp.CurrNB()) +} + +func TestBorrowedVectorMaterializeOwnedCopiesValidityIntoAccount(t *testing.T) { + data := types.EncodeSlice([]int64{11, 22, 33}) + dataLease, err := NewRefCountedBufferLease(data, int64(cap(data)), nil) + require.NoError(t, err) + validity := []byte{0b00000101} // row one is NULL + validityLease, err := NewRefCountedBufferLease(validity, int64(cap(validity)), nil) + require.NoError(t, err) + + registry, err := mpool.NewAllocationAccountRegistry(1, 64) + require.NoError(t, err) + account, err := registry.Open(1 << 20) + require.NoError(t, err) + selection, err := NewAllocationAccountSelection( + account, mpool.AllocationOwnerExternal, 1, 2, 3, 4, + ) + require.NoError(t, err) + vec, err := NewBorrowedFixedVectorWithAllocation( + types.T_int64.ToType(), 3, data, dataLease, selection, + ) + require.NoError(t, err) + require.NoError(t, vec.GetNulls().InstallBorrowedValidity( + validity, 0, 3, 1, validityLease, + )) + dataLease.Release() + validityLease.Release() + + mp := mpool.MustNewZero() + clone, err := vec.CloneWindow(0, vec.Length(), mp) + require.NoError(t, err) + require.False(t, clone.HasBorrowedBacking()) + require.Equal(t, []uint64{1}, clone.GetNulls().ToArray()) + clone.Free(mp) + + require.NoError(t, vec.MaterializeOwned(mp)) + require.False(t, vec.HasBorrowedBacking()) + require.Equal(t, []uint64{1}, vec.GetNulls().ToArray()) + require.Equal(t, []int64{11, 22, 33}, MustFixedColNoTypeCheck[int64](vec)) + require.Greater(t, account.Snapshot().Used, uint64(0)) + vec.Free(mp) + require.Zero(t, account.Snapshot().Used) + require.Zero(t, mp.CurrNB()) + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) +} + +func TestBorrowedValidityLegacyMaterializationUsesReservedMPoolStorage(t *testing.T) { + registry, err := mpool.NewAllocationAccountRegistry(1, 16) + require.NoError(t, err) + account, err := registry.Open(1 << 20) + require.NoError(t, err) + selection, err := NewAllocationAccountSelection( + account, mpool.AllocationOwnerExternal, 1, 2, 3, 4, + ) + require.NoError(t, err) + mp := mpool.MustNewZero() + vec, err := NewOffHeapVecWithTypeAndAllocation(types.T_int64.ToType(), selection) + require.NoError(t, err) + require.NoError(t, vec.PrepareBorrowedValidity(3, mp)) + admittedBytes := mp.CurrNB() + require.Positive(t, admittedBytes) + require.Positive(t, account.Snapshot().Used) + + validity := []byte{0b00000101} // row one is NULL + var releases atomic.Int32 + lease, err := NewRefCountedBufferLease(validity, int64(cap(validity)), func() { + releases.Add(1) + }) + require.NoError(t, err) + require.NoError(t, vec.GetNulls().InstallBorrowedValidity(validity, 0, 3, 1, lease)) + lease.Release() + require.True(t, vec.GetNulls().HasBorrowedValidity()) + + bitmap := vec.GetNulls().GetBitmap() + require.False(t, vec.GetNulls().HasBorrowedValidity()) + require.Equal(t, int32(1), releases.Load()) + require.True(t, bitmap.HasExternalStorage()) + require.Equal(t, []uint64{1}, vec.GetNulls().ToArray()) + require.Equal(t, admittedBytes, mp.CurrNB(), "legacy materialization must not allocate outside admission") + + vec.Free(mp) + require.Zero(t, account.Snapshot().Used) + require.Zero(t, mp.CurrNB()) + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) +} + +func TestRetainedReadonlyViewWithMPCopiesOwnedDescriptorsAndRetainsArea(t *testing.T) { + mp := mpool.MustNewZero() + vec := NewOffHeapVecWithType(types.T_varchar.ToType()) + require.NoError(t, vec.PreExtend(2, mp)) + vec.SetLength(2) + first := []byte("first payload longer than twenty three bytes") + second := []byte("second payload longer than twenty three bytes") + area := append(append([]byte(nil), first...), second...) + descriptors := MustFixedColNoTypeCheck[types.Varlena](vec) + descriptors[0].SetOffsetLen(0, uint32(len(first))) + descriptors[1].SetOffsetLen(uint32(len(first)), uint32(len(second))) + lease, err := NewRefCountedBufferLease(area, int64(cap(area)), nil) + require.NoError(t, err) + require.NoError(t, vec.InstallBorrowedArea(area, lease)) + lease.Release() + + dataPointer := uintptr(unsafe.Pointer(unsafe.SliceData(vec.GetData()))) + areaPointer := uintptr(unsafe.Pointer(unsafe.SliceData(vec.GetArea()))) + view, err := vec.RetainedReadonlyViewWithMP(mp) + require.NoError(t, err) + require.NotEqual(t, dataPointer, uintptr(unsafe.Pointer(unsafe.SliceData(view.GetData())))) + require.Equal(t, areaPointer, uintptr(unsafe.Pointer(unsafe.SliceData(view.GetArea())))) + require.Equal(t, BorrowedLease, view.AreaBackingKind()) + require.Equal(t, OwnedMPoolUnique, view.DataBackingKind()) + + vec.Free(mp) + require.Equal(t, string(first), view.GetStringAt(0)) + require.Equal(t, string(second), view.GetStringAt(1)) + require.NotNil(t, lease.Bytes()) + view.Free(mp) + require.Nil(t, lease.Bytes()) + require.Zero(t, mp.CurrNB()) +} + +func TestBorrowedVectorResetReleasesEachBacking(t *testing.T) { + dataLease, err := NewRefCountedBufferLease([]byte{1}, 1, nil) + require.NoError(t, err) + areaLease, err := NewRefCountedBufferLease([]byte("long-value"), 10, nil) + require.NoError(t, err) + vec := NewOffHeapVecWithType(types.T_varchar.ToType()) + require.NoError(t, vec.InstallBorrowedData([]byte{1}, dataLease)) + require.NoError(t, vec.InstallBorrowedArea([]byte("long-value"), areaLease)) + dataLease.Release() + areaLease.Release() + + vec.ResetWithSameType() + require.Equal(t, OwnedMPoolUnique, vec.DataBackingKind()) + require.Equal(t, OwnedMPoolUnique, vec.AreaBackingKind()) + require.Nil(t, dataLease.Bytes()) + require.Nil(t, areaLease.Bytes()) +} + +func TestBorrowedAccountedBytesDoesNotCompareNonComparableLease(t *testing.T) { + lease := nonComparableBufferLease(make([]byte, 0, 7)) + vec := NewOffHeapVecWithType(types.T_varchar.ToType()) + require.NoError(t, vec.InstallBorrowedData(lease, lease)) + require.NoError(t, vec.InstallBorrowedArea(lease, lease)) + require.Equal(t, int64(14), vec.BorrowedAccountedBytes(), + "unknown lease identity must be accounted conservatively") + vec.Free(nil) +} + +func TestBorrowedVarlenMarshalUsesCanonicalCompactedArea(t *testing.T) { + first := []byte("first payload longer than twenty three bytes") + second := []byte("second payload longer than twenty three bytes") + area := append([]byte("unused-prefix"), first...) + secondOffset := len(area) + area = append(area, second...) + area = append(area, []byte("unused-suffix")...) + descriptors := make([]types.Varlena, 3) + descriptors[0].SetOffsetLen(uint32(len("unused-prefix")), uint32(len(first))) + descriptors[1][0] = 5 + copy(descriptors[1][1:], "short") + descriptors[2].SetOffsetLen(uint32(secondOffset), uint32(len(second))) + data := types.EncodeSlice(descriptors) + dataLease, err := NewRefCountedBufferLease(data, int64(cap(data)), nil) + require.NoError(t, err) + areaLease, err := NewRefCountedBufferLease(area, int64(cap(area)), nil) + require.NoError(t, err) + vec := NewOffHeapVecWithType(types.T_varchar.ToType()) + require.NoError(t, vec.InstallBorrowedData(data, dataLease)) + require.NoError(t, vec.InstallBorrowedArea(area, areaLease)) + dataLease.Release() + areaLease.Release() + vec.SetLength(3) + vec.GetNulls().Add(2) + + plan, err := vec.PrepareMarshalBinary() + require.NoError(t, err) + fullAreaPlanSize := 1 + types.TSize + 4 + 4 + len(data) + 4 + len(area) + 4 + vec.GetNulls().MarshalSize() + 1 + require.Equal(t, fullAreaPlanSize-len(area)+len(first), plan.Size(), + "unused prefix/suffix and the NULL row payload must not enter canonical wire bytes") + encoded, err := vec.MarshalBinary() + require.NoError(t, err) + var streamed bytes.Buffer + require.NoError(t, plan.MarshalTo(&streamed)) + require.Equal(t, encoded, streamed.Bytes()) + + decoded := NewVecFromReuse() + require.NoError(t, decoded.UnmarshalBinary(encoded)) + require.Equal(t, string(first), decoded.GetStringAt(0)) + require.Equal(t, "short", decoded.GetStringAt(1)) + require.True(t, decoded.IsNull(2)) + require.Equal(t, first, decoded.GetArea()) + decoded.Free(nil) + vec.Free(nil) + require.Nil(t, dataLease.Bytes()) + require.Nil(t, areaLease.Bytes()) +} diff --git a/pkg/container/vector/pSpoolTools.go b/pkg/container/vector/pSpoolTools.go index 27dd7f90c5991..a625940089f65 100644 --- a/pkg/container/vector/pSpoolTools.go +++ b/pkg/container/vector/pSpoolTools.go @@ -33,7 +33,7 @@ const ( ) func DetachVectorData(v *Vector) DetachedBuffer { - if v == nil { + if v == nil || !v.CanDetach(BackingData) { return DetachedBuffer{} } buffer := DetachedBuffer{ @@ -48,7 +48,7 @@ func DetachVectorData(v *Vector) DetachedBuffer { } func DetachVectorArea(v *Vector) DetachedBuffer { - if v == nil { + if v == nil || !v.CanDetach(BackingArea) { return DetachedBuffer{} } buffer := DetachedBuffer{ diff --git a/pkg/container/vector/vector.go b/pkg/container/vector/vector.go index 008f623c36719..8194bc8e56e6e 100644 --- a/pkg/container/vector/vector.go +++ b/pkg/container/vector/vector.go @@ -105,6 +105,10 @@ type Vector struct { cantFreeData bool cantFreeArea bool + // Borrowed leases are independent release owners for data and area. The + // cantFree bits remain only as the legacy-alias compatibility marker. + dataLease BufferLease + areaLease BufferLease sorted bool // for some optimization @@ -223,6 +227,7 @@ func (v *Vector) SetSorted(b bool) { // Reset update vector's fields with a specific type. // we should redefine the value of capacity and values-ptr because of the possible change in type. func (v *Vector) Reset(typ types.Type) { + v.releaseBorrowedBacking() v.typ = typ v.resetPrepareParamKind() v.resetStringSource() @@ -245,6 +250,7 @@ func (v *Vector) Reset(typ types.Type) { } func (v *Vector) ResetWithSameType() { + v.releaseBorrowedBacking() v.resetPrepareParamKind() v.resetStringSource() v.class = FLAT @@ -264,12 +270,14 @@ func (v *Vector) ResetWithSameType() { } func (v *Vector) ResetArea() { + v.releaseBorrowedArea() v.area = v.area[:0] v.areaDisjoint = v.length == 0 } // TODO: It is semantically same as Reset, need to merge them later. func (v *Vector) ResetWithNewType(t *types.Type) { + v.releaseBorrowedBacking() v.typ = *t v.resetPrepareParamKind() v.resetStringSource() @@ -3725,7 +3733,9 @@ func (v *Vector) propagateBinaryStringBatch(w *Vector, oldLength int, offset int } func (v *Vector) NeedDup() bool { - return v.cantFreeArea || v.cantFreeData + return v.AreaBackingKind() != OwnedMPoolUnique || + v.DataBackingKind() != OwnedMPoolUnique || + v.nsp.HasBorrowedValidity() } // make sure the type check is done before calling this function @@ -3787,7 +3797,9 @@ func (v *Vector) GetRawBytesAt(i int) []byte { } func (v *Vector) CleanOnlyData() { - if v.data != nil { + hadData := v.data != nil + v.releaseBorrowedBacking() + if hadData { v.length = 0 } if v.area != nil { @@ -4134,6 +4146,9 @@ func (v *Vector) UnsetNull(i uint64) { // call this function if type already checked func SetFixedAtNoTypeCheck[T types.FixedSizeT](v *Vector, idx int, t T) error { + if v.HasBorrowedBacking() { + return moerr.NewInternalErrorNoCtx("borrowed vector must be materialized before mutation") + } if v.typ.IsVarlen() { // A caller-provided varlena descriptor can alias an existing area range. v.areaDisjoint = false @@ -4152,6 +4167,9 @@ func SetFixedAtNoTypeCheck[T types.FixedSizeT](v *Vector, idx int, t T) error { // Note: // it is 10x slower than SetFixedAtNoTypeCheck func SetFixedAtWithTypeCheck[T types.FixedSizeT](v *Vector, idx int, t T) error { + if v.HasBorrowedBacking() { + return moerr.NewInternalErrorNoCtx("borrowed vector must be materialized before mutation") + } if v.typ.IsVarlen() { // A caller-provided varlena descriptor can alias an existing area range. v.areaDisjoint = false @@ -4170,6 +4188,9 @@ func SetFixedAtWithTypeCheck[T types.FixedSizeT](v *Vector, idx int, t T) error } func SetBytesAt(v *Vector, idx int, bs []byte, mp *mpool.MPool) error { + if err := v.MaterializeOwned(mp); err != nil { + return err + } disjoint := v.areaDisjoint var va types.Varlena err := BuildVarlenaFromByteSlice(v, &va, &bs, mp) @@ -4349,10 +4370,14 @@ func (v *Vector) Free(mp *mpool.MPool) { return } - if !v.cantFreeData { + if v.dataLease != nil { + v.releaseBorrowedData() + } else if !v.cantFreeData { mp.Free(v.data) } - if !v.cantFreeArea { + if v.areaLease != nil { + v.releaseBorrowedArea() + } else if !v.cantFreeArea { mp.Free(v.area) } v.freeBitmapStorage(mp) @@ -4362,6 +4387,8 @@ func (v *Vector) Free(mp *mpool.MPool) { v.length = 0 v.cantFreeData = false v.cantFreeArea = false + v.dataLease = nil + v.areaLease = nil v.nsp.Reset() v.gsp.Reset() @@ -4406,14 +4433,17 @@ func (v *Vector) MarshalBinaryWithBuffer(buf *bytes.Buffer) error { return v.MarshalBinaryTo(buf) } -// MarshalBinaryPlan is a validated, allocation-free snapshot of one Vector's -// wire lengths. It lets batch writers size once and encode once. +// MarshalBinaryPlan is a validated snapshot of one Vector's wire layout. It +// lets batch writers size once and encode once. type MarshalBinaryPlan struct { - vector *Vector - size int - dataLength uint32 - areaLength uint32 - nullLength uint32 + vector *Vector + size int + dataLength uint32 + areaLength uint32 + nullLength uint32 + canonicalVarlen bool + canonicalOffset []uint32 + canonicalFirst []bool } func (p MarshalBinaryPlan) Size() int { @@ -4449,6 +4479,58 @@ func (v *Vector) PrepareMarshalBinary() (MarshalBinaryPlan, error) { dataLength = 0 } areaLength := uint64(len(v.area)) + canonicalVarlen := isVarlenaMarshalType(v.typ.Oid) && dataLength > 0 + if dataLength > uint64(len(v.data)) { + return MarshalBinaryPlan{}, moerr.NewInvalidInputNoCtx( + "vector data is shorter than its marshal length", + ) + } + var canonicalOffset []uint32 + var canonicalFirst []bool + if canonicalVarlen { + if dataLength%types.VarlenaSize != 0 { + return MarshalBinaryPlan{}, moerr.NewInvalidInputNoCtx( + "varlen vector data is not descriptor aligned", + ) + } + areaLength = 0 + descriptors := MustFixedColNoTypeCheck[types.Varlena](v) + if uint64(len(descriptors))*types.VarlenaSize != dataLength { + return MarshalBinaryPlan{}, moerr.NewInvalidInputNoCtx( + "varlen vector descriptor count does not match marshal length", + ) + } + canonicalOffset = make([]uint32, len(descriptors)) + canonicalFirst = make([]bool, len(descriptors)) + type areaSpan struct{ offset, length uint32 } + seen := make(map[areaSpan]uint32, len(descriptors)) + for index := range descriptors { + if v.IsNull(uint64(index)) || descriptors[index].IsSmall() { + continue + } + offset, length := descriptors[index].OffsetLen() + end := uint64(offset) + uint64(length) + if end > uint64(len(v.area)) { + return MarshalBinaryPlan{}, moerr.NewInvalidInputNoCtx( + "varlen vector descriptor is outside its area", + ) + } + span := areaSpan{offset: offset, length: length} + if compactOffset, ok := seen[span]; ok { + canonicalOffset[index] = compactOffset + continue + } + if uint64(length) > maxWireBuffer-areaLength { + return MarshalBinaryPlan{}, moerr.NewInvalidInputNoCtx( + "canonical varlen area exceeds marshal format", + ) + } + canonicalOffset[index] = uint32(areaLength) + canonicalFirst[index] = true + seen[span] = uint32(areaLength) + areaLength += uint64(length) + } + } nullLength := uint64(v.nsp.MarshalSize()) if dataLength > maxWireBuffer || areaLength > maxWireBuffer || @@ -4457,11 +4539,6 @@ func (v *Vector) PrepareMarshalBinary() (MarshalBinaryPlan, error) { "vector buffer exceeds marshal format", ) } - if dataLength > uint64(len(v.data)) { - return MarshalBinaryPlan{}, moerr.NewInvalidInputNoCtx( - "vector data is shorter than its marshal length", - ) - } total := uint64(1+types.TSize+4+4+4+4+1) + dataLength + areaLength + nullLength if total > uint64(^uint(0)>>1) { @@ -4470,14 +4547,29 @@ func (v *Vector) PrepareMarshalBinary() (MarshalBinaryPlan, error) { ) } return MarshalBinaryPlan{ - vector: v, - size: int(total), - dataLength: uint32(dataLength), - areaLength: uint32(areaLength), - nullLength: uint32(nullLength), + vector: v, + size: int(total), + dataLength: uint32(dataLength), + areaLength: uint32(areaLength), + nullLength: uint32(nullLength), + canonicalVarlen: canonicalVarlen, + canonicalOffset: canonicalOffset, + canonicalFirst: canonicalFirst, }, nil } +func isVarlenaMarshalType(oid types.T) bool { + switch oid { + case types.T_char, types.T_varchar, types.T_blob, types.T_json, types.T_text, + types.T_binary, types.T_varbinary, types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + types.T_datalink, types.T_geometry, types.T_geometry32: + return true + default: + return false + } +} + func (v *Vector) MarshalBinarySize() (int, error) { plan, err := v.PrepareMarshalBinary() return plan.Size(), err @@ -4511,8 +4603,24 @@ func (p MarshalBinaryPlan) MarshalTo(w io.Writer) error { return err } if p.dataLength > 0 { - if err := writeVectorMarshalBytes(w, v.data[:p.dataLength]); err != nil { - return err + if p.canonicalVarlen { + for index, descriptor := range MustFixedColNoTypeCheck[types.Varlena](v) { + canonical := descriptor + if v.IsNull(uint64(index)) { + canonical = types.Varlena{} + } else if !descriptor.IsSmall() { + _, length := descriptor.OffsetLen() + canonical.SetOffsetLen(p.canonicalOffset[index], length) + } + bytes := unsafe.Slice((*byte)(unsafe.Pointer(&canonical)), types.VarlenaSize) + if err := writeVectorMarshalBytes(w, bytes); err != nil { + return err + } + } + } else { + if err := writeVectorMarshalBytes(w, v.data[:p.dataLength]); err != nil { + return err + } } } @@ -4520,8 +4628,20 @@ func (p MarshalBinaryPlan) MarshalTo(w io.Writer) error { return err } if p.areaLength > 0 { - if err := writeVectorMarshalBytes(w, v.area); err != nil { - return err + if p.canonicalVarlen { + for index, descriptor := range MustFixedColNoTypeCheck[types.Varlena](v) { + if v.IsNull(uint64(index)) || descriptor.IsSmall() || !p.canonicalFirst[index] { + continue + } + offset, length := descriptor.OffsetLen() + if err := writeVectorMarshalBytes(w, v.area[offset:offset+length]); err != nil { + return err + } + } + } else { + if err := writeVectorMarshalBytes(w, v.area); err != nil { + return err + } } } @@ -5143,6 +5263,9 @@ func (v *Vector) ToConst() { // PreExtend use to expand the capacity of the vector. // PreExtend does not change the length of the vector. func (v *Vector) PreExtend(rows int, mp *mpool.MPool) error { + if err := v.MaterializeOwned(mp); err != nil { + return err + } if v.class == CONSTANT { return nil } @@ -5153,12 +5276,18 @@ func (v *Vector) PreExtend(rows int, mp *mpool.MPool) error { // represent rows without allocating vector data. Unaccounted vectors are // unchanged. func (v *Vector) PreExtendBitmap(rows int, mp *mpool.MPool) error { + if err := v.MaterializeOwned(mp); err != nil { + return err + } return v.ensureBitmapCapacity(rows, mp) } // PreExtendNulls ensures allocation-accounted null storage can represent rows. // Unaccounted vectors are unchanged. func (v *Vector) PreExtendNulls(rows int, mp *mpool.MPool) error { + if err := v.MaterializeOwned(mp); err != nil { + return err + } return v.ensureNullCapacity(rows, mp) } @@ -5567,10 +5696,12 @@ func (v *Vector) dup( } // A bitmap may be shorter than a sparse vector or longer than a reused vector // that was shortened with SetLength. Preserve both the complete row domain - // and the source bitmap extent before InitWith copies its storage. - if v.GetNulls().GetBitmap().Len() > 0 { + // and the source bitmap extent. Len does not materialize borrowed Arrow + // validity, which lets the destination admit its owned bitmap first. + nullLength := v.GetNulls().Len() + if nullLength > 0 { if err := w.ensureNullCapacity( - max(v.length, int(v.GetNulls().GetBitmap().Len())), + max(v.length, int(nullLength)), mp, ); err != nil { w.Free(mp) @@ -5587,7 +5718,15 @@ func (v *Vector) dup( } } w.length = v.length - w.GetNulls().InitWith(v.GetNulls()) + if v.GetNulls().HasBorrowedValidity() { + w.GetNulls().GetBitmap().InitWithSize(nullLength) + v.GetNulls().Foreach(func(row uint64) bool { + w.GetNulls().GetBitmap().Add(row) + return true + }) + } else { + w.GetNulls().InitWith(v.GetNulls()) + } w.GetGrouping().InitWith(v.GetGrouping()) if err := v.copyBinaryStringTo(w, mp); err != nil { w.Free(mp) @@ -8871,6 +9010,9 @@ func (v *Vector) RowToString(idx int) string { } func SetConstNull(vec *Vector, length int, mp *mpool.MPool) error { + if err := vec.MaterializeOwned(mp); err != nil { + return err + } if vec.typ.IsVarlen() { vec.areaDisjoint = false } @@ -8883,6 +9025,9 @@ func SetConstNull(vec *Vector, length int, mp *mpool.MPool) error { } func SetConstFixed[T any](vec *Vector, val T, length int, mp *mpool.MPool) error { + if err := vec.MaterializeOwned(mp); err != nil { + return err + } if vec.typ.IsVarlen() { vec.areaDisjoint = false } @@ -8898,6 +9043,9 @@ func SetConstFixed[T any](vec *Vector, val T, length int, mp *mpool.MPool) error } func SetConstBytes(vec *Vector, val []byte, length int, mp *mpool.MPool) error { + if err := vec.MaterializeOwned(mp); err != nil { + return err + } vec.areaDisjoint = false if err := extend(vec, 1, mp); err != nil { return err @@ -8912,6 +9060,9 @@ func SetConstBytes(vec *Vector, val []byte, length int, mp *mpool.MPool) error { } func SetConstByteJson(vec *Vector, bj bytejson.ByteJson, length int, mp *mpool.MPool) error { + if err := vec.MaterializeOwned(mp); err != nil { + return err + } vec.areaDisjoint = false if err := extend(vec, 1, mp); err != nil { return err @@ -8931,6 +9082,9 @@ func SetConstByteJsonEncoded( length int, mp *mpool.MPool, ) error { + if err := vec.MaterializeOwned(mp); err != nil { + return err + } vec.areaDisjoint = false oldAreaLen := len(vec.area) var value types.Varlena @@ -8950,6 +9104,9 @@ func SetConstByteJsonEncoded( // SetConstArray set current vector as Constant_Array vector of given length. func SetConstArray[T types.ArrayElement](vec *Vector, val []T, length int, mp *mpool.MPool) error { + if err := vec.MaterializeOwned(mp); err != nil { + return err + } vec.areaDisjoint = false var err error @@ -10035,6 +10192,20 @@ func (v *Vector) window( } w.data = v.data w.area = v.area + if v.dataLease != nil { + if !v.dataLease.Retain() { + w.Free(mp) + return nil, moerr.NewInternalErrorNoCtx("buffer lease is already released") + } + w.dataLease = v.dataLease + } + if v.areaLease != nil { + if !v.areaLease.Retain() { + w.Free(mp) + return nil, moerr.NewInternalErrorNoCtx("buffer lease is already released") + } + w.areaLease = v.areaLease + } // Const-null is a scalar property. In particular, an offset logical // window must not lose it merely because the physical null marker (when // present) lives at row zero. @@ -10048,9 +10219,23 @@ func (v *Vector) window( if start != end { w.data = v.data[start*v.typ.TypeSize() : end*v.typ.TypeSize()] } + if v.dataLease != nil { + if !v.dataLease.Retain() { + w.Free(mp) + return nil, moerr.NewInternalErrorNoCtx("buffer lease is already released") + } + w.dataLease = v.dataLease + } if v.typ.IsVarlen() { w.area = v.area w.areaDisjoint = v.areaDisjoint + if v.areaLease != nil { + if !v.areaLease.Retain() { + w.Free(mp) + return nil, moerr.NewInternalErrorNoCtx("buffer lease is already released") + } + w.areaLease = v.areaLease + } } w.cantFreeData = true w.cantFreeArea = true @@ -10092,14 +10277,50 @@ func (v *Vector) CoversLogicalRows(start, rows int) bool { func (v *Vector) copyWindowBitmaps(w *Vector, start, end int, mp *mpool.MPool) error { length := end - start - hasNull := v.nsp.GetBitmap().CountRange(uint64(start), uint64(end)) > 0 + if v.nsp.HasBorrowedValidity() { + if v.nsp.CountRange(uint64(start), uint64(end)) > 0 { + // Retained/asynchronous windows provide an MPool and must reserve + // their independent COW destination before sharing the source view. + if mp != nil { + if err := w.PrepareBorrowedValidity(length, mp); err != nil { + return err + } + } + if _, err := v.nsp.InitBorrowedWindow(&w.nsp, start, end); err != nil { + return err + } + } + } else { + hasNull := v.nsp.GetBitmap().CountRange(uint64(start), uint64(end)) > 0 + if hasNull { + if err := w.PreExtendNulls(length, mp); err != nil { + return err + } + nulls.Range(&v.nsp, uint64(start), uint64(end), uint64(start), &w.nsp) + } + } hasGrouping := v.gsp.GetBitmap().CountRange(uint64(start), uint64(end)) > 0 - if hasNull { + if hasGrouping { + if err := w.PreExtendGrouping(length, mp); err != nil { + return err + } + nulls.Range(&v.gsp, uint64(start), uint64(end), uint64(start), &w.gsp) + } + return nil +} + +// copyWindowBitmapsOwned is the deep-copy counterpart of copyWindowBitmaps. +// It never lets a borrowed Arrow validity lease cross an owning Clone/Dup +// boundary. +func (v *Vector) copyWindowBitmapsOwned(w *Vector, start, end int, mp *mpool.MPool) error { + length := end - start + if v.nsp.CountRange(uint64(start), uint64(end)) > 0 { if err := w.PreExtendNulls(length, mp); err != nil { return err } nulls.Range(&v.nsp, uint64(start), uint64(end), uint64(start), &w.nsp) } + hasGrouping := v.gsp.GetBitmap().CountRange(uint64(start), uint64(end)) > 0 if hasGrouping { if err := w.PreExtendGrouping(length, mp); err != nil { return err @@ -10174,7 +10395,7 @@ func (v *Vector) CloneWindowTo(w *Vector, start, end int, mp *mpool.MPool) error if err := v.copyStringSourceWindowToWithMP(w, start, end, mp); err != nil { return err } - if err := v.copyWindowBitmaps(w, start, end, mp); err != nil { + if err := v.copyWindowBitmapsOwned(w, start, end, mp); err != nil { return err } if v.IsConstNull() { diff --git a/pkg/defines/const.go b/pkg/defines/const.go index 5c599fbf75069..0eddb48af78fc 100644 --- a/pkg/defines/const.go +++ b/pkg/defines/const.go @@ -91,7 +91,8 @@ const ( MORPCVersion53 int64 = 53 // ordered-stream distributed Top-N merge MORPCVersion54 int64 = 54 // catalog-authenticated proxy cache reuse MORPCVersion55 int64 = 55 // session-owned temporary DDL with transactional data - MORPCLatestVersion = MORPCVersion55 + MORPCVersion56 int64 = 56 // Arrow LOAD external-scan pipeline payload + MORPCLatestVersion = MORPCVersion56 ) // DefaultLockWaitTimeoutSeconds is shared by the frontend default and by diff --git a/pkg/embed/config.go b/pkg/embed/config.go index 11126c5e14b50..36ae8c0ebd5e5 100644 --- a/pkg/embed/config.go +++ b/pkg/embed/config.go @@ -151,6 +151,7 @@ func newServiceConfig() ServiceConfig { Frontend: config.FrontendParameters{ KeyEncryptionKey: "JlxRbXjFGnCsvbsFQSJFvhMhDLaAXq5y", MongoDB: *config.NewMongoDBParameters(), + ArrowLoad: *config.NewArrowLoadParameters(), }, }, } diff --git a/pkg/fileservice/aliyun_sdk.go b/pkg/fileservice/aliyun_sdk.go index d0d4d4b954523..7ff0532c443dd 100644 --- a/pkg/fileservice/aliyun_sdk.go +++ b/pkg/fileservice/aliyun_sdk.go @@ -50,6 +50,7 @@ type AliyunSDK struct { } var _ objectStorageCopier = new(AliyunSDK) +var _ objectStorageIdentityReader = new(AliyunSDK) func (a *AliyunSDK) CopyObject( ctx context.Context, @@ -235,6 +236,69 @@ func (a *AliyunSDK) Stat( return } +func (a *AliyunSDK) StatObjectIdentity(ctx context.Context, key string) (ObjectIdentity, error) { + info, err := a.statObject(ctx, key) + if err != nil { + if a.is404(err) { + return ObjectIdentity{}, moerr.NewFileNotFoundNoCtx(key) + } + return ObjectIdentity{}, err + } + size, err := strconv.ParseInt(info.Get(oss.HTTPHeaderContentLength), 10, 64) + if err != nil { + return ObjectIdentity{}, err + } + identity := ObjectIdentity{ + VersionID: oss.GetVersionId(info), + ETag: info.Get(oss.HTTPHeaderEtag), + Size: size, + } + if modified := info.Get(oss.HTTPHeaderLastModified); modified != "" { + identity.LastModified, err = http.ParseTime(modified) + if err != nil { + return ObjectIdentity{}, err + } + } + return identity, identity.Validate() +} + +func (a *AliyunSDK) ReadObjectWithIdentity( + ctx context.Context, + key string, + min *int64, + max *int64, + expected ObjectIdentity, +) (io.ReadCloser, error) { + if err := expected.Validate(); err != nil { + return nil, err + } + var options []oss.Option + if expected.VersionID != "" { + options = append(options, oss.VersionId(expected.VersionID)) + } else { + options = append(options, oss.IfMatch(expected.ETag)) + } + r, err := a.getObject(ctx, key, min, max, options...) + if err != nil { + return nil, mapAliyunConditionalReadError(err) + } + r = mapReadCloserErrors(r, mapAliyunConditionalReadError) + if max == nil { + return r, nil + } + return &readCloser{r: io.LimitReader(r, *max-*min), closeFunc: r.Close}, nil +} + +func mapAliyunConditionalReadError(err error) error { + var serviceError oss.ServiceError + if errors.As(err, &serviceError) && + (serviceError.StatusCode == http.StatusNotFound || + serviceError.StatusCode == http.StatusPreconditionFailed) { + return errors.Join(ErrObjectChanged, moerr.NewInternalErrorNoCtx("conditional OSS read failed")) + } + return err +} + func (a *AliyunSDK) Exists( ctx context.Context, key string, @@ -492,7 +556,13 @@ func (a *AliyunSDK) putObject( return err } -func (a *AliyunSDK) getObject(ctx context.Context, key string, min *int64, max *int64) (io.ReadCloser, error) { +func (a *AliyunSDK) getObject( + ctx context.Context, + key string, + min *int64, + max *int64, + extraOptions ...oss.Option, +) (io.ReadCloser, error) { ctx, task := gotrace.NewTask(ctx, "AliyunSDK.getObject") defer task.End() if min == nil { @@ -503,9 +573,10 @@ func (a *AliyunSDK) getObject(ctx context.Context, key string, min *int64, max * opts := []oss.Option{ oss.WithContext(ctx), } + opts = append(opts, extraOptions...) var rang string if max != nil { - rang = fmt.Sprintf("%d-%d", offset, *max) + rang = fmt.Sprintf("%d-%d", offset, *max-1) } else { rang = fmt.Sprintf("%d-", offset) } diff --git a/pkg/fileservice/aliyun_sdk_test.go b/pkg/fileservice/aliyun_sdk_test.go index aad59baf2a40c..7bcb32e8e8d4a 100644 --- a/pkg/fileservice/aliyun_sdk_test.go +++ b/pkg/fileservice/aliyun_sdk_test.go @@ -42,6 +42,91 @@ func TestOSSCredential(t *testing.T) { assert.Equal(t, "", token) } +func TestAliyunSDKConditionalObjectIdentityReads(t *testing.T) { + const lastModRaw = "Wed, 02 Sep 2026 03:04:05 GMT" + var requests []*http.Request + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Clone(context.Background())) + if r.URL.Query().Get("versionId") == "gone" { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `NoSuchVersionplanned version was deleted`) + return + } + if r.URL.Query().Get("versionId") == "stale" || r.Header.Get("If-Match") == `"stale"` { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusPreconditionFailed) + _, _ = io.WriteString(w, `PreconditionFailedobject changed`) + return + } + if r.Method == http.MethodHead { + w.Header().Set("Content-Length", "6") + w.Header().Set("ETag", `"etag-v1"`) + w.Header().Set("x-oss-version-id", "version-v1") + w.Header().Set("Last-Modified", lastModRaw) + w.WriteHeader(http.StatusOK) + return + } + w.Header().Set("Content-Length", "3") + w.Header().Set("Content-Range", "bytes 1-3/6") + w.Header().Set("ETag", `"etag-v1"`) + w.Header().Set("Last-Modified", lastModRaw) + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, "bcd") + })) + defer server.Close() + + client, err := oss.New(server.URL, "id", "secret", + oss.ForcePathStyle(true), oss.HTTPClient(server.Client())) + require.NoError(t, err) + bucket, err := client.Bucket("bucket") + require.NoError(t, err) + sdk := &AliyunSDK{bucket: bucket} + + identity, err := sdk.StatObjectIdentity(context.Background(), "object") + require.NoError(t, err) + wantLastModified, err := time.Parse(http.TimeFormat, lastModRaw) + require.NoError(t, err) + require.Equal(t, ObjectIdentity{ + VersionID: "version-v1", ETag: `"etag-v1"`, Size: 6, LastModified: wantLastModified, + }, identity) + + min, max := int64(1), int64(4) + reader, err := sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, identity) + require.NoError(t, err) + data, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + require.Equal(t, "bcd", string(data)) + versionRequest := requests[len(requests)-1] + require.Equal(t, "version-v1", versionRequest.URL.Query().Get("versionId")) + require.Empty(t, versionRequest.Header.Get("If-Match")) + require.Equal(t, "bytes=1-3", versionRequest.Header.Get("Range")) + + etagIdentity := identity + etagIdentity.VersionID = "" + reader, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, etagIdentity) + require.NoError(t, err) + require.NoError(t, reader.Close()) + etagRequest := requests[len(requests)-1] + require.Empty(t, etagRequest.URL.Query().Get("versionId")) + require.Equal(t, `"etag-v1"`, etagRequest.Header.Get("If-Match")) + + stale := identity + stale.VersionID = "stale" + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, stale) + require.ErrorIs(t, err, ErrObjectChanged) + stale.VersionID = "" + stale.ETag = `"stale"` + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, stale) + require.ErrorIs(t, err, ErrObjectChanged) + + gone := identity + gone.VersionID = "gone" + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, gone) + require.ErrorIs(t, err, ErrObjectChanged) +} + func TestAliyunSDKCopyObjectPropagatesCancellation(t *testing.T) { started := make(chan struct{}) server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { diff --git a/pkg/fileservice/aws_sdk_v2.go b/pkg/fileservice/aws_sdk_v2.go index 9695924067a09..98fea587cf941 100644 --- a/pkg/fileservice/aws_sdk_v2.go +++ b/pkg/fileservice/aws_sdk_v2.go @@ -23,6 +23,7 @@ import ( "io" "iter" "math" + nethttp "net/http" "net/url" gotrace "runtime/trace" "slices" @@ -62,6 +63,7 @@ type AwsSDKv2 struct { } var _ objectStorageCopier = new(AwsSDKv2) +var _ objectStorageIdentityReader = new(AwsSDKv2) func (a *AwsSDKv2) CopyObject( ctx context.Context, @@ -326,6 +328,67 @@ func (a *AwsSDKv2) Stat( return } +func (a *AwsSDKv2) StatObjectIdentity(ctx context.Context, key string) (ObjectIdentity, error) { + output, err := a.headObject(ctx, &s3.HeadObjectInput{ + Bucket: ptrTo(a.bucket), + Key: ptrTo(key), + }) + if err != nil { + return ObjectIdentity{}, a.mapError(err, key) + } + identity := ObjectIdentity{ + Size: aws.ToInt64(output.ContentLength), + ETag: aws.ToString(output.ETag), + VersionID: aws.ToString(output.VersionId), + } + if output.LastModified != nil { + identity.LastModified = *output.LastModified + } + return identity, identity.Validate() +} + +func (a *AwsSDKv2) ReadObjectWithIdentity( + ctx context.Context, + key string, + min *int64, + max *int64, + expected ObjectIdentity, +) (io.ReadCloser, error) { + if err := expected.Validate(); err != nil { + return nil, err + } + params := &s3.GetObjectInput{Bucket: ptrTo(a.bucket), Key: ptrTo(key)} + if expected.VersionID != "" { + params.VersionId = ptrTo(expected.VersionID) + } else { + params.IfMatch = ptrTo(expected.ETag) + } + r, err := a.getObject(ctx, min, max, params) + if err != nil { + // Preserve the conditional-read contract before the ordinary S3 mapper + // turns a deleted planned version into a generic file-not-found error. + return nil, a.mapError(mapAWSConditionalReadError(err), key) + } + r = mapReadCloserErrors(r, mapAWSConditionalReadError) + if max == nil { + return r, nil + } + return &readCloser{ + r: io.LimitReader(r, *max-*min), + closeFunc: r.Close, + }, nil +} + +func mapAWSConditionalReadError(err error) error { + var responseError *http.ResponseError + if errors.As(err, &responseError) && responseError.Response != nil && + (responseError.Response.StatusCode == nethttp.StatusNotFound || + responseError.Response.StatusCode == nethttp.StatusPreconditionFailed) { + return errors.Join(ErrObjectChanged, moerr.NewInternalErrorNoCtx("conditional S3 read failed")) + } + return err +} + func (a *AwsSDKv2) Exists( ctx context.Context, key string, @@ -974,7 +1037,7 @@ func (a *AwsSDKv2) getObject(ctx context.Context, min *int64, max *int64, params defer LogEvent(ctx, str_retryable_reader_new_reader_end) var rang string if max != nil { - rang = fmt.Sprintf("bytes=%d-%d", offset, *max) + rang = fmt.Sprintf("bytes=%d-%d", offset, *max-1) } else { rang = fmt.Sprintf("bytes=%d-", offset) } diff --git a/pkg/fileservice/aws_sdk_v2_test.go b/pkg/fileservice/aws_sdk_v2_test.go index 1235163c2e025..3e75bea60e90b 100644 --- a/pkg/fileservice/aws_sdk_v2_test.go +++ b/pkg/fileservice/aws_sdk_v2_test.go @@ -23,12 +23,92 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/stretchr/testify/require" "github.com/matrixorigin/matrixone/pkg/common/moerr" ) +func TestAwsSDKv2ConditionalObjectIdentityReads(t *testing.T) { + const ( + body = "abcdef" + lastModRaw = "Wed, 02 Sep 2026 03:04:05 GMT" + ) + var requests []*http.Request + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Clone(context.Background())) + if r.Method == http.MethodHead { + w.Header().Set("Content-Length", "6") + w.Header().Set("ETag", `"etag-v1"`) + w.Header().Set("x-amz-version-id", "version-v1") + w.Header().Set("Last-Modified", lastModRaw) + w.WriteHeader(http.StatusOK) + return + } + if r.URL.Query().Get("versionId") == "gone" { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, awsS3ErrorXML("NoSuchVersion", "planned version was deleted")) + return + } + if r.URL.Query().Get("versionId") == "stale" || r.Header.Get("If-Match") == `"stale"` { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusPreconditionFailed) + _, _ = io.WriteString(w, awsS3ErrorXML("PreconditionFailed", "object changed")) + return + } + w.Header().Set("Content-Length", "3") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, body[1:4]) + })) + defer server.Close() + + sdk := newTestAWSClient(t, server) + identity, err := sdk.StatObjectIdentity(context.Background(), "object") + require.NoError(t, err) + wantLastModified, err := time.Parse(http.TimeFormat, lastModRaw) + require.NoError(t, err) + require.Equal(t, ObjectIdentity{ + VersionID: "version-v1", ETag: `"etag-v1"`, Size: 6, LastModified: wantLastModified, + }, identity) + + min, max := int64(1), int64(4) + reader, err := sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, identity) + require.NoError(t, err) + data, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + require.Equal(t, "bcd", string(data)) + versionRequest := requests[len(requests)-1] + require.Equal(t, "version-v1", versionRequest.URL.Query().Get("versionId")) + require.Empty(t, versionRequest.Header.Get("If-Match")) + require.Equal(t, "bytes=1-3", versionRequest.Header.Get("Range")) + + etagIdentity := identity + etagIdentity.VersionID = "" + reader, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, etagIdentity) + require.NoError(t, err) + require.NoError(t, reader.Close()) + etagRequest := requests[len(requests)-1] + require.Empty(t, etagRequest.URL.Query().Get("versionId")) + require.Equal(t, `"etag-v1"`, etagRequest.Header.Get("If-Match")) + + stale := identity + stale.VersionID = "stale" + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, stale) + require.ErrorIs(t, err, ErrObjectChanged) + stale.VersionID = "" + stale.ETag = `"stale"` + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, stale) + require.ErrorIs(t, err, ErrObjectChanged) + + gone := identity + gone.VersionID = "gone" + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, gone) + require.ErrorIs(t, err, ErrObjectChanged) +} + func Test_NewAwsSDKv2(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusForbidden) diff --git a/pkg/fileservice/fifocache/data_cache.go b/pkg/fileservice/fifocache/data_cache.go index 26539a7a838e9..6cce0a3835b97 100644 --- a/pkg/fileservice/fifocache/data_cache.go +++ b/pkg/fileservice/fifocache/data_cache.go @@ -98,6 +98,7 @@ func shardCacheKey(key fscache.CacheKey) uint64 { } var _ fscache.DataCache = new(DataCache) +var _ fscache.DataCacheWithPinAdmission = new(DataCache) func (d *DataCache) Available() int64 { ret := d.fifo.capacity() - d.fifo.Used() @@ -172,6 +173,28 @@ func (d *DataCache) Get(ctx context.Context, key query.CacheKey) (fscache.Data, return value.data, true } +func (d *DataCache) GetWithPinAdmission( + ctx context.Context, + key query.CacheKey, + admit fscache.DataCachePinAdmission, +) (data fscache.Data, release func(), ok bool, err error) { + if admit == nil { + data, ok = d.Get(ctx, key) + return data, nil, ok, nil + } + value, release, ok, err := d.fifo.GetWithAdmission( + ctx, + key, + func(value dataCacheValue, size int64) (func(), error) { + return admit(size) + }, + ) + if !ok || err != nil { + return nil, release, ok, err + } + return value.data, release, true, nil +} + func (d *DataCache) Contains(key query.CacheKey) bool { return d.fifo.Contains(key) } diff --git a/pkg/fileservice/fifocache/data_cache_test.go b/pkg/fileservice/fifocache/data_cache_test.go index 6c53152505eef..2c76c5bd3a0b4 100644 --- a/pkg/fileservice/fifocache/data_cache_test.go +++ b/pkg/fileservice/fifocache/data_cache_test.go @@ -16,6 +16,7 @@ package fifocache import ( "context" + "errors" "fmt" "strconv" "strings" @@ -197,3 +198,58 @@ func TestDataCacheCallbacksReceiveCapturedLogicalSize(t *testing.T) { t.Fatalf("post-evict sizes = %+v, want %+v", postEvict, want) } } + +func TestDataCachePinAdmissionPrecedesRetain(t *testing.T) { + var events []string + cache := NewDataCache( + fscache.ConstCapacity(1024), + nil, + func(context.Context, fscache.CacheKey, fscache.Data, int64) { + events = append(events, "retain") + }, + nil, + ) + key := fscache.CacheKey{Path: "foo", Sz: 3} + if _, err := cache.Set(context.Background(), key, testBytes(make([]byte, 3, 8))); err != nil { + t.Fatal(err) + } + + data, release, ok, err := cache.GetWithPinAdmission( + context.Background(), key, + func(capacity int64) (func(), error) { + if capacity != 8 { + t.Fatalf("admitted capacity = %d, want 8", capacity) + } + events = append(events, "admit") + return func() { events = append(events, "release") }, nil + }, + ) + if err != nil || !ok || data == nil { + t.Fatalf("admitted get = (%v, %v, %v), want cache hit", data, ok, err) + } + if fmt.Sprint(events) != "[admit retain]" { + t.Fatalf("callback order = %v, want admission before retain", events) + } + release() + if fmt.Sprint(events) != "[admit retain release]" { + t.Fatalf("callback order after release = %v", events) + } + + rejected := errors.New("rejected") + rejectedRelease := 0 + _, _, ok, err = cache.GetWithPinAdmission( + context.Background(), key, + func(int64) (func(), error) { + return func() { rejectedRelease++ }, rejected + }, + ) + if !errors.Is(err, rejected) || ok { + t.Fatalf("rejected get = (ok=%v, err=%v)", ok, err) + } + if rejectedRelease != 1 { + t.Fatalf("rejected admission release count = %d, want 1", rejectedRelease) + } + if fmt.Sprint(events) != "[admit retain release]" { + t.Fatalf("rejected admission unexpectedly retained data: %v", events) + } +} diff --git a/pkg/fileservice/fifocache/fifo.go b/pkg/fileservice/fifocache/fifo.go index 42008ad44bd82..ab871a8993e5b 100644 --- a/pkg/fileservice/fifocache/fifo.go +++ b/pkg/fileservice/fifocache/fifo.go @@ -466,20 +466,30 @@ func (c *Cache[K, V]) enqueuePendingItem(item *_CacheItem[K, V], queue uint8) { } func (c *Cache[K, V]) Get(ctx context.Context, key K) (value V, ok bool) { - var item *_CacheItem[K, V] - defer func() { - // item ok, increase count - if item != nil { - item.inc() - } - }() + value, _, ok, err := c.GetWithAdmission(ctx, key, nil) + if err != nil { + panic(err) + } + return value, ok +} + +// GetWithAdmission performs admission while the matching item is protected by +// its shard lock, before postGet retains the value. The admission callback and +// any release it returns must complete in bounded time and must not call back +// into this cache or panic. A release returned together with an error is invoked +// here because the value has not yet been published to a caller. +func (c *Cache[K, V]) GetWithAdmission( + ctx context.Context, + key K, + admit func(value V, size int64) (release func(), err error), +) (value V, release func(), ok bool, err error) { shard := &c.shards[c.keyShardFunc(key)%numShards] shard.Lock() defer shard.Unlock() - item, ok = shard.values[key] - if !ok { + item, found := shard.values[key] + if !found { // not exist return } @@ -490,10 +500,33 @@ func (c *Cache[K, V]) Get(ctx context.Context, key K) (value V, ok bool) { return } + published := false + if admit != nil { + release, err = admit(item.value, item.size) + if err != nil { + // Be defensive about a callback that acquired a partial reservation + // before returning an error. No cache retain has happened yet, so this + // function remains the only possible cleanup owner. + if release != nil { + release() + release = nil + } + ok = false + return + } + defer func() { + if !published && release != nil { + release() + release = nil + } + }() + } if c.postGet != nil { c.postGet(ctx, item.key, item.value, item.size) } - return item.value, true + item.inc() + published = true + return item.value, release, true, nil } func (c *Cache[K, V]) Contains(key K) bool { diff --git a/pkg/fileservice/file_service.go b/pkg/fileservice/file_service.go index f53a223eb2fda..79aeddc403e7a 100644 --- a/pkg/fileservice/file_service.go +++ b/pkg/fileservice/file_service.go @@ -144,6 +144,12 @@ type IOEntry struct { // Data, WriterForRead, ReadCloserForRead may be empty if CachedData is not null // if ToCacheData is provided, caller should always read CachedData instead of Data, WriterForRead or ReadCloserForRead CachedData fscache.Data + // admitCachedData reserves caller-owned capacity before a memory-cache hit + // retains its backing. releaseCachedData drops that reservation after the + // retained cache reference is released. Both are internal FileService + // ownership hooks and must move with the entry. + admitCachedData fscache.DataCachePinAdmission + releaseCachedData func() // CachedDataSize is the expected size of the final cache representation. // Zero means Size. It may differ from Size when ToCacheData decompresses the // storage extent before cache admission. diff --git a/pkg/fileservice/fscache/data_cache.go b/pkg/fileservice/fscache/data_cache.go index 055fc8b9a7518..e33afc0e5ec56 100644 --- a/pkg/fileservice/fscache/data_cache.go +++ b/pkg/fileservice/fscache/data_cache.go @@ -23,6 +23,10 @@ import ( type CacheKey = pb.CacheKey +// ErrCacheAdmissionRejected reports that a cache entry exists but a caller's +// separate retention budget or pin policy declined it. A cache coordinator may +// treat this sentinel as a miss and retry through an uncached authoritative +// read; cancellation and backend errors must remain distinct. var ErrCacheAdmissionRejected = moerr.NewInternalErrorNoCtx("cache admission rejected") type DataCache interface { @@ -37,3 +41,23 @@ type DataCache interface { Evict(ctx context.Context, done chan int64) EvictToTargetWithWait(ctx context.Context, target int64) int64 } + +// DataCachePinAdmission runs before a cache hit retains its backing. It executes +// under the cache key's shard lock and therefore must be bounded, non-blocking, +// and must not call back into the cache. Its returned release function owns the +// admitted capacity for exactly as long as the caller owns the retained Data +// reference. The cache may also invoke release under the shard lock when an +// admission attempt reports an error, so release has the same bounded and +// non-reentrant requirements. +type DataCachePinAdmission func(capacity int64) (release func(), err error) + +// DataCacheWithPinAdmission is the cache capability required by consumers +// that account retained cache backing against a separate statement budget. +type DataCacheWithPinAdmission interface { + DataCache + GetWithPinAdmission( + context.Context, + CacheKey, + DataCachePinAdmission, + ) (data Data, release func(), ok bool, err error) +} diff --git a/pkg/fileservice/io_vector.go b/pkg/fileservice/io_vector.go index 65d785514d017..536036eedcc0e 100644 --- a/pkg/fileservice/io_vector.go +++ b/pkg/fileservice/io_vector.go @@ -35,6 +35,9 @@ func (i *IOVector) Release() { if entry.CachedData != nil { entry.CachedData.Release() } + if entry.releaseCachedData != nil { + entry.releaseCachedData() + } if entry.releaseData != nil { entry.releaseData() } @@ -48,6 +51,10 @@ func (i *IOVector) ReleaseReadResultOnError() { entry.CachedData.Release() entry.CachedData = nil } + if entry.releaseCachedData != nil { + entry.releaseCachedData() + entry.releaseCachedData = nil + } if entry.done && entry.releaseData != nil { entry.releaseData() entry.releaseData = nil diff --git a/pkg/fileservice/io_vector_test.go b/pkg/fileservice/io_vector_test.go index 61b90b12e06ee..2f9cc4f7d7f95 100644 --- a/pkg/fileservice/io_vector_test.go +++ b/pkg/fileservice/io_vector_test.go @@ -53,6 +53,7 @@ func (r *releaseCountingData) Release() { func TestIOVectorReleaseReadResultOnErrorSkipsUndoneReleaseData(t *testing.T) { var cacheReleases atomic.Int32 + var pinReleases atomic.Int32 var doneReleaseData atomic.Int32 var undoneReleaseData atomic.Int32 @@ -60,6 +61,9 @@ func TestIOVectorReleaseReadResultOnErrorSkipsUndoneReleaseData(t *testing.T) { Entries: []IOEntry{ { CachedData: &releaseCountingData{releases: &cacheReleases}, + releaseCachedData: func() { + pinReleases.Add(1) + }, releaseData: func() { doneReleaseData.Add(1) }, @@ -68,6 +72,9 @@ func TestIOVectorReleaseReadResultOnErrorSkipsUndoneReleaseData(t *testing.T) { }, { CachedData: &releaseCountingData{releases: &cacheReleases}, + releaseCachedData: func() { + pinReleases.Add(1) + }, releaseData: func() { undoneReleaseData.Add(1) }, @@ -79,13 +86,16 @@ func TestIOVectorReleaseReadResultOnErrorSkipsUndoneReleaseData(t *testing.T) { vector.ReleaseReadResultOnError() require.Equal(t, int32(2), cacheReleases.Load()) + require.Equal(t, int32(2), pinReleases.Load()) require.Equal(t, int32(1), doneReleaseData.Load()) require.Equal(t, int32(0), undoneReleaseData.Load()) require.Nil(t, vector.Entries[0].CachedData) + require.Nil(t, vector.Entries[0].releaseCachedData) require.Nil(t, vector.Entries[0].releaseData) require.False(t, vector.Entries[0].done) require.Nil(t, vector.Entries[0].fromCache) require.Nil(t, vector.Entries[1].CachedData) + require.Nil(t, vector.Entries[1].releaseCachedData) require.NotNil(t, vector.Entries[1].releaseData) require.False(t, vector.Entries[1].done) require.Nil(t, vector.Entries[1].fromCache) diff --git a/pkg/fileservice/mem_cache.go b/pkg/fileservice/mem_cache.go index b52bc0ff79c1c..8f1808013dd99 100644 --- a/pkg/fileservice/mem_cache.go +++ b/pkg/fileservice/mem_cache.go @@ -706,7 +706,32 @@ func (m *MemCache) Read( Offset: entry.Offset, Sz: entry.Size, } - bs, ok := m.cache.Get(ctx, key) + var bs fscache.Data + var ok bool + if entry.admitCachedData != nil { + cache, supportsAdmission := m.cache.(fscache.DataCacheWithPinAdmission) + if !supportsAdmission { + continue + } + var release func() + bs, release, ok, err = cache.GetWithPinAdmission(ctx, key, entry.admitCachedData) + if errors.Is(err, fscache.ErrCacheAdmissionRejected) { + // The key exists, but retaining its physical backing would violate + // the caller's pin policy or budget. Treat that as a cache miss so + // cache warmth cannot turn an otherwise valid read into a failure. + numRead++ + err = nil + continue + } + if err != nil { + return err + } + if ok { + vector.Entries[i].releaseCachedData = release + } + } else { + bs, ok = m.cache.Get(ctx, key) + } numRead++ if ok { vector.Entries[i].CachedData = bs diff --git a/pkg/fileservice/minio_sdk.go b/pkg/fileservice/minio_sdk.go index 653cf5bfc411d..3b7ecc72b2208 100644 --- a/pkg/fileservice/minio_sdk.go +++ b/pkg/fileservice/minio_sdk.go @@ -17,6 +17,7 @@ package fileservice import ( "bytes" "context" + "errors" "io" "iter" "net/http" @@ -48,6 +49,7 @@ type MinioSDK struct { } var _ objectStorageCopier = new(MinioSDK) +var _ objectStorageIdentityReader = new(MinioSDK) func (a *MinioSDK) CopyObject( ctx context.Context, @@ -319,6 +321,60 @@ func (a *MinioSDK) Stat( return } +func (a *MinioSDK) StatObjectIdentity(ctx context.Context, key string) (ObjectIdentity, error) { + info, err := a.statObject(ctx, key) + if err != nil { + if a.is404(err) { + return ObjectIdentity{}, moerr.NewFileNotFoundNoCtx(key) + } + return ObjectIdentity{}, err + } + identity := ObjectIdentity{ + VersionID: info.VersionID, ETag: info.ETag, Size: info.Size, LastModified: info.LastModified, + } + return identity, identity.Validate() +} + +func (a *MinioSDK) ReadObjectWithIdentity( + ctx context.Context, + key string, + min *int64, + max *int64, + expected ObjectIdentity, +) (io.ReadCloser, error) { + if err := expected.Validate(); err != nil { + return nil, err + } + options := minio.GetObjectOptions{} + if expected.VersionID != "" { + options.VersionID = expected.VersionID + } else if err := options.SetMatchETag(expected.ETag); err != nil { + return nil, err + } + r, err := a.getObjectWithOptions(ctx, key, min, max, options) + if err != nil { + return nil, mapMinioConditionalReadError(err) + } + // MinIO defers the GET until the first operation on Object. + if _, err = r.Read(nil); err != nil { + r.Close() + return nil, mapMinioConditionalReadError(err) + } + r = mapReadCloserErrors(r, mapMinioConditionalReadError) + if max == nil { + return r, nil + } + return &readCloser{r: io.LimitReader(r, *max-*min), closeFunc: r.Close}, nil +} + +func mapMinioConditionalReadError(err error) error { + response := minio.ToErrorResponse(err) + if response.StatusCode == http.StatusNotFound || response.StatusCode == http.StatusPreconditionFailed { + return errors.Join(ErrObjectChanged, moerr.NewInternalErrorNoCtx("conditional S3-compatible read failed")) + } + return err +} + func (a *MinioSDK) Exists( ctx context.Context, key string, @@ -573,6 +629,16 @@ func (a *MinioSDK) putObject( } func (a *MinioSDK) getObject(ctx context.Context, key string, min *int64, max *int64) (io.ReadCloser, error) { + return a.getObjectWithOptions(ctx, key, min, max, minio.GetObjectOptions{}) +} + +func (a *MinioSDK) getObjectWithOptions( + ctx context.Context, + key string, + min *int64, + max *int64, + options minio.GetObjectOptions, +) (io.ReadCloser, error) { ctx, task := gotrace.NewTask(ctx, "MinioSDK.getObject") defer task.End() if min == nil { @@ -587,7 +653,7 @@ func (a *MinioSDK) getObject(ctx context.Context, key string, min *int64, max *i perfcounter.Update(ctx, func(counter *perfcounter.CounterSet) { counter.FileService.S3.Get.Add(1) }, a.perfCounterSets...) - return a.client.GetObject(ctx, a.bucket, key, minio.GetObjectOptions{}) + return a.client.GetObject(ctx, a.bucket, key, options) }, maxRetryAttemps, IsRetryableError, diff --git a/pkg/fileservice/minio_sdk_test.go b/pkg/fileservice/minio_sdk_test.go index 58516930a9722..ce141aa41e41b 100644 --- a/pkg/fileservice/minio_sdk_test.go +++ b/pkg/fileservice/minio_sdk_test.go @@ -86,6 +86,8 @@ func TestMinioSDK(t *testing.T) { Name: name, Endpoint: "http://localhost:9007", Bucket: "test", + KeyID: "minioadmin", + KeySecret: "minioadmin", KeyPrefix: time.Now().Format("2006-01-02.15:04:05.000000"), IsMinio: true, }, @@ -105,6 +107,96 @@ func TestMinioSDK(t *testing.T) { } +func TestMinioSDKConditionalObjectIdentityReads(t *testing.T) { + const lastModRaw = "Wed, 02 Sep 2026 03:04:05 GMT" + var requests []*http.Request + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Query().Has("location") { + w.Header().Set("Content-Type", "application/xml") + _, _ = io.WriteString(w, `us-east-1`) + return + } + requests = append(requests, r.Clone(context.Background())) + if r.URL.Query().Get("versionId") == "gone" { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `NoSuchVersionplanned version was deleted`) + return + } + if r.URL.Query().Get("versionId") == "stale" || r.Header.Get("If-Match") == `"stale"` { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusPreconditionFailed) + _, _ = io.WriteString(w, `PreconditionFailedobject changed`) + return + } + if r.Method == http.MethodHead { + w.Header().Set("Content-Length", "6") + w.Header().Set("ETag", `"etag-v1"`) + w.Header().Set("x-amz-version-id", "version-v1") + w.Header().Set("Last-Modified", lastModRaw) + w.WriteHeader(http.StatusOK) + return + } + w.Header().Set("Content-Length", "3") + w.Header().Set("ETag", `"etag-v1"`) + w.Header().Set("Last-Modified", lastModRaw) + w.Header().Set("Content-Range", "bytes 1-3/6") + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, "bcd") + })) + defer server.Close() + + endpoint, err := url.Parse(server.URL) + require.NoError(t, err) + client, err := minio.New(endpoint.Host, &minio.Options{ + Creds: credentials.NewStaticV4("id", "secret", ""), Secure: false, + Transport: server.Client().Transport, + Region: "us-east-1", + }) + require.NoError(t, err) + sdk := &MinioSDK{bucket: "bucket", client: client} + + identity, err := sdk.StatObjectIdentity(context.Background(), "object") + require.NoError(t, err) + wantLastModified, err := time.Parse(http.TimeFormat, lastModRaw) + require.NoError(t, err) + require.Equal(t, ObjectIdentity{ + VersionID: "version-v1", ETag: "etag-v1", Size: 6, LastModified: wantLastModified, + }, identity) + + min, max := int64(1), int64(4) + reader, err := sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, identity) + require.NoError(t, err) + data, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + require.Equal(t, "bcd", string(data)) + versionRequest := requests[len(requests)-1] + require.Equal(t, "version-v1", versionRequest.URL.Query().Get("versionId")) + + etagIdentity := identity + etagIdentity.VersionID = "" + reader, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, etagIdentity) + require.NoError(t, err) + require.NoError(t, reader.Close()) + etagRequest := requests[len(requests)-1] + require.Equal(t, `"etag-v1"`, etagRequest.Header.Get("If-Match")) + + stale := identity + stale.VersionID = "stale" + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, stale) + require.ErrorIs(t, err, ErrObjectChanged) + stale.VersionID = "" + stale.ETag = "stale" + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, stale) + require.ErrorIs(t, err, ErrObjectChanged) + + gone := identity + gone.VersionID = "gone" + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, gone) + require.ErrorIs(t, err, ErrObjectChanged) +} + func TestMinioPutObjectPhysicalAccounting(t *testing.T) { var fail bool server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/fileservice/object_storage.go b/pkg/fileservice/object_storage.go index b9f85d556a17b..160cf31edcf9f 100644 --- a/pkg/fileservice/object_storage.go +++ b/pkg/fileservice/object_storage.go @@ -194,6 +194,49 @@ type ObjectStorage interface { ) } +// objectStorageIdentityReader is an optional provider capability used by +// format-neutral conditional FileService reads. Implementations must preserve +// expected.VersionID or expected.ETag across every internal retry. +type objectStorageIdentityReader interface { + StatObjectIdentity(ctx context.Context, key string) (ObjectIdentity, error) + ReadObjectWithIdentity( + ctx context.Context, + key string, + min *int64, + max *int64, + expected ObjectIdentity, + ) (io.ReadCloser, error) +} + +// mappedErrorReadCloser keeps provider error normalization outside the +// retryable reader. A conditional GET may succeed initially and then reopen +// after a transport failure; errors from that later reopen are returned by +// Read rather than by the constructor and must obey the same public contract. +type mappedErrorReadCloser struct { + io.ReadCloser + mapError func(error) error +} + +func (r *mappedErrorReadCloser) Read(buffer []byte) (int, error) { + n, err := r.ReadCloser.Read(buffer) + if err != nil { + err = r.mapError(err) + } + return n, err +} + +func (r *mappedErrorReadCloser) Close() error { + err := r.ReadCloser.Close() + if err != nil { + err = r.mapError(err) + } + return err +} + +func mapReadCloserErrors(reader io.ReadCloser, mapError func(error) error) io.ReadCloser { + return &mappedErrorReadCloser{ReadCloser: reader, mapError: mapError} +} + // objectStorageCopier is implemented by object-store SDK adapters that can // ask the provider to copy an object without downloading it through CN. type objectStorageCopier interface { diff --git a/pkg/fileservice/object_storage_close_test.go b/pkg/fileservice/object_storage_close_test.go index d0f5f36084407..5c08e2fef8d56 100644 --- a/pkg/fileservice/object_storage_close_test.go +++ b/pkg/fileservice/object_storage_close_test.go @@ -27,6 +27,14 @@ type testCloser struct { err error } +type testErrorReadCloser struct { + readErr error + closeErr error +} + +func (r *testErrorReadCloser) Read([]byte) (int, error) { return 0, r.readErr } +func (r *testErrorReadCloser) Close() error { return r.closeErr } + func (c *testCloser) Close() error { c.calls++ return c.err @@ -68,3 +76,20 @@ func TestCloseOnError(t *testing.T) { }) } } + +func TestMappedErrorReadCloserMapsDeferredProviderErrors(t *testing.T) { + providerErr := errors.New("provider object disappeared") + mapper := func(err error) error { + if errors.Is(err, providerErr) { + return errors.Join(ErrObjectChanged, err) + } + return err + } + + reader := mapReadCloserErrors(&testErrorReadCloser{ + readErr: providerErr, closeErr: providerErr, + }, mapper) + _, err := reader.Read(make([]byte, 1)) + require.ErrorIs(t, err, ErrObjectChanged) + require.ErrorIs(t, reader.Close(), ErrObjectChanged) +} diff --git a/pkg/fileservice/object_storage_http_trace.go b/pkg/fileservice/object_storage_http_trace.go index 58d927c50ecf8..a520857ff2892 100644 --- a/pkg/fileservice/object_storage_http_trace.go +++ b/pkg/fileservice/object_storage_http_trace.go @@ -38,6 +38,7 @@ func newObjectStorageHTTPTrace(upstream ObjectStorage) *objectStorageHTTPTrace { var _ ObjectStorage = new(objectStorageHTTPTrace) var _ ParallelMultipartWriter = new(objectStorageHTTPTrace) var _ objectStorageCopier = new(objectStorageHTTPTrace) +var _ objectStorageIdentityReader = new(objectStorageHTTPTrace) func (o *objectStorageHTTPTrace) CopyObject( ctx context.Context, @@ -83,6 +84,34 @@ func (o *objectStorageHTTPTrace) Read(ctx context.Context, key string, min *int6 return o.upstream.Read(ctx, key, min, max) } +func (o *objectStorageHTTPTrace) StatObjectIdentity(ctx context.Context, key string) (ObjectIdentity, error) { + upstream, ok := o.upstream.(objectStorageIdentityReader) + if !ok { + return ObjectIdentity{}, moerr.NewNotSupported(ctx, "object storage identity") + } + traceInfo := o.newTraceInfo() + defer o.closeTraceInfo(traceInfo) + ctx = httptrace.WithClientTrace(ctx, traceInfo.trace) + return upstream.StatObjectIdentity(ctx, key) +} + +func (o *objectStorageHTTPTrace) ReadObjectWithIdentity( + ctx context.Context, + key string, + min *int64, + max *int64, + expected ObjectIdentity, +) (io.ReadCloser, error) { + upstream, ok := o.upstream.(objectStorageIdentityReader) + if !ok { + return nil, moerr.NewNotSupported(ctx, "conditional object storage read") + } + traceInfo := o.newTraceInfo() + defer o.closeTraceInfo(traceInfo) + ctx = httptrace.WithClientTrace(ctx, traceInfo.trace) + return upstream.ReadObjectWithIdentity(ctx, key, min, max, expected) +} + func (o *objectStorageHTTPTrace) Stat(ctx context.Context, key string) (size int64, err error) { traceInfo := o.newTraceInfo() defer o.closeTraceInfo(traceInfo) diff --git a/pkg/fileservice/object_storage_metrics.go b/pkg/fileservice/object_storage_metrics.go index cf5232c886af6..aa75a55586a65 100644 --- a/pkg/fileservice/object_storage_metrics.go +++ b/pkg/fileservice/object_storage_metrics.go @@ -58,6 +58,7 @@ func newObjectStorageMetrics( var _ ObjectStorage = new(objectStorageMetrics) var _ ParallelMultipartWriter = new(objectStorageMetrics) var _ objectStorageCopier = new(objectStorageMetrics) +var _ objectStorageIdentityReader = new(objectStorageMetrics) func (o *objectStorageMetrics) CopyObject( ctx context.Context, @@ -107,6 +108,39 @@ func (o *objectStorageMetrics) Read(ctx context.Context, key string, min *int64, }, nil } +func (o *objectStorageMetrics) StatObjectIdentity(ctx context.Context, key string) (ObjectIdentity, error) { + upstream, ok := o.upstream.(objectStorageIdentityReader) + if !ok { + return ObjectIdentity{}, moerr.NewNotSupported(ctx, "object storage identity") + } + o.numStat.Inc() + return upstream.StatObjectIdentity(ctx, key) +} + +func (o *objectStorageMetrics) ReadObjectWithIdentity( + ctx context.Context, + key string, + min *int64, + max *int64, + expected ObjectIdentity, +) (io.ReadCloser, error) { + upstream, ok := o.upstream.(objectStorageIdentityReader) + if !ok { + return nil, moerr.NewNotSupported(ctx, "conditional object storage read") + } + o.numRead.Inc() + o.numActiveRead.Inc() + r, err := upstream.ReadObjectWithIdentity(ctx, key, min, max, expected) + if err != nil { + o.numActiveRead.Dec() + return nil, err + } + return &readCloser{r: r, closeFunc: func() error { + o.numActiveRead.Dec() + return r.Close() + }}, nil +} + func (o *objectStorageMetrics) Stat(ctx context.Context, key string) (size int64, err error) { o.numStat.Inc() return o.upstream.Stat(ctx, key) diff --git a/pkg/fileservice/object_storage_semaphore.go b/pkg/fileservice/object_storage_semaphore.go index f810a73e0b2b6..ca88bb71c332a 100644 --- a/pkg/fileservice/object_storage_semaphore.go +++ b/pkg/fileservice/object_storage_semaphore.go @@ -55,6 +55,7 @@ func (o *objectStorageSemaphore) release() { var _ ObjectStorage = new(objectStorageSemaphore) var _ ParallelMultipartWriter = new(objectStorageSemaphore) var _ objectStorageCopier = new(objectStorageSemaphore) +var _ objectStorageIdentityReader = new(objectStorageSemaphore) func (o *objectStorageSemaphore) CopyObject( ctx context.Context, @@ -131,6 +132,53 @@ func (o *objectStorageSemaphore) Read(ctx context.Context, key string, min *int6 }, nil } +func (o *objectStorageSemaphore) StatObjectIdentity(ctx context.Context, key string) (ObjectIdentity, error) { + upstream, ok := o.upstream.(objectStorageIdentityReader) + if !ok { + return ObjectIdentity{}, moerr.NewNotSupported(ctx, "object storage identity") + } + if err := o.acquireContext(ctx); err != nil { + return ObjectIdentity{}, err + } + defer o.release() + return upstream.StatObjectIdentity(ctx, key) +} + +func (o *objectStorageSemaphore) ReadObjectWithIdentity( + ctx context.Context, + key string, + min *int64, + max *int64, + expected ObjectIdentity, +) (io.ReadCloser, error) { + upstream, ok := o.upstream.(objectStorageIdentityReader) + if !ok { + return nil, moerr.NewNotSupported(ctx, "conditional object storage read") + } + if err := o.acquireContext(ctx); err != nil { + return nil, err + } + r, err := upstream.ReadObjectWithIdentity(ctx, key, min, max, expected) + if err != nil { + o.release() + return nil, err + } + release := sync.OnceFunc(o.release) + return &readCloser{ + r: readerFunc(func(buffer []byte) (int, error) { + n, readErr := r.Read(buffer) + if readErr != nil { + release() + } + return n, readErr + }), + closeFunc: func() error { + release() + return r.Close() + }, + }, nil +} + func (o *objectStorageSemaphore) Stat(ctx context.Context, key string) (size int64, err error) { if err := o.acquireContext(ctx); err != nil { return 0, err diff --git a/pkg/fileservice/policy.go b/pkg/fileservice/policy.go index 4500e6c864e87..9087ef7444797 100644 --- a/pkg/fileservice/policy.go +++ b/pkg/fileservice/policy.go @@ -24,14 +24,17 @@ const ( SkipDiskCacheReads SkipDiskCacheWrites SkipFullFilePreloads + SkipRemoteCacheReads + SkipRemoteCacheWrites ) const ( - SkipCacheReads = SkipMemoryCacheReads | SkipDiskCacheReads - SkipCacheWrites = SkipMemoryCacheWrites | SkipDiskCacheWrites + SkipCacheReads = SkipMemoryCacheReads | SkipDiskCacheReads | SkipRemoteCacheReads + SkipCacheWrites = SkipMemoryCacheWrites | SkipDiskCacheWrites | SkipRemoteCacheWrites SkipDiskCache = SkipDiskCacheReads | SkipDiskCacheWrites SkipMemoryCache = SkipMemoryCacheReads | SkipMemoryCacheWrites - SkipAllCache = SkipDiskCache | SkipMemoryCache + SkipRemoteCache = SkipRemoteCacheReads | SkipRemoteCacheWrites + SkipAllCache = SkipDiskCache | SkipMemoryCache | SkipRemoteCache ) func (c Policy) Any(policies ...Policy) bool { diff --git a/pkg/fileservice/qcloud_sdk.go b/pkg/fileservice/qcloud_sdk.go index 5ad76440961bb..b96c6d82bc908 100644 --- a/pkg/fileservice/qcloud_sdk.go +++ b/pkg/fileservice/qcloud_sdk.go @@ -150,6 +150,7 @@ func NewQCloudSDK( } var _ objectStorageCopier = new(QCloudSDK) +var _ objectStorageIdentityReader = new(QCloudSDK) func (a *QCloudSDK) CopyObject( ctx context.Context, @@ -251,6 +252,63 @@ func (a *QCloudSDK) Stat( return } +func (a *QCloudSDK) StatObjectIdentity(ctx context.Context, key string) (ObjectIdentity, error) { + header, err := a.statObject(ctx, key) + if err != nil { + if a.is404(err) { + return ObjectIdentity{}, moerr.NewFileNotFoundNoCtx(key) + } + return ObjectIdentity{}, err + } + size, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64) + if err != nil { + return ObjectIdentity{}, err + } + identity := ObjectIdentity{ + VersionID: header.Get("x-cos-version-id"), + ETag: header.Get("ETag"), + Size: size, + } + if modified := header.Get("Last-Modified"); modified != "" { + identity.LastModified, err = http.ParseTime(modified) + if err != nil { + return ObjectIdentity{}, err + } + } + return identity, identity.Validate() +} + +func (a *QCloudSDK) ReadObjectWithIdentity( + ctx context.Context, + key string, + min *int64, + max *int64, + expected ObjectIdentity, +) (io.ReadCloser, error) { + if err := expected.Validate(); err != nil { + return nil, err + } + r, err := a.getObjectWithIdentity(ctx, key, min, max, &expected) + if err != nil { + return nil, mapQCloudConditionalReadError(err) + } + r = mapReadCloserErrors(r, mapQCloudConditionalReadError) + if max == nil { + return r, nil + } + return &readCloser{r: io.LimitReader(r, *max-*min), closeFunc: r.Close}, nil +} + +func mapQCloudConditionalReadError(err error) error { + var response *cos.ErrorResponse + if errors.As(err, &response) && response.Response != nil && + (response.Response.StatusCode == http.StatusNotFound || + response.Response.StatusCode == http.StatusPreconditionFailed) { + return errors.Join(ErrObjectChanged, moerr.NewInternalErrorNoCtx("conditional COS read failed")) + } + return err +} + func (a *QCloudSDK) Exists( ctx context.Context, key string, @@ -887,6 +945,16 @@ func (a *QCloudSDK) putObject( } func (a *QCloudSDK) getObject(ctx context.Context, key string, min *int64, max *int64) (io.ReadCloser, error) { + return a.getObjectWithIdentity(ctx, key, min, max, nil) +} + +func (a *QCloudSDK) getObjectWithIdentity( + ctx context.Context, + key string, + min *int64, + max *int64, + expected *ObjectIdentity, +) (io.ReadCloser, error) { ctx, task := gotrace.NewTask(ctx, "QCloudSDK.getObject") defer task.End() @@ -898,13 +966,18 @@ func (a *QCloudSDK) getObject(ctx context.Context, key string, min *int64, max * func(offset int64) (io.ReadCloser, error) { var rang string if max != nil { - rang = fmt.Sprintf("bytes=%d-%d", offset, *max) + rang = fmt.Sprintf("bytes=%d-%d", offset, *max-1) } else { rang = fmt.Sprintf("bytes=%d-", offset) } opts := &cos.ObjectGetOptions{ Range: rang, } + if expected != nil && expected.VersionID == "" { + headers := make(http.Header) + headers.Set("If-Match", expected.ETag) + opts.XOptionHeader = &headers + } return doQCloudReadWithRetry( ctx, @@ -913,7 +986,13 @@ func (a *QCloudSDK) getObject(ctx context.Context, key string, min *int64, max * perfcounter.Update(ctx, func(counter *perfcounter.CounterSet) { counter.FileService.S3.Get.Add(1) }, a.perfCounterSets...) - resp, err := a.client.Object.Get(ctx, key, opts) + var resp *cos.Response + var err error + if expected != nil && expected.VersionID != "" { + resp, err = a.client.Object.Get(ctx, key, opts, expected.VersionID) + } else { + resp, err = a.client.Object.Get(ctx, key, opts) + } if err != nil { return nil, err } diff --git a/pkg/fileservice/qcloud_sdk_test.go b/pkg/fileservice/qcloud_sdk_test.go index f650d5576fd81..86791940e77ba 100644 --- a/pkg/fileservice/qcloud_sdk_test.go +++ b/pkg/fileservice/qcloud_sdk_test.go @@ -156,6 +156,85 @@ func TestQCloudSDKCopyObject(t *testing.T) { require.False(t, copied) } +func TestQCloudSDKConditionalObjectIdentityReads(t *testing.T) { + const lastModRaw = "Wed, 02 Sep 2026 03:04:05 GMT" + var requests []*http.Request + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Clone(context.Background())) + if r.URL.Query().Get("versionId") == "gone" { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `NoSuchVersionplanned version was deleted`) + return + } + if r.URL.Query().Get("versionId") == "stale" || r.Header.Get("If-Match") == `"stale"` { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusPreconditionFailed) + _, _ = io.WriteString(w, `PreconditionFailedobject changed`) + return + } + if r.Method == http.MethodHead { + w.Header().Set("Content-Length", "6") + w.Header().Set("ETag", `"etag-v1"`) + w.Header().Set("x-cos-version-id", "version-v1") + w.Header().Set("Last-Modified", lastModRaw) + w.WriteHeader(http.StatusOK) + return + } + w.Header().Set("Content-Length", "3") + w.Header().Set("Content-Range", "bytes 1-3/6") + w.Header().Set("ETag", `"etag-v1"`) + w.Header().Set("Last-Modified", lastModRaw) + w.WriteHeader(http.StatusPartialContent) + _, _ = io.WriteString(w, "bcd") + })) + defer server.Close() + + sdk := newTestCOSClient(t, server) + identity, err := sdk.StatObjectIdentity(context.Background(), "object") + require.NoError(t, err) + wantLastModified, err := time.Parse(http.TimeFormat, lastModRaw) + require.NoError(t, err) + require.Equal(t, ObjectIdentity{ + VersionID: "version-v1", ETag: `"etag-v1"`, Size: 6, LastModified: wantLastModified, + }, identity) + + min, max := int64(1), int64(4) + reader, err := sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, identity) + require.NoError(t, err) + data, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + require.Equal(t, "bcd", string(data)) + versionRequest := requests[len(requests)-1] + require.Equal(t, "version-v1", versionRequest.URL.Query().Get("versionId")) + require.Empty(t, versionRequest.Header.Get("If-Match")) + require.Equal(t, "bytes=1-3", versionRequest.Header.Get("Range")) + + etagIdentity := identity + etagIdentity.VersionID = "" + reader, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, etagIdentity) + require.NoError(t, err) + require.NoError(t, reader.Close()) + etagRequest := requests[len(requests)-1] + require.Empty(t, etagRequest.URL.Query().Get("versionId")) + require.Equal(t, `"etag-v1"`, etagRequest.Header.Get("If-Match")) + + stale := identity + stale.VersionID = "stale" + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, stale) + require.ErrorIs(t, err, ErrObjectChanged) + stale.VersionID = "" + stale.ETag = `"stale"` + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, stale) + require.ErrorIs(t, err, ErrObjectChanged) + + gone := identity + gone.VersionID = "gone" + _, err = sdk.ReadObjectWithIdentity(context.Background(), "object", &min, &max, gone) + require.ErrorIs(t, err, ErrObjectChanged) +} + func TestQCloudSDKWriteRetriesSeekablePut(t *testing.T) { data := bytes.Repeat([]byte("x"), int(smallObjectThreshold)) size := int64(len(data)) diff --git a/pkg/fileservice/range_lease.go b/pkg/fileservice/range_lease.go new file mode 100644 index 0000000000000..ed84ba547853d --- /dev/null +++ b/pkg/fileservice/range_lease.go @@ -0,0 +1,461 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileservice + +import ( + "context" + "errors" + "io" + "math" + "sync/atomic" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/fileservice/fscache" +) + +// ErrObjectChanged is returned when a conditional object read can no longer +// address the identity fixed by planning. Callers must fail the whole +// statement; retrying without the same identity could combine object versions. +var ErrObjectChanged = moerr.NewInternalErrorNoCtx("fileservice: object changed") + +// ObjectIdentity is a format-neutral immutable object reference. VersionID is +// preferred when the backend exposes versioning; otherwise ETag is required +// for conditional reads. Size participates in every identity check. +type ObjectIdentity struct { + VersionID string + ETag string + Size int64 + LastModified time.Time +} + +func (i ObjectIdentity) Validate() error { + if i.Size < 0 || (i.VersionID == "" && i.ETag == "") { + return moerr.NewInvalidInputNoCtx("object identity requires a non-negative size and version ID or ETag") + } + return nil +} + +// ObjectIdentityFileService is the optional backend capability beneath a +// ConditionalLeasedRangeReader. OpenReadWithIdentity performs one conditional +// read; size -1 means through end of object. +type ObjectIdentityFileService interface { + StatFileIdentity(ctx context.Context, path string) (ObjectIdentity, error) + OpenReadWithIdentity( + ctx context.Context, + path string, + offset, size int64, + expected ObjectIdentity, + ) (io.ReadCloser, error) +} + +// CapacityLease is the format-independent release side of a committed read +// reservation. +type CapacityLease interface { + Release() +} + +// CapacityReservation reserves an upper bound before a read can allocate or +// pin its authoritative backing. +type CapacityReservation interface { + Commit(actualCapacity int64) (CapacityLease, error) + Abort() +} + +// RangeReadAdmission connects FileService range reads to an execution-owned +// capacity account without making FileService depend on SQL or file formats. +type RangeReadAdmission interface { + Reserve(ctx context.Context, upperBound int64) (CapacityReservation, error) +} + +type allocationAccountRangeAdmission struct { + account *mpool.AllocationAccount + owner mpool.AllocationOwner + site mpool.AllocationSite + capacityClass mpool.AllocationCapacityClass +} + +// NewAllocationAccountRangeAdmission binds non-MPool range capacity to the +// same statement account used by execution allocations. +func NewAllocationAccountRangeAdmission( + account *mpool.AllocationAccount, + owner mpool.AllocationOwner, + site mpool.AllocationSite, + capacityClass mpool.AllocationCapacityClass, +) (RangeReadAdmission, error) { + if account == nil || site < mpool.AllocationSiteMin || site > mpool.AllocationSiteMax { + return nil, mpool.ErrAllocationAccountInvalid + } + return &allocationAccountRangeAdmission{ + account: account, owner: owner, site: site, capacityClass: capacityClass, + }, nil +} + +func (a *allocationAccountRangeAdmission) Reserve( + ctx context.Context, + upperBound int64, +) (CapacityReservation, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if a == nil || a.account == nil || upperBound <= 0 { + return nil, mpool.ErrAllocationAccountInvalid + } + reservation, err := a.account.ReserveCapacityWithClass( + uint64(upperBound), a.capacityClass, a.owner, a.site, + ) + if err != nil { + return nil, err + } + return allocationCapacityReservation{reservation: reservation}, nil +} + +type allocationCapacityReservation struct { + reservation *mpool.CapacityReservation +} + +func (r allocationCapacityReservation) Commit(actualCapacity int64) (CapacityLease, error) { + if r.reservation == nil || actualCapacity < 0 { + return nil, mpool.ErrAllocationAccountInvalid + } + return r.reservation.Commit(uint64(actualCapacity)) +} + +func (r allocationCapacityReservation) Abort() { + if r.reservation != nil { + r.reservation.Abort() + } +} + +// RangeLease keeps the authoritative read result and its committed capacity +// reservation alive under one idempotent release owner. +type RangeLease interface { + Bytes() []byte + Capacity() int64 + Release() +} + +// LeasedRangeReader is the generic range capability used by columnar readers. +type LeasedRangeReader interface { + ReadRangeLease( + ctx context.Context, + path string, + offset, size int64, + admission RangeReadAdmission, + ) (RangeLease, error) +} + +// ConditionalLeasedRangeReader prevents one logical scan from combining +// ranges belonging to different versions of a mutable object. +type ConditionalLeasedRangeReader interface { + LeasedRangeReader + StatIdentity(ctx context.Context, path string) (ObjectIdentity, error) + ReadRangeLeaseWithIdentity( + ctx context.Context, + path string, + offset, size int64, + expected ObjectIdentity, + admission RangeReadAdmission, + ) (RangeLease, error) +} + +type fileServiceRangeReader struct { + fs FileService +} + +const maxRangeLeasePinAmplification = int64(4) + +// NewLeasedRangeReader adapts every FileService through its ordinary Read +// contract. Backend-specific implementations may replace this adapter while +// preserving the same ownership and admission semantics. +func NewLeasedRangeReader(fs FileService) LeasedRangeReader { + base := &fileServiceRangeReader{fs: fs} + if identityFS, ok := fs.(ObjectIdentityFileService); ok { + return &conditionalFileServiceRangeReader{fileServiceRangeReader: base, identityFS: identityFS} + } + return base +} + +type conditionalFileServiceRangeReader struct { + *fileServiceRangeReader + identityFS ObjectIdentityFileService +} + +func (r *conditionalFileServiceRangeReader) StatIdentity( + ctx context.Context, + path string, +) (ObjectIdentity, error) { + if r == nil || r.identityFS == nil { + return ObjectIdentity{}, moerr.NewNotSupported(ctx, "object identity") + } + identity, err := r.identityFS.StatFileIdentity(ctx, path) + if err != nil { + return ObjectIdentity{}, err + } + if err := identity.Validate(); err != nil { + return ObjectIdentity{}, err + } + return identity, nil +} + +func (r *conditionalFileServiceRangeReader) ReadRangeLeaseWithIdentity( + ctx context.Context, + path string, + offset, size int64, + expected ObjectIdentity, + admission RangeReadAdmission, +) (RangeLease, error) { + if r == nil || r.identityFS == nil { + return nil, moerr.NewNotSupported(ctx, "conditional object range read") + } + if err := expected.Validate(); err != nil { + return nil, err + } + if offset < 0 || size <= 0 || offset > expected.Size || size > expected.Size-offset { + return nil, moerr.NewInvalidInput(ctx, "conditional range is outside the fixed object identity") + } + return readRangeLease(ctx, path, offset, size, admission, func(destination []byte) error { + reader, err := r.identityFS.OpenReadWithIdentity(ctx, path, offset, size, expected) + if err != nil { + return err + } + var readErr error + if _, readErr = io.ReadFull(reader, destination); readErr == nil { + var probe [1]byte + if n, probeErr := reader.Read(probe[:]); n != 0 || (probeErr != nil && !errors.Is(probeErr, io.EOF)) { + readErr = moerr.NewUnexpectedEOFNoCtx(path) + } + } + // Conditional providers can report an object-generation mismatch only + // while finalizing the response. Do not commit the range reservation + // until that Close result has been observed. + return errors.Join(readErr, reader.Close()) + }) +} + +type ioVectorRangeLease struct { + data []byte + capacity int64 + vector *IOVector + capacityLease CapacityLease + released atomic.Bool +} + +func (l *ioVectorRangeLease) Bytes() []byte { + if l == nil || l.released.Load() { + return nil + } + return l.data +} + +func (l *ioVectorRangeLease) Capacity() int64 { + if l == nil { + return 0 + } + return l.capacity +} + +func (l *ioVectorRangeLease) Release() { + if l == nil || !l.released.CompareAndSwap(false, true) { + return + } + // Release backing before its capacity charge. Observing zero usage then + // implies that no range backing remains reachable through this owner. + if l.vector != nil { + l.vector.Release() + l.vector = nil + } + l.data = nil + if l.capacityLease != nil { + l.capacityLease.Release() + l.capacityLease = nil + } +} + +func (r *fileServiceRangeReader) ReadRangeLease( + ctx context.Context, + path string, + offset, size int64, + admission RangeReadAdmission, +) (_ RangeLease, retErr error) { + if r == nil || r.fs == nil || admission == nil || path == "" || + offset < 0 || size <= 0 || offset > math.MaxInt64-size || + uint64(size) > uint64(^uint(0)>>1) { + return nil, moerr.NewInvalidInput(ctx, "invalid leased range read") + } + + vector := &IOVector{ + FilePath: path, + // Only an exact memory-cache entry can transfer its backing into this + // lease. Disk and remote cache tiers allocate while reading and cannot + // run the pre-retain statement admission hook. + Policy: SkipDiskCache | SkipRemoteCache | SkipFullFilePreloads, + Entries: []IOEntry{{ + Offset: offset, + Size: size, + }}, + } + entry := &vector.Entries[0] + entry.admitCachedData = func(capacity int64) (func(), error) { + return admitRangeCachePin(ctx, admission, size, capacity) + } + published := false + defer func() { + if !published { + vector.ReleaseReadResultOnError() + } + }() + + // Avoid allocating a raw destination when the exact range is already in + // memory cache. MemCache performs admission before retaining the hit. + if err := r.fs.ReadCache(ctx, vector); err != nil { + return nil, err + } + if entry.CachedData != nil { + lease, err := rangeLeaseFromCachedData(ctx, vector, size) + if err != nil { + return nil, err + } + published = true + return lease, nil + } + + // A miss keeps the established exact-destination path: it introduces no + // cache copy and reserves exactly the raw backing before allocation. Other + // FileService consumers may populate the memory cache for a later leased + // read, but a range lease never creates two authoritative copies itself. + published = true + vector.Release() + lease, err := readRangeLease(ctx, path, offset, size, admission, func(destination []byte) error { + readVector := &IOVector{ + FilePath: path, + Policy: SkipAllCache, + Entries: []IOEntry{{ + Offset: offset, + Size: size, + Data: destination, + }}, + } + if err := r.fs.Read(ctx, readVector); err != nil { + readVector.ReleaseReadResultOnError() + return err + } + readEntry := &readVector.Entries[0] + if readEntry.CachedData != nil || readEntry.releaseData != nil || + len(readEntry.Data) != len(destination) || + (cap(readEntry.Data) > 0 && &readEntry.Data[0] != &destination[0]) { + readVector.Release() + return moerr.NewInvalidInput(ctx, "leased range backend replaced exact destination") + } + readEntry.Data = nil + readVector.Release() + return nil + }) + if err != nil { + return nil, err + } + published = true + return lease, nil +} + +func admitRangeCachePin( + ctx context.Context, + admission RangeReadAdmission, + logicalSize, capacity int64, +) (func(), error) { + if err := validateRangePinAmplification(ctx, logicalSize, capacity); err != nil { + return nil, errors.Join(fscache.ErrCacheAdmissionRejected, err) + } + reservation, err := admission.Reserve(ctx, capacity) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return nil, err + } + return nil, errors.Join(fscache.ErrCacheAdmissionRejected, err) + } + capacityLease, err := reservation.Commit(capacity) + if err != nil { + reservation.Abort() + return nil, errors.Join(fscache.ErrCacheAdmissionRejected, err) + } + return capacityLease.Release, nil +} + +func validateRangePinAmplification(ctx context.Context, logicalSize, capacity int64) error { + if logicalSize <= 0 || capacity < logicalSize || + (logicalSize <= math.MaxInt64/maxRangeLeasePinAmplification && + capacity > logicalSize*maxRangeLeasePinAmplification) { + return moerr.NewInvalidInput(ctx, "leased range cache pin amplification exceeds limit") + } + return nil +} + +func rangeLeaseFromCachedData( + ctx context.Context, + vector *IOVector, + logicalSize int64, +) (RangeLease, error) { + entry := &vector.Entries[0] + if entry.CachedData == nil || entry.releaseCachedData == nil || + entry.CachedData.Size() != logicalSize || int64(len(entry.CachedData.Bytes())) != logicalSize { + return nil, moerr.NewInvalidInput(ctx, "leased range backend did not return admitted exact cache data") + } + capacity := entry.CachedData.Capacity() + if err := validateRangePinAmplification(ctx, logicalSize, capacity); err != nil { + return nil, err + } + return &ioVectorRangeLease{ + data: entry.CachedData.Bytes(), capacity: capacity, vector: vector, + }, nil +} + +func readRangeLease( + ctx context.Context, + path string, + offset, size int64, + admission RangeReadAdmission, + read func(destination []byte) error, +) (RangeLease, error) { + if admission == nil || read == nil || path == "" || offset < 0 || size <= 0 || + offset > math.MaxInt64-size || uint64(size) > uint64(^uint(0)>>1) { + return nil, moerr.NewInvalidInput(ctx, "invalid leased range read") + } + reservation, err := admission.Reserve(ctx, size) + if err != nil { + return nil, err + } + committed := false + defer func() { + // This also runs while unwinding an allocation/backend panic. Until a + // capacity lease is published, the reservation remains our sole owner. + if !committed { + reservation.Abort() + } + }() + destination := make([]byte, int(size)) + if err = read(destination); err != nil { + return nil, err + } + capacity := int64(cap(destination)) + capacityLease, err := reservation.Commit(capacity) + if err != nil { + return nil, err + } + committed = true + return &ioVectorRangeLease{ + data: destination, capacity: capacity, capacityLease: capacityLease, + }, nil +} diff --git a/pkg/fileservice/range_lease_test.go b/pkg/fileservice/range_lease_test.go new file mode 100644 index 0000000000000..bfc2e17c815a3 --- /dev/null +++ b/pkg/fileservice/range_lease_test.go @@ -0,0 +1,384 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileservice + +import ( + "bytes" + "context" + "errors" + "io" + "sync/atomic" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/fileservice/fscache" + "github.com/matrixorigin/matrixone/pkg/util/toml" + "github.com/stretchr/testify/require" +) + +type conditionalMemoryFS struct { + *MemoryFS + identity ObjectIdentity +} + +type closeErrorReadCloser struct { + *bytes.Reader + err error +} + +func (r *closeErrorReadCloser) Close() error { return r.err } + +type conditionalCloseErrorFS struct { + *conditionalMemoryFS + closeErr error +} + +func (f *conditionalCloseErrorFS) OpenReadWithIdentity( + ctx context.Context, + path string, + offset, size int64, + expected ObjectIdentity, +) (io.ReadCloser, error) { + if expected != f.identity { + return nil, ErrObjectChanged + } + vector := &IOVector{FilePath: path, Policy: SkipAllCache, Entries: []IOEntry{{Offset: offset, Size: size}}} + if err := f.MemoryFS.Read(ctx, vector); err != nil { + return nil, err + } + data := append([]byte(nil), vector.Entries[0].Data...) + vector.Release() + return &closeErrorReadCloser{Reader: bytes.NewReader(data), err: f.closeErr}, nil +} + +func (f *conditionalMemoryFS) StatFileIdentity(ctx context.Context, path string) (ObjectIdentity, error) { + if err := ctx.Err(); err != nil { + return ObjectIdentity{}, err + } + return f.identity, nil +} + +func (f *conditionalMemoryFS) OpenReadWithIdentity( + ctx context.Context, + path string, + offset, size int64, + expected ObjectIdentity, +) (io.ReadCloser, error) { + if expected != f.identity { + return nil, ErrObjectChanged + } + vector := &IOVector{FilePath: path, Policy: SkipAllCache, Entries: []IOEntry{{Offset: offset, Size: size}}} + if err := f.Read(ctx, vector); err != nil { + return nil, err + } + data := append([]byte(nil), vector.Entries[0].Data...) + vector.Release() + return io.NopCloser(bytes.NewReader(data)), nil +} + +type testRangeAdmission struct { + reject error + reserved atomic.Int64 + committed atomic.Int64 + released atomic.Int64 + aborted atomic.Int64 + commitReject error +} + +type rejectFirstRangeAdmission struct { + first error + calls atomic.Int64 + inner testRangeAdmission +} + +func (a *rejectFirstRangeAdmission) Reserve( + ctx context.Context, + upper int64, +) (CapacityReservation, error) { + if a.calls.Add(1) == 1 { + return nil, a.first + } + return a.inner.Reserve(ctx, upper) +} + +func (a *testRangeAdmission) Reserve(_ context.Context, upper int64) (CapacityReservation, error) { + if a.reject != nil { + return nil, a.reject + } + a.reserved.Add(upper) + return &testRangeReservation{admission: a, upper: upper}, nil +} + +type testRangeReservation struct { + admission *testRangeAdmission + upper int64 + done atomic.Bool +} + +func (r *testRangeReservation) Commit(actual int64) (CapacityLease, error) { + if r.admission.commitReject != nil { + return nil, r.admission.commitReject + } + if actual > r.upper || !r.done.CompareAndSwap(false, true) { + return nil, errors.New("invalid commit") + } + r.admission.committed.Add(actual) + return &testRangeCapacityLease{admission: r.admission, capacity: actual}, nil +} + +func (r *testRangeReservation) Abort() { + if r.done.CompareAndSwap(false, true) { + r.admission.aborted.Add(r.upper) + } +} + +type testRangeCapacityLease struct { + admission *testRangeAdmission + capacity int64 + released atomic.Bool +} + +func (l *testRangeCapacityLease) Release() { + if l.released.CompareAndSwap(false, true) { + l.admission.released.Add(l.capacity) + } +} + +func TestReadRangeLeaseLifecycle(t *testing.T) { + ctx := context.Background() + fs, err := NewMemoryFS("range", DisabledCacheConfig, nil) + require.NoError(t, err) + require.NoError(t, fs.Write(ctx, IOVector{ + FilePath: "range:file", + Entries: []IOEntry{{Offset: 0, Size: 6, Data: []byte("abcdef")}}, + })) + admission := new(testRangeAdmission) + lease, err := NewLeasedRangeReader(fs).ReadRangeLease(ctx, "range:file", 2, 3, admission) + require.NoError(t, err) + require.Equal(t, []byte("cde"), lease.Bytes()) + require.Equal(t, int64(3), lease.Capacity()) + require.Equal(t, int64(3), admission.reserved.Load()) + require.Equal(t, int64(3), admission.committed.Load()) + + lease.Release() + lease.Release() + require.Nil(t, lease.Bytes()) + require.Equal(t, int64(3), admission.released.Load()) +} + +func TestReadRangeLeaseMemoryCacheAdmissionLifecycle(t *testing.T) { + ctx := context.Background() + cacheCapacity := toml.ByteSize(1 << 20) + fs, err := NewLocalFS2(ctx, "range-cache", t.TempDir(), CacheConfig{ + MemoryCapacity: &cacheCapacity, + }, nil) + require.NoError(t, err) + t.Cleanup(func() { fs.Close(ctx) }) + require.NoError(t, fs.Write(ctx, IOVector{ + FilePath: "range-cache:file", + Entries: []IOEntry{{Offset: 0, Size: 6, Data: []byte("abcdef")}}, + Policy: SkipAllCache, + })) + reader := NewLeasedRangeReader(fs) + + missAdmission := new(testRangeAdmission) + miss, err := reader.ReadRangeLease(ctx, "range-cache:file", 2, 3, missAdmission) + require.NoError(t, err) + require.Equal(t, []byte("cde"), miss.Bytes()) + require.Equal(t, int64(3), miss.Capacity()) + require.Equal(t, int64(3), missAdmission.reserved.Load()) + require.Zero(t, missAdmission.aborted.Load()) + require.Equal(t, int64(3), missAdmission.committed.Load()) + miss.Release() + require.Equal(t, missAdmission.committed.Load(), missAdmission.released.Load()) + + // Populate the exact memory-cache key through the ordinary FileService + // cache path. The next range read must pin this backing without allocating + // or reserving a raw destination. + warm := &IOVector{ + FilePath: "range-cache:file", + Policy: SkipDiskCache | SkipRemoteCache | SkipFullFilePreloads, + Entries: []IOEntry{{ + Offset: 2, + Size: 3, + ToCacheData: CacheOriginalData, + }}, + } + require.NoError(t, fs.Read(ctx, warm)) + require.NotNil(t, warm.Entries[0].CachedData) + warm.Release() + + hitAdmission := new(testRangeAdmission) + hit, err := reader.ReadRangeLease(ctx, "range-cache:file", 2, 3, hitAdmission) + require.NoError(t, err) + require.Equal(t, []byte("cde"), hit.Bytes()) + require.Equal(t, hit.Capacity(), hitAdmission.reserved.Load(), + "cache hit must not allocate or reserve a raw destination") + require.Zero(t, hitAdmission.aborted.Load()) + require.Equal(t, hit.Capacity(), hitAdmission.committed.Load()) + hit.Release() + require.Equal(t, hitAdmission.committed.Load(), hitAdmission.released.Load()) + + // A cache-only rejection is a miss, not a statement failure. The fallback + // reserves the exact raw range and avoids making success depend on whether + // another reader happened to warm a larger cache backing first. + fallbackAdmission := &rejectFirstRangeAdmission{first: errors.New("pin budget rejected")} + fallback, err := reader.ReadRangeLease(ctx, "range-cache:file", 2, 3, fallbackAdmission) + require.NoError(t, err) + require.Equal(t, []byte("cde"), fallback.Bytes()) + require.Equal(t, int64(2), fallbackAdmission.calls.Load()) + require.Equal(t, int64(3), fallbackAdmission.inner.committed.Load()) + fallback.Release() + require.Equal(t, int64(3), fallbackAdmission.inner.released.Load()) + + reject := errors.New("pin budget rejected") + _, err = reader.ReadRangeLease(ctx, "range-cache:file", 2, 3, &testRangeAdmission{reject: reject}) + require.ErrorIs(t, err, reject) + + // A rejected pin did not retain or corrupt the cache entry. + recoveryAdmission := new(testRangeAdmission) + recovered, err := reader.ReadRangeLease(ctx, "range-cache:file", 2, 3, recoveryAdmission) + require.NoError(t, err) + require.Equal(t, []byte("cde"), recovered.Bytes()) + recovered.Release() + require.Equal(t, recoveryAdmission.committed.Load(), recoveryAdmission.released.Load()) +} + +func TestReadRangeLeaseRejectsPinAmplificationBeforeAdmission(t *testing.T) { + admission := new(testRangeAdmission) + _, err := admitRangeCachePin(context.Background(), admission, 3, 13) + require.ErrorIs(t, err, fscache.ErrCacheAdmissionRejected) + require.ErrorContains(t, err, "pin amplification") + require.Zero(t, admission.reserved.Load()) +} + +func TestReadRangeLeaseAdmissionAndCommitFailure(t *testing.T) { + ctx := context.Background() + fs, err := NewMemoryFS("range-errors", DisabledCacheConfig, nil) + require.NoError(t, err) + require.NoError(t, fs.Write(ctx, IOVector{ + FilePath: "range-errors:file", + Entries: []IOEntry{{Offset: 0, Size: 3, Data: []byte("abc")}}, + })) + + reject := errors.New("capacity rejected") + _, err = NewLeasedRangeReader(fs).ReadRangeLease(ctx, "range-errors:file", 0, 3, &testRangeAdmission{reject: reject}) + require.ErrorIs(t, err, reject) + + admission := &testRangeAdmission{commitReject: reject} + _, err = NewLeasedRangeReader(fs).ReadRangeLease(ctx, "range-errors:file", 0, 3, admission) + require.ErrorIs(t, err, reject) + require.Equal(t, int64(3), admission.aborted.Load()) +} + +func TestReadRangeLeaseBackendPanicAbortsReservation(t *testing.T) { + admission := new(testRangeAdmission) + require.PanicsWithValue(t, "injected range read panic", func() { + _, _ = readRangeLease( + context.Background(), "panic:file", 0, 8, admission, + func([]byte) error { panic("injected range read panic") }, + ) + }) + require.Equal(t, int64(8), admission.reserved.Load()) + require.Equal(t, int64(8), admission.aborted.Load()) + require.Zero(t, admission.committed.Load()) +} + +func TestReadRangeLeaseRejectsInvalidAndCanceledReads(t *testing.T) { + fs, err := NewMemoryFS("range-invalid", DisabledCacheConfig, nil) + require.NoError(t, err) + reader := NewLeasedRangeReader(fs) + admission := new(testRangeAdmission) + _, err = reader.ReadRangeLease(context.Background(), "range-invalid:file", 0, 0, admission) + require.Error(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = reader.ReadRangeLease(ctx, "range-invalid:file", 0, 1, admission) + require.ErrorIs(t, err, context.Canceled) + require.Zero(t, admission.reserved.Load(), "canceled cache probe must fail before raw admission") + require.Zero(t, admission.aborted.Load()) +} + +func TestConditionalRangeLeaseFixesObjectIdentity(t *testing.T) { + ctx := context.Background() + memoryFS, err := NewMemoryFS("conditional", DisabledCacheConfig, nil) + require.NoError(t, err) + require.NoError(t, memoryFS.Write(ctx, IOVector{ + FilePath: "conditional:file", + Entries: []IOEntry{{Offset: 0, Size: 6, Data: []byte("abcdef")}}, + })) + identity := ObjectIdentity{ + ETag: "etag-v1", Size: 6, LastModified: time.Unix(10, 0).UTC(), + } + fs := &conditionalMemoryFS{MemoryFS: memoryFS, identity: identity} + reader, ok := NewLeasedRangeReader(fs).(ConditionalLeasedRangeReader) + require.True(t, ok) + gotIdentity, err := reader.StatIdentity(ctx, "conditional:file") + require.NoError(t, err) + require.Equal(t, identity, gotIdentity) + + admission := new(testRangeAdmission) + lease, err := reader.ReadRangeLeaseWithIdentity(ctx, "conditional:file", 1, 3, identity, admission) + require.NoError(t, err) + require.Equal(t, []byte("bcd"), lease.Bytes()) + require.Equal(t, int64(3), admission.committed.Load()) + lease.Release() + require.Equal(t, int64(3), admission.released.Load()) + + fs.identity.ETag = "etag-v2" + _, err = reader.ReadRangeLeaseWithIdentity(ctx, "conditional:file", 1, 3, identity, admission) + require.ErrorIs(t, err, ErrObjectChanged) + require.Equal(t, int64(3), admission.aborted.Load()) +} + +func TestConditionalRangeLeaseAbortsAdmissionWhenCloseReportsObjectChanged(t *testing.T) { + ctx := context.Background() + memoryFS, err := NewMemoryFS("conditional-close", DisabledCacheConfig, nil) + require.NoError(t, err) + require.NoError(t, memoryFS.Write(ctx, IOVector{ + FilePath: "conditional-close:file", + Entries: []IOEntry{{Offset: 0, Size: 3, Data: []byte("abc")}}, + })) + identity := ObjectIdentity{ETag: "etag-v1", Size: 3, LastModified: time.Unix(10, 0).UTC()} + fs := &conditionalCloseErrorFS{ + conditionalMemoryFS: &conditionalMemoryFS{MemoryFS: memoryFS, identity: identity}, + closeErr: errors.Join(ErrObjectChanged, errors.New("conditional response finalized stale")), + } + reader := NewLeasedRangeReader(fs).(ConditionalLeasedRangeReader) + admission := new(testRangeAdmission) + + lease, err := reader.ReadRangeLeaseWithIdentity(ctx, "conditional-close:file", 0, 3, identity, admission) + require.Nil(t, lease) + require.ErrorIs(t, err, ErrObjectChanged) + require.Equal(t, int64(3), admission.reserved.Load()) + require.Equal(t, int64(3), admission.aborted.Load()) + require.Zero(t, admission.committed.Load()) + require.Zero(t, admission.released.Load()) +} + +func TestConditionalRangeRejectsIncompleteIdentityAndOutOfBounds(t *testing.T) { + ctx := context.Background() + memoryFS, err := NewMemoryFS("conditional-invalid", DisabledCacheConfig, nil) + require.NoError(t, err) + fs := &conditionalMemoryFS{MemoryFS: memoryFS, identity: ObjectIdentity{ETag: "etag", Size: 2}} + reader := NewLeasedRangeReader(fs).(ConditionalLeasedRangeReader) + admission := new(testRangeAdmission) + + _, err = reader.ReadRangeLeaseWithIdentity(ctx, "conditional-invalid:file", 0, 1, ObjectIdentity{Size: 2}, admission) + require.ErrorContains(t, err, "version ID or ETag") + _, err = reader.ReadRangeLeaseWithIdentity(ctx, "conditional-invalid:file", 1, 2, fs.identity, admission) + require.ErrorContains(t, err, "outside") +} diff --git a/pkg/fileservice/remote_cache.go b/pkg/fileservice/remote_cache.go index 29721d5857178..1645c8fa00362 100644 --- a/pkg/fileservice/remote_cache.go +++ b/pkg/fileservice/remote_cache.go @@ -60,6 +60,9 @@ func NewRemoteCache(client client.QueryClient, factory KeyRouterFactory[query.Ca } func (r *RemoteCache) Read(ctx context.Context, vector *IOVector) error { + if vector.Policy.Any(SkipRemoteCacheReads) { + return nil + } if r.keyRouterFactory == nil { return nil } diff --git a/pkg/fileservice/s3_fs.go b/pkg/fileservice/s3_fs.go index d1206c97b77ac..fd412ba3bf361 100644 --- a/pkg/fileservice/s3_fs.go +++ b/pkg/fileservice/s3_fs.go @@ -63,6 +63,7 @@ type S3FS struct { // / -> file content var _ FileService = new(S3FS) +var _ ObjectIdentityFileService = new(S3FS) func NewS3FS( ctx context.Context, @@ -246,6 +247,19 @@ func resolveS3CopySource(fs FileService, filePath string) (*S3FS, string, error) } } +// IsS3BackedFileService reports whether filePath resolves to an S3FS through +// the supplied FileService. Callers that enforce storage-specific policy must +// resolve FileServices and SubPath wrappers instead of relying on the visible +// service name: deployments may give an S3-backed service an arbitrary name. +// Resolution is metadata-only and never opens the object store. +func IsS3BackedFileService(fs FileService, filePath string) bool { + if fs == nil { + return false + } + s3, _, err := resolveS3CopySource(fs, filePath) + return err == nil && s3 != nil +} + func (s *S3FS) AllocateCacheData(ctx context.Context, size int) fscache.Data { if s.memCache != nil { return s.memCache.AllocateCacheData(ctx, size) @@ -381,6 +395,58 @@ func (s *S3FS) StatFile(ctx context.Context, filePath string) (*DirEntry, error) }, nil } +func (s *S3FS) StatFileIdentity(ctx context.Context, filePath string) (ObjectIdentity, error) { + if err := ctx.Err(); err != nil { + return ObjectIdentity{}, err + } + path, err := parseFilePathAtService(filePath, s.name) + if err != nil { + return ObjectIdentity{}, err + } + storage, ok := s.storage.(objectStorageIdentityReader) + if !ok { + return ObjectIdentity{}, moerr.NewNotSupported(ctx, "object storage identity") + } + identity, err := storage.StatObjectIdentity(ctx, s.pathToKey(path.File)) + if err != nil { + return ObjectIdentity{}, err + } + if err := identity.Validate(); err != nil { + return ObjectIdentity{}, err + } + return identity, nil +} + +func (s *S3FS) OpenReadWithIdentity( + ctx context.Context, + filePath string, + offset, size int64, + expected ObjectIdentity, +) (io.ReadCloser, error) { + if err := expected.Validate(); err != nil { + return nil, err + } + if offset < 0 || size == 0 || size < -1 || offset > expected.Size || + (size > 0 && size > expected.Size-offset) { + return nil, moerr.NewInvalidInput(ctx, "conditional read is outside the fixed object identity") + } + path, err := parseFilePathAtService(filePath, s.name) + if err != nil { + return nil, err + } + storage, ok := s.storage.(objectStorageIdentityReader) + if !ok { + return nil, moerr.NewNotSupported(ctx, "conditional object storage read") + } + min := offset + var max *int64 + if size > 0 { + end := offset + size + max = &end + } + return storage.ReadObjectWithIdentity(ctx, s.pathToKey(path.File), &min, max, expected) +} + func (s *S3FS) PrefetchFile(ctx context.Context, filePath string) error { if err := ctx.Err(); err != nil { return err diff --git a/pkg/fileservice/s3_fs_test.go b/pkg/fileservice/s3_fs_test.go index 4d03371ee43a3..0c967cb1fe86b 100644 --- a/pkg/fileservice/s3_fs_test.go +++ b/pkg/fileservice/s3_fs_test.go @@ -173,6 +173,19 @@ func TestObjectCopyCapabilityFallbacks(t *testing.T) { require.Error(t, err) } +func TestIsS3BackedFileServiceResolvesWrappers(t *testing.T) { + s3 := &S3FS{name: "archive"} + local := dummyFileService{name: "local"} + services, err := NewFileServices("local", local, s3) + require.NoError(t, err) + + require.True(t, IsS3BackedFileService(services, "archive:input.arrow")) + require.True(t, IsS3BackedFileService(SubPath(s3, "tenant"), "input.arrow")) + require.False(t, IsS3BackedFileService(services, "local:input.arrow")) + require.False(t, IsS3BackedFileService(services, "missing:input.arrow")) + require.False(t, IsS3BackedFileService(nil, "archive:input.arrow")) +} + func TestObjectCopyRejectsIncompatibleEndpoints(t *testing.T) { copied, err := (&AwsSDKv2{endpoint: "https://s3-b.example.com"}).CopyObject( context.Background(), &AwsSDKv2{endpoint: "https://s3-a.example.com"}, "src", "dst", diff --git a/pkg/frontend/util.go b/pkg/frontend/util.go index aaa1f743ab1dd..4b2ac78ab1939 100644 --- a/pkg/frontend/util.go +++ b/pkg/frontend/util.go @@ -122,7 +122,8 @@ var PathExists = func(path string) (bool, bool, error) { func getSystemVariables(configFile string) (*mo_config.FrontendParameters, error) { sv := &mo_config.FrontendParameters{ - MongoDB: *mo_config.NewMongoDBParameters(), + MongoDB: *mo_config.NewMongoDBParameters(), + ArrowLoad: *mo_config.NewArrowLoadParameters(), } var err error _, err = toml.DecodeFile(configFile, sv) diff --git a/pkg/iceberg/api/import_boundary_test.go b/pkg/iceberg/api/import_boundary_test.go index 3e9fae3d76717..d628a98ecfcb1 100644 --- a/pkg/iceberg/api/import_boundary_test.go +++ b/pkg/iceberg/api/import_boundary_test.go @@ -36,7 +36,7 @@ func TestIcebergAdapterImportBoundary(t *testing.T) { } if d.IsDir() { switch d.Name() { - case ".git", "vendor", "node_modules": + case ".git", ".proto-vendor", "vendor", "node_modules": return filepath.SkipDir } return nil @@ -44,17 +44,27 @@ func TestIcebergAdapterImportBoundary(t *testing.T) { if !strings.HasSuffix(path, ".go") { return nil } + if strings.HasSuffix(path, "_test.go") { + return nil + } fset := token.NewFileSet() file, parseErr := parser.ParseFile(fset, path, nil, parser.ImportsOnly) if parseErr != nil { return parseErr } slashPath := filepath.ToSlash(path) - adapterAllowed := strings.Contains(slashPath, "/pkg/iceberg/adapter/iceberggo/") + // Arrow has two approved MatrixOne-owned boundaries: the Iceberg + // adapter and the static-file ingestion bridge. Everywhere else remains + // forbidden so an implementation detail cannot leak into SQL layers. + arrowAllowed := strings.Contains(slashPath, "/pkg/iceberg/adapter/iceberggo/") || + strings.Contains(slashPath, "/pkg/container/arrowbridge/") || + strings.Contains(slashPath, "/pkg/sql/colexec/external/arrowio/") || + strings.HasSuffix(slashPath, "/pkg/sql/colexec/external/reader_arrow.go") || + strings.HasSuffix(slashPath, "/pkg/sql/compile/compile.go") for _, imp := range file.Imports { importPath := strings.Trim(imp.Path.Value, `"`) for _, prefix := range forbidden { - if strings.HasPrefix(importPath, prefix) && !adapterAllowed { + if strings.HasPrefix(importPath, prefix) && !arrowAllowed { t.Fatalf("forbidden Iceberg/Arrow import outside adapter: %s imports %s", slashPath, importPath) } } diff --git a/pkg/objectio/ioutil/sinker.go b/pkg/objectio/ioutil/sinker.go index c16b3f426816c..efbd4f3a25cab 100644 --- a/pkg/objectio/ioutil/sinker.go +++ b/pkg/objectio/ioutil/sinker.go @@ -1056,6 +1056,19 @@ func (sinker *Sinker) WriteOwned( ctx context.Context, data *batch.Batch, ) (owned bool, err error) { + if data == nil { + return false, moerr.NewInvalidInput(ctx, "Sinker.WriteOwned requires a batch") + } + for index, vec := range data.Vecs { + if vec == nil { + return false, moerr.NewInvalidInputf(ctx, + "Sinker.WriteOwned batch has a nil vector at column %d", index) + } + if vec.HasBorrowedBacking() || vec.NeedDup() { + return false, moerr.NewInvalidInputf(ctx, + "Sinker.WriteOwned requires unique-owned vector backing at column %d", index) + } + } if data.RowCount() != objectio.BlockMaxRows { return false, sinker.Write(ctx, data) } diff --git a/pkg/objectio/ioutil/sinker_test.go b/pkg/objectio/ioutil/sinker_test.go index b52bfd3679b40..d0c6e4da249d6 100644 --- a/pkg/objectio/ioutil/sinker_test.go +++ b/pkg/objectio/ioutil/sinker_test.go @@ -22,7 +22,9 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/testutil" @@ -44,6 +46,25 @@ func mockSchema(colCnt int, pkIdx int) ([]string, []types.Type, []uint16) { return attrs, typs, seq } +func TestSinkerWriteOwnedRejectsBorrowedBackingBeforeStaging(t *testing.T) { + data := types.EncodeSlice([]int64{7}) + lease, err := vector.NewRefCountedBufferLease(data, int64(cap(data)), nil) + require.NoError(t, err) + vec, err := vector.NewBorrowedFixedVector(types.T_int64.ToType(), 1, data, lease) + require.NoError(t, err) + lease.Release() + bat := batch.NewOffHeap([]string{"v"}) + bat.Vecs[0] = vec + bat.SetRowCount(1) + + owned, err := new(Sinker).WriteOwned(context.Background(), bat) + require.False(t, owned) + require.ErrorContains(t, err, "unique-owned vector backing") + require.True(t, vec.HasBorrowedBacking()) + bat.Clean(nil) + require.Nil(t, lease.Bytes()) +} + func TestNewSinker(t *testing.T) { proc := testutil.NewProc(t) fs, err := fileservice.Get[fileservice.FileService]( diff --git a/pkg/pb/pipeline/pipeline.pb.go b/pkg/pb/pipeline/pipeline.pb.go index 769befbaed6b2..44d582c58a485 100644 --- a/pkg/pb/pipeline/pipeline.pb.go +++ b/pkg/pb/pipeline/pipeline.pb.go @@ -131,6 +131,33 @@ func (Status) EnumDescriptor() ([]byte, []int) { return fileDescriptor_7ac67a7adf3df9c7, []int{2} } +// ArrowExecutionScope is compile-produced positive authorization for Arrow +// ingestion. Its zero value is deliberately fail-closed on every CN. +type ArrowExecutionScope int32 + +const ( + ArrowExecutionScope_UnknownArrowExecutionScope ArrowExecutionScope = 0 + ArrowExecutionScope_ArrowLoadData ArrowExecutionScope = 1 +) + +var ArrowExecutionScope_name = map[int32]string{ + 0: "UnknownArrowExecutionScope", + 1: "ArrowLoadData", +} + +var ArrowExecutionScope_value = map[string]int32{ + "UnknownArrowExecutionScope": 0, + "ArrowLoadData": 1, +} + +func (x ArrowExecutionScope) String() string { + return proto.EnumName(ArrowExecutionScope_name, int32(x)) +} + +func (ArrowExecutionScope) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_7ac67a7adf3df9c7, []int{3} +} + type SampleFunc_SampleType int32 const ( @@ -4100,9 +4127,21 @@ type ExternalScan struct { ForeignScan *plan.ForeignScan `protobuf:"bytes,25,opt,name=foreign_scan,json=foreignScan,proto3" json:"foreign_scan,omitempty"` KafkaScan *plan.KafkaScan `protobuf:"bytes,26,opt,name=kafka_scan,json=kafkaScan,proto3" json:"kafka_scan,omitempty"` ParquetWholeFileFanout bool `protobuf:"varint,27,opt,name=parquet_whole_file_fanout,json=parquetWholeFileFanout,proto3" json:"parquet_whole_file_fanout,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ArrowExecutionScope ArrowExecutionScope `protobuf:"varint,28,opt,name=arrow_execution_scope,json=arrowExecutionScope,proto3,enum=pipeline.ArrowExecutionScope" json:"arrow_execution_scope,omitempty"` + ArrowObjectIdentities []*ArrowObjectIdentity `protobuf:"bytes,29,rep,name=arrow_object_identities,json=arrowObjectIdentities,proto3" json:"arrow_object_identities,omitempty"` + ArrowRecordBatchShards []*ArrowRecordBatchShard `protobuf:"bytes,30,rep,name=arrow_record_batch_shards,json=arrowRecordBatchShards,proto3" json:"arrow_record_batch_shards,omitempty"` + ArrowSchemaFingerprint []byte `protobuf:"bytes,31,opt,name=arrow_schema_fingerprint,json=arrowSchemaFingerprint,proto3" json:"arrow_schema_fingerprint,omitempty"` + ArrowConversionPlanVersion uint32 `protobuf:"varint,32,opt,name=arrow_conversion_plan_version,json=arrowConversionPlanVersion,proto3" json:"arrow_conversion_plan_version,omitempty"` + // Compile-time policy snapshot. Older CNs ignore this additive field and + // still reject Arrow format before execution, so mixed versions fail closed. + ArrowForceMaterialize bool `protobuf:"varint,33,opt,name=arrow_force_materialize,json=arrowForceMaterialize,proto3" json:"arrow_force_materialize,omitempty"` + // This records actual fanout placement independently of the SQL PARALLEL + // request. Fanout scopes clear that request after splitting, but every + // receiving CN must still enforce its own distributed Arrow rollout gate. + ArrowDistributedExecution bool `protobuf:"varint,34,opt,name=arrow_distributed_execution,json=arrowDistributedExecution,proto3" json:"arrow_distributed_execution,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ExternalScan) Reset() { *m = ExternalScan{} } @@ -4326,6 +4365,55 @@ func (m *ExternalScan) GetParquetWholeFileFanout() bool { return false } +func (m *ExternalScan) GetArrowExecutionScope() ArrowExecutionScope { + if m != nil { + return m.ArrowExecutionScope + } + return ArrowExecutionScope_UnknownArrowExecutionScope +} + +func (m *ExternalScan) GetArrowObjectIdentities() []*ArrowObjectIdentity { + if m != nil { + return m.ArrowObjectIdentities + } + return nil +} + +func (m *ExternalScan) GetArrowRecordBatchShards() []*ArrowRecordBatchShard { + if m != nil { + return m.ArrowRecordBatchShards + } + return nil +} + +func (m *ExternalScan) GetArrowSchemaFingerprint() []byte { + if m != nil { + return m.ArrowSchemaFingerprint + } + return nil +} + +func (m *ExternalScan) GetArrowConversionPlanVersion() uint32 { + if m != nil { + return m.ArrowConversionPlanVersion + } + return 0 +} + +func (m *ExternalScan) GetArrowForceMaterialize() bool { + if m != nil { + return m.ArrowForceMaterialize + } + return false +} + +func (m *ExternalScan) GetArrowDistributedExecution() bool { + if m != nil { + return m.ArrowDistributedExecution + } + return false +} + type TableScan struct { Types []plan.Type `protobuf:"bytes,1,rep,name=types,proto3" json:"types"` FilterExprs []*plan.Expr `protobuf:"bytes,2,rep,name=filter_exprs,json=filterExprs,proto3" json:"filter_exprs,omitempty"` @@ -6573,10 +6661,177 @@ func (m *ODKUForeignKeyCheck) GetEligibilityResultPos() int32 { return 0 } +type ArrowObjectIdentity struct { + FileIndex int32 `protobuf:"varint,1,opt,name=file_index,json=fileIndex,proto3" json:"file_index,omitempty"` + VersionId string `protobuf:"bytes,2,opt,name=version_id,json=versionId,proto3" json:"version_id,omitempty"` + Etag string `protobuf:"bytes,3,opt,name=etag,proto3" json:"etag,omitempty"` + Size int64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` + LastModifiedUnixNano int64 `protobuf:"varint,5,opt,name=last_modified_unix_nano,json=lastModifiedUnixNano,proto3" json:"last_modified_unix_nano,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ArrowObjectIdentity) Reset() { *m = ArrowObjectIdentity{} } +func (m *ArrowObjectIdentity) String() string { return proto.CompactTextString(m) } +func (*ArrowObjectIdentity) ProtoMessage() {} +func (*ArrowObjectIdentity) Descriptor() ([]byte, []int) { + return fileDescriptor_7ac67a7adf3df9c7, []int{58} +} +func (m *ArrowObjectIdentity) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ArrowObjectIdentity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ArrowObjectIdentity.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ArrowObjectIdentity) XXX_Merge(src proto.Message) { + xxx_messageInfo_ArrowObjectIdentity.Merge(m, src) +} +func (m *ArrowObjectIdentity) XXX_Size() int { + return m.ProtoSize() +} +func (m *ArrowObjectIdentity) XXX_DiscardUnknown() { + xxx_messageInfo_ArrowObjectIdentity.DiscardUnknown(m) +} + +var xxx_messageInfo_ArrowObjectIdentity proto.InternalMessageInfo + +func (m *ArrowObjectIdentity) GetFileIndex() int32 { + if m != nil { + return m.FileIndex + } + return 0 +} + +func (m *ArrowObjectIdentity) GetVersionId() string { + if m != nil { + return m.VersionId + } + return "" +} + +func (m *ArrowObjectIdentity) GetEtag() string { + if m != nil { + return m.Etag + } + return "" +} + +func (m *ArrowObjectIdentity) GetSize() int64 { + if m != nil { + return m.Size + } + return 0 +} + +func (m *ArrowObjectIdentity) GetLastModifiedUnixNano() int64 { + if m != nil { + return m.LastModifiedUnixNano + } + return 0 +} + +type ArrowRecordBatchShard struct { + FileIndex int32 `protobuf:"varint,1,opt,name=file_index,json=fileIndex,proto3" json:"file_index,omitempty"` + RecordBatchStart int32 `protobuf:"varint,2,opt,name=record_batch_start,json=recordBatchStart,proto3" json:"record_batch_start,omitempty"` + RecordBatchEnd int32 `protobuf:"varint,3,opt,name=record_batch_end,json=recordBatchEnd,proto3" json:"record_batch_end,omitempty"` + RequiredDictionaryBlockIndices []int32 `protobuf:"varint,4,rep,packed,name=required_dictionary_block_indices,json=requiredDictionaryBlockIndices,proto3" json:"required_dictionary_block_indices,omitempty"` + EstimatedRows int64 `protobuf:"varint,5,opt,name=estimated_rows,json=estimatedRows,proto3" json:"estimated_rows,omitempty"` + EstimatedWireBytes int64 `protobuf:"varint,6,opt,name=estimated_wire_bytes,json=estimatedWireBytes,proto3" json:"estimated_wire_bytes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ArrowRecordBatchShard) Reset() { *m = ArrowRecordBatchShard{} } +func (m *ArrowRecordBatchShard) String() string { return proto.CompactTextString(m) } +func (*ArrowRecordBatchShard) ProtoMessage() {} +func (*ArrowRecordBatchShard) Descriptor() ([]byte, []int) { + return fileDescriptor_7ac67a7adf3df9c7, []int{59} +} +func (m *ArrowRecordBatchShard) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ArrowRecordBatchShard) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ArrowRecordBatchShard.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ArrowRecordBatchShard) XXX_Merge(src proto.Message) { + xxx_messageInfo_ArrowRecordBatchShard.Merge(m, src) +} +func (m *ArrowRecordBatchShard) XXX_Size() int { + return m.ProtoSize() +} +func (m *ArrowRecordBatchShard) XXX_DiscardUnknown() { + xxx_messageInfo_ArrowRecordBatchShard.DiscardUnknown(m) +} + +var xxx_messageInfo_ArrowRecordBatchShard proto.InternalMessageInfo + +func (m *ArrowRecordBatchShard) GetFileIndex() int32 { + if m != nil { + return m.FileIndex + } + return 0 +} + +func (m *ArrowRecordBatchShard) GetRecordBatchStart() int32 { + if m != nil { + return m.RecordBatchStart + } + return 0 +} + +func (m *ArrowRecordBatchShard) GetRecordBatchEnd() int32 { + if m != nil { + return m.RecordBatchEnd + } + return 0 +} + +func (m *ArrowRecordBatchShard) GetRequiredDictionaryBlockIndices() []int32 { + if m != nil { + return m.RequiredDictionaryBlockIndices + } + return nil +} + +func (m *ArrowRecordBatchShard) GetEstimatedRows() int64 { + if m != nil { + return m.EstimatedRows + } + return 0 +} + +func (m *ArrowRecordBatchShard) GetEstimatedWireBytes() int64 { + if m != nil { + return m.EstimatedWireBytes + } + return 0 +} + func init() { proto.RegisterEnum("pipeline.Method", Method_name, Method_value) proto.RegisterEnum("pipeline.StreamTeardownMode", StreamTeardownMode_name, StreamTeardownMode_value) proto.RegisterEnum("pipeline.Status", Status_name, Status_value) + proto.RegisterEnum("pipeline.ArrowExecutionScope", ArrowExecutionScope_name, ArrowExecutionScope_value) proto.RegisterEnum("pipeline.SampleFunc_SampleType", SampleFunc_SampleType_name, SampleFunc_SampleType_value) proto.RegisterEnum("pipeline.SessionLoggerInfo_LogLevel", SessionLoggerInfo_LogLevel_name, SessionLoggerInfo_LogLevel_value) proto.RegisterEnum("pipeline.Pipeline_PipelineType", Pipeline_PipelineType_name, Pipeline_PipelineType_value) @@ -6642,560 +6897,587 @@ func init() { proto.RegisterType((*UuidToRegIdx)(nil), "pipeline.UuidToRegIdx") proto.RegisterType((*Apply)(nil), "pipeline.Apply") proto.RegisterType((*ODKUForeignKeyCheck)(nil), "pipeline.ODKUForeignKeyCheck") + proto.RegisterType((*ArrowObjectIdentity)(nil), "pipeline.ArrowObjectIdentity") + proto.RegisterType((*ArrowRecordBatchShard)(nil), "pipeline.ArrowRecordBatchShard") } func init() { proto.RegisterFile("pipeline.proto", fileDescriptor_7ac67a7adf3df9c7) } var fileDescriptor_7ac67a7adf3df9c7 = []byte{ - // 8763 bytes of a gzipped FileDescriptorProto + // 9155 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbc, 0x4d, 0x6c, 0x1d, 0x47, - 0xb6, 0x18, 0xec, 0x4b, 0xf2, 0xfe, 0x9d, 0xfb, 0xcb, 0x22, 0x45, 0x5d, 0xfd, 0xd8, 0xa2, 0xaf, - 0x2d, 0x99, 0x96, 0x65, 0x4a, 0xa6, 0xed, 0x19, 0xdb, 0x33, 0x1e, 0x3f, 0x8a, 0x92, 0x6c, 0x8e, - 0x45, 0x89, 0xd3, 0xa4, 0x9e, 0x01, 0x03, 0xdf, 0xd7, 0x68, 0x76, 0xd7, 0xbd, 0x6c, 0xb3, 0x6f, - 0x77, 0xab, 0xab, 0x5b, 0x22, 0xb5, 0x4a, 0x96, 0x09, 0xb2, 0xcf, 0x43, 0x56, 0x13, 0x24, 0x8b, - 0x04, 0x59, 0x06, 0x2f, 0xcb, 0xac, 0x1f, 0x82, 0x2c, 0x66, 0x95, 0x55, 0x10, 0x04, 0x33, 0xcb, - 0x00, 0x49, 0x10, 0x20, 0x41, 0x80, 0xe0, 0x01, 0xc1, 0x39, 0xa7, 0xaa, 0xbb, 0xef, 0xe5, 0x95, - 0x64, 0x3b, 0xc1, 0xdb, 0xe4, 0xed, 0xba, 0xcf, 0x39, 0x55, 0x5d, 0x5d, 0x75, 0x7e, 0xeb, 0x9c, - 0x2a, 0xe8, 0xc6, 0x7e, 0x2c, 0x03, 0x3f, 0x94, 0x9b, 0x71, 0x12, 0xa5, 0x91, 0x68, 0x98, 0xf7, - 0xcb, 0x1f, 0x8e, 0xfd, 0xf4, 0x38, 0x3b, 0xda, 0x74, 0xa3, 0xc9, 0xed, 0x71, 0x34, 0x8e, 0x6e, - 0x13, 0xc1, 0x51, 0x36, 0xa2, 0x37, 0x7a, 0xa1, 0x27, 0x6e, 0x78, 0x19, 0x82, 0xc8, 0x3d, 0x31, - 0xcf, 0x71, 0xe0, 0x84, 0xfa, 0xb9, 0x97, 0xfa, 0x13, 0xa9, 0x52, 0x67, 0x12, 0x6b, 0x40, 0x33, - 0x3d, 0xd5, 0xb8, 0xe1, 0xef, 0x6b, 0x50, 0xdf, 0x93, 0x4a, 0x39, 0x63, 0x29, 0x86, 0xb0, 0xa8, - 0x7c, 0x6f, 0x50, 0x59, 0xaf, 0x6c, 0x74, 0xb7, 0xfa, 0x9b, 0xf9, 0xb0, 0x0e, 0x52, 0x27, 0xcd, - 0x94, 0x85, 0x48, 0xa4, 0x71, 0x27, 0xde, 0x60, 0x61, 0x96, 0x66, 0x4f, 0xa6, 0xc7, 0x91, 0x67, - 0x21, 0x52, 0xf4, 0x61, 0x51, 0x26, 0xc9, 0x60, 0x71, 0xbd, 0xb2, 0xd1, 0xb6, 0xf0, 0x51, 0x08, - 0x58, 0xf2, 0x9c, 0xd4, 0x19, 0x2c, 0x11, 0x88, 0x9e, 0xc5, 0xbb, 0xd0, 0x8d, 0x93, 0xc8, 0xb5, - 0xfd, 0x70, 0x14, 0xd9, 0x84, 0xad, 0x12, 0xb6, 0x8d, 0xd0, 0xdd, 0x70, 0x14, 0xdd, 0x43, 0xaa, - 0x01, 0xd4, 0x9d, 0xd0, 0x09, 0xce, 0x94, 0x1c, 0xd4, 0x08, 0x6d, 0x5e, 0x45, 0x17, 0x16, 0x7c, - 0x6f, 0x50, 0x5f, 0xaf, 0x6c, 0x2c, 0x59, 0x0b, 0xbe, 0x87, 0xdf, 0xc8, 0x32, 0xdf, 0x1b, 0x34, - 0xf8, 0x1b, 0xf8, 0x2c, 0x86, 0xd0, 0x0e, 0xa5, 0xf4, 0x1e, 0x45, 0xa9, 0x25, 0xe3, 0xe0, 0x6c, - 0xd0, 0x5c, 0xaf, 0x6c, 0x34, 0xac, 0x29, 0x98, 0xb8, 0x0c, 0x0d, 0x4f, 0x1e, 0x65, 0xe3, 0x3d, - 0x35, 0x1e, 0xc0, 0x7a, 0x65, 0xa3, 0x69, 0xe5, 0xef, 0xe2, 0x10, 0x2e, 0x26, 0xf2, 0x69, 0x26, - 0x55, 0x2a, 0x3d, 0x3b, 0x95, 0x4e, 0xe2, 0x45, 0xcf, 0x43, 0x7b, 0x12, 0x79, 0x72, 0xd0, 0xa2, - 0x19, 0xb8, 0x5a, 0x9e, 0xa5, 0x44, 0x3a, 0x93, 0x43, 0x4d, 0xb4, 0x17, 0x79, 0xd2, 0xba, 0x90, - 0x37, 0x2e, 0x83, 0x85, 0x05, 0x6b, 0x8e, 0xeb, 0xca, 0xf8, 0x7c, 0xa7, 0xed, 0x1f, 0xd1, 0xe9, - 0xaa, 0x69, 0x3b, 0xd5, 0xe7, 0x57, 0x70, 0xb5, 0x18, 0xe9, 0x91, 0x93, 0xba, 0xc7, 0xb6, 0x9b, - 0x48, 0xcf, 0x4f, 0x6d, 0x37, 0xca, 0xc2, 0x74, 0xd0, 0x59, 0xaf, 0x6c, 0x74, 0xac, 0x4b, 0x39, - 0xcd, 0x5d, 0x24, 0xd9, 0x21, 0x8a, 0x1d, 0x24, 0x78, 0x45, 0x07, 0x47, 0x67, 0xa9, 0x54, 0x83, - 0x2e, 0x4d, 0xf4, 0xdc, 0x0e, 0xee, 0x22, 0x81, 0xf8, 0x12, 0xae, 0xe4, 0x7f, 0x35, 0x67, 0x00, - 0x3d, 0x1a, 0xc0, 0xc0, 0x90, 0x9c, 0xfb, 0xfe, 0x4b, 0x9b, 0xf3, 0xe7, 0xfb, 0xf4, 0xf9, 0x79, - 0xcd, 0xf9, 0xeb, 0xd7, 0xa1, 0xcb, 0xad, 0x14, 0x0e, 0x30, 0x74, 0xe5, 0x60, 0x99, 0x5a, 0x74, - 0x08, 0x7a, 0xa0, 0x81, 0xe2, 0x16, 0x08, 0x26, 0x73, 0xdc, 0x93, 0x82, 0x54, 0x10, 0x69, 0x9f, - 0x30, 0xdb, 0xee, 0x89, 0xa1, 0xfe, 0x62, 0xe9, 0x2f, 0x7e, 0x7f, 0xed, 0x8d, 0xe1, 0x13, 0x68, - 0xee, 0x44, 0x61, 0x28, 0xdd, 0x34, 0x4a, 0xc4, 0x35, 0x68, 0x99, 0xc5, 0xb1, 0xb5, 0xac, 0x54, - 0x2d, 0x30, 0xa0, 0x5d, 0x4f, 0xbc, 0x07, 0x3d, 0xd7, 0x50, 0xdb, 0x7e, 0xe8, 0xc9, 0x53, 0x12, - 0x96, 0xaa, 0xd5, 0xcd, 0xc1, 0xbb, 0x08, 0x1d, 0xfe, 0xe7, 0x45, 0xa8, 0x1f, 0x1c, 0x67, 0xa3, - 0x51, 0x20, 0xc5, 0xbb, 0xd0, 0xd1, 0x8f, 0x3b, 0x51, 0xb0, 0xeb, 0x9d, 0xea, 0x7e, 0xa7, 0x81, - 0x62, 0x1d, 0x5a, 0x1a, 0x70, 0x78, 0x16, 0x4b, 0xdd, 0x6d, 0x19, 0x34, 0xdd, 0xcf, 0x9e, 0x1f, - 0x92, 0x0c, 0x2e, 0x5a, 0xd3, 0xc0, 0x19, 0x2a, 0xe7, 0x94, 0xc4, 0x72, 0x9a, 0xca, 0xa1, 0xaf, - 0x6d, 0x07, 0xfe, 0x33, 0x69, 0xc9, 0xf1, 0x4e, 0x98, 0x92, 0x70, 0x56, 0xad, 0x32, 0x48, 0x6c, - 0xc1, 0x05, 0xc5, 0x4d, 0xec, 0xc4, 0x09, 0xc7, 0x52, 0xd9, 0x99, 0x1f, 0xa6, 0xbf, 0xf8, 0x64, - 0x50, 0x5b, 0x5f, 0xdc, 0x58, 0xb2, 0x56, 0x34, 0xd2, 0x22, 0xdc, 0x13, 0x42, 0x89, 0x3b, 0xb0, - 0x3a, 0xd3, 0x86, 0x9b, 0xd4, 0xd7, 0x17, 0x37, 0x16, 0x2d, 0x31, 0xd5, 0x64, 0x97, 0x5a, 0xdc, - 0x87, 0xe5, 0x24, 0x0b, 0x51, 0x85, 0x3d, 0xf0, 0x83, 0x54, 0x26, 0x07, 0xb1, 0x74, 0x49, 0xc8, - 0x5b, 0x5b, 0x17, 0x37, 0x49, 0xcb, 0x59, 0xb3, 0x68, 0xeb, 0x7c, 0x0b, 0x71, 0x2b, 0x9f, 0xbc, - 0xfb, 0xa7, 0x71, 0x42, 0x9a, 0xa0, 0xb5, 0x05, 0xdc, 0x01, 0x42, 0xac, 0x32, 0x5a, 0xdc, 0x84, - 0x65, 0x2f, 0x71, 0xfc, 0xd0, 0x76, 0x82, 0xc0, 0x3e, 0xca, 0xdc, 0x13, 0x99, 0x2a, 0xd2, 0x0e, - 0x0d, 0xab, 0x47, 0x88, 0xed, 0x20, 0xb8, 0xcb, 0x60, 0x71, 0x03, 0x7a, 0x2a, 0x4d, 0xfc, 0x70, - 0x6c, 0x1f, 0x3b, 0xea, 0xd8, 0x3e, 0x91, 0x67, 0xa4, 0x1c, 0x1a, 0x56, 0x87, 0xc1, 0xdf, 0x38, - 0xea, 0xf8, 0x5b, 0x79, 0x36, 0xfc, 0x9f, 0x0b, 0xd0, 0xb8, 0xe7, 0xab, 0x18, 0xb9, 0x4c, 0x5c, - 0x84, 0xfa, 0x28, 0x0b, 0xdd, 0x82, 0x87, 0x6a, 0xf8, 0xba, 0xeb, 0x89, 0x5f, 0x43, 0x2f, 0x88, - 0x5c, 0x27, 0xb0, 0x73, 0x76, 0x19, 0x2c, 0xac, 0x2f, 0x6e, 0xb4, 0xb6, 0x56, 0x0a, 0xad, 0x90, - 0xb3, 0xa3, 0xd5, 0x25, 0xda, 0x82, 0x3d, 0xbf, 0x84, 0x7e, 0x22, 0x27, 0x51, 0x2a, 0x4b, 0xcd, - 0x17, 0xa9, 0xb9, 0x28, 0x9a, 0x7f, 0x97, 0x38, 0xf1, 0x23, 0x54, 0x25, 0x3d, 0xa6, 0x2d, 0x9a, - 0x7f, 0x54, 0x5a, 0x51, 0x39, 0xb6, 0x7d, 0xef, 0xd4, 0xa6, 0x0f, 0x0c, 0x96, 0xd6, 0x17, 0x37, - 0xaa, 0xc5, 0xf2, 0xc8, 0xf1, 0xae, 0x77, 0xfa, 0x10, 0x31, 0xe2, 0x63, 0x58, 0x9b, 0x6d, 0xc2, - 0xbd, 0x0e, 0xaa, 0xd4, 0x66, 0x65, 0xaa, 0x8d, 0x45, 0x28, 0xf1, 0x36, 0xb4, 0x4d, 0xa3, 0x14, - 0x59, 0xb9, 0xc6, 0xcc, 0xa5, 0x4a, 0xac, 0x7c, 0x11, 0xea, 0xbe, 0xb2, 0x95, 0x1f, 0x9e, 0x90, - 0x8e, 0x6f, 0x58, 0x35, 0x5f, 0x1d, 0xf8, 0xe1, 0x89, 0xb8, 0x04, 0x8d, 0x44, 0xba, 0x8c, 0x69, - 0x10, 0xa6, 0x9e, 0x48, 0x97, 0x50, 0x17, 0x01, 0x1f, 0x6d, 0x37, 0x95, 0x5a, 0xd3, 0xd7, 0x12, - 0xe9, 0xee, 0xa4, 0x72, 0xa8, 0xa0, 0xba, 0x27, 0x93, 0xb1, 0x44, 0x65, 0x8f, 0x0d, 0x0f, 0x5c, - 0x27, 0xa4, 0x79, 0x6f, 0x58, 0xf9, 0x3b, 0x9a, 0x9a, 0xd8, 0x49, 0x52, 0xdf, 0x09, 0x48, 0xb4, - 0x1a, 0x96, 0x79, 0x15, 0x57, 0xa0, 0xa9, 0x52, 0x27, 0x49, 0xf1, 0xef, 0x48, 0xa4, 0xaa, 0x56, - 0x83, 0x00, 0x28, 0x95, 0x17, 0xa1, 0x2e, 0x43, 0x8f, 0x50, 0x4b, 0xbc, 0x92, 0x32, 0xf4, 0x76, - 0xbd, 0xd3, 0xe1, 0xbf, 0xac, 0x40, 0x67, 0x2f, 0x0b, 0x52, 0x7f, 0x3b, 0x19, 0x67, 0x72, 0x12, - 0xa6, 0x68, 0xa2, 0xee, 0xf9, 0x2a, 0xd5, 0x5f, 0xa6, 0x67, 0xb1, 0x01, 0xcd, 0xaf, 0x93, 0x28, - 0x8b, 0x89, 0x2b, 0x79, 0xa5, 0xcb, 0x5c, 0x59, 0x20, 0x91, 0x83, 0x1f, 0x27, 0x9e, 0x4c, 0xee, - 0x9e, 0x11, 0xed, 0xe2, 0x39, 0xda, 0x32, 0x5a, 0x5c, 0x85, 0xe6, 0x81, 0x8c, 0x9d, 0xc4, 0x41, - 0x16, 0x58, 0x22, 0xbb, 0x56, 0x00, 0xf0, 0x5f, 0x89, 0x78, 0xd7, 0xd3, 0x82, 0x6d, 0x5e, 0x87, - 0xff, 0xa4, 0x02, 0xcd, 0xed, 0xf1, 0x38, 0x91, 0x63, 0x27, 0x25, 0x23, 0x1b, 0xc5, 0x34, 0xde, - 0x45, 0x6b, 0x21, 0x8a, 0xc9, 0x90, 0xe3, 0x1f, 0xf0, 0x04, 0xd1, 0xb3, 0x78, 0x0b, 0x96, 0xe4, - 0xfc, 0x01, 0x11, 0x5c, 0xac, 0x41, 0xcd, 0x8d, 0xc2, 0x91, 0x3f, 0xd6, 0xe6, 0x5f, 0xbf, 0x89, - 0x2f, 0xa0, 0xc5, 0x4f, 0xcc, 0x03, 0x55, 0xb2, 0x7d, 0x97, 0xb8, 0x79, 0x3e, 0x82, 0x1d, 0xa2, - 0x40, 0x8e, 0xb0, 0xc0, 0xcd, 0x9f, 0x87, 0x7f, 0x5a, 0x82, 0x2a, 0xcd, 0x0c, 0xae, 0x0d, 0x9a, - 0x73, 0x5b, 0x3e, 0x73, 0x02, 0xb3, 0xa4, 0x08, 0xb8, 0xff, 0xcc, 0x09, 0xc4, 0x3a, 0x54, 0x71, - 0x08, 0x6a, 0xce, 0xc4, 0x32, 0x42, 0xdc, 0x80, 0x2a, 0x7e, 0x5d, 0x4d, 0x8f, 0x1e, 0xbf, 0x71, - 0x77, 0xe9, 0xaf, 0xfe, 0xc3, 0xb5, 0x37, 0x2c, 0x46, 0x8b, 0xf7, 0x60, 0xc9, 0x19, 0x8f, 0x15, - 0x09, 0xc2, 0x94, 0x2c, 0xe6, 0x23, 0xb5, 0x88, 0x40, 0x7c, 0x0a, 0x4d, 0x5e, 0x74, 0xa4, 0xae, - 0x12, 0xf5, 0xc5, 0x92, 0x9b, 0x54, 0xe6, 0x07, 0xab, 0xa0, 0xc4, 0xe5, 0xf2, 0x95, 0xd6, 0x40, - 0x24, 0x0e, 0x0d, 0xab, 0x00, 0xa0, 0x1f, 0x13, 0x27, 0x72, 0x3b, 0x08, 0x22, 0xf7, 0xc0, 0x7f, - 0x21, 0xb5, 0xd7, 0x33, 0x05, 0x13, 0x37, 0xa0, 0xbb, 0xcf, 0xfc, 0x6a, 0x49, 0x95, 0x05, 0xa9, - 0xd2, 0x9e, 0xd0, 0x0c, 0x54, 0x6c, 0x82, 0x98, 0x82, 0x1c, 0xd2, 0xef, 0x37, 0xd7, 0x17, 0x37, - 0x3a, 0xd6, 0x1c, 0x8c, 0x78, 0x07, 0x3a, 0x63, 0x9c, 0x69, 0x54, 0x70, 0xa3, 0xc0, 0x41, 0x27, - 0x69, 0x11, 0x9d, 0x28, 0x03, 0x7c, 0x10, 0x38, 0x63, 0x92, 0x90, 0xd8, 0x0f, 0x02, 0x7b, 0x22, - 0x27, 0xa4, 0xfd, 0x16, 0xad, 0x06, 0x01, 0xf6, 0xe4, 0x44, 0xbc, 0x0f, 0xcb, 0x44, 0x6c, 0x1f, - 0x9d, 0x15, 0x2a, 0xb2, 0x4d, 0xda, 0xa1, 0x4b, 0x88, 0xbb, 0x67, 0x5a, 0x47, 0x8a, 0xf7, 0xa1, - 0xef, 0x9d, 0x85, 0xce, 0xc4, 0x77, 0x6d, 0xd3, 0x3f, 0xb9, 0x2e, 0xa8, 0x76, 0x19, 0xfe, 0xb5, - 0x06, 0xa3, 0xe2, 0x91, 0x93, 0x38, 0x3d, 0xcb, 0x09, 0x6d, 0x25, 0x51, 0x42, 0xd1, 0x55, 0x41, - 0x5b, 0xb2, 0x42, 0x58, 0x43, 0x7e, 0x20, 0xd3, 0x5d, 0x4f, 0xa1, 0xfd, 0x3f, 0xdf, 0x88, 0x7c, - 0x93, 0x86, 0xd5, 0x9f, 0x6d, 0x30, 0xfc, 0x17, 0x4b, 0x50, 0xdb, 0x0d, 0x95, 0x4c, 0x52, 0x54, - 0x1c, 0xce, 0x68, 0x24, 0xdd, 0x54, 0xb2, 0xc2, 0x5e, 0xb2, 0xf2, 0x77, 0x5c, 0xbb, 0xc3, 0xe8, - 0xbb, 0xc4, 0x4f, 0xe5, 0xc1, 0xc7, 0x5a, 0x32, 0x0a, 0x00, 0x9a, 0x12, 0xc7, 0xf3, 0x6c, 0x43, - 0x6d, 0x27, 0xd1, 0x73, 0x45, 0x4a, 0xa4, 0x61, 0xf5, 0x1c, 0xcf, 0xdb, 0xd6, 0x70, 0x2b, 0x7a, - 0xae, 0xc4, 0xdb, 0xb0, 0x98, 0xc8, 0x11, 0xc9, 0x49, 0x6b, 0xab, 0xc7, 0xbc, 0xf8, 0xf8, 0xe8, - 0x07, 0xe9, 0xa6, 0x96, 0x1c, 0x59, 0x88, 0x13, 0xab, 0x50, 0x75, 0xd2, 0x34, 0x61, 0xde, 0x6a, - 0x5a, 0xfc, 0x22, 0x36, 0x61, 0x85, 0x94, 0x55, 0xea, 0x47, 0xa1, 0x9d, 0x3a, 0x47, 0x81, 0xa4, - 0x99, 0x60, 0x43, 0xbc, 0x9c, 0xa3, 0x0e, 0x11, 0x83, 0xf3, 0xb0, 0x05, 0x17, 0x66, 0xe9, 0x43, - 0x67, 0x22, 0x15, 0xd9, 0xe1, 0xa6, 0xb5, 0x32, 0xdd, 0xe2, 0x11, 0xa2, 0x90, 0x11, 0x8a, 0x36, - 0xa8, 0xee, 0x1a, 0xa4, 0x39, 0xda, 0x39, 0x10, 0xb5, 0xe1, 0x05, 0xa8, 0xf9, 0xca, 0x96, 0xa1, - 0xa7, 0x35, 0x70, 0xd5, 0x57, 0xf7, 0x43, 0x4f, 0x7c, 0x00, 0x4d, 0xfe, 0x8a, 0x27, 0x47, 0x64, - 0x47, 0x5b, 0x5b, 0x5d, 0x2d, 0x6a, 0x08, 0xbe, 0x27, 0x47, 0x56, 0x23, 0xd5, 0x4f, 0xe8, 0x63, - 0xa5, 0x91, 0x2d, 0x4f, 0x53, 0x99, 0x84, 0x4e, 0xa0, 0x8d, 0x29, 0xa4, 0xd1, 0x7d, 0x0d, 0x11, - 0x9f, 0xc2, 0x45, 0x83, 0xb5, 0x55, 0x3a, 0x49, 0xed, 0x2c, 0xf4, 0x4f, 0xed, 0xd0, 0x09, 0x23, - 0xf2, 0xa0, 0x17, 0xad, 0x55, 0x83, 0x3e, 0x48, 0x27, 0xe9, 0x93, 0xd0, 0x3f, 0x7d, 0xe4, 0x84, - 0x91, 0xd8, 0x80, 0x7e, 0xde, 0x2c, 0x7d, 0x41, 0x3f, 0x4c, 0xcc, 0xd5, 0xb4, 0xba, 0x06, 0x7e, - 0xf8, 0x02, 0xff, 0x95, 0x78, 0xab, 0x44, 0x19, 0x8d, 0x46, 0xc8, 0x5b, 0x4a, 0xba, 0xe4, 0x06, - 0x57, 0xad, 0x95, 0x82, 0xfe, 0x31, 0xe1, 0x0e, 0xa4, 0x3b, 0xfc, 0xcb, 0x0a, 0xb4, 0x48, 0xa0, - 0x9f, 0xc4, 0x1e, 0xea, 0xce, 0x77, 0xa0, 0x33, 0xbd, 0xe8, 0xcc, 0x37, 0x6d, 0xa7, 0xbc, 0xe2, - 0x6b, 0x50, 0xdb, 0x76, 0x71, 0xf2, 0x88, 0x71, 0x3a, 0x96, 0x7e, 0x13, 0xbf, 0x84, 0x5e, 0x46, - 0xdd, 0xd8, 0x6e, 0x7a, 0x6a, 0x07, 0xa8, 0x73, 0x59, 0x43, 0x69, 0xae, 0xe0, 0x6f, 0xec, 0xa4, - 0xa7, 0x56, 0x27, 0x33, 0x8f, 0x0f, 0x51, 0x1b, 0xdf, 0x81, 0xd5, 0x44, 0x22, 0xc7, 0xd8, 0x2f, - 0x64, 0x12, 0xd9, 0xa9, 0x9c, 0xc4, 0x51, 0x42, 0x16, 0x1c, 0x67, 0x51, 0x30, 0xee, 0x7b, 0x99, - 0x44, 0x87, 0x1a, 0x33, 0x7c, 0x13, 0xaa, 0xdb, 0x49, 0xe2, 0x9c, 0x11, 0x6b, 0xe1, 0xc3, 0xa0, - 0x42, 0xb2, 0xc9, 0x2f, 0x43, 0x17, 0x16, 0xf7, 0x9c, 0x58, 0x5c, 0x87, 0x85, 0x49, 0x4c, 0x98, - 0xd6, 0xd6, 0x85, 0x92, 0x42, 0x73, 0xe2, 0xcd, 0xbd, 0xf8, 0x7e, 0x98, 0x26, 0x67, 0xd6, 0xc2, - 0x24, 0xbe, 0xfc, 0x29, 0xd4, 0xf5, 0x2b, 0x86, 0x81, 0x28, 0xe8, 0x15, 0x9a, 0x61, 0x7c, 0xc4, - 0x0f, 0x3c, 0x73, 0x82, 0xcc, 0xb8, 0xae, 0xfc, 0xf2, 0xc5, 0xc2, 0x67, 0x95, 0xe1, 0x7f, 0x5f, - 0x82, 0xc6, 0x3d, 0x19, 0x48, 0xfa, 0xf7, 0x21, 0xb4, 0xcb, 0x52, 0x61, 0xe6, 0x6d, 0x4a, 0x52, - 0x86, 0xd0, 0x66, 0x5f, 0x82, 0x5a, 0x49, 0x2d, 0x76, 0x53, 0x30, 0x34, 0x72, 0xbb, 0xec, 0xa4, - 0x91, 0xbc, 0x75, 0x2c, 0xf3, 0x8a, 0x98, 0x47, 0x1a, 0xb3, 0xc4, 0x18, 0xfd, 0x2a, 0xae, 0x02, - 0x24, 0xd1, 0x73, 0xdb, 0x67, 0x83, 0xce, 0xb6, 0xb1, 0x91, 0x44, 0xcf, 0x77, 0xd1, 0xa4, 0xff, - 0x8d, 0x88, 0xd9, 0x2f, 0x61, 0x50, 0x12, 0x33, 0x0c, 0x15, 0x6c, 0x3f, 0xe4, 0x90, 0x48, 0x4b, - 0x5c, 0xd1, 0x27, 0x45, 0x12, 0xbb, 0x21, 0x45, 0x43, 0x46, 0x79, 0x34, 0x5f, 0xa1, 0x3c, 0xe6, - 0xea, 0x22, 0x98, 0xaf, 0x8b, 0xee, 0x02, 0x1c, 0xc8, 0xf1, 0x44, 0x86, 0xe9, 0x9e, 0x13, 0x0f, - 0x5a, 0xb4, 0xf0, 0xc3, 0x62, 0xe1, 0xcd, 0x6a, 0x6d, 0x16, 0x44, 0xcc, 0x05, 0xa5, 0x56, 0xe8, - 0xe7, 0xb9, 0x4e, 0x68, 0xa7, 0x49, 0x16, 0xba, 0x4e, 0xca, 0xf1, 0x6d, 0xc3, 0x6a, 0xb9, 0x4e, - 0x78, 0xa8, 0x41, 0x25, 0x85, 0xd1, 0x29, 0x2b, 0x8c, 0x1b, 0xd0, 0x8b, 0x13, 0x7f, 0xe2, 0x24, - 0x67, 0x68, 0x2d, 0x68, 0x31, 0x58, 0xf4, 0x3a, 0x1a, 0xfc, 0xad, 0x3c, 0xdb, 0xf5, 0x4e, 0x2f, - 0x7f, 0x09, 0xbd, 0x99, 0x01, 0xfc, 0x24, 0xbe, 0xfb, 0x07, 0x55, 0x68, 0xee, 0x27, 0x52, 0x2b, - 0xf9, 0x6b, 0xd0, 0x52, 0xee, 0xb1, 0x9c, 0x38, 0xac, 0x1b, 0xb8, 0x07, 0x60, 0x10, 0xe9, 0x85, - 0x29, 0x35, 0xb6, 0xf0, 0x1a, 0x35, 0xd6, 0x87, 0x45, 0xf6, 0x17, 0x51, 0x98, 0xf0, 0xb1, 0xd0, - 0xdd, 0x4b, 0x65, 0xdd, 0xbd, 0x0e, 0xed, 0x63, 0x47, 0xd9, 0x4e, 0x96, 0x46, 0xb6, 0x1b, 0x05, - 0xc4, 0x74, 0x0d, 0x0b, 0x8e, 0x1d, 0xb5, 0x9d, 0xa5, 0xd1, 0x4e, 0x14, 0x88, 0x37, 0x01, 0xdc, - 0x28, 0xd0, 0x6a, 0x48, 0x3b, 0xcb, 0x4d, 0x37, 0x0a, 0x58, 0xf7, 0x20, 0x57, 0x4a, 0x95, 0xfa, - 0x13, 0x47, 0x2f, 0xa9, 0x8e, 0xb8, 0xeb, 0xa4, 0x0a, 0x97, 0x73, 0x94, 0x15, 0x3d, 0xe7, 0x50, - 0xfb, 0x0e, 0x74, 0xdd, 0x68, 0x12, 0xdb, 0x31, 0xce, 0x2c, 0xb9, 0x6e, 0x8d, 0x73, 0xd1, 0x50, - 0x1b, 0x29, 0xf6, 0x4f, 0x24, 0x3b, 0x93, 0x5b, 0xd0, 0x73, 0x83, 0x4c, 0xa5, 0x32, 0x41, 0x1b, - 0x2e, 0xe7, 0x07, 0x50, 0x1d, 0x4d, 0xa2, 0x1d, 0xd0, 0x21, 0x74, 0x7c, 0x65, 0x47, 0x81, 0x67, - 0xb3, 0x82, 0xd2, 0x7c, 0xd6, 0xf2, 0xd5, 0xe3, 0xc0, 0xd3, 0x2a, 0x92, 0x69, 0x42, 0xf9, 0xdc, - 0xd0, 0xb4, 0x0c, 0xcd, 0x23, 0xf9, 0x5c, 0xd3, 0xbc, 0x4c, 0xa1, 0xb5, 0x5f, 0xa6, 0xd0, 0x70, - 0x3e, 0x70, 0x42, 0x53, 0x27, 0x19, 0x93, 0xd6, 0x0e, 0x38, 0x0e, 0x62, 0xfe, 0x5a, 0x3e, 0x76, - 0xd4, 0x21, 0x61, 0x0e, 0x34, 0x02, 0xa3, 0x1e, 0x4d, 0x8b, 0x93, 0x17, 0x66, 0x93, 0x23, 0x99, - 0xd0, 0x4a, 0x30, 0xc7, 0x09, 0x46, 0x5a, 0xd1, 0xf3, 0x47, 0x84, 0xc2, 0x15, 0xb9, 0x09, 0xcb, - 0xba, 0x89, 0xe3, 0xa6, 0xfe, 0x33, 0x49, 0xe4, 0x3d, 0x22, 0xef, 0x31, 0x62, 0x9b, 0xe0, 0x48, - 0xfb, 0x7e, 0x4e, 0xab, 0x35, 0x0b, 0xd2, 0xf6, 0x79, 0x4f, 0x20, 0xef, 0x7a, 0xd7, 0xdb, 0x89, - 0x82, 0xe1, 0xbf, 0x5b, 0x80, 0xfa, 0x7e, 0xa4, 0xd2, 0x7b, 0x93, 0xc0, 0x88, 0x73, 0xe5, 0xa7, - 0x8a, 0xf3, 0xc2, 0x7c, 0x71, 0x9e, 0x23, 0x50, 0x8b, 0x73, 0x04, 0x0a, 0x8d, 0x64, 0x99, 0x8e, - 0x04, 0x81, 0xc3, 0x87, 0x6e, 0x41, 0x48, 0xc2, 0x70, 0x05, 0x5d, 0x56, 0xdb, 0x63, 0xfd, 0xcb, - 0x4c, 0xdb, 0xf0, 0x95, 0xd6, 0xbd, 0x8c, 0xf4, 0x49, 0xae, 0xb4, 0x3f, 0xdb, 0xf0, 0x95, 0x96, - 0xb3, 0xcf, 0xe1, 0x52, 0xde, 0xd2, 0x7e, 0xee, 0xa7, 0xc7, 0x51, 0x96, 0xda, 0x23, 0x8a, 0xd5, - 0x95, 0x8e, 0xf6, 0xd6, 0x4c, 0x4f, 0xdf, 0x31, 0x9a, 0x23, 0x79, 0x72, 0xaf, 0x47, 0x59, 0x10, - 0xd8, 0xa9, 0x3c, 0x4d, 0x35, 0xdb, 0x0e, 0x78, 0x6e, 0xf4, 0xbc, 0x3d, 0xc8, 0x82, 0xe0, 0x50, - 0x9e, 0xa6, 0x68, 0x1a, 0x1b, 0x23, 0xfd, 0x32, 0xfc, 0x87, 0x4b, 0x00, 0x0f, 0x23, 0xf7, 0x84, - 0x57, 0x1e, 0x63, 0x48, 0xa3, 0xbd, 0xb5, 0x75, 0xa9, 0xa7, 0xac, 0xb3, 0xc5, 0x16, 0xac, 0x99, - 0xff, 0x47, 0x99, 0xc3, 0x78, 0x96, 0xd5, 0xaf, 0x56, 0x1e, 0x42, 0x63, 0x79, 0x4f, 0x86, 0x74, - 0xaf, 0xf8, 0xac, 0x98, 0x5b, 0x6c, 0x93, 0x9e, 0xc5, 0x34, 0xb7, 0xf3, 0xc2, 0x89, 0x4e, 0xd1, - 0xfc, 0xf0, 0x2c, 0x16, 0x77, 0xe0, 0x42, 0x22, 0x47, 0x89, 0x54, 0xc7, 0x76, 0xaa, 0xca, 0x1f, - 0xe3, 0x50, 0x72, 0x59, 0x23, 0x0f, 0x55, 0xfe, 0xad, 0x3b, 0x70, 0x81, 0x67, 0x6a, 0x76, 0x78, - 0x6c, 0xab, 0x96, 0x19, 0x59, 0x1e, 0xdd, 0x9b, 0x40, 0x1b, 0xc3, 0x6c, 0x7f, 0x4c, 0x6c, 0x11, - 0xd0, 0x64, 0x1c, 0x05, 0x12, 0xbd, 0xd7, 0x9d, 0x63, 0x27, 0x1c, 0xa3, 0xce, 0xd2, 0x93, 0x5f, - 0x00, 0xc4, 0x10, 0x96, 0xf6, 0x22, 0x4f, 0xd2, 0x54, 0x77, 0xb7, 0xba, 0x9b, 0xb4, 0xc5, 0x8c, - 0x33, 0x49, 0x7b, 0x91, 0x84, 0x13, 0xef, 0x01, 0x75, 0xc7, 0xec, 0x77, 0x5e, 0x2f, 0x34, 0x10, - 0x49, 0x3c, 0x78, 0x07, 0x2e, 0x14, 0x23, 0xb1, 0x9d, 0xd4, 0x4e, 0x8f, 0x25, 0xa9, 0x7e, 0x56, - 0x0d, 0xcb, 0xf9, 0xa0, 0xb6, 0xd3, 0xc3, 0x63, 0x89, 0x66, 0x60, 0x03, 0xea, 0xd1, 0xd1, 0x0f, - 0x36, 0x0a, 0x42, 0x6b, 0xbe, 0x20, 0xd4, 0xa2, 0xa3, 0x1f, 0x2c, 0x39, 0x12, 0xbf, 0x28, 0x9b, - 0xcd, 0x99, 0xa9, 0x69, 0xd3, 0xd4, 0xac, 0xe6, 0xf8, 0xd2, 0xec, 0x0c, 0x3f, 0x83, 0x1a, 0xfe, - 0xce, 0xe3, 0x58, 0x6c, 0x42, 0x9d, 0xc5, 0x51, 0x69, 0x37, 0x67, 0xb5, 0xb0, 0x76, 0x05, 0xef, - 0x58, 0x86, 0x68, 0x68, 0x41, 0x2f, 0x37, 0x1d, 0x4f, 0x42, 0xff, 0x69, 0x26, 0xc5, 0x57, 0xb0, - 0x1c, 0x27, 0x52, 0xb3, 0xbd, 0x9d, 0x9d, 0xa0, 0xf3, 0xa6, 0x25, 0x78, 0x55, 0x73, 0x69, 0xde, - 0xe2, 0x04, 0x39, 0xb4, 0x1b, 0x4f, 0xbd, 0x0f, 0xbf, 0x87, 0x8b, 0x39, 0xc5, 0x81, 0x74, 0xa3, - 0xd0, 0x73, 0x92, 0x33, 0xb2, 0xf2, 0x33, 0x7d, 0xab, 0x9f, 0xd2, 0xf7, 0x01, 0xf5, 0xfd, 0x5f, - 0x2b, 0xd0, 0x7a, 0x90, 0xbd, 0x78, 0x71, 0xc6, 0xb2, 0x24, 0xda, 0x50, 0x79, 0x44, 0x1d, 0x2c, - 0x58, 0x95, 0x47, 0xe8, 0x88, 0xee, 0x9f, 0xa0, 0x5c, 0x13, 0x9f, 0x37, 0x2d, 0xfd, 0x86, 0x01, - 0xf2, 0xfe, 0xc9, 0xe1, 0x2b, 0x38, 0x9a, 0xd1, 0x18, 0x20, 0xdd, 0xcd, 0xfc, 0x00, 0xdd, 0x24, - 0xcd, 0xbc, 0xf9, 0x3b, 0x86, 0x9c, 0xbb, 0x23, 0x1e, 0xca, 0x83, 0x24, 0x9a, 0xf0, 0x64, 0x69, - 0x95, 0x31, 0x07, 0x23, 0xbe, 0x86, 0x15, 0xbd, 0x81, 0xa7, 0xb5, 0x82, 0xad, 0x62, 0xe9, 0x12, - 0xeb, 0xfe, 0xa4, 0x4d, 0xbf, 0xe1, 0xdf, 0xad, 0x41, 0x03, 0x43, 0xcb, 0xdf, 0x46, 0x7e, 0x28, - 0xee, 0x40, 0xf3, 0x87, 0xc8, 0x0f, 0x79, 0xb7, 0x81, 0x93, 0x1c, 0x2b, 0xdc, 0xd7, 0xa3, 0xc8, - 0x93, 0x9b, 0x48, 0x43, 0xfb, 0x0c, 0x8d, 0x1f, 0xf4, 0x93, 0x36, 0x4f, 0x89, 0x3f, 0x3e, 0x4e, - 0x6d, 0x04, 0x6a, 0xdd, 0xda, 0xf2, 0x95, 0x85, 0x30, 0xea, 0xf5, 0x2a, 0x00, 0xc5, 0xb4, 0x51, - 0x68, 0xc7, 0x27, 0x3a, 0xae, 0x6b, 0x20, 0xe4, 0x71, 0xb8, 0x7f, 0x82, 0xb2, 0xe7, 0x2b, 0x5b, - 0xef, 0x6b, 0x69, 0x1f, 0xbc, 0x14, 0xd7, 0xbf, 0x0b, 0x5d, 0xf4, 0x8f, 0xd4, 0x89, 0x1f, 0xdb, - 0x71, 0x12, 0x1d, 0x99, 0x49, 0x41, 0xaf, 0xe9, 0xe0, 0xc4, 0x8f, 0xf7, 0x11, 0x46, 0x6e, 0x89, - 0xde, 0x2d, 0x43, 0xb5, 0xcd, 0xf6, 0x1f, 0x34, 0x08, 0xe7, 0x97, 0xb6, 0xc4, 0x02, 0x8e, 0x12, - 0xea, 0xe4, 0x6e, 0xd4, 0x13, 0x19, 0x50, 0x38, 0x70, 0x09, 0x1a, 0x28, 0x0c, 0x84, 0x6a, 0x30, - 0xca, 0x8d, 0x18, 0xf5, 0x3e, 0x40, 0x20, 0x47, 0xa9, 0x8d, 0x5c, 0xc6, 0x1b, 0x00, 0x33, 0x5b, - 0x4f, 0x88, 0xdd, 0x41, 0xa4, 0xf8, 0x00, 0x5a, 0x3c, 0x0b, 0x4c, 0x0b, 0xe7, 0x68, 0x81, 0xd0, - 0x4c, 0x7c, 0x13, 0x5a, 0x61, 0x14, 0xda, 0xf2, 0x29, 0x51, 0x6b, 0xb9, 0x9d, 0xea, 0x38, 0x8c, - 0xc2, 0xfb, 0x4f, 0x91, 0x58, 0xdc, 0xd6, 0x63, 0xe0, 0x3d, 0x98, 0xf6, 0x4b, 0xf6, 0x60, 0x68, - 0x24, 0xbc, 0x1b, 0xf1, 0x91, 0x19, 0x09, 0xb7, 0xe8, 0xbc, 0xa4, 0x05, 0x8f, 0x87, 0x9b, 0xac, - 0x43, 0x9b, 0xd6, 0x7d, 0xe2, 0xc4, 0x76, 0xea, 0x8c, 0xb5, 0x55, 0x07, 0x84, 0xed, 0x39, 0xf1, - 0xa1, 0x33, 0x16, 0x16, 0x5c, 0x9a, 0xe1, 0xb7, 0x23, 0x64, 0x5d, 0x9e, 0xb5, 0x9e, 0xd9, 0xc3, - 0x99, 0xcf, 0x75, 0x6b, 0x53, 0x5c, 0x47, 0x2c, 0x4f, 0xb3, 0xfb, 0x39, 0x5c, 0x92, 0x13, 0xca, - 0x7e, 0x4c, 0xe2, 0x44, 0x2a, 0x35, 0xe5, 0x9a, 0xf5, 0xd9, 0xc6, 0x21, 0xc1, 0x4e, 0x8e, 0xcf, - 0xfd, 0xb3, 0x77, 0xa1, 0xeb, 0xa8, 0x68, 0x64, 0x9b, 0x29, 0x0f, 0x28, 0x97, 0x51, 0xb5, 0xda, - 0x08, 0xb5, 0x78, 0xa2, 0x03, 0x34, 0xe8, 0x44, 0xa5, 0x87, 0x2a, 0x47, 0x29, 0xe5, 0x31, 0x1a, - 0x56, 0x07, 0xc1, 0x3c, 0x10, 0x39, 0x4a, 0x87, 0xff, 0x74, 0x01, 0x1a, 0x0f, 0xa3, 0x28, 0xfe, - 0x99, 0x32, 0x50, 0xe6, 0xad, 0x85, 0x97, 0xf3, 0xd6, 0xe2, 0x34, 0x6f, 0xcd, 0xf0, 0xc0, 0xd2, - 0x8f, 0xe7, 0x81, 0xea, 0x4f, 0xe6, 0x81, 0xda, 0xcf, 0xe0, 0x81, 0xfa, 0x2c, 0x0f, 0x0c, 0xef, - 0x40, 0xf5, 0x40, 0xa6, 0x8f, 0x63, 0xb4, 0x66, 0xc6, 0x2f, 0x36, 0x86, 0x60, 0xca, 0x9a, 0x69, - 0x9f, 0x58, 0x0d, 0xff, 0xba, 0x05, 0xcd, 0x7b, 0xd2, 0xcb, 0x78, 0x66, 0xcb, 0xf3, 0x54, 0x79, - 0xf9, 0x3c, 0x2d, 0x4c, 0xcf, 0x13, 0x9a, 0x4e, 0x23, 0x83, 0x73, 0x36, 0x50, 0x1b, 0x46, 0x04, - 0x51, 0x58, 0x0b, 0x09, 0xd4, 0xbb, 0x90, 0x53, 0xf3, 0x99, 0x0b, 0xe0, 0xab, 0xb9, 0xb9, 0xfa, - 0xf3, 0xb8, 0x79, 0x5a, 0x8f, 0x9d, 0xdb, 0x9f, 0x7c, 0xed, 0xf4, 0xce, 0xea, 0xb0, 0xc6, 0x39, - 0x1d, 0xf6, 0x10, 0x56, 0xa2, 0xd0, 0xf6, 0xb2, 0x38, 0xf0, 0x31, 0x2e, 0x24, 0xbf, 0x3a, 0x0a, - 0xc9, 0x9d, 0xa0, 0x8c, 0x68, 0xce, 0xa3, 0x8f, 0xc3, 0x7b, 0x86, 0x88, 0xf7, 0x4a, 0xac, 0xe5, - 0x68, 0x16, 0x84, 0x22, 0xe4, 0xe1, 0xd2, 0x90, 0x27, 0x40, 0x3e, 0x2c, 0xa7, 0x76, 0xdb, 0x04, - 0xdd, 0x89, 0x02, 0xb2, 0x6d, 0x9f, 0x41, 0xaf, 0xa0, 0x62, 0x66, 0x6a, 0xbd, 0x84, 0x99, 0x3a, - 0xa6, 0x21, 0xf3, 0xd3, 0xdf, 0x84, 0xde, 0xfa, 0x10, 0x56, 0xcc, 0x16, 0x90, 0x76, 0x67, 0x68, - 0x05, 0xbb, 0xc4, 0x41, 0x7d, 0xbd, 0xeb, 0x43, 0x9e, 0x0c, 0x2d, 0xd1, 0xaf, 0x60, 0xb5, 0x44, - 0x8e, 0xec, 0x5b, 0xd6, 0x5f, 0x65, 0x5e, 0x59, 0xce, 0xdb, 0xe2, 0xeb, 0x43, 0xde, 0xc3, 0x6f, - 0x79, 0x32, 0x30, 0x1f, 0xd2, 0xd1, 0x49, 0xd3, 0x93, 0x81, 0x4e, 0x3d, 0xee, 0xc1, 0xbb, 0x18, - 0xc9, 0x21, 0xde, 0x75, 0xe2, 0x34, 0x4b, 0xa4, 0x1d, 0x07, 0x8e, 0x2b, 0x8f, 0xa3, 0xc0, 0x93, - 0x49, 0x31, 0xb8, 0x65, 0x1a, 0xdc, 0xb5, 0x28, 0xc0, 0x70, 0x66, 0x87, 0x29, 0xf7, 0x0b, 0x42, - 0x33, 0xd6, 0x6d, 0x78, 0xeb, 0x5c, 0x77, 0x68, 0xea, 0x8a, 0x8e, 0x04, 0x75, 0x74, 0x69, 0xba, - 0x23, 0x24, 0x31, 0x5d, 0x7c, 0x04, 0x17, 0x78, 0xed, 0x98, 0xb9, 0x4f, 0xa4, 0x8c, 0xed, 0xc0, - 0x51, 0xe9, 0x60, 0x85, 0xdd, 0x0a, 0x42, 0x12, 0x03, 0x7f, 0x2b, 0x65, 0xfc, 0xd0, 0xe1, 0xaf, - 0x72, 0x13, 0x1d, 0x79, 0x50, 0x9b, 0xa9, 0xb9, 0x5d, 0xe5, 0xaf, 0x12, 0x15, 0x87, 0x1f, 0xd8, - 0xb8, 0x34, 0xc9, 0xbf, 0x86, 0x2b, 0x53, 0x5d, 0x4c, 0x9c, 0xe4, 0xa4, 0x70, 0xc5, 0x07, 0x17, - 0x68, 0xde, 0x2e, 0x96, 0xda, 0xef, 0x11, 0x81, 0x9e, 0xc5, 0x8f, 0x61, 0x0d, 0x03, 0xd3, 0xc8, - 0x3b, 0xc9, 0x66, 0x82, 0xb6, 0x35, 0x1a, 0x34, 0x86, 0xad, 0x8f, 0xbd, 0x93, 0x6c, 0x2a, 0x70, - 0xfb, 0x25, 0x0c, 0xa6, 0x68, 0xed, 0x84, 0x36, 0xe7, 0xed, 0x38, 0x52, 0x83, 0x8b, 0xbc, 0x1f, - 0x54, 0xde, 0x51, 0xe4, 0xad, 0xfb, 0xfd, 0x48, 0x89, 0x07, 0xb0, 0x1e, 0x1f, 0x9f, 0x29, 0x9f, - 0x92, 0x89, 0xe4, 0xd0, 0x9f, 0xef, 0x60, 0x40, 0x1d, 0x5c, 0x35, 0x74, 0xec, 0xf7, 0xcf, 0xf4, - 0xf3, 0x19, 0x5c, 0x32, 0x8c, 0x75, 0x2c, 0xdd, 0x93, 0xe9, 0x19, 0xbb, 0x44, 0x33, 0x76, 0x41, - 0x73, 0x14, 0xe2, 0x4b, 0xb3, 0xb5, 0x01, 0x7d, 0xb2, 0x77, 0xf6, 0x28, 0xca, 0x42, 0xfd, 0xa7, - 0x97, 0xe9, 0x4f, 0xbb, 0x04, 0x7f, 0x80, 0x60, 0xfa, 0xc9, 0x0d, 0xe8, 0x93, 0xb5, 0x64, 0xa9, - 0x67, 0xca, 0x2b, 0x4c, 0x89, 0x70, 0x2d, 0xe8, 0x48, 0xf9, 0x29, 0x5c, 0xd4, 0x44, 0x23, 0x3f, - 0x74, 0x82, 0xf2, 0xcf, 0x5c, 0x65, 0x37, 0x9f, 0xd1, 0x0f, 0x10, 0x5b, 0xfc, 0xc4, 0xef, 0x40, - 0x8c, 0xa2, 0x44, 0xfa, 0xe3, 0x90, 0xc2, 0x5a, 0xfa, 0x13, 0x35, 0x78, 0x93, 0x64, 0xe3, 0xcd, - 0xc2, 0xcf, 0x7f, 0x7c, 0xef, 0xdb, 0x27, 0x0f, 0x98, 0xee, 0x5b, 0x79, 0x46, 0xff, 0xa3, 0x25, - 0xb3, 0x3f, 0x9a, 0x06, 0xab, 0xe1, 0x3f, 0xaa, 0x41, 0x97, 0xac, 0xf1, 0xdf, 0x1a, 0x81, 0xbf, - 0x35, 0x02, 0xff, 0x2f, 0x18, 0x81, 0x9b, 0xb0, 0xec, 0x87, 0x71, 0x96, 0xa2, 0x04, 0x29, 0x3b, - 0xe3, 0x28, 0x6e, 0x99, 0xb7, 0x9b, 0x08, 0xf1, 0xad, 0x3c, 0x53, 0x1c, 0xc2, 0x0d, 0xff, 0x4e, - 0x05, 0xea, 0xfb, 0x49, 0xe4, 0x65, 0x6e, 0xfa, 0x33, 0xa5, 0x62, 0x9a, 0xdb, 0x16, 0x5f, 0xc7, - 0x6d, 0x4b, 0xe7, 0x3c, 0xba, 0x7f, 0x5e, 0x81, 0xa6, 0x1e, 0xc2, 0xc3, 0xad, 0x9f, 0x39, 0x88, - 0x22, 0xb7, 0x5d, 0x99, 0x9b, 0xdb, 0x7e, 0xed, 0x28, 0x90, 0x09, 0x9f, 0x71, 0x31, 0x50, 0x14, - 0x17, 0x89, 0xee, 0xa6, 0xd5, 0x66, 0xe8, 0xe3, 0x98, 0xf2, 0xd9, 0xcf, 0xa1, 0x49, 0x51, 0x3e, - 0x69, 0x91, 0x35, 0xa8, 0xb1, 0x56, 0xd3, 0x03, 0xd5, 0x6f, 0xaf, 0x96, 0xe9, 0x85, 0x9f, 0x25, - 0xd3, 0xc3, 0x7f, 0xb3, 0x08, 0x1d, 0xda, 0x72, 0x79, 0x90, 0x85, 0x2c, 0x35, 0xf9, 0x26, 0x75, - 0x65, 0x7a, 0x93, 0x7a, 0x29, 0x91, 0xa9, 0x49, 0xa4, 0xb7, 0xf9, 0x33, 0x3b, 0x51, 0x70, 0x4f, - 0x8e, 0x2c, 0xc2, 0xe0, 0x54, 0x39, 0xc9, 0x58, 0xcd, 0x2b, 0x03, 0x40, 0x38, 0xfe, 0x55, 0xec, - 0x24, 0xce, 0x44, 0x99, 0x32, 0x00, 0x7e, 0x13, 0x02, 0x96, 0x48, 0x36, 0x79, 0x5a, 0xe8, 0x59, - 0xef, 0x1e, 0x2a, 0x3f, 0x1c, 0xe7, 0x8a, 0xa6, 0x41, 0xe5, 0x1f, 0xe3, 0x40, 0x8a, 0x7b, 0x20, - 0x38, 0x2d, 0x92, 0x48, 0x07, 0x9d, 0x0f, 0xea, 0x87, 0xb4, 0x4d, 0x6b, 0x6b, 0x8d, 0x3f, 0x4b, - 0x73, 0x69, 0x11, 0x7a, 0x1f, 0xb1, 0x56, 0xdf, 0x9f, 0x81, 0xcc, 0x99, 0x4c, 0xf6, 0x40, 0xf2, - 0x48, 0xf9, 0x47, 0x4f, 0x26, 0xb9, 0x25, 0xc4, 0x2d, 0x5f, 0xc1, 0xca, 0x28, 0x0b, 0x82, 0x54, - 0x9e, 0xa6, 0xb6, 0x8a, 0xb2, 0xc4, 0x95, 0xf6, 0x2b, 0x32, 0x32, 0xcb, 0x86, 0xf6, 0x80, 0x48, - 0x2d, 0x39, 0x12, 0x5f, 0x82, 0xc8, 0x3b, 0x30, 0xff, 0x68, 0xf2, 0xa5, 0xe7, 0xda, 0xf7, 0x0d, - 0xa9, 0xfe, 0xdb, 0xd1, 0x70, 0x1b, 0x2e, 0x98, 0x1c, 0x29, 0xaa, 0xb6, 0x2d, 0x94, 0x5b, 0xda, - 0x3b, 0x32, 0x73, 0x5c, 0x29, 0xcd, 0xf1, 0x2a, 0x54, 0xcb, 0xe5, 0x69, 0xfc, 0x32, 0xbc, 0x0e, - 0xad, 0x91, 0x1f, 0x48, 0x9d, 0x6b, 0xc0, 0x45, 0xd3, 0x59, 0x87, 0x0a, 0x25, 0xd5, 0xf5, 0xdb, - 0xf0, 0x2f, 0x2b, 0x70, 0x31, 0x76, 0x92, 0xa7, 0x99, 0xde, 0xd5, 0xe6, 0xfc, 0xbe, 0x3a, 0x76, - 0x12, 0x0f, 0x05, 0x97, 0xba, 0xe0, 0xde, 0xb9, 0xba, 0xa9, 0x89, 0x10, 0x1e, 0xcb, 0x0d, 0xe8, - 0x95, 0x5a, 0xa4, 0x4e, 0x62, 0x76, 0x56, 0x3b, 0x49, 0xf4, 0x9c, 0xb2, 0xef, 0x07, 0x08, 0x14, - 0x43, 0xe8, 0x14, 0x74, 0x92, 0x2c, 0x23, 0x15, 0x09, 0x19, 0xaa, 0xfb, 0xa1, 0x87, 0x92, 0x1b, - 0x66, 0x13, 0x76, 0x17, 0xb8, 0x88, 0xad, 0x1e, 0x66, 0x13, 0xf2, 0x13, 0x56, 0xa1, 0xca, 0x95, - 0x83, 0x55, 0x82, 0xf3, 0xcb, 0xf0, 0x0f, 0x55, 0x58, 0xd9, 0x75, 0xe5, 0x91, 0x4c, 0xc6, 0xf7, - 0x9c, 0xd4, 0x79, 0xe0, 0x07, 0xf2, 0xd0, 0x51, 0x27, 0xc8, 0x70, 0x34, 0xe6, 0xd8, 0x49, 0x8f, - 0xf5, 0x2c, 0x35, 0x10, 0xb0, 0xef, 0xa4, 0xc7, 0x68, 0xb6, 0x08, 0x39, 0x8a, 0x92, 0x89, 0xde, - 0x07, 0x6e, 0x5a, 0xf4, 0x8f, 0x0f, 0x08, 0x92, 0xb7, 0x56, 0xfe, 0x0b, 0xa9, 0x4b, 0xee, 0xa8, - 0x35, 0xd5, 0x65, 0xbc, 0x0d, 0xed, 0x44, 0xba, 0x51, 0xe2, 0xe9, 0xd8, 0x9f, 0xc7, 0xd9, 0x62, - 0x18, 0x07, 0xfc, 0x37, 0xa1, 0xc8, 0x1d, 0xd2, 0x56, 0x97, 0xed, 0x9b, 0xba, 0x9c, 0x5e, 0x8e, - 0x40, 0xce, 0xdb, 0xf5, 0xc4, 0xff, 0x07, 0xfd, 0x82, 0x96, 0x12, 0x59, 0x26, 0x02, 0xde, 0x2a, - 0xdc, 0x98, 0x39, 0xbf, 0xb8, 0xb9, 0x6f, 0x5a, 0xfd, 0x39, 0x35, 0xe2, 0x64, 0x5d, 0xd1, 0x3d, - 0x43, 0xc5, 0x3b, 0xd0, 0x51, 0x71, 0xe0, 0xa7, 0x9a, 0x01, 0x94, 0x2e, 0xcc, 0x6b, 0x13, 0x90, - 0xf3, 0x4d, 0x6a, 0xde, 0x12, 0x36, 0x7e, 0xd4, 0x12, 0x36, 0xcf, 0x2f, 0xe1, 0xfb, 0xd0, 0x77, - 0x13, 0xe9, 0xc9, 0x30, 0xf5, 0x9d, 0xc0, 0x56, 0x6e, 0x14, 0x1b, 0x33, 0xdd, 0x2b, 0xe0, 0x07, - 0x08, 0x16, 0xbf, 0x80, 0x8b, 0x6e, 0x14, 0xa6, 0x32, 0x4c, 0xf3, 0xd2, 0x4d, 0x9d, 0xad, 0xd1, - 0x25, 0x27, 0x17, 0x34, 0xda, 0x14, 0x70, 0x72, 0xbe, 0x46, 0xdc, 0x81, 0x55, 0x5e, 0x9e, 0x99, - 0x46, 0x5c, 0x2b, 0x20, 0x68, 0xa5, 0xa6, 0x5b, 0xe8, 0x0c, 0x52, 0x22, 0x95, 0xef, 0x65, 0x4e, - 0xa0, 0x35, 0x44, 0x29, 0x83, 0x64, 0x69, 0x8c, 0xde, 0x4a, 0xa5, 0x1c, 0xd5, 0x14, 0x2d, 0x15, - 0xba, 0xd0, 0x56, 0x53, 0xd3, 0x12, 0xc9, 0x14, 0xf5, 0x37, 0x8e, 0x3a, 0xbe, 0x7c, 0x17, 0x56, - 0xe7, 0x2d, 0xc8, 0xeb, 0x92, 0x97, 0xcd, 0x52, 0xf2, 0x52, 0x97, 0xa7, 0xfe, 0xb7, 0x05, 0xb8, - 0x60, 0xd6, 0x9b, 0x42, 0x8e, 0x9c, 0xa9, 0xaf, 0x91, 0x3d, 0xc7, 0x30, 0x25, 0xdf, 0xee, 0x69, - 0x5a, 0xc0, 0x20, 0xda, 0xdb, 0xd9, 0x80, 0xbe, 0x26, 0x28, 0x98, 0x9f, 0xbf, 0xd2, 0xf5, 0xf2, - 0xae, 0x48, 0x04, 0xe8, 0x07, 0x47, 0x32, 0xc1, 0x39, 0xf2, 0xa8, 0x5a, 0x9b, 0x9a, 0x10, 0xb3, - 0xd3, 0x0f, 0x1a, 0x9c, 0x61, 0x39, 0xaa, 0xb4, 0x79, 0x9a, 0x39, 0x81, 0x9f, 0x9e, 0xd9, 0x23, - 0x5f, 0x06, 0x1e, 0x65, 0xca, 0xb9, 0x8e, 0xb0, 0x6f, 0x30, 0x0f, 0x10, 0xb1, 0xeb, 0xa9, 0xd2, - 0x48, 0x74, 0x02, 0x36, 0x17, 0x00, 0x3d, 0x92, 0x03, 0x02, 0xef, 0x7a, 0xf3, 0x65, 0xa5, 0x36, - 0x5f, 0x56, 0xde, 0x83, 0xde, 0xec, 0x9a, 0x73, 0x52, 0xb4, 0xab, 0xa6, 0xd7, 0x7b, 0x1e, 0x13, - 0x36, 0xe6, 0x32, 0xa1, 0x9e, 0xf4, 0xff, 0xb1, 0x00, 0xab, 0x7a, 0xd2, 0x77, 0xa2, 0x20, 0x9b, - 0xa0, 0xb5, 0xa7, 0xaa, 0xa4, 0x75, 0x68, 0x4f, 0x22, 0x76, 0xa1, 0x4a, 0xea, 0x0f, 0x26, 0x51, - 0xae, 0x8b, 0x37, 0xa0, 0xef, 0x73, 0xcb, 0x7c, 0x5e, 0x4c, 0x85, 0xb0, 0x86, 0xeb, 0x59, 0x41, - 0x2e, 0x54, 0xa1, 0x13, 0xab, 0xe3, 0x28, 0xd5, 0xa4, 0xa4, 0xc4, 0x79, 0xce, 0x97, 0x0d, 0x8a, - 0xa8, 0xc9, 0x93, 0xbd, 0x05, 0xc2, 0xcd, 0x92, 0x04, 0xe5, 0xa3, 0x44, 0xce, 0xc9, 0xbb, 0xbe, - 0xc6, 0x14, 0xd4, 0xef, 0x40, 0x7d, 0x12, 0x15, 0x1e, 0xc9, 0x94, 0x23, 0x6a, 0xd5, 0x26, 0x11, - 0x71, 0xc8, 0x65, 0xf4, 0x9a, 0x9e, 0x66, 0x7e, 0x22, 0x3d, 0x63, 0x87, 0xcd, 0xbb, 0x36, 0xd2, - 0xc7, 0xbe, 0xe7, 0xc9, 0x50, 0x27, 0x8e, 0x1a, 0xbe, 0xfa, 0x86, 0xde, 0xa9, 0x80, 0x56, 0x8e, - 0x1c, 0x0c, 0xcd, 0xc2, 0x2c, 0x20, 0xa9, 0x08, 0x74, 0xb9, 0x66, 0x4f, 0x23, 0x1e, 0x65, 0x01, - 0x4a, 0x44, 0xa0, 0x97, 0x94, 0x6c, 0x09, 0xb2, 0xa0, 0x7d, 0xec, 0x87, 0x29, 0xa9, 0x8a, 0x26, - 0x2d, 0x29, 0x22, 0x90, 0x09, 0xbf, 0xf1, 0xc3, 0x74, 0xf8, 0xcf, 0x16, 0x60, 0x4d, 0x4f, 0xfc, - 0x81, 0x9e, 0x00, 0x6d, 0x9f, 0x29, 0xba, 0x30, 0xd3, 0xa5, 0xf3, 0x7a, 0x8b, 0x16, 0x18, 0xd0, - 0x2e, 0x0d, 0xb8, 0xe0, 0xae, 0x05, 0x5d, 0xc6, 0x69, 0xf8, 0xea, 0x16, 0x88, 0x73, 0x7c, 0xa5, - 0xf4, 0xb6, 0x66, 0x7f, 0x86, 0xb1, 0x94, 0xf8, 0x04, 0xd6, 0x26, 0x32, 0x75, 0x48, 0x10, 0x82, - 0xc8, 0x75, 0xa8, 0x15, 0x89, 0x3c, 0x4f, 0xf7, 0xaa, 0xc1, 0x3e, 0xd4, 0x48, 0x14, 0x7a, 0xfc, - 0xc6, 0xc4, 0x09, 0xfd, 0x91, 0x54, 0x29, 0xf9, 0x19, 0xdc, 0x82, 0x1d, 0x9f, 0xbe, 0xc1, 0xa0, - 0x27, 0x41, 0xd4, 0xe4, 0xb1, 0x8e, 0x78, 0x11, 0x6b, 0x44, 0x53, 0x4f, 0xe4, 0x48, 0xaf, 0x5d, - 0x07, 0xd7, 0x2a, 0xf4, 0xc3, 0x31, 0x1f, 0x1c, 0xa8, 0xb3, 0x4f, 0x69, 0x80, 0x7b, 0x91, 0x27, - 0x87, 0x7f, 0xb1, 0x94, 0xf3, 0xe8, 0xbe, 0x86, 0x1f, 0xa4, 0x4e, 0x4a, 0xb5, 0xf2, 0xf9, 0xe0, - 0xd9, 0x46, 0xf2, 0x5c, 0x75, 0x0c, 0x94, 0x4b, 0xea, 0x37, 0x61, 0x65, 0x7a, 0xb4, 0x4c, 0xbb, - 0xc0, 0x65, 0x05, 0xe5, 0xe1, 0xe6, 0x25, 0xf8, 0x39, 0x3d, 0x93, 0xea, 0xea, 0x73, 0x03, 0x65, - 0xb2, 0x0f, 0x8b, 0x49, 0x50, 0x3a, 0x39, 0x2f, 0x3d, 0x6d, 0x15, 0xf3, 0x5e, 0xd5, 0x81, 0x46, - 0xa0, 0x68, 0x16, 0xe4, 0x71, 0x92, 0x85, 0xd2, 0xd3, 0x26, 0xbd, 0x97, 0xc3, 0xf7, 0x09, 0x8c, - 0x03, 0xce, 0x35, 0x53, 0xa9, 0xeb, 0x1a, 0x77, 0xed, 0x69, 0xcd, 0x54, 0x74, 0x8d, 0x3c, 0x5a, - 0xd0, 0xeb, 0xbe, 0x59, 0x41, 0xf4, 0x72, 0x6a, 0xdd, 0xf7, 0x2f, 0x61, 0x90, 0xd3, 0xf2, 0xdf, - 0x15, 0x1f, 0x68, 0xb0, 0xf1, 0x31, 0x4d, 0xe8, 0x37, 0xf3, 0x8f, 0x7c, 0x0c, 0x6b, 0xb3, 0x0d, - 0xf5, 0x97, 0x9a, 0xd4, 0x6c, 0x65, 0xaa, 0x59, 0xf1, 0x27, 0xf9, 0xfa, 0xba, 0x8e, 0x7b, 0x2c, - 0xed, 0x63, 0x5f, 0x17, 0xa0, 0x2f, 0x5a, 0xcb, 0x06, 0xb5, 0x83, 0x98, 0x6f, 0xfc, 0x54, 0xcd, - 0xa1, 0x9f, 0xf8, 0x4a, 0x69, 0xab, 0x38, 0x4d, 0xbf, 0xe7, 0x2b, 0x35, 0xfc, 0x2f, 0x2d, 0x68, - 0x1b, 0x4f, 0x91, 0x6a, 0x9f, 0x6f, 0x95, 0x9d, 0xfe, 0xd6, 0x56, 0xdf, 0x78, 0xef, 0x48, 0xb2, - 0x9d, 0xa6, 0x89, 0xc9, 0xf5, 0x71, 0x30, 0x30, 0xe5, 0xef, 0x2c, 0x90, 0x83, 0x50, 0xf8, 0x3b, - 0xdb, 0xb0, 0x5c, 0xf2, 0x20, 0xed, 0x34, 0x4a, 0x9d, 0x40, 0x07, 0x05, 0xa5, 0xba, 0xb1, 0x12, - 0x89, 0xd5, 0xc3, 0x17, 0xf6, 0x2d, 0x0e, 0x91, 0x1a, 0x83, 0x0d, 0x37, 0x0a, 0x4c, 0xb1, 0xed, - 0x4c, 0xb0, 0x81, 0x18, 0xaa, 0x88, 0x49, 0x24, 0xc6, 0xb9, 0xea, 0x69, 0xa0, 0x25, 0xa8, 0xc9, - 0x90, 0x83, 0xa7, 0x41, 0x3e, 0x40, 0x72, 0xe6, 0x6b, 0x14, 0xc7, 0xd0, 0x00, 0xc9, 0x4b, 0xff, - 0x10, 0x5a, 0x51, 0xe2, 0x8f, 0x7d, 0x4a, 0x13, 0xb3, 0x83, 0x33, 0xfb, 0x11, 0x60, 0x82, 0x1d, - 0xfc, 0xd4, 0x10, 0x6a, 0xda, 0xfc, 0x9f, 0xaf, 0x92, 0xd1, 0x18, 0x73, 0x04, 0xc0, 0x4d, 0x71, - 0x38, 0x2c, 0x91, 0xcd, 0xe2, 0x08, 0x80, 0x9b, 0x1e, 0x3c, 0x0d, 0x28, 0x53, 0x7e, 0x03, 0x7a, - 0x2e, 0x99, 0x0b, 0x16, 0xa8, 0x40, 0x86, 0xb4, 0xa6, 0x55, 0xab, 0xc3, 0x60, 0x1c, 0xdf, 0x43, - 0x19, 0xea, 0x52, 0x4b, 0x27, 0x08, 0x30, 0x62, 0x8d, 0x1c, 0x4f, 0xd7, 0xc5, 0xb4, 0x0d, 0xf0, - 0x61, 0xe4, 0x78, 0xe2, 0x0b, 0xb8, 0x8c, 0x38, 0x9b, 0x0b, 0x5a, 0xc3, 0x6c, 0x22, 0x13, 0xdf, - 0xb5, 0x1d, 0x45, 0x75, 0x32, 0xba, 0x3c, 0x66, 0x0d, 0x29, 0xee, 0x23, 0xc1, 0x23, 0xc6, 0x6f, - 0xab, 0xef, 0x65, 0x12, 0x89, 0xef, 0x29, 0x5b, 0x3e, 0xcf, 0x7d, 0x37, 0xdb, 0x12, 0x6f, 0x17, - 0x6b, 0xf5, 0x12, 0x4a, 0xaa, 0x43, 0x43, 0x84, 0x65, 0x9c, 0x3e, 0x6a, 0x2f, 0xbe, 0x05, 0x61, - 0x0c, 0x1c, 0x71, 0x7e, 0xea, 0xa8, 0x13, 0x2e, 0xca, 0x9d, 0xda, 0x6a, 0x9b, 0xe3, 0xa3, 0x5a, - 0xc6, 0x32, 0x22, 0x10, 0x01, 0x4a, 0xfc, 0x0e, 0x56, 0xf3, 0xce, 0xb4, 0x2f, 0x43, 0xdd, 0xf1, - 0x86, 0xc6, 0xb5, 0xf3, 0xdd, 0x4d, 0xb9, 0x40, 0x96, 0x19, 0x09, 0x83, 0xb9, 0xcb, 0xaf, 0xa1, - 0x67, 0xba, 0xe4, 0x59, 0x57, 0x83, 0x3e, 0xf5, 0xf6, 0xd6, 0xb9, 0xde, 0xa6, 0x6c, 0x7b, 0x6e, - 0x9f, 0x19, 0x8a, 0x3f, 0x9a, 0x5b, 0x72, 0x63, 0x65, 0x68, 0x3b, 0xa4, 0xb5, 0xb5, 0x7e, 0xae, - 0xa7, 0x19, 0x63, 0x65, 0x99, 0x21, 0x18, 0xb8, 0xf8, 0x08, 0x2e, 0x98, 0xce, 0x22, 0x0a, 0xf1, - 0x6c, 0x3f, 0xa2, 0xe8, 0x4f, 0xb0, 0x8b, 0xa5, 0x91, 0x1c, 0xfe, 0xed, 0x46, 0x1c, 0x2d, 0x5e, - 0x31, 0x4d, 0xd8, 0x0a, 0x53, 0x44, 0x9c, 0xff, 0xd4, 0x0a, 0xd9, 0xae, 0x81, 0x26, 0x61, 0xbb, - 0x8c, 0x11, 0xb0, 0x19, 0xfe, 0x06, 0xf4, 0xa9, 0x72, 0x1e, 0x97, 0x35, 0x4a, 0x3c, 0x3f, 0x74, - 0x82, 0xc1, 0x2a, 0xef, 0xb9, 0x22, 0xdc, 0x8a, 0x9e, 0x3f, 0x66, 0xa8, 0x38, 0x84, 0x35, 0xf3, - 0xa1, 0x5c, 0xcd, 0x28, 0x34, 0x25, 0xb4, 0xe1, 0x3d, 0x6f, 0xe2, 0xa6, 0x0c, 0x8e, 0x65, 0x96, - 0x70, 0xda, 0x0c, 0xdd, 0x83, 0x6b, 0x33, 0x4b, 0x3b, 0x71, 0x4e, 0xed, 0x89, 0x9c, 0x44, 0xc9, - 0x99, 0x36, 0x20, 0x6b, 0xa4, 0xc0, 0xae, 0x4c, 0x2d, 0xe2, 0x9e, 0x73, 0xba, 0x47, 0x34, 0x6c, - 0x4e, 0xbe, 0x82, 0xab, 0x33, 0xbd, 0x70, 0x21, 0xba, 0x0c, 0x9d, 0xa3, 0x40, 0x7a, 0xb4, 0x45, - 0xde, 0xb0, 0x2e, 0x4d, 0x75, 0x71, 0x80, 0x14, 0xf7, 0x99, 0x40, 0x7c, 0x09, 0xa4, 0xec, 0x15, - 0x9d, 0xb4, 0xb3, 0x95, 0xeb, 0x84, 0xb4, 0x2b, 0x9e, 0x57, 0x55, 0x20, 0x2f, 0xf2, 0x31, 0x3c, - 0xd4, 0x94, 0x56, 0xb7, 0x20, 0x26, 0xcd, 0xf9, 0x09, 0xb4, 0xcd, 0xc6, 0x32, 0xb5, 0xbd, 0x44, - 0x6d, 0x97, 0xb9, 0xad, 0xde, 0x4a, 0xa6, 0x86, 0xad, 0x51, 0xf1, 0x22, 0x36, 0x01, 0x4e, 0x9c, - 0xd1, 0x89, 0xc3, 0x6d, 0x2e, 0x97, 0x03, 0xfc, 0x6f, 0x11, 0x4e, 0x2d, 0x9a, 0x27, 0xe6, 0x51, - 0x7c, 0x0e, 0x97, 0x8c, 0x14, 0x3e, 0x3f, 0x8e, 0x02, 0xed, 0xb0, 0x8f, 0x9c, 0x30, 0xca, 0x52, - 0xbd, 0x51, 0xbe, 0xa6, 0x09, 0xbe, 0x43, 0x3c, 0x0a, 0xc0, 0x03, 0xc2, 0x6a, 0x87, 0xf5, 0x08, - 0x9a, 0xb4, 0xcd, 0x43, 0xbd, 0xe5, 0x87, 0x1e, 0x2a, 0xaf, 0x3e, 0xf4, 0xf0, 0x21, 0xb4, 0x75, - 0x34, 0xf3, 0xb2, 0x53, 0x14, 0x2d, 0xc6, 0x73, 0x42, 0xf4, 0x16, 0x34, 0x29, 0x94, 0xa1, 0x6f, - 0x5c, 0x83, 0x16, 0x9f, 0xb4, 0x3b, 0x0a, 0x22, 0xf7, 0xc4, 0x04, 0x1f, 0x04, 0xba, 0x8b, 0x90, - 0x21, 0x40, 0xe3, 0x49, 0xe8, 0x47, 0xe1, 0x76, 0x10, 0x0c, 0xff, 0x54, 0x83, 0x26, 0xfa, 0x3c, - 0xb4, 0x2f, 0x85, 0x61, 0x23, 0x31, 0x26, 0xd5, 0x55, 0x4c, 0x9c, 0x58, 0x1f, 0xeb, 0x68, 0x21, - 0x10, 0xa9, 0xf6, 0x9c, 0x78, 0xa6, 0xec, 0x62, 0x61, 0xa6, 0xec, 0xe2, 0x6d, 0x3e, 0xf7, 0xc9, - 0x55, 0xb3, 0xd2, 0x94, 0xdb, 0x53, 0x07, 0x77, 0x19, 0x84, 0xbe, 0x18, 0x91, 0x38, 0x01, 0xf9, - 0x6f, 0x18, 0x1d, 0x06, 0x4a, 0x57, 0x68, 0x90, 0x5c, 0x6c, 0x6b, 0xc4, 0x81, 0x64, 0x7b, 0x53, - 0xda, 0x8c, 0xac, 0xce, 0x6e, 0x46, 0xde, 0x04, 0x70, 0xa3, 0xd0, 0x23, 0x17, 0x71, 0x26, 0x21, - 0xcd, 0xe5, 0x11, 0x05, 0xf6, 0x47, 0x6c, 0x93, 0xbf, 0x07, 0xfd, 0x9c, 0x02, 0x3d, 0x40, 0x37, - 0xcc, 0xe3, 0x6b, 0x4d, 0x65, 0xc9, 0xd1, 0x4e, 0x98, 0xce, 0xee, 0xa7, 0x37, 0xcf, 0xed, 0xa7, - 0xbf, 0xa4, 0x90, 0x06, 0x7e, 0xf2, 0xe9, 0xb9, 0x4b, 0xd0, 0xa0, 0x8a, 0x3d, 0x2f, 0x8b, 0xb5, - 0x2d, 0xaa, 0xfb, 0x8a, 0xf2, 0x1e, 0x2f, 0xdb, 0xb3, 0x6f, 0xff, 0xdf, 0xda, 0xb3, 0xef, 0xfc, - 0xb8, 0x3d, 0xfb, 0xee, 0x8f, 0xdb, 0xb3, 0x9f, 0xd9, 0xe3, 0xee, 0xcd, 0xee, 0x71, 0xbf, 0x34, - 0xad, 0xd8, 0x7f, 0x69, 0x5a, 0xf1, 0x35, 0x39, 0xc1, 0xe5, 0x57, 0xe7, 0x04, 0x5f, 0x9f, 0x94, - 0x14, 0xaf, 0x4b, 0x4a, 0xde, 0x80, 0x5e, 0x9a, 0x38, 0xee, 0x09, 0x47, 0x5a, 0x27, 0xf2, 0x4c, - 0xe9, 0x24, 0x68, 0x87, 0xc0, 0x18, 0x67, 0x7d, 0x2b, 0xcf, 0xd4, 0xf0, 0x09, 0x00, 0x85, 0xa0, - 0xf4, 0x6b, 0x2f, 0xe3, 0x8d, 0xca, 0x4f, 0x2e, 0xb2, 0xfa, 0xeb, 0x0a, 0xc0, 0x81, 0x33, 0x89, - 0x79, 0x0f, 0x59, 0xfc, 0x19, 0xb4, 0x14, 0xbd, 0x95, 0x8b, 0x4c, 0x4a, 0x86, 0xba, 0x20, 0xd5, - 0x8f, 0x7c, 0xb8, 0x4b, 0xe5, 0xcf, 0xc4, 0xd6, 0xdc, 0x43, 0x5e, 0xd0, 0x5a, 0x35, 0x04, 0xb4, - 0xb7, 0x77, 0x1d, 0xba, 0x9a, 0x20, 0x96, 0x89, 0x2b, 0x43, 0xae, 0xef, 0xaf, 0x58, 0x1d, 0x86, - 0xee, 0x33, 0x50, 0x7c, 0x94, 0x93, 0x19, 0x93, 0x78, 0x3e, 0x65, 0xa6, 0x9b, 0x68, 0x9b, 0x38, - 0xdc, 0x32, 0xbf, 0x42, 0x03, 0x69, 0xc0, 0x12, 0x7e, 0xaf, 0xff, 0x86, 0x68, 0x41, 0x5d, 0xf7, - 0xda, 0xaf, 0x88, 0x0e, 0x34, 0xe9, 0x34, 0x21, 0xe1, 0x16, 0x86, 0x7f, 0x5a, 0x85, 0xd6, 0x6e, - 0xa8, 0xd2, 0x24, 0x63, 0x16, 0x2e, 0xce, 0xcc, 0x55, 0xe9, 0xcc, 0x9c, 0xae, 0x03, 0xe7, 0xdf, - 0xa0, 0x3a, 0xf0, 0x0f, 0xa1, 0xae, 0x8f, 0x67, 0xea, 0xc4, 0xc2, 0xdc, 0xb3, 0x9d, 0x86, 0x46, - 0x6c, 0x42, 0xc3, 0xd3, 0xe7, 0x46, 0x75, 0x25, 0x4d, 0xe9, 0x30, 0xa7, 0x39, 0x51, 0x6a, 0xe5, - 0x34, 0xe2, 0x6d, 0x58, 0x74, 0xc6, 0x63, 0x1d, 0xd5, 0xf7, 0x0a, 0x52, 0x72, 0xd2, 0x2c, 0xc4, - 0x89, 0xdb, 0xd0, 0x24, 0xf5, 0x49, 0x55, 0x6d, 0xb5, 0xd9, 0x3e, 0x4d, 0xc9, 0x1c, 0x6b, 0x54, - 0xca, 0x49, 0xdc, 0x86, 0x66, 0x10, 0x45, 0x31, 0x37, 0xa8, 0xcf, 0x36, 0x30, 0xf5, 0x45, 0x56, - 0x23, 0x30, 0x95, 0x46, 0x37, 0xa0, 0x86, 0xee, 0x7f, 0x14, 0x6b, 0xb7, 0xb9, 0x34, 0x0e, 0xaa, - 0xb3, 0xb1, 0xaa, 0x8a, 0xca, 0x6d, 0xb6, 0x00, 0x98, 0xff, 0xa9, 0xe7, 0xe6, 0xec, 0x74, 0xe4, - 0xb9, 0x55, 0x14, 0x52, 0x93, 0x66, 0xbd, 0x0b, 0x7d, 0xce, 0xa3, 0x95, 0x5a, 0x82, 0xa9, 0x05, - 0x36, 0x2d, 0xa7, 0x53, 0xb3, 0x56, 0x37, 0x99, 0x4e, 0xd5, 0x7e, 0x00, 0xf5, 0x98, 0x93, 0x43, - 0xa4, 0x61, 0xc8, 0x64, 0x9b, 0xa6, 0x3a, 0x6b, 0x64, 0x19, 0x0a, 0xf1, 0x1b, 0xe8, 0x72, 0xcd, - 0xea, 0x48, 0x67, 0x49, 0x68, 0x67, 0x6f, 0xea, 0x64, 0xdf, 0x54, 0x12, 0xc5, 0xea, 0xa4, 0x53, - 0x39, 0x95, 0x5f, 0x41, 0xa7, 0x38, 0xb0, 0x84, 0x16, 0xbf, 0x67, 0xb2, 0x15, 0xa6, 0x79, 0x39, - 0x1a, 0xb3, 0xda, 0xb2, 0x1c, 0x9b, 0x6d, 0x40, 0x4d, 0xd7, 0x51, 0xf7, 0xa9, 0x55, 0xe9, 0xd6, - 0x05, 0xae, 0x9c, 0xb4, 0x34, 0x1e, 0xe7, 0xb2, 0x28, 0x11, 0x25, 0xc7, 0x71, 0x6a, 0x2e, 0xf3, - 0xfa, 0x50, 0xab, 0x99, 0x97, 0x86, 0x8a, 0xfb, 0xd3, 0x25, 0xab, 0x9c, 0xd4, 0x5b, 0xa1, 0xa6, - 0x97, 0xe6, 0x34, 0xe5, 0xf4, 0x9e, 0xd5, 0x8b, 0x67, 0x2a, 0x5f, 0x6f, 0x41, 0x23, 0x4a, 0x3c, - 0x3a, 0x1f, 0x40, 0x55, 0x14, 0xb9, 0x0b, 0xa4, 0xcf, 0xa4, 0x92, 0xf2, 0xa8, 0x47, 0xfc, 0x82, - 0x8e, 0x45, 0x9c, 0x44, 0xe4, 0xe5, 0x92, 0x8a, 0xbb, 0x70, 0xde, 0xb1, 0xd0, 0x78, 0x52, 0x70, - 0xef, 0x42, 0xdd, 0x54, 0x87, 0xaf, 0x9d, 0xa3, 0x34, 0x28, 0xf1, 0x31, 0xf4, 0xa6, 0x15, 0x9a, - 0x1a, 0x5c, 0x3c, 0x47, 0xdd, 0x9d, 0xd2, 0x5f, 0x68, 0x8d, 0xab, 0x81, 0x3f, 0xf1, 0x53, 0xed, - 0xf3, 0x4d, 0x9d, 0x10, 0x25, 0x04, 0xc6, 0x7f, 0x3a, 0x05, 0x72, 0xe9, 0x7c, 0xfc, 0xa7, 0xd3, - 0x24, 0x03, 0xa8, 0xfb, 0xea, 0x81, 0x9f, 0xa8, 0x54, 0xd7, 0x37, 0x98, 0x57, 0xb1, 0x06, 0x35, - 0x5f, 0xa1, 0x99, 0xd0, 0x5e, 0x9a, 0x7e, 0x13, 0x37, 0xa1, 0xa6, 0x2b, 0xe7, 0xd7, 0xcf, 0x49, - 0xb4, 0x3e, 0x59, 0x63, 0x69, 0x0a, 0xf1, 0x3e, 0xd4, 0xa9, 0x6c, 0x3a, 0x8a, 0x07, 0x6f, 0xcf, - 0x72, 0x00, 0xd7, 0x2e, 0x5b, 0xb5, 0x80, 0x6b, 0x98, 0x3f, 0x80, 0xba, 0x71, 0x52, 0x86, 0xb3, - 0x5c, 0xad, 0x9d, 0x15, 0xcb, 0x50, 0x88, 0xeb, 0x50, 0x9d, 0xa0, 0x1e, 0x1b, 0xbc, 0x33, 0x2b, - 0xa1, 0xac, 0xde, 0x18, 0x2b, 0xfe, 0x7f, 0xb8, 0x5c, 0x2e, 0x3c, 0x36, 0x55, 0xc9, 0x7a, 0x83, - 0xf3, 0x3a, 0xb5, 0x7d, 0x7b, 0x0e, 0xab, 0x4c, 0xd7, 0x2f, 0x5b, 0x17, 0xe3, 0x97, 0x14, 0x36, - 0x7f, 0x9a, 0xab, 0x7b, 0x94, 0xae, 0xc1, 0x0d, 0xe3, 0x7c, 0x9f, 0x37, 0x18, 0xc6, 0x08, 0x90, - 0x9d, 0xf9, 0x0c, 0xda, 0xa3, 0xec, 0xc5, 0x8b, 0x33, 0xb3, 0x39, 0xff, 0x1e, 0xb5, 0x2b, 0x6d, - 0x31, 0x94, 0x6a, 0x9d, 0xad, 0xd6, 0xa8, 0x54, 0xf8, 0x7c, 0x11, 0xea, 0x6e, 0x68, 0x3b, 0x9e, - 0x97, 0x0c, 0x36, 0xb8, 0xd6, 0xd9, 0x0d, 0xb7, 0x3d, 0x8f, 0x2e, 0x77, 0x88, 0x62, 0x49, 0x27, - 0xa4, 0x6d, 0xdf, 0x1b, 0xbc, 0xcf, 0x86, 0xc7, 0x80, 0x76, 0x3d, 0xba, 0xfd, 0xc1, 0xc4, 0xe5, - 0xbe, 0x37, 0xb8, 0xa9, 0x6f, 0x7f, 0xd0, 0xa0, 0x5d, 0x0f, 0x1d, 0x4f, 0x0c, 0x62, 0x0c, 0x64, - 0xf0, 0x01, 0x27, 0x3c, 0x26, 0xce, 0xe9, 0xbe, 0x06, 0xa1, 0x90, 0x72, 0x6a, 0x8f, 0xd4, 0xd6, - 0xad, 0x59, 0x21, 0xcd, 0xd3, 0xc0, 0x56, 0xd3, 0xcf, 0x33, 0xc2, 0x24, 0xd8, 0xa4, 0x8a, 0xec, - 0x60, 0x6b, 0xf0, 0xe1, 0x79, 0xc1, 0xd6, 0x59, 0x6e, 0x14, 0x6c, 0x93, 0xf0, 0xde, 0x02, 0x60, - 0x9d, 0x45, 0x0a, 0x67, 0x73, 0xb6, 0x4d, 0x1e, 0x0d, 0x58, 0x7c, 0xa4, 0x89, 0x54, 0xcd, 0x16, - 0x00, 0xa5, 0x17, 0xb8, 0xcd, 0xed, 0xd9, 0x36, 0xb9, 0x77, 0x6f, 0x35, 0x9f, 0xe5, 0x8e, 0xfe, - 0x6d, 0x68, 0x66, 0xe8, 0xc7, 0xa3, 0x27, 0x3d, 0xb8, 0x33, 0xcb, 0xcc, 0xc6, 0xc5, 0xb7, 0x1a, - 0x99, 0x7e, 0xc2, 0x8f, 0x90, 0xed, 0x21, 0x37, 0x64, 0xf0, 0xd1, 0xec, 0x47, 0xf2, 0x38, 0xc0, - 0x22, 0x13, 0xc5, 0x21, 0xc1, 0xa7, 0xd0, 0xe2, 0x49, 0xe3, 0x46, 0x5b, 0xb3, 0x3c, 0x52, 0xf8, - 0x35, 0x16, 0xcf, 0x2e, 0x37, 0xbb, 0x0e, 0x55, 0x27, 0x8e, 0x83, 0xb3, 0xc1, 0xc7, 0xb3, 0x1c, - 0xbe, 0x8d, 0x60, 0x8b, 0xb1, 0xc8, 0x4a, 0x93, 0x2c, 0x48, 0x7d, 0x73, 0x0a, 0xe9, 0x93, 0x59, - 0x56, 0x2a, 0x1d, 0xeb, 0xb4, 0x5a, 0x93, 0xd2, 0x19, 0xcf, 0x5b, 0xd0, 0x88, 0x23, 0x95, 0xda, - 0xde, 0x24, 0x18, 0x7c, 0x7a, 0xce, 0x8c, 0xf0, 0x89, 0x14, 0xab, 0x1e, 0xeb, 0x23, 0x3d, 0x53, - 0xa7, 0xa4, 0x7f, 0x31, 0x73, 0x4a, 0x7a, 0x0b, 0xda, 0x93, 0x28, 0x1c, 0x47, 0xde, 0x11, 0xcf, - 0xfe, 0x2f, 0xcb, 0x41, 0xe1, 0x1e, 0x62, 0x38, 0x8c, 0xd4, 0x44, 0xda, 0x34, 0xf4, 0xb5, 0xef, - 0xe6, 0x2b, 0xdb, 0x51, 0xa4, 0xf6, 0x3f, 0xe3, 0x10, 0x9e, 0xe1, 0xbb, 0x6a, 0x9b, 0xa0, 0xd3, - 0xdb, 0xdb, 0x47, 0x67, 0x3a, 0x17, 0xf9, 0x39, 0xb1, 0x67, 0xb1, 0xbd, 0x7d, 0xf7, 0x8c, 0x13, - 0x92, 0x5f, 0xc0, 0xe5, 0xd2, 0xb9, 0xc5, 0x28, 0xb6, 0x43, 0x1b, 0x55, 0x40, 0x22, 0xbd, 0xcc, - 0x95, 0x83, 0x2f, 0xf2, 0x78, 0x53, 0x1f, 0x5e, 0x8c, 0xe2, 0x47, 0xfb, 0x89, 0xb4, 0x08, 0x2b, - 0x1e, 0x95, 0xcf, 0x48, 0x3a, 0xc1, 0x38, 0x4a, 0xfc, 0xf4, 0x78, 0x32, 0xf8, 0x15, 0xf9, 0x81, - 0x6f, 0x96, 0xe2, 0x81, 0x3c, 0xff, 0xb5, 0x6d, 0x88, 0xac, 0x62, 0x8c, 0x39, 0x4c, 0x7c, 0x01, - 0x97, 0xb4, 0x2d, 0xc0, 0x0e, 0xa7, 0x8e, 0xa2, 0xab, 0xc1, 0xaf, 0xe9, 0x2c, 0xfa, 0xc5, 0x82, - 0xe0, 0xeb, 0xd2, 0xa9, 0x74, 0x25, 0xb6, 0xe1, 0xcd, 0x79, 0x6d, 0xd1, 0x31, 0xe1, 0x09, 0xf8, - 0x92, 0x26, 0xe0, 0xf2, 0xf9, 0xf6, 0x07, 0x52, 0xdf, 0x4b, 0x73, 0x0b, 0x04, 0x4e, 0x00, 0x59, - 0x2e, 0xe9, 0xd9, 0x51, 0x96, 0xc6, 0x59, 0x3a, 0xf8, 0x0d, 0xc7, 0x89, 0x69, 0x14, 0x3f, 0x66, - 0xc4, 0x63, 0x82, 0x63, 0x9c, 0x4e, 0x4a, 0xd3, 0x2e, 0xb7, 0xe1, 0xdd, 0x02, 0x35, 0xf8, 0x8a, - 0xe7, 0x8d, 0x08, 0x0e, 0xf3, 0x96, 0xbc, 0xb7, 0xa0, 0x38, 0x4e, 0xff, 0xed, 0x52, 0x63, 0xb9, - 0x2f, 0x7e, 0xbb, 0xd4, 0x78, 0xb7, 0x7f, 0xdd, 0x6a, 0x95, 0xf6, 0x24, 0x86, 0x9f, 0x42, 0x7b, - 0x9b, 0x6e, 0x3e, 0xf2, 0x15, 0xd9, 0xc4, 0xeb, 0xb0, 0x94, 0xd7, 0xb2, 0xe4, 0xc6, 0x96, 0x28, - 0x5e, 0xc8, 0xdd, 0x70, 0x14, 0x59, 0x84, 0x1e, 0xfe, 0xab, 0x25, 0xa8, 0x71, 0x7d, 0xc1, 0xeb, - 0x4f, 0x37, 0xbe, 0x69, 0x34, 0x46, 0x58, 0x1c, 0x03, 0x61, 0xe5, 0x40, 0xe8, 0xd9, 0x72, 0xef, - 0x66, 0x51, 0x26, 0xb3, 0x0a, 0x55, 0x0e, 0xf3, 0x39, 0xfb, 0xc1, 0x2f, 0xa4, 0x2d, 0x33, 0x75, - 0x4c, 0xd7, 0x1b, 0xe9, 0x7c, 0xde, 0x92, 0x05, 0x06, 0xb4, 0xeb, 0xd1, 0x36, 0xa7, 0x21, 0x20, - 0x75, 0x5c, 0xd3, 0x69, 0x0c, 0x0d, 0x24, 0xa5, 0x6c, 0x4a, 0x70, 0xea, 0x2f, 0x29, 0xc1, 0x79, - 0x0b, 0x96, 0x42, 0x73, 0x42, 0x29, 0xc7, 0xd3, 0xf5, 0x26, 0x04, 0x17, 0x37, 0x21, 0x3f, 0x92, - 0xa9, 0xdd, 0xcb, 0x97, 0x1f, 0xd9, 0xdc, 0x82, 0x66, 0x7e, 0x57, 0x96, 0xf6, 0x28, 0x57, 0x37, - 0x8b, 0xdb, 0xb3, 0x0e, 0xcd, 0x93, 0x55, 0x90, 0xbd, 0xba, 0x90, 0xa4, 0xf5, 0xf3, 0x0a, 0x49, - 0x38, 0xdc, 0x76, 0xa3, 0x50, 0xa5, 0x7a, 0x23, 0xb7, 0xee, 0xab, 0x1d, 0x7c, 0x15, 0x9f, 0x43, - 0x27, 0x91, 0xee, 0x33, 0x7b, 0xa2, 0xc6, 0xfc, 0x89, 0x4e, 0xf9, 0x58, 0xf8, 0x44, 0x8d, 0xbf, - 0xa1, 0x22, 0x17, 0x1d, 0xfd, 0xb6, 0x90, 0x76, 0x4f, 0x8d, 0xa9, 0xd7, 0x0f, 0x60, 0x79, 0x22, - 0x27, 0x47, 0x32, 0x51, 0xc7, 0x7e, 0x6c, 0xcc, 0x66, 0x97, 0x8a, 0x71, 0xfa, 0x05, 0x82, 0xc7, - 0x32, 0xfc, 0xfb, 0x15, 0x68, 0xe0, 0x2c, 0x22, 0x2f, 0x09, 0x01, 0x4b, 0x13, 0x37, 0xce, 0x74, - 0x50, 0x43, 0xcf, 0xfa, 0xfe, 0x2d, 0xe6, 0x12, 0x7d, 0xff, 0x16, 0xad, 0x21, 0xa7, 0x27, 0xe9, - 0x99, 0xaf, 0x54, 0x39, 0xa3, 0x1d, 0x6c, 0xe6, 0x0c, 0xf3, 0x2a, 0x2e, 0x40, 0xcd, 0x0d, 0x69, - 0x67, 0x83, 0xd3, 0xbc, 0x55, 0x37, 0xdc, 0x09, 0x53, 0x0d, 0x2e, 0x4e, 0xb9, 0x54, 0xdd, 0x70, - 0xd7, 0x3b, 0x1d, 0xfe, 0xdb, 0x0a, 0x2c, 0xef, 0x27, 0x91, 0x2b, 0x95, 0x7a, 0x88, 0x4e, 0x19, - 0xa5, 0xd4, 0xf0, 0x8b, 0x94, 0x81, 0xe0, 0xec, 0x15, 0x3d, 0x23, 0x0f, 0xf3, 0xb6, 0x53, 0x1e, - 0x3a, 0x2e, 0x5a, 0x4d, 0x82, 0x50, 0xe4, 0x98, 0xa3, 0x4b, 0xa5, 0x1a, 0x8c, 0xa6, 0xdc, 0xc5, - 0x75, 0xe8, 0x16, 0xba, 0xab, 0x54, 0x55, 0x52, 0x5c, 0x7c, 0x40, 0xbd, 0x5c, 0x83, 0x96, 0xae, - 0x3d, 0xa2, 0x6e, 0x38, 0x1d, 0x05, 0x0c, 0x3a, 0xd0, 0xa3, 0x60, 0x45, 0x4f, 0x78, 0x4e, 0x40, - 0xb1, 0xea, 0x47, 0xf4, 0xf0, 0x0f, 0x15, 0xe8, 0xef, 0x27, 0x32, 0x76, 0x12, 0x49, 0xc5, 0x48, - 0x34, 0xc7, 0x6b, 0x50, 0x0b, 0x64, 0x38, 0xd6, 0xf5, 0x27, 0x8b, 0x96, 0x7e, 0xcb, 0xef, 0x4e, - 0x5b, 0x28, 0xdd, 0x9d, 0x86, 0x73, 0x9d, 0x48, 0x47, 0x5f, 0xb1, 0x46, 0xcf, 0x28, 0x83, 0x18, - 0xff, 0x73, 0x90, 0xdb, 0xb0, 0xf8, 0x45, 0x1f, 0xaf, 0x3e, 0xf2, 0x43, 0x2a, 0xfa, 0xa4, 0xe3, - 0xd5, 0x77, 0x7d, 0x32, 0x1c, 0x0c, 0x46, 0x47, 0x8e, 0xaf, 0x29, 0xa2, 0x6d, 0xab, 0x86, 0xd5, - 0x25, 0x02, 0x27, 0x39, 0x3b, 0x20, 0x28, 0xc5, 0xda, 0x7c, 0xbb, 0x11, 0x57, 0x34, 0x71, 0xc2, - 0xa4, 0x63, 0x2e, 0x37, 0x62, 0xdd, 0xa2, 0x86, 0xff, 0xb8, 0x0e, 0x2d, 0xbd, 0x42, 0xf4, 0x37, - 0xcc, 0x1d, 0x95, 0x9c, 0x3b, 0xfa, 0xb0, 0xa8, 0x9e, 0x06, 0x9a, 0x5d, 0xf0, 0x51, 0x7c, 0x0c, - 0x8b, 0x81, 0x3f, 0xd1, 0x01, 0xf0, 0x95, 0x29, 0x67, 0x66, 0x7a, 0x9d, 0x35, 0x2b, 0x23, 0x35, - 0x5a, 0x50, 0xba, 0xeb, 0x01, 0x85, 0x46, 0xaf, 0x0d, 0x3a, 0x16, 0xa7, 0x28, 0x99, 0x38, 0xeb, - 0x8e, 0xcb, 0x05, 0xc7, 0x5a, 0xdd, 0x74, 0xac, 0xa6, 0x86, 0xec, 0x7a, 0xe2, 0x13, 0x68, 0xe4, - 0xdb, 0xf4, 0x26, 0xe4, 0x4d, 0x4f, 0xc3, 0xcd, 0x9d, 0x47, 0x87, 0xa7, 0xa1, 0xd9, 0x87, 0xd7, - 0x1f, 0xcb, 0x29, 0xc5, 0x6f, 0xa0, 0xad, 0xa4, 0x52, 0x7c, 0x18, 0x7f, 0x14, 0x69, 0x35, 0x74, - 0xa1, 0x1c, 0xcd, 0x12, 0x16, 0xff, 0xda, 0x08, 0x9d, 0x2a, 0x40, 0xe2, 0x1b, 0xe8, 0x9a, 0xf6, - 0x41, 0x34, 0x1e, 0xe7, 0x69, 0xa4, 0x2b, 0xe7, 0x7a, 0x78, 0x48, 0xe8, 0x52, 0x3f, 0x1d, 0x55, - 0x46, 0x88, 0xaf, 0xa1, 0x1b, 0x33, 0xd3, 0xd8, 0xba, 0x90, 0x8e, 0xd5, 0xd9, 0xe5, 0x29, 0xdf, - 0x7b, 0x8a, 0xa9, 0x8a, 0x43, 0xa7, 0x05, 0x5c, 0x9d, 0xbf, 0x98, 0x82, 0xf3, 0x8a, 0xd3, 0x17, - 0x53, 0x48, 0x58, 0xd3, 0x37, 0x49, 0x8d, 0x12, 0x87, 0x0e, 0xd8, 0xb3, 0xc9, 0x34, 0xd5, 0xb1, - 0xb7, 0xcf, 0xad, 0x18, 0x7e, 0x70, 0x93, 0x2f, 0x60, 0x78, 0xa0, 0x9b, 0x90, 0x09, 0xd5, 0xa5, - 0x46, 0xab, 0xc9, 0x1c, 0x94, 0xd8, 0x84, 0x15, 0xfd, 0x19, 0x79, 0x2a, 0xdd, 0x4c, 0xdf, 0x2d, - 0x42, 0x4a, 0xaf, 0x6d, 0x2d, 0x33, 0xea, 0xbe, 0xc1, 0xec, 0x7a, 0xe2, 0x33, 0x18, 0xa8, 0xd4, - 0x49, 0x25, 0x0d, 0xc8, 0xe8, 0x5d, 0x7f, 0x1c, 0x46, 0x89, 0xd4, 0xe5, 0x39, 0x6b, 0x39, 0x5e, - 0xeb, 0xdb, 0x5d, 0xc2, 0x8a, 0xdf, 0x40, 0x1f, 0x55, 0x64, 0x9e, 0xaa, 0xb1, 0x53, 0xa5, 0xa3, - 0xf8, 0xf9, 0x2a, 0xbe, 0x8b, 0xd4, 0x86, 0x2d, 0x0e, 0x29, 0xe5, 0x4f, 0xed, 0xc7, 0x32, 0xc4, - 0x78, 0x80, 0x34, 0x84, 0xcc, 0x94, 0xf4, 0xf4, 0xf5, 0x31, 0xab, 0x88, 0xfd, 0x3a, 0x47, 0x5a, - 0x84, 0x43, 0x0f, 0xc4, 0x88, 0x8f, 0xde, 0xa9, 0x25, 0xdf, 0xb7, 0xf0, 0x8b, 0xfa, 0xc4, 0xa6, - 0x97, 0xb5, 0x34, 0x31, 0x0d, 0xba, 0xc0, 0xb9, 0x03, 0x74, 0xf9, 0x6b, 0xb8, 0xf4, 0xd2, 0x59, - 0x7d, 0x5d, 0xbd, 0x50, 0xa7, 0x7c, 0xd9, 0xc1, 0xff, 0x5a, 0x84, 0x56, 0x89, 0x5b, 0xe9, 0xc6, - 0x44, 0x25, 0x13, 0x53, 0x15, 0x88, 0xcf, 0x08, 0x3b, 0x8e, 0x94, 0x29, 0x72, 0xa3, 0x67, 0x84, - 0x25, 0x51, 0x5e, 0xec, 0x43, 0xcf, 0xc8, 0x43, 0x7a, 0x7b, 0x4a, 0xaf, 0xd8, 0x12, 0x5f, 0xd2, - 0x51, 0x00, 0x77, 0x3d, 0xba, 0x5a, 0xd1, 0x49, 0x9d, 0x23, 0x47, 0x99, 0xf2, 0xce, 0xfc, 0x1d, - 0x4d, 0xc3, 0x33, 0x99, 0xe0, 0x58, 0x4c, 0x71, 0x83, 0x7e, 0x45, 0x19, 0xa7, 0x55, 0x7d, 0x11, - 0x85, 0x5c, 0xd8, 0xd0, 0xb6, 0x1a, 0x08, 0xf8, 0x3e, 0x0a, 0xa9, 0x99, 0x96, 0x68, 0x5d, 0xa0, - 0x63, 0x5e, 0xd1, 0x66, 0x3e, 0xcd, 0x24, 0xc6, 0xa5, 0x1e, 0x1d, 0x66, 0x6c, 0x5a, 0x75, 0x7a, - 0xe7, 0x9a, 0x21, 0x0a, 0xa0, 0x9f, 0x3b, 0x7e, 0x4a, 0xaa, 0x23, 0xca, 0x52, 0xcd, 0xf4, 0x3d, - 0x44, 0x7c, 0xe7, 0xf8, 0xe9, 0x21, 0x83, 0xc5, 0x47, 0xfa, 0x8c, 0x72, 0x99, 0x96, 0x2e, 0x09, - 0xe2, 0x6d, 0x6f, 0x31, 0x43, 0x7f, 0x20, 0xe9, 0x1e, 0xbc, 0x89, 0x93, 0x26, 0xfe, 0x69, 0x14, - 0xa2, 0xef, 0x44, 0xf7, 0x01, 0xe4, 0xd7, 0x39, 0x36, 0xac, 0x95, 0x1c, 0xf9, 0x88, 0x70, 0x94, - 0x09, 0x7e, 0x02, 0x1b, 0xf2, 0x34, 0x0e, 0x7c, 0xd7, 0x9f, 0xb9, 0xd7, 0xc0, 0x76, 0x1d, 0x95, - 0xda, 0x89, 0x4c, 0xb3, 0x24, 0x54, 0xb4, 0xa3, 0xab, 0xf9, 0xfa, 0x1d, 0x43, 0x5f, 0xbe, 0xeb, - 0x60, 0xc7, 0x51, 0xa9, 0xc5, 0xb4, 0x8f, 0xb2, 0x20, 0xc0, 0x49, 0xc8, 0x33, 0xd0, 0x5c, 0x7c, - 0x56, 0x57, 0x9c, 0x7b, 0x1e, 0xfe, 0xa7, 0x0a, 0x2c, 0x9f, 0xd3, 0x34, 0x18, 0x0b, 0xa3, 0x96, - 0x31, 0x05, 0x33, 0x6d, 0xab, 0x86, 0xaf, 0xbb, 0x1e, 0x21, 0xd2, 0x49, 0x6a, 0x4a, 0x65, 0x10, - 0x91, 0x4e, 0x50, 0x8d, 0x5e, 0x80, 0x5a, 0x7a, 0x4a, 0x4b, 0xce, 0xd6, 0xa7, 0x9a, 0x9e, 0xe2, - 0x5a, 0x6f, 0x43, 0x33, 0x88, 0xc6, 0x76, 0x20, 0x9f, 0x49, 0xbe, 0x6c, 0xa6, 0xbb, 0xf5, 0xee, - 0x2b, 0x54, 0xdc, 0xe6, 0xc3, 0x68, 0xfc, 0x10, 0x69, 0xad, 0x46, 0xa0, 0x9f, 0x86, 0xbf, 0x85, - 0x86, 0x81, 0x8a, 0x26, 0x54, 0xef, 0xc9, 0xa3, 0x6c, 0xdc, 0x7f, 0x43, 0x34, 0x60, 0x09, 0x5b, - 0xf4, 0x2b, 0xf8, 0xf4, 0x9d, 0x93, 0x84, 0xfd, 0x05, 0x44, 0xdf, 0x4f, 0x92, 0x28, 0xe9, 0x2f, - 0xe2, 0xe3, 0xbe, 0x13, 0xfa, 0x6e, 0x7f, 0x09, 0x1f, 0x1f, 0x38, 0xa9, 0x13, 0xf4, 0xab, 0xc3, - 0x7f, 0x5f, 0x85, 0xc6, 0xbe, 0xfe, 0xba, 0xb8, 0x07, 0x9d, 0xfc, 0xd2, 0xc6, 0xf9, 0x9b, 0xd2, - 0xfb, 0xb3, 0x0f, 0xb4, 0x29, 0xdd, 0x8e, 0x4b, 0x6f, 0xb3, 0x57, 0x3f, 0x2e, 0x9c, 0xbb, 0xfa, - 0xf1, 0x2a, 0x2c, 0x3e, 0x4d, 0xce, 0xa6, 0x6b, 0xc5, 0xf7, 0x03, 0x27, 0xb4, 0x10, 0x2c, 0x3e, - 0x82, 0x16, 0xa5, 0xc3, 0xd9, 0x8c, 0xea, 0x8d, 0xdc, 0xf2, 0x2d, 0xab, 0x5c, 0x05, 0x0c, 0x48, - 0xa4, 0x3d, 0xf6, 0x4d, 0x68, 0xb8, 0xc7, 0x7e, 0xe0, 0x25, 0x32, 0xd4, 0x67, 0x36, 0xc4, 0xf9, - 0x21, 0x5b, 0x39, 0x8d, 0xf8, 0x33, 0xe8, 0xfb, 0xc5, 0x46, 0x74, 0x51, 0xfd, 0x30, 0x65, 0xaf, - 0x4a, 0x5b, 0xd5, 0x56, 0xaf, 0x44, 0x4e, 0x2e, 0x62, 0x71, 0x1b, 0x4b, 0xbd, 0x7c, 0x1b, 0x0b, - 0x5f, 0xc6, 0x47, 0x7e, 0x5c, 0x23, 0xdf, 0xc6, 0x42, 0x37, 0xee, 0x86, 0x76, 0xbe, 0x9b, 0xb3, - 0x71, 0xbf, 0x71, 0x1d, 0xb5, 0x13, 0xfe, 0x2e, 0x74, 0xd1, 0xa9, 0xb7, 0x39, 0x16, 0x40, 0x3b, - 0x0a, 0xfa, 0xf2, 0xa8, 0x4c, 0x1d, 0xdf, 0xc3, 0x68, 0x00, 0x99, 0xf1, 0x3a, 0x74, 0xcd, 0xbf, - 0xe8, 0xf8, 0xac, 0xa5, 0xab, 0x23, 0x34, 0x94, 0x43, 0xb2, 0x4d, 0x58, 0x71, 0x8f, 0x9d, 0x30, - 0x94, 0x81, 0x7d, 0x94, 0x8d, 0x46, 0xc6, 0x0d, 0xe3, 0x1b, 0xc5, 0x96, 0x35, 0xea, 0x2e, 0x61, - 0xc8, 0x1b, 0x1b, 0x42, 0x27, 0xf4, 0x03, 0x73, 0xab, 0x68, 0xc8, 0x2e, 0x73, 0xd5, 0x6a, 0x85, - 0x7e, 0xc0, 0xf7, 0x88, 0xd2, 0xf5, 0xa7, 0xfd, 0x2c, 0xf3, 0x3d, 0x65, 0xa7, 0x91, 0xb9, 0xc7, - 0x50, 0xa7, 0x94, 0x4a, 0x9b, 0xb4, 0x4f, 0x32, 0xdf, 0x3b, 0x8c, 0xf4, 0x4d, 0x86, 0x1d, 0xa2, - 0x37, 0xaf, 0x38, 0xf6, 0xe9, 0x78, 0x8f, 0x4a, 0x14, 0x1a, 0x56, 0x27, 0x2a, 0x87, 0x79, 0xc3, - 0xaf, 0xa0, 0x5d, 0x66, 0x31, 0x64, 0x59, 0xda, 0x6c, 0xeb, 0xbf, 0x21, 0x00, 0x6a, 0x8f, 0xa2, - 0x64, 0xe2, 0x04, 0xfd, 0x0a, 0x3e, 0xb3, 0xce, 0xef, 0x2f, 0x88, 0x36, 0x34, 0xcc, 0xe6, 0x51, - 0x7f, 0x51, 0xa7, 0x73, 0x7f, 0x05, 0x0d, 0x73, 0x8b, 0x23, 0xdd, 0x80, 0x17, 0x79, 0x92, 0x23, - 0x28, 0x5d, 0xbb, 0x8c, 0x00, 0x8a, 0x9e, 0xcc, 0xad, 0xb8, 0x0b, 0xc5, 0xad, 0xb8, 0xc3, 0xdf, - 0x41, 0xbb, 0xfc, 0x27, 0x26, 0x41, 0x51, 0x29, 0x12, 0x14, 0x73, 0x5a, 0x51, 0x4d, 0x4d, 0x12, - 0x4d, 0xec, 0x92, 0x93, 0xdf, 0x40, 0x00, 0x7e, 0x66, 0xf8, 0xf7, 0x16, 0xa0, 0x4a, 0x5b, 0x2a, - 0xe4, 0x84, 0xe1, 0x43, 0x21, 0x68, 0x55, 0xab, 0x49, 0x90, 0xff, 0x83, 0xe3, 0xc4, 0x79, 0xc2, - 0x7a, 0xe9, 0xd5, 0x09, 0xeb, 0x6d, 0x58, 0x7e, 0x56, 0xba, 0x79, 0x95, 0x37, 0x52, 0xaa, 0xc6, - 0x63, 0xc3, 0x36, 0x7f, 0x5e, 0xdc, 0xc0, 0x4a, 0xdb, 0x29, 0xbd, 0x67, 0xd3, 0x00, 0xf1, 0x36, - 0xe8, 0x93, 0x19, 0x36, 0x17, 0x44, 0x71, 0xf5, 0x50, 0x8b, 0x61, 0xdb, 0x54, 0xfe, 0x84, 0x71, - 0xf2, 0x69, 0x68, 0xae, 0xe3, 0xe1, 0x82, 0xb1, 0x66, 0x7a, 0x1a, 0x72, 0x09, 0xd3, 0x70, 0x02, - 0x2b, 0x73, 0x8e, 0x91, 0x89, 0x75, 0x68, 0x4f, 0xa5, 0xf5, 0xf8, 0x6c, 0x07, 0xb8, 0x45, 0x1e, - 0xef, 0x13, 0x58, 0x93, 0x81, 0x3f, 0xf6, 0x8f, 0x7c, 0xaa, 0x9a, 0x2d, 0x9d, 0x6c, 0x63, 0x5d, - 0xb3, 0x5a, 0xc2, 0xe6, 0x27, 0xdb, 0x6e, 0xfe, 0xeb, 0x0a, 0xd4, 0xf8, 0xf6, 0x65, 0xb1, 0x0c, - 0x9d, 0x27, 0xe1, 0x49, 0x18, 0x3d, 0x0f, 0x19, 0xd0, 0x7f, 0x43, 0xac, 0x40, 0xcf, 0xf0, 0x9b, - 0xbe, 0xe6, 0xb9, 0x5f, 0x11, 0x7d, 0x68, 0x13, 0xe3, 0x1b, 0xc8, 0x82, 0xb8, 0x0a, 0x03, 0xed, - 0x3b, 0xde, 0x43, 0x3b, 0x15, 0xa5, 0xfe, 0xe8, 0xcc, 0x60, 0x17, 0x45, 0x0f, 0x5a, 0x07, 0x69, - 0x14, 0x1f, 0xc8, 0xd0, 0xf3, 0xc3, 0x71, 0x7f, 0x49, 0x0c, 0x60, 0xd5, 0xf4, 0xca, 0x7c, 0xfd, - 0xc0, 0x0f, 0x7d, 0x75, 0xdc, 0xaf, 0x8a, 0x2b, 0x70, 0x71, 0x1e, 0x66, 0xdb, 0x3d, 0xe9, 0xd7, - 0xc4, 0x2a, 0xf4, 0x0d, 0xf2, 0xae, 0xbe, 0x6b, 0xb7, 0x5f, 0xbf, 0xf9, 0x09, 0x88, 0xf3, 0xd7, - 0x1c, 0xe3, 0x37, 0x1f, 0xca, 0xb1, 0xe3, 0x9e, 0xed, 0x04, 0x91, 0x42, 0xf1, 0xe8, 0x40, 0xb3, - 0xe8, 0xab, 0x72, 0xf3, 0x01, 0xd4, 0xf8, 0x5e, 0xea, 0xd2, 0x5f, 0x33, 0xa0, 0xff, 0x06, 0x36, - 0x46, 0x1b, 0xed, 0x87, 0xe3, 0x47, 0xf2, 0x34, 0x65, 0xcb, 0xf1, 0xd0, 0x51, 0x69, 0x7f, 0x41, - 0x74, 0x01, 0xf4, 0x8f, 0xdd, 0x0f, 0xbd, 0xfe, 0xe2, 0xdd, 0x9d, 0xbf, 0xfa, 0xe3, 0x5b, 0x95, - 0x3f, 0xfc, 0xf1, 0xad, 0xca, 0x7f, 0xfc, 0xe3, 0x5b, 0x6f, 0xfc, 0xfe, 0x4f, 0x6f, 0x55, 0xbe, - 0xff, 0xa8, 0x74, 0xeb, 0xb6, 0x36, 0xdd, 0x54, 0x0d, 0x76, 0x3b, 0xb7, 0xe3, 0xb7, 0xe3, 0x93, - 0xf1, 0xed, 0xf8, 0xe8, 0xb6, 0x51, 0x0c, 0x47, 0x35, 0xba, 0x4c, 0xfb, 0xe3, 0xff, 0x1d, 0x00, - 0x00, 0xff, 0xff, 0xb4, 0xc1, 0xbc, 0x3b, 0xcb, 0x5b, 0x00, 0x00, + 0xb6, 0x18, 0xec, 0x4b, 0xf2, 0xfe, 0x9d, 0xfb, 0xc3, 0xcb, 0x22, 0x45, 0x5e, 0xfd, 0x58, 0xa2, + 0xaf, 0x2d, 0x9b, 0x96, 0x65, 0x4a, 0xa6, 0xed, 0x19, 0xdb, 0x33, 0x1e, 0x3f, 0x8a, 0x12, 0x6d, + 0x8e, 0x45, 0x89, 0xd3, 0xa4, 0xc6, 0x80, 0x81, 0xef, 0x6b, 0x34, 0xbb, 0xeb, 0x5e, 0xf6, 0xb0, + 0x6f, 0x77, 0xab, 0xab, 0x5b, 0x22, 0xb5, 0x4a, 0x96, 0x09, 0xb2, 0xcf, 0x43, 0x56, 0x13, 0x24, + 0x8b, 0x04, 0xd9, 0xbd, 0xe0, 0x65, 0x99, 0xf5, 0x43, 0x90, 0xc5, 0xac, 0xb2, 0x0a, 0x82, 0x60, + 0x66, 0x97, 0x00, 0x41, 0x10, 0x20, 0x41, 0x80, 0xe0, 0x01, 0xc1, 0x39, 0xa7, 0xaa, 0xbb, 0xef, + 0xe5, 0x95, 0x64, 0x3b, 0xc1, 0xdb, 0xe4, 0xed, 0xba, 0xcf, 0x39, 0x55, 0x5d, 0x5d, 0x75, 0x7e, + 0xeb, 0x9c, 0x2a, 0xe8, 0xc6, 0x7e, 0x2c, 0x03, 0x3f, 0x94, 0x9b, 0x71, 0x12, 0xa5, 0x91, 0x68, + 0x98, 0xf7, 0x2b, 0x1f, 0x8e, 0xfc, 0xf4, 0x24, 0x3b, 0xde, 0x74, 0xa3, 0xf1, 0x9d, 0x51, 0x34, + 0x8a, 0xee, 0x10, 0xc1, 0x71, 0x36, 0xa4, 0x37, 0x7a, 0xa1, 0x27, 0x6e, 0x78, 0x05, 0x82, 0xc8, + 0x3d, 0x35, 0xcf, 0x71, 0xe0, 0x84, 0xfa, 0x79, 0x31, 0xf5, 0xc7, 0x52, 0xa5, 0xce, 0x38, 0xd6, + 0x80, 0x66, 0x7a, 0xa6, 0x71, 0x83, 0xdf, 0xd7, 0xa0, 0xbe, 0x2f, 0x95, 0x72, 0x46, 0x52, 0x0c, + 0x60, 0x5e, 0xf9, 0x5e, 0xbf, 0xb2, 0x5e, 0xd9, 0xe8, 0x6e, 0xf5, 0x36, 0xf3, 0x61, 0x1d, 0xa6, + 0x4e, 0x9a, 0x29, 0x0b, 0x91, 0x48, 0xe3, 0x8e, 0xbd, 0xfe, 0xdc, 0x34, 0xcd, 0xbe, 0x4c, 0x4f, + 0x22, 0xcf, 0x42, 0xa4, 0xe8, 0xc1, 0xbc, 0x4c, 0x92, 0xfe, 0xfc, 0x7a, 0x65, 0xa3, 0x6d, 0xe1, + 0xa3, 0x10, 0xb0, 0xe0, 0x39, 0xa9, 0xd3, 0x5f, 0x20, 0x10, 0x3d, 0x8b, 0x77, 0xa0, 0x1b, 0x27, + 0x91, 0x6b, 0xfb, 0xe1, 0x30, 0xb2, 0x09, 0x5b, 0x25, 0x6c, 0x1b, 0xa1, 0x7b, 0xe1, 0x30, 0xba, + 0x8f, 0x54, 0x7d, 0xa8, 0x3b, 0xa1, 0x13, 0x9c, 0x2b, 0xd9, 0xaf, 0x11, 0xda, 0xbc, 0x8a, 0x2e, + 0xcc, 0xf9, 0x5e, 0xbf, 0xbe, 0x5e, 0xd9, 0x58, 0xb0, 0xe6, 0x7c, 0x0f, 0xbf, 0x91, 0x65, 0xbe, + 0xd7, 0x6f, 0xf0, 0x37, 0xf0, 0x59, 0x0c, 0xa0, 0x1d, 0x4a, 0xe9, 0x3d, 0x8a, 0x52, 0x4b, 0xc6, + 0xc1, 0x79, 0xbf, 0xb9, 0x5e, 0xd9, 0x68, 0x58, 0x13, 0x30, 0x71, 0x05, 0x1a, 0x9e, 0x3c, 0xce, + 0x46, 0xfb, 0x6a, 0xd4, 0x87, 0xf5, 0xca, 0x46, 0xd3, 0xca, 0xdf, 0xc5, 0x11, 0xac, 0x25, 0xf2, + 0x69, 0x26, 0x55, 0x2a, 0x3d, 0x3b, 0x95, 0x4e, 0xe2, 0x45, 0xcf, 0x43, 0x7b, 0x1c, 0x79, 0xb2, + 0xdf, 0xa2, 0x19, 0xb8, 0x56, 0x9e, 0xa5, 0x44, 0x3a, 0xe3, 0x23, 0x4d, 0xb4, 0x1f, 0x79, 0xd2, + 0xba, 0x94, 0x37, 0x2e, 0x83, 0x85, 0x05, 0xab, 0x8e, 0xeb, 0xca, 0xf8, 0x62, 0xa7, 0xed, 0x1f, + 0xd0, 0xe9, 0x8a, 0x69, 0x3b, 0xd1, 0xe7, 0x57, 0x70, 0xad, 0x18, 0xe9, 0xb1, 0x93, 0xba, 0x27, + 0xb6, 0x9b, 0x48, 0xcf, 0x4f, 0x6d, 0x37, 0xca, 0xc2, 0xb4, 0xdf, 0x59, 0xaf, 0x6c, 0x74, 0xac, + 0xcb, 0x39, 0xcd, 0x3d, 0x24, 0xd9, 0x21, 0x8a, 0x1d, 0x24, 0x78, 0x45, 0x07, 0xc7, 0xe7, 0xa9, + 0x54, 0xfd, 0x2e, 0x4d, 0xf4, 0xcc, 0x0e, 0xee, 0x21, 0x81, 0xf8, 0x12, 0xae, 0xe6, 0x7f, 0x35, + 0x63, 0x00, 0x8b, 0x34, 0x80, 0xbe, 0x21, 0xb9, 0xf0, 0xfd, 0x97, 0x36, 0xe7, 0xcf, 0xf7, 0xe8, + 0xf3, 0xb3, 0x9a, 0xf3, 0xd7, 0x6f, 0x42, 0x97, 0x5b, 0x29, 0x1c, 0x60, 0xe8, 0xca, 0xfe, 0x12, + 0xb5, 0xe8, 0x10, 0xf4, 0x50, 0x03, 0xc5, 0x6d, 0x10, 0x4c, 0xe6, 0xb8, 0xa7, 0x05, 0xa9, 0x20, + 0xd2, 0x1e, 0x61, 0xb6, 0xdd, 0x53, 0x43, 0xfd, 0xc5, 0xc2, 0x9f, 0xff, 0xfe, 0xc6, 0x1b, 0x83, + 0x27, 0xd0, 0xdc, 0x89, 0xc2, 0x50, 0xba, 0x69, 0x94, 0x88, 0x1b, 0xd0, 0x32, 0x8b, 0x63, 0x6b, + 0x59, 0xa9, 0x5a, 0x60, 0x40, 0x7b, 0x9e, 0x78, 0x0f, 0x16, 0x5d, 0x43, 0x6d, 0xfb, 0xa1, 0x27, + 0xcf, 0x48, 0x58, 0xaa, 0x56, 0x37, 0x07, 0xef, 0x21, 0x74, 0xf0, 0x5f, 0xe6, 0xa1, 0x7e, 0x78, + 0x92, 0x0d, 0x87, 0x81, 0x14, 0xef, 0x40, 0x47, 0x3f, 0xee, 0x44, 0xc1, 0x9e, 0x77, 0xa6, 0xfb, + 0x9d, 0x04, 0x8a, 0x75, 0x68, 0x69, 0xc0, 0xd1, 0x79, 0x2c, 0x75, 0xb7, 0x65, 0xd0, 0x64, 0x3f, + 0xfb, 0x7e, 0x48, 0x32, 0x38, 0x6f, 0x4d, 0x02, 0xa7, 0xa8, 0x9c, 0x33, 0x12, 0xcb, 0x49, 0x2a, + 0x87, 0xbe, 0xb6, 0x1d, 0xf8, 0xcf, 0xa4, 0x25, 0x47, 0x3b, 0x61, 0x4a, 0xc2, 0x59, 0xb5, 0xca, + 0x20, 0xb1, 0x05, 0x97, 0x14, 0x37, 0xb1, 0x13, 0x27, 0x1c, 0x49, 0x65, 0x67, 0x7e, 0x98, 0xfe, + 0xec, 0x93, 0x7e, 0x6d, 0x7d, 0x7e, 0x63, 0xc1, 0x5a, 0xd6, 0x48, 0x8b, 0x70, 0x4f, 0x08, 0x25, + 0xee, 0xc2, 0xca, 0x54, 0x1b, 0x6e, 0x52, 0x5f, 0x9f, 0xdf, 0x98, 0xb7, 0xc4, 0x44, 0x93, 0x3d, + 0x6a, 0xf1, 0x00, 0x96, 0x92, 0x2c, 0x44, 0x15, 0xb6, 0xeb, 0x07, 0xa9, 0x4c, 0x0e, 0x63, 0xe9, + 0x92, 0x90, 0xb7, 0xb6, 0xd6, 0x36, 0x49, 0xcb, 0x59, 0xd3, 0x68, 0xeb, 0x62, 0x0b, 0x71, 0x3b, + 0x9f, 0xbc, 0x07, 0x67, 0x71, 0x42, 0x9a, 0xa0, 0xb5, 0x05, 0xdc, 0x01, 0x42, 0xac, 0x32, 0x5a, + 0xdc, 0x82, 0x25, 0x2f, 0x71, 0xfc, 0xd0, 0x76, 0x82, 0xc0, 0x3e, 0xce, 0xdc, 0x53, 0x99, 0x2a, + 0xd2, 0x0e, 0x0d, 0x6b, 0x91, 0x10, 0xdb, 0x41, 0x70, 0x8f, 0xc1, 0xe2, 0x5d, 0x58, 0x54, 0x69, + 0xe2, 0x87, 0x23, 0xfb, 0xc4, 0x51, 0x27, 0xf6, 0xa9, 0x3c, 0x27, 0xe5, 0xd0, 0xb0, 0x3a, 0x0c, + 0xfe, 0xc6, 0x51, 0x27, 0xdf, 0xca, 0xf3, 0xc1, 0xff, 0x9c, 0x83, 0xc6, 0x7d, 0x5f, 0xc5, 0xc8, + 0x65, 0x62, 0x0d, 0xea, 0xc3, 0x2c, 0x74, 0x0b, 0x1e, 0xaa, 0xe1, 0xeb, 0x9e, 0x27, 0x7e, 0x09, + 0x8b, 0x41, 0xe4, 0x3a, 0x81, 0x9d, 0xb3, 0x4b, 0x7f, 0x6e, 0x7d, 0x7e, 0xa3, 0xb5, 0xb5, 0x5c, + 0x68, 0x85, 0x9c, 0x1d, 0xad, 0x2e, 0xd1, 0x16, 0xec, 0xf9, 0x25, 0xf4, 0x12, 0x39, 0x8e, 0x52, + 0x59, 0x6a, 0x3e, 0x4f, 0xcd, 0x45, 0xd1, 0xfc, 0xbb, 0xc4, 0x89, 0x1f, 0xa1, 0x2a, 0x59, 0x64, + 0xda, 0xa2, 0xf9, 0x47, 0xa5, 0x15, 0x95, 0x23, 0xdb, 0xf7, 0xce, 0x6c, 0xfa, 0x40, 0x7f, 0x61, + 0x7d, 0x7e, 0xa3, 0x5a, 0x2c, 0x8f, 0x1c, 0xed, 0x79, 0x67, 0x0f, 0x11, 0x23, 0x3e, 0x86, 0xd5, + 0xe9, 0x26, 0xdc, 0x6b, 0xbf, 0x4a, 0x6d, 0x96, 0x27, 0xda, 0x58, 0x84, 0x12, 0x6f, 0x41, 0xdb, + 0x34, 0x4a, 0x91, 0x95, 0x6b, 0xcc, 0x5c, 0xaa, 0xc4, 0xca, 0x6b, 0x50, 0xf7, 0x95, 0xad, 0xfc, + 0xf0, 0x94, 0x74, 0x7c, 0xc3, 0xaa, 0xf9, 0xea, 0xd0, 0x0f, 0x4f, 0xc5, 0x65, 0x68, 0x24, 0xd2, + 0x65, 0x4c, 0x83, 0x30, 0xf5, 0x44, 0xba, 0x84, 0x5a, 0x03, 0x7c, 0xb4, 0xdd, 0x54, 0x6a, 0x4d, + 0x5f, 0x4b, 0xa4, 0xbb, 0x93, 0xca, 0x81, 0x82, 0xea, 0xbe, 0x4c, 0x46, 0x12, 0x95, 0x3d, 0x36, + 0x3c, 0x74, 0x9d, 0x90, 0xe6, 0xbd, 0x61, 0xe5, 0xef, 0x68, 0x6a, 0x62, 0x27, 0x49, 0x7d, 0x27, + 0x20, 0xd1, 0x6a, 0x58, 0xe6, 0x55, 0x5c, 0x85, 0xa6, 0x4a, 0x9d, 0x24, 0xc5, 0xbf, 0x23, 0x91, + 0xaa, 0x5a, 0x0d, 0x02, 0xa0, 0x54, 0xae, 0x41, 0x5d, 0x86, 0x1e, 0xa1, 0x16, 0x78, 0x25, 0x65, + 0xe8, 0xed, 0x79, 0x67, 0x83, 0x7f, 0x59, 0x81, 0xce, 0x7e, 0x16, 0xa4, 0xfe, 0x76, 0x32, 0xca, + 0xe4, 0x38, 0x4c, 0xd1, 0x44, 0xdd, 0xf7, 0x55, 0xaa, 0xbf, 0x4c, 0xcf, 0x62, 0x03, 0x9a, 0x5f, + 0x27, 0x51, 0x16, 0x13, 0x57, 0xf2, 0x4a, 0x97, 0xb9, 0xb2, 0x40, 0x22, 0x07, 0x3f, 0x4e, 0x3c, + 0x99, 0xdc, 0x3b, 0x27, 0xda, 0xf9, 0x0b, 0xb4, 0x65, 0xb4, 0xb8, 0x06, 0xcd, 0x43, 0x19, 0x3b, + 0x89, 0x83, 0x2c, 0xb0, 0x40, 0x76, 0xad, 0x00, 0xe0, 0xbf, 0x12, 0xf1, 0x9e, 0xa7, 0x05, 0xdb, + 0xbc, 0x0e, 0xfe, 0x49, 0x05, 0x9a, 0xdb, 0xa3, 0x51, 0x22, 0x47, 0x4e, 0x4a, 0x46, 0x36, 0x8a, + 0x69, 0xbc, 0xf3, 0xd6, 0x5c, 0x14, 0x93, 0x21, 0xc7, 0x3f, 0xe0, 0x09, 0xa2, 0x67, 0x71, 0x1d, + 0x16, 0xe4, 0xec, 0x01, 0x11, 0x5c, 0xac, 0x42, 0xcd, 0x8d, 0xc2, 0xa1, 0x3f, 0xd2, 0xe6, 0x5f, + 0xbf, 0x89, 0x2f, 0xa0, 0xc5, 0x4f, 0xcc, 0x03, 0x55, 0xb2, 0x7d, 0x97, 0xb9, 0x79, 0x3e, 0x82, + 0x1d, 0xa2, 0x40, 0x8e, 0xb0, 0xc0, 0xcd, 0x9f, 0x07, 0x7f, 0x5a, 0x80, 0x2a, 0xcd, 0x0c, 0xae, + 0x0d, 0x9a, 0x73, 0x5b, 0x3e, 0x73, 0x02, 0xb3, 0xa4, 0x08, 0x78, 0xf0, 0xcc, 0x09, 0xc4, 0x3a, + 0x54, 0x71, 0x08, 0x6a, 0xc6, 0xc4, 0x32, 0x42, 0xbc, 0x0b, 0x55, 0xfc, 0xba, 0x9a, 0x1c, 0x3d, + 0x7e, 0xe3, 0xde, 0xc2, 0x5f, 0xfd, 0x87, 0x1b, 0x6f, 0x58, 0x8c, 0x16, 0xef, 0xc1, 0x82, 0x33, + 0x1a, 0x29, 0x12, 0x84, 0x09, 0x59, 0xcc, 0x47, 0x6a, 0x11, 0x81, 0xf8, 0x14, 0x9a, 0xbc, 0xe8, + 0x48, 0x5d, 0x25, 0xea, 0xb5, 0x92, 0x9b, 0x54, 0xe6, 0x07, 0xab, 0xa0, 0xc4, 0xe5, 0xf2, 0x95, + 0xd6, 0x40, 0x24, 0x0e, 0x0d, 0xab, 0x00, 0xa0, 0x1f, 0x13, 0x27, 0x72, 0x3b, 0x08, 0x22, 0xf7, + 0xd0, 0x7f, 0x21, 0xb5, 0xd7, 0x33, 0x01, 0x13, 0xef, 0x42, 0xf7, 0x80, 0xf9, 0xd5, 0x92, 0x2a, + 0x0b, 0x52, 0xa5, 0x3d, 0xa1, 0x29, 0xa8, 0xd8, 0x04, 0x31, 0x01, 0x39, 0xa2, 0xdf, 0x6f, 0xae, + 0xcf, 0x6f, 0x74, 0xac, 0x19, 0x18, 0xf1, 0x36, 0x74, 0x46, 0x38, 0xd3, 0xa8, 0xe0, 0x86, 0x81, + 0x83, 0x4e, 0xd2, 0x3c, 0x3a, 0x51, 0x06, 0xb8, 0x1b, 0x38, 0x23, 0x92, 0x90, 0xd8, 0x0f, 0x02, + 0x7b, 0x2c, 0xc7, 0xa4, 0xfd, 0xe6, 0xad, 0x06, 0x01, 0xf6, 0xe5, 0x58, 0xbc, 0x0f, 0x4b, 0x44, + 0x6c, 0x1f, 0x9f, 0x17, 0x2a, 0xb2, 0x4d, 0xda, 0xa1, 0x4b, 0x88, 0x7b, 0xe7, 0x5a, 0x47, 0x8a, + 0xf7, 0xa1, 0xe7, 0x9d, 0x87, 0xce, 0xd8, 0x77, 0x6d, 0xd3, 0x3f, 0xb9, 0x2e, 0xa8, 0x76, 0x19, + 0xfe, 0xb5, 0x06, 0xa3, 0xe2, 0x91, 0xe3, 0x38, 0x3d, 0xcf, 0x09, 0x6d, 0x25, 0x51, 0x42, 0xd1, + 0x55, 0x41, 0x5b, 0xb2, 0x4c, 0x58, 0x43, 0x7e, 0x28, 0xd3, 0x3d, 0x4f, 0xa1, 0xfd, 0xbf, 0xd8, + 0x88, 0x7c, 0x93, 0x86, 0xd5, 0x9b, 0x6e, 0x30, 0xf8, 0x17, 0x0b, 0x50, 0xdb, 0x0b, 0x95, 0x4c, + 0x52, 0x54, 0x1c, 0xce, 0x70, 0x28, 0xdd, 0x54, 0xb2, 0xc2, 0x5e, 0xb0, 0xf2, 0x77, 0x5c, 0xbb, + 0xa3, 0xe8, 0xbb, 0xc4, 0x4f, 0xe5, 0xe1, 0xc7, 0x5a, 0x32, 0x0a, 0x00, 0x9a, 0x12, 0xc7, 0xf3, + 0x6c, 0x43, 0x6d, 0x27, 0xd1, 0x73, 0x45, 0x4a, 0xa4, 0x61, 0x2d, 0x3a, 0x9e, 0xb7, 0xad, 0xe1, + 0x56, 0xf4, 0x5c, 0x89, 0xb7, 0x60, 0x3e, 0x91, 0x43, 0x92, 0x93, 0xd6, 0xd6, 0x22, 0xf3, 0xe2, + 0xe3, 0xe3, 0xdf, 0x49, 0x37, 0xb5, 0xe4, 0xd0, 0x42, 0x9c, 0x58, 0x81, 0xaa, 0x93, 0xa6, 0x09, + 0xf3, 0x56, 0xd3, 0xe2, 0x17, 0xb1, 0x09, 0xcb, 0xa4, 0xac, 0x52, 0x3f, 0x0a, 0xed, 0xd4, 0x39, + 0x0e, 0x24, 0xcd, 0x04, 0x1b, 0xe2, 0xa5, 0x1c, 0x75, 0x84, 0x18, 0x9c, 0x87, 0x2d, 0xb8, 0x34, + 0x4d, 0x1f, 0x3a, 0x63, 0xa9, 0xc8, 0x0e, 0x37, 0xad, 0xe5, 0xc9, 0x16, 0x8f, 0x10, 0x85, 0x8c, + 0x50, 0xb4, 0x41, 0x75, 0xd7, 0x20, 0xcd, 0xd1, 0xce, 0x81, 0xa8, 0x0d, 0x2f, 0x41, 0xcd, 0x57, + 0xb6, 0x0c, 0x3d, 0xad, 0x81, 0xab, 0xbe, 0x7a, 0x10, 0x7a, 0xe2, 0x03, 0x68, 0xf2, 0x57, 0x3c, + 0x39, 0x24, 0x3b, 0xda, 0xda, 0xea, 0x6a, 0x51, 0x43, 0xf0, 0x7d, 0x39, 0xb4, 0x1a, 0xa9, 0x7e, + 0x42, 0x1f, 0x2b, 0x8d, 0x6c, 0x79, 0x96, 0xca, 0x24, 0x74, 0x02, 0x6d, 0x4c, 0x21, 0x8d, 0x1e, + 0x68, 0x88, 0xf8, 0x14, 0xd6, 0x0c, 0xd6, 0x56, 0xe9, 0x38, 0xb5, 0xb3, 0xd0, 0x3f, 0xb3, 0x43, + 0x27, 0x8c, 0xc8, 0x83, 0x9e, 0xb7, 0x56, 0x0c, 0xfa, 0x30, 0x1d, 0xa7, 0x4f, 0x42, 0xff, 0xec, + 0x91, 0x13, 0x46, 0x62, 0x03, 0x7a, 0x79, 0xb3, 0xf4, 0x05, 0xfd, 0x30, 0x31, 0x57, 0xd3, 0xea, + 0x1a, 0xf8, 0xd1, 0x0b, 0xfc, 0x57, 0xe2, 0xad, 0x12, 0x65, 0x34, 0x1c, 0x22, 0x6f, 0x29, 0xe9, + 0x92, 0x1b, 0x5c, 0xb5, 0x96, 0x0b, 0xfa, 0xc7, 0x84, 0x3b, 0x94, 0xee, 0xe0, 0x2f, 0x2b, 0xd0, + 0x22, 0x81, 0x7e, 0x12, 0x7b, 0xa8, 0x3b, 0xdf, 0x86, 0xce, 0xe4, 0xa2, 0x33, 0xdf, 0xb4, 0x9d, + 0xf2, 0x8a, 0xaf, 0x42, 0x6d, 0xdb, 0xc5, 0xc9, 0x23, 0xc6, 0xe9, 0x58, 0xfa, 0x4d, 0xfc, 0x1c, + 0x16, 0x33, 0xea, 0xc6, 0x76, 0xd3, 0x33, 0x3b, 0x40, 0x9d, 0xcb, 0x1a, 0x4a, 0x73, 0x05, 0x7f, + 0x63, 0x27, 0x3d, 0xb3, 0x3a, 0x99, 0x79, 0x7c, 0x88, 0xda, 0xf8, 0x2e, 0xac, 0x24, 0x12, 0x39, + 0xc6, 0x7e, 0x21, 0x93, 0xc8, 0x4e, 0xe5, 0x38, 0x8e, 0x12, 0xb2, 0xe0, 0x38, 0x8b, 0x82, 0x71, + 0xdf, 0xcb, 0x24, 0x3a, 0xd2, 0x98, 0xc1, 0x9b, 0x50, 0xdd, 0x4e, 0x12, 0xe7, 0x9c, 0x58, 0x0b, + 0x1f, 0xfa, 0x15, 0x92, 0x4d, 0x7e, 0x19, 0xb8, 0x30, 0xbf, 0xef, 0xc4, 0xe2, 0x26, 0xcc, 0x8d, + 0x63, 0xc2, 0xb4, 0xb6, 0x2e, 0x95, 0x14, 0x9a, 0x13, 0x6f, 0xee, 0xc7, 0x0f, 0xc2, 0x34, 0x39, + 0xb7, 0xe6, 0xc6, 0xf1, 0x95, 0x4f, 0xa1, 0xae, 0x5f, 0x31, 0x0c, 0x44, 0x41, 0xaf, 0xd0, 0x0c, + 0xe3, 0x23, 0x7e, 0xe0, 0x99, 0x13, 0x64, 0xc6, 0x75, 0xe5, 0x97, 0x2f, 0xe6, 0x3e, 0xab, 0x0c, + 0xfe, 0xfb, 0x02, 0x34, 0xee, 0xcb, 0x40, 0xd2, 0xbf, 0x0f, 0xa0, 0x5d, 0x96, 0x0a, 0x33, 0x6f, + 0x13, 0x92, 0x32, 0x80, 0x36, 0xfb, 0x12, 0xd4, 0x4a, 0x6a, 0xb1, 0x9b, 0x80, 0xa1, 0x91, 0xdb, + 0x63, 0x27, 0x8d, 0xe4, 0xad, 0x63, 0x99, 0x57, 0xc4, 0x3c, 0xd2, 0x98, 0x05, 0xc6, 0xe8, 0x57, + 0x71, 0x0d, 0x20, 0x89, 0x9e, 0xdb, 0x3e, 0x1b, 0x74, 0xb6, 0x8d, 0x8d, 0x24, 0x7a, 0xbe, 0x87, + 0x26, 0xfd, 0x6f, 0x44, 0xcc, 0x7e, 0x0e, 0xfd, 0x92, 0x98, 0x61, 0xa8, 0x60, 0xfb, 0x21, 0x87, + 0x44, 0x5a, 0xe2, 0x8a, 0x3e, 0x29, 0x92, 0xd8, 0x0b, 0x29, 0x1a, 0x32, 0xca, 0xa3, 0xf9, 0x0a, + 0xe5, 0x31, 0x53, 0x17, 0xc1, 0x6c, 0x5d, 0x74, 0x0f, 0xe0, 0x50, 0x8e, 0xc6, 0x32, 0x4c, 0xf7, + 0x9d, 0xb8, 0xdf, 0xa2, 0x85, 0x1f, 0x14, 0x0b, 0x6f, 0x56, 0x6b, 0xb3, 0x20, 0x62, 0x2e, 0x28, + 0xb5, 0x42, 0x3f, 0xcf, 0x75, 0x42, 0x3b, 0x4d, 0xb2, 0xd0, 0x75, 0x52, 0x8e, 0x6f, 0x1b, 0x56, + 0xcb, 0x75, 0xc2, 0x23, 0x0d, 0x2a, 0x29, 0x8c, 0x4e, 0x59, 0x61, 0xbc, 0x0b, 0x8b, 0x71, 0xe2, + 0x8f, 0x9d, 0xe4, 0x1c, 0xad, 0x05, 0x2d, 0x06, 0x8b, 0x5e, 0x47, 0x83, 0xbf, 0x95, 0xe7, 0x7b, + 0xde, 0xd9, 0x95, 0x2f, 0x61, 0x71, 0x6a, 0x00, 0x3f, 0x8a, 0xef, 0xfe, 0x41, 0x15, 0x9a, 0x07, + 0x89, 0xd4, 0x4a, 0xfe, 0x06, 0xb4, 0x94, 0x7b, 0x22, 0xc7, 0x0e, 0xeb, 0x06, 0xee, 0x01, 0x18, + 0x44, 0x7a, 0x61, 0x42, 0x8d, 0xcd, 0xbd, 0x46, 0x8d, 0xf5, 0x60, 0x9e, 0xfd, 0x45, 0x14, 0x26, + 0x7c, 0x2c, 0x74, 0xf7, 0x42, 0x59, 0x77, 0xaf, 0x43, 0xfb, 0xc4, 0x51, 0xb6, 0x93, 0xa5, 0x91, + 0xed, 0x46, 0x01, 0x31, 0x5d, 0xc3, 0x82, 0x13, 0x47, 0x6d, 0x67, 0x69, 0xb4, 0x13, 0x05, 0xe2, + 0x4d, 0x00, 0x37, 0x0a, 0xb4, 0x1a, 0xd2, 0xce, 0x72, 0xd3, 0x8d, 0x02, 0xd6, 0x3d, 0xc8, 0x95, + 0x52, 0xa5, 0xfe, 0xd8, 0xd1, 0x4b, 0xaa, 0x23, 0xee, 0x3a, 0xa9, 0xc2, 0xa5, 0x1c, 0x65, 0x45, + 0xcf, 0x39, 0xd4, 0xbe, 0x0b, 0x5d, 0x37, 0x1a, 0xc7, 0x76, 0x8c, 0x33, 0x4b, 0xae, 0x5b, 0xe3, + 0x42, 0x34, 0xd4, 0x46, 0x8a, 0x83, 0x53, 0xc9, 0xce, 0xe4, 0x16, 0x2c, 0xba, 0x41, 0xa6, 0x52, + 0x99, 0xa0, 0x0d, 0x97, 0xb3, 0x03, 0xa8, 0x8e, 0x26, 0xd1, 0x0e, 0xe8, 0x00, 0x3a, 0xbe, 0xb2, + 0xa3, 0xc0, 0xb3, 0x59, 0x41, 0x69, 0x3e, 0x6b, 0xf9, 0xea, 0x71, 0xe0, 0x69, 0x15, 0xc9, 0x34, + 0xa1, 0x7c, 0x6e, 0x68, 0x5a, 0x86, 0xe6, 0x91, 0x7c, 0xae, 0x69, 0x5e, 0xa6, 0xd0, 0xda, 0x2f, + 0x53, 0x68, 0x38, 0x1f, 0x38, 0xa1, 0xa9, 0x93, 0x8c, 0x48, 0x6b, 0x07, 0x1c, 0x07, 0x31, 0x7f, + 0x2d, 0x9d, 0x38, 0xea, 0x88, 0x30, 0x87, 0x1a, 0x81, 0x51, 0x8f, 0xa6, 0xc5, 0xc9, 0x0b, 0xb3, + 0xf1, 0xb1, 0x4c, 0x68, 0x25, 0x98, 0xe3, 0x04, 0x23, 0xad, 0xe8, 0xf9, 0x23, 0x42, 0xe1, 0x8a, + 0xdc, 0x82, 0x25, 0xdd, 0xc4, 0x71, 0x53, 0xff, 0x99, 0x24, 0xf2, 0x45, 0x22, 0x5f, 0x64, 0xc4, + 0x36, 0xc1, 0x91, 0xf6, 0xfd, 0x9c, 0x56, 0x6b, 0x16, 0xa4, 0xed, 0xf1, 0x9e, 0x40, 0xde, 0xf5, + 0x9e, 0xb7, 0x13, 0x05, 0x83, 0x7f, 0x37, 0x07, 0xf5, 0x83, 0x48, 0xa5, 0xf7, 0xc7, 0x81, 0x11, + 0xe7, 0xca, 0x8f, 0x15, 0xe7, 0xb9, 0xd9, 0xe2, 0x3c, 0x43, 0xa0, 0xe6, 0x67, 0x08, 0x14, 0x1a, + 0xc9, 0x32, 0x1d, 0x09, 0x02, 0x87, 0x0f, 0xdd, 0x82, 0x90, 0x84, 0xe1, 0x2a, 0xba, 0xac, 0xb6, + 0xc7, 0xfa, 0x97, 0x99, 0xb6, 0xe1, 0x2b, 0xad, 0x7b, 0x19, 0xe9, 0x93, 0x5c, 0x69, 0x7f, 0xb6, + 0xe1, 0x2b, 0x2d, 0x67, 0x9f, 0xc3, 0xe5, 0xbc, 0xa5, 0xfd, 0xdc, 0x4f, 0x4f, 0xa2, 0x2c, 0xb5, + 0x87, 0x14, 0xab, 0x2b, 0x1d, 0xed, 0xad, 0x9a, 0x9e, 0xbe, 0x63, 0x34, 0x47, 0xf2, 0xe4, 0x5e, + 0x0f, 0xb3, 0x20, 0xb0, 0x53, 0x79, 0x96, 0x6a, 0xb6, 0xed, 0xf3, 0xdc, 0xe8, 0x79, 0xdb, 0xcd, + 0x82, 0xe0, 0x48, 0x9e, 0xa5, 0x68, 0x1a, 0x1b, 0x43, 0xfd, 0x32, 0xf8, 0x87, 0x0b, 0x00, 0x0f, + 0x23, 0xf7, 0x94, 0x57, 0x1e, 0x63, 0x48, 0xa3, 0xbd, 0xb5, 0x75, 0xa9, 0xa7, 0xac, 0xb3, 0xc5, + 0x16, 0xac, 0x9a, 0xff, 0x47, 0x99, 0xc3, 0x78, 0x96, 0xd5, 0xaf, 0x56, 0x1e, 0x42, 0x63, 0x79, + 0x4f, 0x86, 0x74, 0xaf, 0xf8, 0xac, 0x98, 0x5b, 0x6c, 0x93, 0x9e, 0xc7, 0x34, 0xb7, 0xb3, 0xc2, + 0x89, 0x4e, 0xd1, 0xfc, 0xe8, 0x3c, 0x16, 0x77, 0xe1, 0x52, 0x22, 0x87, 0x89, 0x54, 0x27, 0x76, + 0xaa, 0xca, 0x1f, 0xe3, 0x50, 0x72, 0x49, 0x23, 0x8f, 0x54, 0xfe, 0xad, 0xbb, 0x70, 0x89, 0x67, + 0x6a, 0x7a, 0x78, 0x6c, 0xab, 0x96, 0x18, 0x59, 0x1e, 0xdd, 0x9b, 0x40, 0x1b, 0xc3, 0x6c, 0x7f, + 0x4c, 0x6c, 0x11, 0xd0, 0x64, 0x1c, 0x07, 0x12, 0xbd, 0xd7, 0x9d, 0x13, 0x27, 0x1c, 0xa1, 0xce, + 0xd2, 0x93, 0x5f, 0x00, 0xc4, 0x00, 0x16, 0xf6, 0x23, 0x4f, 0xd2, 0x54, 0x77, 0xb7, 0xba, 0x9b, + 0xb4, 0xc5, 0x8c, 0x33, 0x49, 0x7b, 0x91, 0x84, 0x13, 0xef, 0x01, 0x75, 0xc7, 0xec, 0x77, 0x51, + 0x2f, 0x34, 0x10, 0x49, 0x3c, 0x78, 0x17, 0x2e, 0x15, 0x23, 0xb1, 0x9d, 0xd4, 0x4e, 0x4f, 0x24, + 0xa9, 0x7e, 0x56, 0x0d, 0x4b, 0xf9, 0xa0, 0xb6, 0xd3, 0xa3, 0x13, 0x89, 0x66, 0x60, 0x03, 0xea, + 0xd1, 0xf1, 0xef, 0x6c, 0x14, 0x84, 0xd6, 0x6c, 0x41, 0xa8, 0x45, 0xc7, 0xbf, 0xb3, 0xe4, 0x50, + 0xfc, 0xac, 0x6c, 0x36, 0xa7, 0xa6, 0xa6, 0x4d, 0x53, 0xb3, 0x92, 0xe3, 0x4b, 0xb3, 0x33, 0xf8, + 0x0c, 0x6a, 0xf8, 0x3b, 0x8f, 0x63, 0xb1, 0x09, 0x75, 0x16, 0x47, 0xa5, 0xdd, 0x9c, 0x95, 0xc2, + 0xda, 0x15, 0xbc, 0x63, 0x19, 0xa2, 0x81, 0x05, 0x8b, 0xb9, 0xe9, 0x78, 0x12, 0xfa, 0x4f, 0x33, + 0x29, 0xbe, 0x82, 0xa5, 0x38, 0x91, 0x9a, 0xed, 0xed, 0xec, 0x14, 0x9d, 0x37, 0x2d, 0xc1, 0x2b, + 0x9a, 0x4b, 0xf3, 0x16, 0xa7, 0xc8, 0xa1, 0xdd, 0x78, 0xe2, 0x7d, 0xf0, 0x3d, 0xac, 0xe5, 0x14, + 0x87, 0xd2, 0x8d, 0x42, 0xcf, 0x49, 0xce, 0xc9, 0xca, 0x4f, 0xf5, 0xad, 0x7e, 0x4c, 0xdf, 0x87, + 0xd4, 0xf7, 0x7f, 0xad, 0x40, 0x6b, 0x37, 0x7b, 0xf1, 0xe2, 0x9c, 0x65, 0x49, 0xb4, 0xa1, 0xf2, + 0x88, 0x3a, 0x98, 0xb3, 0x2a, 0x8f, 0xd0, 0x11, 0x3d, 0x38, 0x45, 0xb9, 0x26, 0x3e, 0x6f, 0x5a, + 0xfa, 0x0d, 0x03, 0xe4, 0x83, 0xd3, 0xa3, 0x57, 0x70, 0x34, 0xa3, 0x31, 0x40, 0xba, 0x97, 0xf9, + 0x01, 0xba, 0x49, 0x9a, 0x79, 0xf3, 0x77, 0x0c, 0x39, 0xf7, 0x86, 0x3c, 0x94, 0xdd, 0x24, 0x1a, + 0xf3, 0x64, 0x69, 0x95, 0x31, 0x03, 0x23, 0xbe, 0x86, 0x65, 0xbd, 0x81, 0xa7, 0xb5, 0x82, 0xad, + 0x62, 0xe9, 0x12, 0xeb, 0xfe, 0xa8, 0x4d, 0xbf, 0xc1, 0xdf, 0xad, 0x41, 0x03, 0x43, 0xcb, 0x5f, + 0x47, 0x7e, 0x28, 0xee, 0x42, 0xf3, 0x77, 0x91, 0x1f, 0xf2, 0x6e, 0x03, 0x27, 0x39, 0x96, 0xb9, + 0xaf, 0x47, 0x91, 0x27, 0x37, 0x91, 0x86, 0xf6, 0x19, 0x1a, 0xbf, 0xd3, 0x4f, 0xda, 0x3c, 0x25, + 0xfe, 0xe8, 0x24, 0xb5, 0x11, 0xa8, 0x75, 0x6b, 0xcb, 0x57, 0x16, 0xc2, 0xa8, 0xd7, 0x6b, 0x00, + 0x14, 0xd3, 0x46, 0xa1, 0x1d, 0x9f, 0xea, 0xb8, 0xae, 0x81, 0x90, 0xc7, 0xe1, 0xc1, 0x29, 0xca, + 0x9e, 0xaf, 0x6c, 0xbd, 0xaf, 0xa5, 0x7d, 0xf0, 0x52, 0x5c, 0xff, 0x0e, 0x74, 0xd1, 0x3f, 0x52, + 0xa7, 0x7e, 0x6c, 0xc7, 0x49, 0x74, 0x6c, 0x26, 0x05, 0xbd, 0xa6, 0xc3, 0x53, 0x3f, 0x3e, 0x40, + 0x18, 0xb9, 0x25, 0x7a, 0xb7, 0x0c, 0xd5, 0x36, 0xdb, 0x7f, 0xd0, 0x20, 0x9c, 0x5f, 0xda, 0x12, + 0x0b, 0x38, 0x4a, 0xa8, 0x93, 0xbb, 0x51, 0x4f, 0x64, 0x40, 0xe1, 0xc0, 0x65, 0x68, 0xa0, 0x30, + 0x10, 0xaa, 0xc1, 0x28, 0x37, 0x62, 0xd4, 0xfb, 0x00, 0x81, 0x1c, 0xa6, 0x36, 0x72, 0x19, 0x6f, + 0x00, 0x4c, 0x6d, 0x3d, 0x21, 0x76, 0x07, 0x91, 0xe2, 0x03, 0x68, 0xf1, 0x2c, 0x30, 0x2d, 0x5c, + 0xa0, 0x05, 0x42, 0x33, 0xf1, 0x2d, 0x68, 0x85, 0x51, 0x68, 0xcb, 0xa7, 0x44, 0xad, 0xe5, 0x76, + 0xa2, 0xe3, 0x30, 0x0a, 0x1f, 0x3c, 0x45, 0x62, 0x71, 0x47, 0x8f, 0x81, 0xf7, 0x60, 0xda, 0x2f, + 0xd9, 0x83, 0xa1, 0x91, 0xf0, 0x6e, 0xc4, 0x47, 0x66, 0x24, 0xdc, 0xa2, 0xf3, 0x92, 0x16, 0x3c, + 0x1e, 0x6e, 0xb2, 0x0e, 0x6d, 0x5a, 0xf7, 0xb1, 0x13, 0xdb, 0xa9, 0x33, 0xd2, 0x56, 0x1d, 0x10, + 0xb6, 0xef, 0xc4, 0x47, 0xce, 0x48, 0x58, 0x70, 0x79, 0x8a, 0xdf, 0x8e, 0x91, 0x75, 0x79, 0xd6, + 0x16, 0xcd, 0x1e, 0xce, 0x6c, 0xae, 0x5b, 0x9d, 0xe0, 0x3a, 0x62, 0x79, 0x9a, 0xdd, 0xcf, 0xe1, + 0xb2, 0x1c, 0x53, 0xf6, 0x63, 0x1c, 0x27, 0x52, 0xa9, 0x09, 0xd7, 0xac, 0xc7, 0x36, 0x0e, 0x09, + 0x76, 0x72, 0x7c, 0xee, 0x9f, 0xbd, 0x03, 0x5d, 0x47, 0x45, 0x43, 0xdb, 0x4c, 0x79, 0x40, 0xb9, + 0x8c, 0xaa, 0xd5, 0x46, 0xa8, 0xc5, 0x13, 0x1d, 0xa0, 0x41, 0x27, 0x2a, 0x3d, 0x54, 0x39, 0x4c, + 0x29, 0x8f, 0xd1, 0xb0, 0x3a, 0x08, 0xe6, 0x81, 0xc8, 0x61, 0x3a, 0xf8, 0xa7, 0x73, 0xd0, 0x78, + 0x18, 0x45, 0xf1, 0x4f, 0x94, 0x81, 0x32, 0x6f, 0xcd, 0xbd, 0x9c, 0xb7, 0xe6, 0x27, 0x79, 0x6b, + 0x8a, 0x07, 0x16, 0x7e, 0x38, 0x0f, 0x54, 0x7f, 0x34, 0x0f, 0xd4, 0x7e, 0x02, 0x0f, 0xd4, 0xa7, + 0x79, 0x60, 0x70, 0x17, 0xaa, 0x87, 0x32, 0x7d, 0x1c, 0xa3, 0x35, 0x33, 0x7e, 0xb1, 0x31, 0x04, + 0x13, 0xd6, 0x4c, 0xfb, 0xc4, 0x6a, 0xf0, 0xd7, 0x2d, 0x68, 0xde, 0x97, 0x5e, 0xc6, 0x33, 0x5b, + 0x9e, 0xa7, 0xca, 0xcb, 0xe7, 0x69, 0x6e, 0x72, 0x9e, 0xd0, 0x74, 0x1a, 0x19, 0x9c, 0xb1, 0x81, + 0xda, 0x30, 0x22, 0x88, 0xc2, 0x5a, 0x48, 0xa0, 0xde, 0x85, 0x9c, 0x98, 0xcf, 0x5c, 0x00, 0x5f, + 0xcd, 0xcd, 0xd5, 0x9f, 0xc6, 0xcd, 0x93, 0x7a, 0xec, 0xc2, 0xfe, 0xe4, 0x6b, 0xa7, 0x77, 0x5a, + 0x87, 0x35, 0x2e, 0xe8, 0xb0, 0x87, 0xb0, 0x1c, 0x85, 0xb6, 0x97, 0xc5, 0x81, 0x8f, 0x71, 0x21, + 0xf9, 0xd5, 0x51, 0x48, 0xee, 0x04, 0x65, 0x44, 0x73, 0x1e, 0x7d, 0x1c, 0xde, 0x37, 0x44, 0xbc, + 0x57, 0x62, 0x2d, 0x45, 0xd3, 0x20, 0x14, 0x21, 0x0f, 0x97, 0x86, 0x3c, 0x01, 0xf2, 0x61, 0x39, + 0xb5, 0xdb, 0x26, 0xe8, 0x4e, 0x14, 0x90, 0x6d, 0xfb, 0x0c, 0x16, 0x0b, 0x2a, 0x66, 0xa6, 0xd6, + 0x4b, 0x98, 0xa9, 0x63, 0x1a, 0x32, 0x3f, 0xfd, 0x4d, 0xe8, 0xad, 0x0f, 0x61, 0xd9, 0x6c, 0x01, + 0x69, 0x77, 0x86, 0x56, 0xb0, 0x4b, 0x1c, 0xd4, 0xd3, 0xbb, 0x3e, 0xe4, 0xc9, 0xd0, 0x12, 0xfd, + 0x02, 0x56, 0x4a, 0xe4, 0xc8, 0xbe, 0x65, 0xfd, 0x55, 0xe6, 0x95, 0xa5, 0xbc, 0x2d, 0xbe, 0x3e, + 0xe4, 0x3d, 0xfc, 0x96, 0x27, 0x03, 0xf3, 0x21, 0x1d, 0x9d, 0x34, 0x3d, 0x19, 0xe8, 0xd4, 0xe3, + 0x3e, 0xbc, 0x83, 0x91, 0x1c, 0xe2, 0x5d, 0x27, 0x4e, 0xb3, 0x44, 0xda, 0x71, 0xe0, 0xb8, 0xf2, + 0x24, 0x0a, 0x3c, 0x99, 0x14, 0x83, 0x5b, 0xa2, 0xc1, 0xdd, 0x88, 0x02, 0x0c, 0x67, 0x76, 0x98, + 0xf2, 0xa0, 0x20, 0x34, 0x63, 0xdd, 0x86, 0xeb, 0x17, 0xba, 0x43, 0x53, 0x57, 0x74, 0x24, 0xa8, + 0xa3, 0xcb, 0x93, 0x1d, 0x21, 0x89, 0xe9, 0xe2, 0x23, 0xb8, 0xc4, 0x6b, 0xc7, 0xcc, 0x7d, 0x2a, + 0x65, 0x6c, 0x07, 0x8e, 0x4a, 0xfb, 0xcb, 0xec, 0x56, 0x10, 0x92, 0x18, 0xf8, 0x5b, 0x29, 0xe3, + 0x87, 0x0e, 0x7f, 0x95, 0x9b, 0xe8, 0xc8, 0x83, 0xda, 0x4c, 0xcc, 0xed, 0x0a, 0x7f, 0x95, 0xa8, + 0x38, 0xfc, 0xc0, 0xc6, 0xa5, 0x49, 0xfe, 0x25, 0x5c, 0x9d, 0xe8, 0x62, 0xec, 0x24, 0xa7, 0x85, + 0x2b, 0xde, 0xbf, 0x44, 0xf3, 0xb6, 0x56, 0x6a, 0xbf, 0x4f, 0x04, 0x7a, 0x16, 0x3f, 0x86, 0x55, + 0x0c, 0x4c, 0x23, 0xef, 0x34, 0x9b, 0x0a, 0xda, 0x56, 0x69, 0xd0, 0x18, 0xb6, 0x3e, 0xf6, 0x4e, + 0xb3, 0x89, 0xc0, 0xed, 0xe7, 0xd0, 0x9f, 0xa0, 0xb5, 0x13, 0xda, 0x9c, 0xb7, 0xe3, 0x48, 0xf5, + 0xd7, 0x78, 0x3f, 0xa8, 0xbc, 0xa3, 0xc8, 0x5b, 0xf7, 0x07, 0x91, 0x12, 0xbb, 0xb0, 0x1e, 0x9f, + 0x9c, 0x2b, 0x9f, 0x92, 0x89, 0xe4, 0xd0, 0x5f, 0xec, 0xa0, 0x4f, 0x1d, 0x5c, 0x33, 0x74, 0xec, + 0xf7, 0x4f, 0xf5, 0xf3, 0x19, 0x5c, 0x36, 0x8c, 0x75, 0x22, 0xdd, 0xd3, 0xc9, 0x19, 0xbb, 0x4c, + 0x33, 0x76, 0x49, 0x73, 0x14, 0xe2, 0x4b, 0xb3, 0xb5, 0x01, 0x3d, 0xb2, 0x77, 0xf6, 0x30, 0xca, + 0x42, 0xfd, 0xa7, 0x57, 0xe8, 0x4f, 0xbb, 0x04, 0xdf, 0x45, 0x30, 0xfd, 0xe4, 0x06, 0xf4, 0xc8, + 0x5a, 0xb2, 0xd4, 0x33, 0xe5, 0x55, 0xa6, 0x44, 0xb8, 0x16, 0x74, 0xa4, 0xfc, 0x14, 0xd6, 0x34, + 0xd1, 0xd0, 0x0f, 0x9d, 0xa0, 0xfc, 0x33, 0xd7, 0xd8, 0xcd, 0x67, 0xf4, 0x2e, 0x62, 0x8b, 0x9f, + 0xf8, 0x0d, 0x88, 0x61, 0x94, 0x48, 0x7f, 0x14, 0x52, 0x58, 0x4b, 0x7f, 0xa2, 0xfa, 0x6f, 0x92, + 0x6c, 0xbc, 0x59, 0xf8, 0xf9, 0x8f, 0xef, 0x7f, 0xfb, 0x64, 0x97, 0xe9, 0xbe, 0x95, 0xe7, 0xf4, + 0x3f, 0x5a, 0x32, 0x7b, 0xc3, 0x49, 0xb0, 0x1a, 0xfc, 0xa3, 0x1a, 0x74, 0xc9, 0x1a, 0xff, 0xad, + 0x11, 0xf8, 0x5b, 0x23, 0xf0, 0xff, 0x82, 0x11, 0xb8, 0x05, 0x4b, 0x7e, 0x18, 0x67, 0x29, 0x4a, + 0x90, 0xb2, 0x33, 0x8e, 0xe2, 0x96, 0x78, 0xbb, 0x89, 0x10, 0xdf, 0xca, 0x73, 0xc5, 0x21, 0xdc, + 0xe0, 0xef, 0x54, 0xa0, 0x7e, 0x90, 0x44, 0x5e, 0xe6, 0xa6, 0x3f, 0x51, 0x2a, 0x26, 0xb9, 0x6d, + 0xfe, 0x75, 0xdc, 0xb6, 0x70, 0xc1, 0xa3, 0xfb, 0xe7, 0x15, 0x68, 0xea, 0x21, 0x3c, 0xdc, 0xfa, + 0x89, 0x83, 0x28, 0x72, 0xdb, 0x95, 0x99, 0xb9, 0xed, 0xd7, 0x8e, 0x02, 0x99, 0xf0, 0x19, 0x17, + 0x03, 0x45, 0x71, 0x91, 0xe8, 0x6e, 0x5a, 0x6d, 0x86, 0x3e, 0x8e, 0x29, 0x9f, 0xfd, 0x1c, 0x9a, + 0x14, 0xe5, 0x93, 0x16, 0x59, 0x85, 0x1a, 0x6b, 0x35, 0x3d, 0x50, 0xfd, 0xf6, 0x6a, 0x99, 0x9e, + 0xfb, 0x49, 0x32, 0x3d, 0xf8, 0x37, 0xf3, 0xd0, 0xa1, 0x2d, 0x97, 0xdd, 0x2c, 0x64, 0xa9, 0xc9, + 0x37, 0xa9, 0x2b, 0x93, 0x9b, 0xd4, 0x0b, 0x89, 0x4c, 0x4d, 0x22, 0xbd, 0xcd, 0x9f, 0xd9, 0x89, + 0x82, 0xfb, 0x72, 0x68, 0x11, 0x06, 0xa7, 0xca, 0x49, 0x46, 0x6a, 0x56, 0x19, 0x00, 0xc2, 0xf1, + 0xaf, 0x62, 0x27, 0x71, 0xc6, 0xca, 0x94, 0x01, 0xf0, 0x9b, 0x10, 0xb0, 0x40, 0xb2, 0xc9, 0xd3, + 0x42, 0xcf, 0x7a, 0xf7, 0x50, 0xf9, 0xe1, 0x28, 0x57, 0x34, 0x0d, 0x2a, 0xff, 0x18, 0x05, 0x52, + 0xdc, 0x07, 0xc1, 0x69, 0x91, 0x44, 0x3a, 0xe8, 0x7c, 0x50, 0x3f, 0xa4, 0x6d, 0x5a, 0x5b, 0xab, + 0xfc, 0x59, 0x9a, 0x4b, 0x8b, 0xd0, 0x07, 0x88, 0xb5, 0x7a, 0xfe, 0x14, 0x64, 0xc6, 0x64, 0xb2, + 0x07, 0x92, 0x47, 0xca, 0x3f, 0x78, 0x32, 0xc9, 0x2d, 0x21, 0x6e, 0xf9, 0x0a, 0x96, 0x87, 0x59, + 0x10, 0xa4, 0xf2, 0x2c, 0xb5, 0x55, 0x94, 0x25, 0xae, 0xb4, 0x5f, 0x91, 0x91, 0x59, 0x32, 0xb4, + 0x87, 0x44, 0x6a, 0xc9, 0xa1, 0xf8, 0x12, 0x44, 0xde, 0x81, 0xf9, 0x47, 0x93, 0x2f, 0xbd, 0xd0, + 0xbe, 0x67, 0x48, 0xf5, 0xdf, 0x0e, 0x07, 0xdb, 0x70, 0xc9, 0xe4, 0x48, 0x51, 0xb5, 0x6d, 0xa1, + 0xdc, 0xd2, 0xde, 0x91, 0x99, 0xe3, 0x4a, 0x69, 0x8e, 0x57, 0xa0, 0x5a, 0x2e, 0x4f, 0xe3, 0x97, + 0xc1, 0x4d, 0x68, 0x0d, 0xfd, 0x40, 0xea, 0x5c, 0x03, 0x2e, 0x9a, 0xce, 0x3a, 0x54, 0x28, 0xa9, + 0xae, 0xdf, 0x06, 0x7f, 0x59, 0x81, 0xb5, 0xd8, 0x49, 0x9e, 0x66, 0x7a, 0x57, 0x9b, 0xf3, 0xfb, + 0xea, 0xc4, 0x49, 0x3c, 0x14, 0x5c, 0xea, 0x82, 0x7b, 0xe7, 0xea, 0xa6, 0x26, 0x42, 0x78, 0x2c, + 0xef, 0xc2, 0x62, 0xa9, 0x45, 0xea, 0x24, 0x66, 0x67, 0xb5, 0x93, 0x44, 0xcf, 0x29, 0xfb, 0x7e, + 0x88, 0x40, 0x31, 0x80, 0x4e, 0x41, 0x27, 0xc9, 0x32, 0x52, 0x91, 0x90, 0xa1, 0x7a, 0x10, 0x7a, + 0x28, 0xb9, 0x61, 0x36, 0x66, 0x77, 0x81, 0x8b, 0xd8, 0xea, 0x61, 0x36, 0x26, 0x3f, 0x61, 0x05, + 0xaa, 0x5c, 0x39, 0x58, 0x25, 0x38, 0xbf, 0x0c, 0xfe, 0x50, 0x85, 0xe5, 0x3d, 0x57, 0x1e, 0xcb, + 0x64, 0x74, 0xdf, 0x49, 0x9d, 0x5d, 0x3f, 0x90, 0x47, 0x8e, 0x3a, 0x45, 0x86, 0xa3, 0x31, 0xc7, + 0x4e, 0x7a, 0xa2, 0x67, 0xa9, 0x81, 0x80, 0x03, 0x27, 0x3d, 0x41, 0xb3, 0x45, 0xc8, 0x61, 0x94, + 0x8c, 0xf5, 0x3e, 0x70, 0xd3, 0xa2, 0x7f, 0xdc, 0x25, 0x48, 0xde, 0x5a, 0xf9, 0x2f, 0xa4, 0x2e, + 0xb9, 0xa3, 0xd6, 0x54, 0x97, 0xf1, 0x16, 0xb4, 0x13, 0xe9, 0x46, 0x89, 0xa7, 0x63, 0x7f, 0x1e, + 0x67, 0x8b, 0x61, 0x1c, 0xf0, 0xdf, 0x82, 0x22, 0x77, 0x48, 0x5b, 0x5d, 0xb6, 0x6f, 0xea, 0x72, + 0x16, 0x73, 0x04, 0x72, 0xde, 0x9e, 0x27, 0xfe, 0x3f, 0xe8, 0x15, 0xb4, 0x94, 0xc8, 0x32, 0x11, + 0xf0, 0x56, 0xe1, 0xc6, 0xcc, 0xf8, 0xc5, 0xcd, 0x03, 0xd3, 0xea, 0xb7, 0xd4, 0x88, 0x93, 0x75, + 0x45, 0xf7, 0x0c, 0x15, 0x6f, 0x43, 0x47, 0xc5, 0x81, 0x9f, 0x6a, 0x06, 0x50, 0xba, 0x30, 0xaf, + 0x4d, 0x40, 0xce, 0x37, 0xa9, 0x59, 0x4b, 0xd8, 0xf8, 0x41, 0x4b, 0xd8, 0xbc, 0xb8, 0x84, 0xef, + 0x43, 0xcf, 0x4d, 0xa4, 0x27, 0xc3, 0xd4, 0x77, 0x02, 0x5b, 0xb9, 0x51, 0x6c, 0xcc, 0xf4, 0x62, + 0x01, 0x3f, 0x44, 0xb0, 0xf8, 0x19, 0xac, 0xb9, 0x51, 0x98, 0xca, 0x30, 0xcd, 0x4b, 0x37, 0x75, + 0xb6, 0x46, 0x97, 0x9c, 0x5c, 0xd2, 0x68, 0x53, 0xc0, 0xc9, 0xf9, 0x1a, 0x71, 0x17, 0x56, 0x78, + 0x79, 0xa6, 0x1a, 0x71, 0xad, 0x80, 0xa0, 0x95, 0x9a, 0x6c, 0xa1, 0x33, 0x48, 0x89, 0x54, 0xbe, + 0x97, 0x39, 0x81, 0xd6, 0x10, 0xa5, 0x0c, 0x92, 0xa5, 0x31, 0x7a, 0x2b, 0x95, 0x72, 0x54, 0x13, + 0xb4, 0x54, 0xe8, 0x42, 0x5b, 0x4d, 0x4d, 0x4b, 0x24, 0x13, 0xd4, 0xdf, 0x38, 0xea, 0xe4, 0xca, + 0x3d, 0x58, 0x99, 0xb5, 0x20, 0xaf, 0x4b, 0x5e, 0x36, 0x4b, 0xc9, 0x4b, 0x5d, 0x9e, 0xfa, 0xdf, + 0xe6, 0xe0, 0x92, 0x59, 0x6f, 0x0a, 0x39, 0x72, 0xa6, 0xbe, 0x41, 0xf6, 0x1c, 0xc3, 0x94, 0x7c, + 0xbb, 0xa7, 0x69, 0x01, 0x83, 0x68, 0x6f, 0x67, 0x03, 0x7a, 0x9a, 0xa0, 0x60, 0x7e, 0xfe, 0x4a, + 0xd7, 0xcb, 0xbb, 0x22, 0x11, 0xa0, 0x1f, 0x1c, 0xca, 0x04, 0xe7, 0xc8, 0xa3, 0x6a, 0x6d, 0x6a, + 0x42, 0xcc, 0x4e, 0x3f, 0x68, 0x70, 0x86, 0xe5, 0xa8, 0xd2, 0xe6, 0x69, 0xe6, 0x04, 0x7e, 0x7a, + 0x6e, 0x0f, 0x7d, 0x19, 0x78, 0x94, 0x29, 0xe7, 0x3a, 0xc2, 0x9e, 0xc1, 0xec, 0x22, 0x62, 0xcf, + 0x53, 0xa5, 0x91, 0xe8, 0x04, 0x6c, 0x2e, 0x00, 0x7a, 0x24, 0x87, 0x04, 0xde, 0xf3, 0x66, 0xcb, + 0x4a, 0x6d, 0xb6, 0xac, 0xbc, 0x07, 0x8b, 0xd3, 0x6b, 0xce, 0x49, 0xd1, 0xae, 0x9a, 0x5c, 0xef, + 0x59, 0x4c, 0xd8, 0x98, 0xc9, 0x84, 0x7a, 0xd2, 0xff, 0xc7, 0x1c, 0xac, 0xe8, 0x49, 0xdf, 0x89, + 0x82, 0x6c, 0x8c, 0xd6, 0x9e, 0xaa, 0x92, 0xd6, 0xa1, 0x3d, 0x8e, 0xd8, 0x85, 0x2a, 0xa9, 0x3f, + 0x18, 0x47, 0xb9, 0x2e, 0xde, 0x80, 0x9e, 0xcf, 0x2d, 0xf3, 0x79, 0x31, 0x15, 0xc2, 0x1a, 0xae, + 0x67, 0x05, 0xb9, 0x50, 0x85, 0x4e, 0xac, 0x4e, 0xa2, 0x54, 0x93, 0x92, 0x12, 0xe7, 0x39, 0x5f, + 0x32, 0x28, 0xa2, 0x26, 0x4f, 0xf6, 0x36, 0x08, 0x37, 0x4b, 0x12, 0x94, 0x8f, 0x12, 0x39, 0x27, + 0xef, 0x7a, 0x1a, 0x53, 0x50, 0xbf, 0x0d, 0xf5, 0x71, 0x54, 0x78, 0x24, 0x13, 0x8e, 0xa8, 0x55, + 0x1b, 0x47, 0xc4, 0x21, 0x57, 0xd0, 0x6b, 0x7a, 0x9a, 0xf9, 0x89, 0xf4, 0x8c, 0x1d, 0x36, 0xef, + 0xda, 0x48, 0x9f, 0xf8, 0x9e, 0x27, 0x43, 0x9d, 0x38, 0x6a, 0xf8, 0xea, 0x1b, 0x7a, 0xa7, 0x02, + 0x5a, 0x39, 0x74, 0x30, 0x34, 0x0b, 0xb3, 0x80, 0xa4, 0x22, 0xd0, 0xe5, 0x9a, 0x8b, 0x1a, 0xf1, + 0x28, 0x0b, 0x50, 0x22, 0x02, 0xbd, 0xa4, 0x64, 0x4b, 0x90, 0x05, 0xed, 0x13, 0x3f, 0x4c, 0x49, + 0x55, 0x34, 0x69, 0x49, 0x11, 0x81, 0x4c, 0xf8, 0x8d, 0x1f, 0xa6, 0x83, 0x7f, 0x36, 0x07, 0xab, + 0x7a, 0xe2, 0x0f, 0xf5, 0x04, 0x68, 0xfb, 0x4c, 0xd1, 0x85, 0x99, 0x2e, 0x9d, 0xd7, 0x9b, 0xb7, + 0xc0, 0x80, 0xf6, 0x68, 0xc0, 0x05, 0x77, 0xcd, 0xe9, 0x32, 0x4e, 0xc3, 0x57, 0xb7, 0x41, 0x5c, + 0xe0, 0x2b, 0xa5, 0xb7, 0x35, 0x7b, 0x53, 0x8c, 0xa5, 0xc4, 0x27, 0xb0, 0x3a, 0x96, 0xa9, 0x43, + 0x82, 0x10, 0x44, 0xae, 0x43, 0xad, 0x48, 0xe4, 0x79, 0xba, 0x57, 0x0c, 0xf6, 0xa1, 0x46, 0xa2, + 0xd0, 0xe3, 0x37, 0xc6, 0x4e, 0xe8, 0x0f, 0xa5, 0x4a, 0xc9, 0xcf, 0xe0, 0x16, 0xec, 0xf8, 0xf4, + 0x0c, 0x06, 0x3d, 0x09, 0xa2, 0x26, 0x8f, 0x75, 0xc8, 0x8b, 0x58, 0x23, 0x9a, 0x7a, 0x22, 0x87, + 0x7a, 0xed, 0x3a, 0xb8, 0x56, 0xa1, 0x1f, 0x8e, 0xf8, 0xe0, 0x40, 0x9d, 0x7d, 0x4a, 0x03, 0xdc, + 0x8f, 0x3c, 0x39, 0xf8, 0xf3, 0x85, 0x9c, 0x47, 0x0f, 0x34, 0xfc, 0x30, 0x75, 0x52, 0xaa, 0x95, + 0xcf, 0x07, 0xcf, 0x36, 0x92, 0xe7, 0xaa, 0x63, 0xa0, 0x5c, 0x52, 0xbf, 0x09, 0xcb, 0x93, 0xa3, + 0x65, 0xda, 0x39, 0x2e, 0x2b, 0x28, 0x0f, 0x37, 0x2f, 0xc1, 0xcf, 0xe9, 0x99, 0x54, 0x57, 0x9f, + 0x1b, 0x28, 0x93, 0x7d, 0x58, 0x4c, 0x82, 0xd2, 0xc9, 0x79, 0xe9, 0x69, 0xab, 0x98, 0xf7, 0xaa, + 0x0e, 0x35, 0x02, 0x45, 0xb3, 0x20, 0x8f, 0x93, 0x2c, 0x94, 0x9e, 0x36, 0xe9, 0x8b, 0x39, 0xfc, + 0x80, 0xc0, 0x38, 0xe0, 0x5c, 0x33, 0x95, 0xba, 0xae, 0x71, 0xd7, 0x9e, 0xd6, 0x4c, 0x45, 0xd7, + 0xc8, 0xa3, 0x05, 0xbd, 0xee, 0x9b, 0x15, 0xc4, 0x62, 0x4e, 0xad, 0xfb, 0xfe, 0x39, 0xf4, 0x73, + 0x5a, 0xfe, 0xbb, 0xe2, 0x03, 0x0d, 0x36, 0x3e, 0xa6, 0x09, 0xfd, 0x66, 0xfe, 0x91, 0x8f, 0x61, + 0x75, 0xba, 0xa1, 0xfe, 0x52, 0x93, 0x9a, 0x2d, 0x4f, 0x34, 0x2b, 0xfe, 0x24, 0x5f, 0x5f, 0xd7, + 0x71, 0x4f, 0xa4, 0x7d, 0xe2, 0xeb, 0x02, 0xf4, 0x79, 0x6b, 0xc9, 0xa0, 0x76, 0x10, 0xf3, 0x8d, + 0x9f, 0xaa, 0x19, 0xf4, 0x63, 0x5f, 0x29, 0x6d, 0x15, 0x27, 0xe9, 0xf7, 0x7d, 0xa5, 0x06, 0xff, + 0x69, 0x11, 0xda, 0xc6, 0x53, 0xa4, 0xda, 0xe7, 0xdb, 0x65, 0xa7, 0xbf, 0xb5, 0xd5, 0x33, 0xde, + 0x3b, 0x92, 0x6c, 0xa7, 0x69, 0x62, 0x72, 0x7d, 0x1c, 0x0c, 0x4c, 0xf8, 0x3b, 0x73, 0xe4, 0x20, + 0x14, 0xfe, 0xce, 0x36, 0x2c, 0x95, 0x3c, 0x48, 0x3b, 0x8d, 0x52, 0x27, 0xd0, 0x41, 0x41, 0xa9, + 0x6e, 0xac, 0x44, 0x62, 0x2d, 0xe2, 0x0b, 0xfb, 0x16, 0x47, 0x48, 0x8d, 0xc1, 0x86, 0x1b, 0x05, + 0xa6, 0xd8, 0x76, 0x2a, 0xd8, 0x40, 0x0c, 0x55, 0xc4, 0x24, 0x12, 0xe3, 0x5c, 0xf5, 0x34, 0xd0, + 0x12, 0xd4, 0x64, 0xc8, 0xe1, 0xd3, 0x20, 0x1f, 0x20, 0x39, 0xf3, 0x35, 0x8a, 0x63, 0x68, 0x80, + 0xe4, 0xa5, 0x7f, 0x08, 0xad, 0x28, 0xf1, 0x47, 0x3e, 0xa5, 0x89, 0xd9, 0xc1, 0x99, 0xfe, 0x08, + 0x30, 0xc1, 0x0e, 0x7e, 0x6a, 0x00, 0x35, 0x6d, 0xfe, 0x2f, 0x56, 0xc9, 0x68, 0x8c, 0x39, 0x02, + 0xe0, 0xa6, 0x38, 0x1c, 0x96, 0xc8, 0x66, 0x71, 0x04, 0xc0, 0x4d, 0x0f, 0x9f, 0x06, 0x94, 0x29, + 0x7f, 0x17, 0x16, 0x5d, 0x32, 0x17, 0x2c, 0x50, 0x81, 0x0c, 0x69, 0x4d, 0xab, 0x56, 0x87, 0xc1, + 0x38, 0xbe, 0x87, 0x32, 0xd4, 0xa5, 0x96, 0x4e, 0x10, 0x60, 0xc4, 0x1a, 0x39, 0x9e, 0xae, 0x8b, + 0x69, 0x1b, 0xe0, 0xc3, 0xc8, 0xf1, 0xc4, 0x17, 0x70, 0x05, 0x71, 0x36, 0x17, 0xb4, 0x86, 0xd9, + 0x58, 0x26, 0xbe, 0x6b, 0x3b, 0x8a, 0xea, 0x64, 0x74, 0x79, 0xcc, 0x2a, 0x52, 0x3c, 0x40, 0x82, + 0x47, 0x8c, 0xdf, 0x56, 0xdf, 0xcb, 0x24, 0x12, 0xdf, 0x53, 0xb6, 0x7c, 0x96, 0xfb, 0x6e, 0xb6, + 0x25, 0xde, 0x2a, 0xd6, 0xea, 0x25, 0x94, 0x54, 0x87, 0x86, 0x08, 0xcb, 0x38, 0x7d, 0xd4, 0x5e, + 0x7c, 0x0b, 0xc2, 0x18, 0x38, 0xe2, 0xfc, 0xd4, 0x51, 0xa7, 0x5c, 0x94, 0x3b, 0xb1, 0xd5, 0x36, + 0xc3, 0x47, 0xb5, 0x8c, 0x65, 0x44, 0x20, 0x02, 0x94, 0xf8, 0x0d, 0xac, 0xe4, 0x9d, 0x69, 0x5f, + 0x86, 0xba, 0xe3, 0x0d, 0x8d, 0x1b, 0x17, 0xbb, 0x9b, 0x70, 0x81, 0x2c, 0x33, 0x12, 0x06, 0x73, + 0x97, 0x5f, 0xc3, 0xa2, 0xe9, 0x92, 0x67, 0x5d, 0xf5, 0x7b, 0xd4, 0xdb, 0xf5, 0x0b, 0xbd, 0x4d, + 0xd8, 0xf6, 0xdc, 0x3e, 0x33, 0x14, 0x7f, 0x34, 0xb7, 0xe4, 0xc6, 0xca, 0xd0, 0x76, 0x48, 0x6b, + 0x6b, 0xfd, 0x42, 0x4f, 0x53, 0xc6, 0xca, 0x32, 0x43, 0x30, 0x70, 0xf1, 0x11, 0x5c, 0x32, 0x9d, + 0x45, 0x14, 0xe2, 0xd9, 0x7e, 0x44, 0xd1, 0x9f, 0x60, 0x17, 0x4b, 0x23, 0x39, 0xfc, 0xdb, 0x8b, + 0x38, 0x5a, 0xbc, 0x6a, 0x9a, 0xb0, 0x15, 0xa6, 0x88, 0x38, 0xff, 0xa9, 0x65, 0xb2, 0x5d, 0x7d, + 0x4d, 0xc2, 0x76, 0x19, 0x23, 0x60, 0x33, 0xfc, 0x0d, 0xe8, 0x51, 0xe5, 0x3c, 0x2e, 0x6b, 0x94, + 0x78, 0x7e, 0xe8, 0x04, 0xfd, 0x15, 0xde, 0x73, 0x45, 0xb8, 0x15, 0x3d, 0x7f, 0xcc, 0x50, 0x71, + 0x04, 0xab, 0xe6, 0x43, 0xb9, 0x9a, 0x51, 0x68, 0x4a, 0x68, 0xc3, 0x7b, 0xd6, 0xc4, 0x4d, 0x18, + 0x1c, 0xcb, 0x2c, 0xe1, 0xa4, 0x19, 0xba, 0x0f, 0x37, 0xa6, 0x96, 0x76, 0xec, 0x9c, 0xd9, 0x63, + 0x39, 0x8e, 0x92, 0x73, 0x6d, 0x40, 0x56, 0x49, 0x81, 0x5d, 0x9d, 0x58, 0xc4, 0x7d, 0xe7, 0x6c, + 0x9f, 0x68, 0xd8, 0x9c, 0x7c, 0x05, 0xd7, 0xa6, 0x7a, 0xe1, 0x42, 0x74, 0x19, 0x3a, 0xc7, 0x81, + 0xf4, 0x68, 0x8b, 0xbc, 0x61, 0x5d, 0x9e, 0xe8, 0xe2, 0x10, 0x29, 0x1e, 0x30, 0x81, 0xf8, 0x12, + 0x48, 0xd9, 0x2b, 0x3a, 0x69, 0x67, 0x2b, 0xd7, 0x09, 0x69, 0x57, 0x3c, 0xaf, 0xaa, 0x40, 0x5e, + 0xe4, 0x63, 0x78, 0xa8, 0x29, 0xad, 0x6e, 0x41, 0x4c, 0x9a, 0xf3, 0x13, 0x68, 0x9b, 0x8d, 0x65, + 0x6a, 0x7b, 0x99, 0xda, 0x2e, 0x71, 0x5b, 0xbd, 0x95, 0x4c, 0x0d, 0x5b, 0xc3, 0xe2, 0x45, 0x6c, + 0x02, 0x9c, 0x3a, 0xc3, 0x53, 0x87, 0xdb, 0x5c, 0x29, 0x07, 0xf8, 0xdf, 0x22, 0x9c, 0x5a, 0x34, + 0x4f, 0xcd, 0xa3, 0xf8, 0x1c, 0x2e, 0x1b, 0x29, 0x7c, 0x7e, 0x12, 0x05, 0xda, 0x61, 0x1f, 0x3a, + 0x61, 0x94, 0xa5, 0x7a, 0xa3, 0x7c, 0x55, 0x13, 0x7c, 0x87, 0x78, 0x14, 0x80, 0x5d, 0xc2, 0x8a, + 0xdf, 0xc0, 0x25, 0x27, 0xc1, 0x35, 0x96, 0x67, 0xd2, 0xcd, 0xd8, 0xbd, 0x21, 0x07, 0xf7, 0x1a, + 0xed, 0xaa, 0x96, 0x24, 0x72, 0x1b, 0xc9, 0x1e, 0x18, 0x2a, 0x72, 0x77, 0xad, 0x65, 0xe7, 0x22, + 0x50, 0x3c, 0x81, 0x35, 0xee, 0xd2, 0x70, 0x2a, 0x79, 0xc8, 0xa9, 0x2f, 0x67, 0xec, 0xa8, 0x53, + 0xa7, 0x9a, 0x6b, 0x99, 0xec, 0xdc, 0xe2, 0x01, 0x4d, 0x00, 0x7d, 0xa9, 0xc4, 0xf7, 0x70, 0x99, + 0xbb, 0xd5, 0xf1, 0xb2, 0x3e, 0xd0, 0xc7, 0x5a, 0xe9, 0xfa, 0xb4, 0xc0, 0x53, 0xc7, 0x16, 0x51, + 0x52, 0xfd, 0x2b, 0x69, 0x1f, 0x6b, 0xd5, 0x99, 0x05, 0x56, 0xe2, 0x33, 0xe8, 0x73, 0xdf, 0xda, + 0x03, 0x1c, 0xfa, 0xe1, 0x48, 0x26, 0x71, 0x82, 0xae, 0xe6, 0x0d, 0xda, 0x8f, 0xe2, 0x96, 0x1c, + 0x67, 0xec, 0x16, 0x58, 0xb1, 0x0d, 0x6f, 0x72, 0x4b, 0x37, 0x0a, 0x9f, 0xc9, 0x44, 0xe1, 0x04, + 0xe2, 0x42, 0xd9, 0xfa, 0xa5, 0xbf, 0x4e, 0x15, 0xc4, 0x57, 0x88, 0x68, 0x27, 0xa7, 0x41, 0x5e, + 0xff, 0x2d, 0x3f, 0x62, 0xe0, 0xca, 0x5d, 0x0c, 0xa3, 0xc4, 0x45, 0x36, 0x4f, 0x65, 0xe2, 0x3b, + 0x01, 0x5a, 0xcf, 0xb7, 0x68, 0xed, 0x78, 0x42, 0x76, 0x11, 0xbb, 0x5f, 0x20, 0xc5, 0xaf, 0xe0, + 0x2a, 0xb7, 0xf3, 0x7c, 0xb4, 0x23, 0xc7, 0x59, 0x2a, 0xbd, 0x62, 0x19, 0xfb, 0x03, 0x66, 0x6d, + 0x22, 0xb9, 0x5f, 0x50, 0xe4, 0x8b, 0xa5, 0x63, 0x95, 0x63, 0x68, 0xd2, 0x0e, 0x1f, 0x31, 0x52, + 0x7e, 0xde, 0xa5, 0xf2, 0xea, 0xf3, 0x2e, 0x1f, 0x42, 0x5b, 0x07, 0xb2, 0x2f, 0x3b, 0x40, 0xd3, + 0x62, 0x3c, 0xe7, 0xc2, 0x6f, 0x43, 0x93, 0xa2, 0x58, 0xfa, 0xc6, 0x0d, 0x68, 0xf1, 0xd2, 0x1d, + 0x07, 0x91, 0x7b, 0x6a, 0xe2, 0x4e, 0x02, 0xdd, 0x43, 0xc8, 0x00, 0xa0, 0xf1, 0x24, 0xf4, 0xa3, + 0x70, 0x3b, 0x08, 0x06, 0x7f, 0xaa, 0x41, 0x13, 0xdd, 0x5d, 0xda, 0x92, 0x14, 0x03, 0xe8, 0x90, + 0x4e, 0xa2, 0x92, 0x9a, 0xb1, 0x13, 0xeb, 0x13, 0x3d, 0x2d, 0x04, 0x22, 0xd5, 0xbe, 0x13, 0x4f, + 0x55, 0xdc, 0xcc, 0x4d, 0x55, 0xdc, 0xbc, 0xc5, 0x47, 0x7e, 0x99, 0x79, 0xa4, 0x39, 0x69, 0x41, + 0x1d, 0xdc, 0x63, 0x10, 0xba, 0xe1, 0x44, 0xe2, 0x04, 0xe4, 0xba, 0x4b, 0xf4, 0xe4, 0x94, 0x2e, + 0xce, 0x21, 0x95, 0xb8, 0xad, 0x11, 0x87, 0x92, 0x5d, 0x8d, 0xd2, 0x3e, 0x74, 0x75, 0x7a, 0x1f, + 0xfa, 0x16, 0x80, 0x1b, 0x85, 0x1e, 0x45, 0x07, 0x53, 0xb5, 0x08, 0x5c, 0x19, 0x53, 0x60, 0x7f, + 0x40, 0x86, 0xe4, 0x3d, 0xe8, 0xe5, 0x14, 0xe8, 0xfc, 0xbb, 0x61, 0xbe, 0xb5, 0xa2, 0xa9, 0x2c, + 0x39, 0xdc, 0x09, 0xd3, 0xe9, 0x54, 0x4a, 0xf3, 0x42, 0x2a, 0xe5, 0x25, 0x35, 0x54, 0xf0, 0xa3, + 0x0f, 0x4e, 0x5e, 0x86, 0x06, 0x15, 0x6b, 0x7a, 0x59, 0xac, 0xdd, 0x90, 0xba, 0xaf, 0x28, 0xe5, + 0xf5, 0xb2, 0x74, 0x4d, 0xfb, 0xff, 0x56, 0xba, 0xa6, 0xf3, 0xc3, 0xd2, 0x35, 0xdd, 0x1f, 0x96, + 0xae, 0x99, 0x4a, 0x6f, 0x2c, 0x4e, 0xa7, 0x37, 0x5e, 0x9a, 0x51, 0xee, 0xbd, 0x34, 0xa3, 0xfc, + 0x9a, 0x74, 0xf0, 0xd2, 0xab, 0xd3, 0xc1, 0xaf, 0xcf, 0x47, 0x8b, 0xd7, 0xe5, 0xa3, 0xdf, 0x85, + 0xc5, 0x34, 0x71, 0xdc, 0x53, 0x0e, 0xb2, 0x4f, 0xe5, 0xb9, 0xd2, 0xf9, 0xef, 0x0e, 0x81, 0x31, + 0xc4, 0xfe, 0x56, 0x9e, 0xab, 0xc1, 0x13, 0x00, 0xda, 0x7d, 0xa0, 0x5f, 0x7b, 0x19, 0x6f, 0x54, + 0x7e, 0x74, 0x7d, 0xdd, 0x5f, 0x57, 0x00, 0x0e, 0x9d, 0x71, 0xcc, 0xe9, 0x03, 0xf1, 0x67, 0xd0, + 0x52, 0xf4, 0x56, 0xae, 0x2f, 0x2a, 0xa9, 0xec, 0x82, 0x54, 0x3f, 0xf2, 0xb9, 0x3e, 0x95, 0x3f, + 0x13, 0x5b, 0x73, 0x0f, 0x79, 0x2d, 0x73, 0xd5, 0x10, 0xd0, 0xb6, 0xee, 0x4d, 0xe8, 0x6a, 0x82, + 0x58, 0x26, 0xae, 0x0c, 0xf9, 0x68, 0x47, 0xc5, 0xea, 0x30, 0xf4, 0x80, 0x81, 0xe2, 0xa3, 0x9c, + 0xcc, 0x78, 0x43, 0x17, 0xb3, 0xa5, 0xba, 0x89, 0x76, 0x87, 0x06, 0x5b, 0xe6, 0x57, 0x68, 0x20, + 0x0d, 0x58, 0xc0, 0xef, 0xf5, 0xde, 0x10, 0x2d, 0xa8, 0xeb, 0x5e, 0x7b, 0x15, 0xd1, 0x81, 0x26, + 0x1d, 0x24, 0x25, 0xdc, 0xdc, 0xe0, 0x4f, 0x2b, 0xd0, 0xda, 0x0b, 0x55, 0x9a, 0x64, 0xcc, 0xc2, + 0xc5, 0x71, 0xc9, 0x2a, 0x1d, 0x97, 0xd4, 0x47, 0x00, 0xf8, 0x37, 0xe8, 0x08, 0xc0, 0x87, 0x50, + 0xd7, 0x27, 0x73, 0x75, 0x4e, 0x69, 0xe6, 0xb1, 0x5e, 0x43, 0x23, 0x36, 0xa1, 0xe1, 0xe9, 0x23, + 0xc3, 0xba, 0x88, 0xaa, 0x74, 0x8e, 0xd7, 0x1c, 0x26, 0xb6, 0x72, 0x1a, 0xf1, 0x16, 0xcc, 0x3b, + 0xa3, 0x91, 0xde, 0xd0, 0x59, 0x2c, 0x48, 0xc9, 0x3f, 0xb7, 0x10, 0x27, 0xee, 0x40, 0x93, 0xd4, + 0x27, 0x15, 0x34, 0xd6, 0xa6, 0xfb, 0x34, 0xd5, 0x92, 0xac, 0x51, 0x29, 0x1d, 0x75, 0x07, 0x9a, + 0x41, 0x14, 0xc5, 0xdc, 0xa0, 0x3e, 0xdd, 0xc0, 0x94, 0x96, 0x59, 0x8d, 0xc0, 0x14, 0x99, 0xbd, + 0x0b, 0x35, 0x8c, 0xfc, 0xa2, 0x58, 0x47, 0x4c, 0xa5, 0x71, 0x50, 0x89, 0x95, 0x55, 0x55, 0x54, + 0x69, 0xb5, 0x05, 0xc0, 0xfc, 0x4f, 0x3d, 0x37, 0xa7, 0xa7, 0x23, 0x4f, 0xab, 0xa3, 0x90, 0x9a, + 0x0c, 0xfb, 0x3d, 0xe8, 0x71, 0x0a, 0xb5, 0xd4, 0x12, 0x4c, 0x19, 0xb8, 0x69, 0x39, 0x99, 0x95, + 0xb7, 0xba, 0xc9, 0x64, 0x96, 0xfe, 0x03, 0xa8, 0xc7, 0x9c, 0x17, 0x24, 0x0d, 0x43, 0xde, 0x9a, + 0x69, 0xaa, 0x13, 0x86, 0x96, 0xa1, 0x10, 0xbf, 0x82, 0x2e, 0x97, 0x2b, 0x0f, 0x75, 0x82, 0x8c, + 0x36, 0x75, 0x27, 0x0e, 0x75, 0x4e, 0xe4, 0xcf, 0xac, 0x4e, 0x3a, 0x91, 0x4e, 0xfb, 0x05, 0x74, + 0x8a, 0xb3, 0x6a, 0xe8, 0xec, 0x2d, 0x9a, 0x44, 0x95, 0x69, 0x5e, 0x0e, 0xc4, 0xad, 0xb6, 0x2c, + 0x87, 0xe5, 0x1b, 0x50, 0xd3, 0x25, 0xf4, 0x3d, 0x6a, 0x55, 0xba, 0x70, 0x83, 0x8b, 0x66, 0x2d, + 0x8d, 0xc7, 0xb9, 0x2c, 0xaa, 0x83, 0x29, 0x66, 0x98, 0x98, 0xcb, 0xbc, 0x34, 0xd8, 0x6a, 0xe6, + 0x55, 0xc1, 0xe2, 0xc1, 0x64, 0xb5, 0x32, 0xe7, 0x73, 0x97, 0xa9, 0xe9, 0xe5, 0x19, 0x4d, 0x39, + 0xb3, 0x6b, 0x2d, 0xc6, 0x53, 0x45, 0xcf, 0xb7, 0xa1, 0x11, 0x25, 0x1e, 0x1d, 0x0d, 0xa1, 0x02, + 0x9a, 0xdc, 0xfb, 0xd5, 0xc7, 0x91, 0x49, 0x79, 0xd4, 0x23, 0x7e, 0x41, 0xc7, 0x22, 0x4e, 0x22, + 0x72, 0x1b, 0x49, 0xc5, 0x5d, 0xba, 0xe8, 0x58, 0x68, 0x3c, 0x29, 0xb8, 0x77, 0xa0, 0x6e, 0x0e, + 0x06, 0xac, 0x5e, 0xa0, 0x34, 0x28, 0xf1, 0x31, 0x2c, 0x4e, 0x2a, 0x34, 0xd5, 0x5f, 0xbb, 0x40, + 0xdd, 0x9d, 0xd0, 0x5f, 0x68, 0x8d, 0xab, 0x81, 0x3f, 0xf6, 0x53, 0xed, 0xee, 0x4f, 0x1c, 0x0e, + 0x26, 0x04, 0x86, 0xfe, 0x3a, 0xfb, 0x75, 0xf9, 0x62, 0xe8, 0xaf, 0x33, 0x64, 0x7d, 0xa8, 0xfb, + 0x6a, 0xd7, 0x4f, 0x54, 0xaa, 0x4b, 0x5b, 0xcc, 0xab, 0x58, 0x85, 0x9a, 0xaf, 0xd0, 0x4c, 0x68, + 0x07, 0x5d, 0xbf, 0x89, 0x5b, 0x50, 0xd3, 0x87, 0x26, 0xd6, 0x2f, 0x48, 0xb4, 0x3e, 0x54, 0x65, + 0x69, 0x0a, 0xf1, 0x3e, 0xd4, 0xa9, 0x62, 0x3e, 0x8a, 0xc9, 0x53, 0x9c, 0xe0, 0x00, 0x2e, 0x5b, + 0xb7, 0x6a, 0x01, 0x97, 0xaf, 0x7f, 0x00, 0x75, 0xe3, 0xa4, 0x0c, 0xa6, 0xb9, 0x5a, 0x3b, 0x2b, + 0x96, 0xa1, 0x10, 0x37, 0xa1, 0x3a, 0x46, 0x3d, 0xd6, 0x7f, 0x7b, 0x5a, 0x42, 0x59, 0xbd, 0x31, + 0x56, 0xfc, 0xff, 0x70, 0xa5, 0x5c, 0x73, 0x6e, 0x0a, 0xd2, 0xf5, 0xde, 0xf6, 0x4d, 0x6a, 0xfb, + 0xd6, 0x0c, 0x56, 0x99, 0x2c, 0x5d, 0xb7, 0xd6, 0xe2, 0x97, 0xd4, 0xb4, 0x7f, 0x9a, 0xab, 0x7b, + 0x94, 0xae, 0xfe, 0xbb, 0x26, 0xee, 0xba, 0x68, 0x30, 0x8c, 0x11, 0x20, 0x3b, 0xf3, 0x19, 0xb4, + 0x87, 0xd9, 0x8b, 0x17, 0xe7, 0x26, 0x2f, 0xf3, 0x1e, 0xb5, 0x2b, 0xed, 0x2e, 0x95, 0xca, 0xdc, + 0xad, 0xd6, 0xb0, 0x54, 0xf3, 0xbe, 0x06, 0x75, 0x37, 0xb4, 0x1d, 0xcf, 0x4b, 0xfa, 0x1b, 0x5c, + 0xe6, 0xee, 0x86, 0xdb, 0x9e, 0x47, 0xf7, 0x7a, 0x44, 0xb1, 0xa4, 0xc3, 0xf1, 0xb6, 0xef, 0xf5, + 0xdf, 0x67, 0xc3, 0x63, 0x40, 0x7b, 0x1e, 0x5d, 0xfc, 0x61, 0xb6, 0x64, 0x7c, 0xaf, 0x7f, 0x4b, + 0x5f, 0xfc, 0xa1, 0x41, 0x7b, 0x1e, 0x3a, 0x9e, 0x18, 0xbf, 0x1a, 0x48, 0xff, 0x03, 0xce, 0x75, + 0x8d, 0x9d, 0xb3, 0x03, 0x0d, 0x42, 0x21, 0xe5, 0xac, 0x2e, 0xa9, 0xad, 0xdb, 0xd3, 0x42, 0x9a, + 0x57, 0x00, 0x58, 0x4d, 0x3f, 0x2f, 0x06, 0x20, 0xc1, 0x26, 0x55, 0x64, 0x07, 0x5b, 0xfd, 0x0f, + 0x2f, 0x0a, 0xb6, 0x2e, 0x70, 0x40, 0xc1, 0x36, 0xb5, 0x0e, 0x5b, 0x00, 0xac, 0xb3, 0x48, 0xe1, + 0x6c, 0x4e, 0xb7, 0xc9, 0xa3, 0x01, 0x8b, 0x4f, 0xb3, 0x91, 0xaa, 0xd9, 0x02, 0xa0, 0xcc, 0x12, + 0xb7, 0xb9, 0x33, 0xdd, 0x26, 0xf7, 0xee, 0xad, 0xe6, 0xb3, 0xdc, 0xd1, 0xbf, 0x03, 0xcd, 0x0c, + 0xfd, 0x78, 0xf4, 0xa4, 0xfb, 0x77, 0xa7, 0x99, 0xd9, 0xb8, 0xf8, 0x56, 0x23, 0xd3, 0x4f, 0xf8, + 0x11, 0xb2, 0x3d, 0xe4, 0x86, 0xf4, 0x3f, 0x9a, 0xfe, 0x48, 0x1e, 0x07, 0x58, 0x64, 0xa2, 0x38, + 0x24, 0xf8, 0x14, 0x5a, 0x3c, 0x69, 0xdc, 0x68, 0x6b, 0x9a, 0x47, 0x0a, 0xbf, 0xc6, 0xe2, 0xd9, + 0xe5, 0x66, 0x37, 0xa1, 0xea, 0xc4, 0x71, 0x70, 0xde, 0xff, 0x78, 0x9a, 0xc3, 0xb7, 0x11, 0x6c, + 0x31, 0x16, 0x59, 0x69, 0x9c, 0x05, 0xa9, 0x6f, 0x0e, 0xa0, 0x7d, 0x32, 0xcd, 0x4a, 0xa5, 0x13, + 0xbd, 0x56, 0x6b, 0x5c, 0x3a, 0xde, 0x7b, 0x1b, 0x1a, 0x71, 0xa4, 0x52, 0xdb, 0x1b, 0x07, 0xfd, + 0x4f, 0x2f, 0x98, 0x11, 0x3e, 0x8c, 0x64, 0xd5, 0x63, 0x7d, 0x9a, 0x6b, 0xe2, 0x80, 0xfc, 0xcf, + 0xa6, 0x0e, 0xc8, 0x6f, 0x41, 0x7b, 0x1c, 0x85, 0xa3, 0xc8, 0x3b, 0xe6, 0xd9, 0xff, 0x79, 0x79, + 0x3f, 0x60, 0x1f, 0x31, 0xbc, 0x83, 0xa0, 0x89, 0xb4, 0x69, 0xe8, 0x69, 0xdf, 0xcd, 0x57, 0xb6, + 0xa3, 0x48, 0xed, 0x7f, 0xc6, 0xbb, 0x37, 0x0c, 0xdf, 0x53, 0xdb, 0x04, 0x9d, 0xcc, 0x6c, 0x1c, + 0x9f, 0xeb, 0x34, 0xf4, 0xe7, 0xc4, 0x9e, 0x45, 0x66, 0xe3, 0xde, 0x39, 0xe7, 0xa2, 0xbf, 0x80, + 0x2b, 0xa5, 0x23, 0xab, 0x51, 0x6c, 0x87, 0x36, 0xaa, 0x80, 0x44, 0x7a, 0x99, 0x2b, 0xfb, 0x5f, + 0xe4, 0x5b, 0x0d, 0xfa, 0xdc, 0x6a, 0x14, 0x3f, 0x3a, 0x48, 0xa4, 0x45, 0x58, 0xf1, 0xa8, 0x7c, + 0x3c, 0xd6, 0x09, 0x46, 0x51, 0xe2, 0xa7, 0x27, 0xe3, 0xfe, 0x2f, 0xcc, 0x46, 0x43, 0x1e, 0x0f, + 0xe4, 0xa9, 0xcf, 0x6d, 0x43, 0x64, 0x15, 0x63, 0xcc, 0x61, 0xe2, 0x0b, 0xb8, 0xac, 0x6d, 0x01, + 0x76, 0x38, 0x71, 0x0b, 0x81, 0xea, 0xff, 0x92, 0xae, 0x21, 0x58, 0x2b, 0x08, 0xbe, 0x2e, 0x5d, + 0x48, 0xa0, 0x30, 0x6c, 0x9f, 0xd5, 0x16, 0x1d, 0x13, 0x9e, 0x80, 0x2f, 0x69, 0x02, 0xae, 0x5c, + 0x6c, 0x7f, 0x28, 0xf5, 0x95, 0x44, 0xb7, 0x41, 0xe0, 0x04, 0x90, 0xe5, 0x92, 0x9e, 0x1d, 0x65, + 0x69, 0x9c, 0xa5, 0xfd, 0x5f, 0x71, 0x9c, 0x98, 0x46, 0xf1, 0x63, 0x46, 0x3c, 0x26, 0xb8, 0xf8, + 0x1c, 0x2e, 0x93, 0xd2, 0xb4, 0xcb, 0x6d, 0x78, 0xa3, 0x48, 0xf5, 0xbf, 0xe2, 0x79, 0x23, 0x82, + 0xa3, 0xbc, 0x25, 0x6f, 0x2b, 0x29, 0x8e, 0xd3, 0x7f, 0xbd, 0xd0, 0x58, 0xea, 0x89, 0x5f, 0x2f, + 0x34, 0xde, 0xe9, 0xdd, 0xb4, 0x5a, 0xa5, 0xed, 0xa8, 0xc1, 0xa7, 0xd0, 0xde, 0xa6, 0x4b, 0xaf, + 0x7c, 0x45, 0x36, 0xf1, 0x26, 0x2c, 0xe4, 0x65, 0x4c, 0xb9, 0xb1, 0x25, 0x8a, 0x17, 0x72, 0x2f, + 0x1c, 0x46, 0x16, 0xa1, 0x07, 0xff, 0x6a, 0x01, 0x6a, 0x5c, 0x5a, 0xf2, 0xfa, 0x83, 0xad, 0x6f, + 0x1a, 0x8d, 0x11, 0x16, 0x27, 0x80, 0x58, 0x39, 0x10, 0x7a, 0xba, 0xd2, 0xbf, 0x59, 0x54, 0x48, + 0xad, 0x40, 0x95, 0xc3, 0x7c, 0x4e, 0x7c, 0xf1, 0x0b, 0x69, 0xcb, 0x4c, 0x9d, 0xd0, 0xcd, 0x56, + 0x3a, 0x95, 0xbb, 0x60, 0x81, 0x01, 0xed, 0x79, 0xb4, 0xc3, 0x6d, 0x08, 0x48, 0x1d, 0xd7, 0x74, + 0x06, 0x4b, 0x03, 0x49, 0x29, 0x9b, 0xea, 0xab, 0xfa, 0x4b, 0xaa, 0xaf, 0xae, 0xc3, 0x42, 0x68, + 0x0e, 0xa7, 0xe5, 0x78, 0xba, 0xd9, 0x86, 0xe0, 0xe2, 0x16, 0xe4, 0xa7, 0x71, 0xb5, 0x7b, 0xf9, + 0xf2, 0xd3, 0xba, 0x5b, 0xd0, 0xcc, 0xaf, 0x49, 0xd3, 0x1e, 0xe5, 0xca, 0x66, 0x71, 0x71, 0xda, + 0x91, 0x79, 0xb2, 0x0a, 0xb2, 0x57, 0xd7, 0x10, 0xb5, 0x7e, 0x5a, 0x0d, 0x11, 0x87, 0xdb, 0x6e, + 0x14, 0xaa, 0x54, 0xef, 0xe1, 0xd7, 0x7d, 0xb5, 0x83, 0xaf, 0xe2, 0x73, 0xe8, 0x24, 0xd2, 0x7d, + 0x66, 0x8f, 0xd5, 0x88, 0x3f, 0xd1, 0x29, 0xdf, 0x08, 0x30, 0x56, 0xa3, 0x6f, 0xa8, 0xbe, 0x49, + 0x47, 0xbf, 0x2d, 0xa4, 0xdd, 0x57, 0x23, 0xea, 0xf5, 0x03, 0x58, 0x1a, 0xcb, 0xf1, 0xb1, 0x4c, + 0xd4, 0x89, 0x1f, 0x1b, 0xb3, 0xd9, 0xa5, 0x7d, 0xaf, 0x5e, 0x81, 0xe0, 0xb1, 0x0c, 0xfe, 0x7e, + 0x05, 0x1a, 0x38, 0x8b, 0xc8, 0x4b, 0x42, 0xc0, 0xc2, 0xd8, 0x8d, 0x33, 0x1d, 0xd4, 0xd0, 0xb3, + 0xbe, 0x7a, 0x8d, 0xb9, 0x44, 0x5f, 0xbd, 0x46, 0x6b, 0xc8, 0x99, 0x69, 0x7a, 0xe6, 0xdb, 0x74, + 0xce, 0x29, 0x79, 0xc1, 0x9c, 0x61, 0x5e, 0xc5, 0x25, 0xa8, 0xb9, 0x21, 0xed, 0x6c, 0x70, 0x86, + 0xbf, 0xea, 0x86, 0x3b, 0x61, 0xaa, 0xc1, 0xc5, 0x01, 0xa7, 0xaa, 0x1b, 0xee, 0x79, 0x67, 0x83, + 0x7f, 0x5b, 0x81, 0xa5, 0x83, 0x24, 0x72, 0xa5, 0x52, 0x0f, 0xd1, 0x29, 0xa3, 0x6c, 0x2a, 0x7e, + 0x91, 0x92, 0x4f, 0x9c, 0xb8, 0xa4, 0x67, 0xe4, 0x61, 0xde, 0x76, 0xca, 0x43, 0xc7, 0x79, 0xab, + 0x49, 0x10, 0x8a, 0x1c, 0x73, 0x74, 0xa9, 0x4a, 0x87, 0xd1, 0x94, 0xb6, 0xba, 0x09, 0xdd, 0x42, + 0x77, 0x95, 0x0a, 0x8a, 0x8a, 0x3b, 0x2f, 0xa8, 0x97, 0x1b, 0xd0, 0xd2, 0x65, 0x67, 0xd4, 0x0d, + 0x67, 0x22, 0x81, 0x41, 0x87, 0x7a, 0x14, 0xac, 0xe8, 0x09, 0xcf, 0xb9, 0x47, 0x56, 0xfd, 0x88, + 0x1e, 0xfc, 0xa1, 0x02, 0xbd, 0x83, 0x44, 0xc6, 0x4e, 0x22, 0xa9, 0x0e, 0x8d, 0xe6, 0x78, 0x15, + 0x6a, 0x81, 0x0c, 0x47, 0xba, 0xf4, 0x68, 0xde, 0xd2, 0x6f, 0xf9, 0xb5, 0x79, 0x73, 0xa5, 0x6b, + 0xf3, 0x70, 0xae, 0x13, 0xe9, 0xe8, 0xdb, 0xf5, 0xe8, 0x19, 0x65, 0x10, 0xe3, 0x7f, 0x0e, 0x72, + 0x1b, 0x16, 0xbf, 0xe8, 0x93, 0xf5, 0xc7, 0x7e, 0x48, 0xf5, 0xbe, 0x74, 0xb2, 0xfe, 0x9e, 0x4f, + 0x86, 0x83, 0xc1, 0xe8, 0xc8, 0xf1, 0x0d, 0x55, 0xb4, 0x6d, 0xd5, 0xb0, 0xba, 0x44, 0xe0, 0x24, + 0xe7, 0x87, 0x04, 0xa5, 0x58, 0x9b, 0x2f, 0xb6, 0xe2, 0x62, 0x36, 0xce, 0x95, 0x75, 0xcc, 0xbd, + 0x56, 0xac, 0x5b, 0xd4, 0xe0, 0x1f, 0xd7, 0xa1, 0xa5, 0x57, 0x88, 0xfe, 0x86, 0xb9, 0xa3, 0x92, + 0x73, 0x47, 0x0f, 0xe6, 0xd5, 0xd3, 0x40, 0xb3, 0x0b, 0x3e, 0x8a, 0x8f, 0x61, 0x3e, 0xf0, 0xc7, + 0x3a, 0x00, 0xbe, 0x3a, 0xe1, 0xcc, 0x4c, 0xae, 0xb3, 0x66, 0x65, 0xa4, 0x46, 0x0b, 0x4a, 0xd7, + 0x7c, 0xa0, 0xd0, 0xe8, 0xb5, 0x41, 0xc7, 0xe2, 0x0c, 0x25, 0x13, 0x67, 0xdd, 0x71, 0xb9, 0xd6, + 0x5c, 0xab, 0x9b, 0x8e, 0xd5, 0xd4, 0x90, 0x3d, 0x4f, 0x7c, 0x02, 0x8d, 0x3c, 0x43, 0x63, 0x42, + 0xde, 0xf4, 0x2c, 0xdc, 0xdc, 0x79, 0x74, 0x74, 0x16, 0x9a, 0x14, 0x8c, 0xfe, 0x58, 0x4e, 0x29, + 0x7e, 0x05, 0x6d, 0x25, 0x95, 0xe2, 0x7b, 0x18, 0x86, 0x91, 0x56, 0x43, 0x97, 0xca, 0xd1, 0x2c, + 0x61, 0xf1, 0xaf, 0x8d, 0xd0, 0xa9, 0x02, 0x24, 0xbe, 0x81, 0xae, 0x69, 0x1f, 0x44, 0xa3, 0x51, + 0x9e, 0x41, 0xbc, 0x7a, 0xa1, 0x87, 0x87, 0x84, 0x2e, 0xf5, 0xd3, 0x51, 0x65, 0x84, 0xf8, 0x1a, + 0xba, 0x31, 0x33, 0x8d, 0xad, 0x6b, 0x28, 0x59, 0x9d, 0x5d, 0x99, 0xf0, 0xbd, 0x27, 0x98, 0xaa, + 0x38, 0x6f, 0x5c, 0xc0, 0xd5, 0xc5, 0x3b, 0x49, 0x38, 0xa5, 0x3c, 0x79, 0x27, 0x89, 0x84, 0x55, + 0x7d, 0x89, 0xd8, 0x30, 0x71, 0xe8, 0x6e, 0x05, 0x36, 0x99, 0xa6, 0x30, 0xfa, 0xce, 0x85, 0x15, + 0xc3, 0x0f, 0x6e, 0xf2, 0xdd, 0x1b, 0xbb, 0xba, 0x09, 0x99, 0x50, 0x5d, 0x65, 0xb6, 0x92, 0xcc, + 0x40, 0x89, 0x4d, 0x58, 0xd6, 0x9f, 0x29, 0x32, 0x13, 0xbe, 0x47, 0x4a, 0xaf, 0x6d, 0x2d, 0x31, + 0x2a, 0xdf, 0xcb, 0xde, 0xf3, 0xc4, 0x67, 0xd0, 0x57, 0xa9, 0x93, 0x4a, 0x1a, 0x90, 0xd1, 0xbb, + 0xfe, 0x28, 0x8c, 0x12, 0xa9, 0x2b, 0xb3, 0x56, 0x73, 0xbc, 0xd6, 0xb7, 0x7b, 0x84, 0x15, 0xbf, + 0x82, 0x1e, 0xed, 0xd8, 0xe7, 0xe5, 0x21, 0xa9, 0xd2, 0x51, 0xfc, 0x6c, 0x15, 0xdf, 0x45, 0x6a, + 0xc3, 0x16, 0x47, 0x54, 0xed, 0x41, 0xed, 0x47, 0x32, 0xc4, 0x78, 0x80, 0x34, 0x84, 0xcc, 0x94, + 0xf4, 0xf4, 0xcd, 0x41, 0x2b, 0x88, 0xfd, 0x3a, 0x47, 0x5a, 0x84, 0x43, 0x0f, 0xc4, 0x88, 0x8f, + 0xde, 0xa9, 0x25, 0xdf, 0xb7, 0xf0, 0x8b, 0x7a, 0x9c, 0x38, 0xd0, 0xd2, 0xc4, 0x34, 0xe8, 0x02, + 0xe7, 0x0e, 0xd0, 0x95, 0xaf, 0xe1, 0xf2, 0x4b, 0x67, 0xf5, 0x75, 0xa5, 0x62, 0x9d, 0xf2, 0x3d, + 0x17, 0xff, 0x6b, 0x1e, 0x5a, 0x25, 0x6e, 0xa5, 0xcb, 0x32, 0x95, 0x4c, 0x4c, 0x41, 0x28, 0x3e, + 0x23, 0xec, 0x24, 0x52, 0xa6, 0xbe, 0x91, 0x9e, 0x11, 0x96, 0x44, 0x79, 0x9d, 0x17, 0x3d, 0x23, + 0x0f, 0xe9, 0xed, 0x29, 0xbd, 0x62, 0x0b, 0x7c, 0x3f, 0x4b, 0x01, 0xdc, 0xf3, 0xe8, 0x56, 0x4d, + 0x27, 0x75, 0x8e, 0x1d, 0x65, 0x2a, 0x7b, 0xf3, 0x77, 0x34, 0x0d, 0x26, 0x77, 0xa2, 0xeb, 0x5a, + 0xf4, 0x2b, 0xca, 0x38, 0xad, 0xea, 0x8b, 0x28, 0xe4, 0x9a, 0x96, 0xb6, 0xd5, 0x40, 0xc0, 0xf7, + 0x51, 0x48, 0xcd, 0xb4, 0x44, 0xeb, 0xda, 0x2c, 0xf3, 0x8a, 0x36, 0xf3, 0x69, 0x26, 0x31, 0x2e, + 0xf5, 0xe8, 0x1c, 0x6b, 0xd3, 0xaa, 0xd3, 0x3b, 0x97, 0x8b, 0x51, 0x00, 0xfd, 0xdc, 0xf1, 0x53, + 0x52, 0x1d, 0x51, 0x96, 0x6a, 0xa6, 0x5f, 0x44, 0xc4, 0x77, 0x8e, 0x9f, 0x1e, 0x31, 0x58, 0x7c, + 0xa4, 0x8f, 0xa7, 0x97, 0x69, 0xe9, 0x7e, 0x28, 0xde, 0xf6, 0x16, 0x53, 0xf4, 0x87, 0x92, 0xae, + 0x40, 0x1c, 0x3b, 0x69, 0xe2, 0x9f, 0x45, 0x21, 0xfa, 0x4e, 0x74, 0x15, 0x44, 0x7e, 0x93, 0x67, + 0xc3, 0x5a, 0xce, 0x91, 0x8f, 0x08, 0x47, 0x45, 0x00, 0x4f, 0x60, 0x43, 0x9e, 0xc5, 0x81, 0xef, + 0xfa, 0x53, 0x57, 0x5a, 0xd8, 0xae, 0xa3, 0x52, 0x3b, 0x91, 0x69, 0x96, 0x84, 0x8a, 0x76, 0x74, + 0x35, 0x5f, 0xbf, 0x6d, 0xe8, 0xcb, 0xd7, 0x5c, 0xec, 0x38, 0x2a, 0xb5, 0x98, 0xf6, 0x51, 0x16, + 0x04, 0x38, 0x09, 0x79, 0xf1, 0x01, 0xd7, 0x1d, 0xd6, 0x15, 0x97, 0x1d, 0x0c, 0xfe, 0x73, 0x05, + 0x96, 0x2e, 0x68, 0x1a, 0x8c, 0x85, 0x51, 0xcb, 0x98, 0x5a, 0xa9, 0xb6, 0x55, 0xc3, 0xd7, 0x3d, + 0x8f, 0x10, 0xe9, 0x38, 0x35, 0x55, 0x52, 0x88, 0x48, 0xc7, 0xa8, 0x46, 0x2f, 0x41, 0x2d, 0x3d, + 0xa3, 0x25, 0x67, 0xeb, 0x53, 0x4d, 0xcf, 0x70, 0xad, 0xb7, 0xa1, 0x19, 0x44, 0x23, 0x3b, 0x90, + 0xcf, 0x24, 0xdf, 0x33, 0xd4, 0xdd, 0x7a, 0xe7, 0x15, 0x2a, 0x6e, 0xf3, 0x61, 0x34, 0x7a, 0x88, + 0xb4, 0x56, 0x23, 0xd0, 0x4f, 0x83, 0x5f, 0x43, 0xc3, 0x40, 0x45, 0x13, 0xaa, 0xf7, 0xe5, 0x71, + 0x36, 0xea, 0xbd, 0x21, 0x1a, 0xb0, 0x80, 0x2d, 0x7a, 0x15, 0x7c, 0xfa, 0xce, 0x49, 0xc2, 0xde, + 0x1c, 0xa2, 0x1f, 0x24, 0x49, 0x94, 0xf4, 0xe6, 0xf1, 0xf1, 0xc0, 0x09, 0x7d, 0xb7, 0xb7, 0x80, + 0x8f, 0xbb, 0x4e, 0xea, 0x04, 0xbd, 0xea, 0xe0, 0xdf, 0x57, 0xa1, 0x71, 0xa0, 0xbf, 0x2e, 0xee, + 0x43, 0x27, 0xbf, 0xaf, 0x73, 0xf6, 0xa6, 0xf4, 0xc1, 0xf4, 0x03, 0x6d, 0x4a, 0xb7, 0xe3, 0xd2, + 0xdb, 0xf4, 0xad, 0x9f, 0x73, 0x17, 0x6e, 0xfd, 0xbc, 0x06, 0xf3, 0x4f, 0x93, 0xf3, 0xc9, 0x63, + 0x02, 0x07, 0x81, 0x13, 0x5a, 0x08, 0x16, 0x1f, 0x41, 0x8b, 0x2a, 0x21, 0xd8, 0x8c, 0xea, 0x8d, + 0xdc, 0xf2, 0x05, 0xbb, 0x5c, 0x00, 0x0e, 0x48, 0xa4, 0x3d, 0xf6, 0x4d, 0x68, 0xb8, 0x27, 0x7e, + 0xe0, 0x25, 0x32, 0xd4, 0xc7, 0x75, 0xc4, 0xc5, 0x21, 0x5b, 0x39, 0x8d, 0xf8, 0x33, 0xe8, 0xf9, + 0xc5, 0x46, 0x74, 0x51, 0xf8, 0x32, 0x61, 0xaf, 0x4a, 0x5b, 0xd5, 0xd6, 0x62, 0x89, 0x9c, 0x5c, + 0xc4, 0xe2, 0x22, 0x9e, 0x7a, 0xf9, 0x22, 0x1e, 0xbe, 0x87, 0x91, 0xfc, 0xb8, 0x46, 0xbe, 0x8d, + 0x85, 0x6e, 0xdc, 0xbb, 0xda, 0xf9, 0x6e, 0x4e, 0xc7, 0xfd, 0xc6, 0x75, 0xd4, 0x4e, 0xf8, 0x3b, + 0xd0, 0x45, 0xa7, 0xde, 0xe6, 0x58, 0x00, 0xed, 0x28, 0xe8, 0x7b, 0xc3, 0x32, 0x75, 0x72, 0x1f, + 0xa3, 0x01, 0x64, 0xc6, 0x9b, 0xd0, 0x35, 0xff, 0xa2, 0xe3, 0xb3, 0x96, 0x2e, 0x8c, 0xd1, 0x50, + 0x0e, 0xc9, 0x36, 0x61, 0xd9, 0x3d, 0x71, 0xc2, 0x50, 0x06, 0xf6, 0x71, 0x36, 0x1c, 0x1a, 0x37, + 0x8c, 0x2f, 0x93, 0x5b, 0xd2, 0xa8, 0x7b, 0x84, 0x21, 0x6f, 0x6c, 0x00, 0x9d, 0xd0, 0x0f, 0xcc, + 0x85, 0xb2, 0x21, 0xbb, 0xcc, 0x55, 0xab, 0x15, 0xfa, 0x01, 0x5f, 0x21, 0x4b, 0x37, 0xdf, 0xf6, + 0xb2, 0xcc, 0xf7, 0x94, 0x9d, 0x46, 0xe6, 0x0a, 0x4b, 0x9d, 0x52, 0x2a, 0x6d, 0xd2, 0x3e, 0xc9, + 0x7c, 0xef, 0x28, 0xd2, 0x97, 0x58, 0x76, 0x88, 0xde, 0xbc, 0xe2, 0xd8, 0x27, 0xe3, 0x3d, 0xaa, + 0x4e, 0x69, 0x58, 0x9d, 0xa8, 0x1c, 0xe6, 0x0d, 0xbe, 0x82, 0x76, 0x99, 0xc5, 0x90, 0x65, 0x69, + 0xb3, 0xad, 0xf7, 0x86, 0x00, 0xa8, 0x3d, 0x8a, 0x92, 0xb1, 0x13, 0xf4, 0x2a, 0xf8, 0xcc, 0x3a, + 0xbf, 0x37, 0x27, 0xda, 0xd0, 0x30, 0x9b, 0x47, 0xbd, 0x79, 0x9d, 0xce, 0xfd, 0x05, 0x34, 0xcc, + 0x05, 0x9e, 0x74, 0xf9, 0x61, 0xe4, 0x49, 0x8e, 0xa0, 0x74, 0xd9, 0x3a, 0x02, 0x28, 0x7a, 0x32, + 0x17, 0x22, 0xcf, 0x15, 0x17, 0x22, 0x0f, 0x7e, 0x03, 0xed, 0xf2, 0x9f, 0x98, 0x04, 0x45, 0xa5, + 0x48, 0x50, 0xcc, 0x68, 0x45, 0xe5, 0x54, 0x49, 0x34, 0xb6, 0x4b, 0x4e, 0x7e, 0x03, 0x01, 0xf8, + 0x99, 0xc1, 0xdf, 0x9b, 0x83, 0x2a, 0x6d, 0xa9, 0x90, 0x13, 0x86, 0x0f, 0x85, 0xa0, 0x55, 0xad, + 0x26, 0x41, 0xfe, 0x0f, 0x4e, 0x92, 0xe7, 0x09, 0xeb, 0x85, 0x57, 0x27, 0xac, 0xb7, 0x61, 0xe9, + 0x59, 0xe9, 0xd2, 0x5d, 0xde, 0x48, 0xa9, 0x1a, 0x8f, 0x0d, 0xdb, 0xfc, 0xb6, 0xb8, 0x7c, 0x97, + 0xb6, 0x53, 0x16, 0x9f, 0x4d, 0x02, 0xc4, 0x5b, 0xa0, 0x0f, 0xe5, 0xd8, 0x5c, 0x0b, 0xc7, 0x85, + 0x63, 0x2d, 0x86, 0x6d, 0x53, 0xe5, 0x1b, 0xc6, 0xc9, 0x67, 0xa1, 0xb9, 0x89, 0x89, 0x6b, 0x05, + 0x9b, 0xe9, 0x59, 0xc8, 0xd5, 0x6b, 0x83, 0x31, 0x2c, 0xcf, 0x38, 0x41, 0x28, 0xd6, 0xa1, 0x3d, + 0x91, 0xd6, 0xe3, 0x63, 0x3d, 0xe0, 0x16, 0x79, 0xbc, 0x4f, 0x60, 0x55, 0x06, 0xfe, 0xc8, 0x3f, + 0xf6, 0xa9, 0x60, 0xba, 0x74, 0xa8, 0x91, 0x75, 0xcd, 0x4a, 0x09, 0x9b, 0x1f, 0x6a, 0x1c, 0xfc, + 0x45, 0x05, 0x96, 0x67, 0xd4, 0x57, 0xbc, 0xee, 0x04, 0xc6, 0x9b, 0x00, 0xa6, 0x90, 0x21, 0x0f, + 0xe3, 0x9a, 0x1a, 0xb2, 0x47, 0xd1, 0x9c, 0x4c, 0x9d, 0x91, 0xb1, 0xf9, 0xf8, 0x9c, 0xc7, 0x5b, + 0x0b, 0xa5, 0x78, 0xeb, 0x53, 0x58, 0x0b, 0xd0, 0x60, 0x8d, 0x23, 0xcf, 0x1f, 0xfa, 0xd2, 0x2b, + 0xdd, 0xc2, 0xc7, 0x61, 0xd1, 0x0a, 0xa2, 0xf7, 0x35, 0xd6, 0xdc, 0xc2, 0x37, 0xf8, 0x8b, 0x39, + 0xb8, 0x34, 0xb3, 0x76, 0xe3, 0x75, 0xc3, 0xbe, 0x0d, 0x62, 0xb2, 0x30, 0xa4, 0x74, 0x76, 0xa4, + 0x97, 0x94, 0x3a, 0xa3, 0xb3, 0x07, 0x1b, 0xd0, 0x9b, 0xa0, 0x2e, 0x4e, 0x90, 0x74, 0x4b, 0xb4, + 0xa8, 0xe1, 0xf6, 0xe0, 0x2d, 0x53, 0xd3, 0x6c, 0x7b, 0x3e, 0x29, 0x44, 0x0c, 0x8d, 0x68, 0x23, + 0x03, 0xc7, 0xe2, 0xbb, 0xd2, 0x14, 0xae, 0x5f, 0x37, 0x84, 0xf7, 0x73, 0x3a, 0xaa, 0x68, 0xd8, + 0x63, 0x2a, 0xd4, 0x04, 0x13, 0x37, 0x71, 0x99, 0xd3, 0x27, 0x9d, 0xf2, 0x25, 0x5c, 0x4a, 0xdc, + 0x85, 0x95, 0x82, 0xec, 0xb9, 0x9f, 0xe8, 0xc2, 0x50, 0x1d, 0x2d, 0x8a, 0x1c, 0xf7, 0x9d, 0x9f, + 0x70, 0x59, 0xe8, 0xad, 0x7f, 0x5d, 0x81, 0x1a, 0x5f, 0xb1, 0x2e, 0x96, 0xa0, 0xf3, 0x24, 0x3c, + 0x0d, 0xa3, 0xe7, 0x21, 0x03, 0x7a, 0x6f, 0x88, 0x65, 0x58, 0x34, 0x9a, 0x45, 0xdf, 0xe5, 0xde, + 0xab, 0x88, 0x1e, 0xb4, 0xe9, 0x17, 0x0d, 0x64, 0x4e, 0x5c, 0x83, 0xbe, 0x8e, 0x12, 0xee, 0xa3, + 0x47, 0x12, 0xa5, 0xfe, 0xf0, 0xdc, 0x60, 0xe7, 0xc5, 0x22, 0xb4, 0x0e, 0xd3, 0x28, 0x3e, 0x94, + 0xa1, 0xe7, 0x87, 0xa3, 0xde, 0x82, 0xe8, 0xc3, 0x8a, 0xe9, 0x95, 0x35, 0xd8, 0xae, 0x1f, 0xfa, + 0xea, 0xa4, 0x57, 0x15, 0x57, 0x61, 0x6d, 0x16, 0x66, 0xdb, 0x3d, 0xed, 0xd5, 0xc4, 0x0a, 0xf4, + 0x0c, 0xf2, 0x9e, 0xbe, 0x50, 0xbb, 0x57, 0xbf, 0xf5, 0x09, 0x88, 0x8b, 0x77, 0x99, 0xe3, 0x37, + 0x1f, 0xca, 0x91, 0xe3, 0x9e, 0xef, 0x04, 0x91, 0x42, 0x45, 0xd8, 0x81, 0x66, 0xd1, 0x57, 0xe5, + 0xd6, 0x2e, 0xd4, 0xf8, 0xf2, 0xf9, 0xd2, 0x5f, 0x33, 0xa0, 0xf7, 0x06, 0x36, 0x46, 0x6f, 0xcc, + 0x0f, 0x47, 0x8f, 0xe4, 0x59, 0xca, 0x3e, 0xc2, 0x43, 0x47, 0xa5, 0xbd, 0x39, 0xd1, 0x05, 0xd0, + 0x3f, 0xf6, 0x20, 0xf4, 0x7a, 0xf3, 0xb7, 0xbe, 0xd1, 0x72, 0x32, 0x55, 0xc7, 0x74, 0x1d, 0xae, + 0xe8, 0x4e, 0x67, 0x60, 0x7b, 0x6f, 0xe0, 0x47, 0x09, 0x81, 0xf6, 0xee, 0xbe, 0x93, 0x3a, 0xbd, + 0xca, 0xbd, 0x9d, 0xbf, 0xfa, 0xe3, 0xf5, 0xca, 0x1f, 0xfe, 0x78, 0xbd, 0xf2, 0x1f, 0xff, 0x78, + 0xfd, 0x8d, 0xdf, 0xff, 0xe9, 0x7a, 0xe5, 0xfb, 0x8f, 0x4a, 0x97, 0xf4, 0x6b, 0x77, 0x8f, 0x8a, + 0x47, 0xef, 0xe4, 0xbe, 0xdf, 0x9d, 0xf8, 0x74, 0x74, 0x27, 0x3e, 0xbe, 0x63, 0x8c, 0xc9, 0x71, + 0x8d, 0xee, 0xde, 0xff, 0xf8, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0x15, 0xbf, 0x32, 0x75, 0xfa, + 0x5f, 0x00, 0x00, } func (m *Message) Marshal() (dAtA []byte, err error) { @@ -11145,6 +11427,85 @@ func (m *ExternalScan) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ArrowDistributedExecution { + i-- + if m.ArrowDistributedExecution { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0x90 + } + if m.ArrowForceMaterialize { + i-- + if m.ArrowForceMaterialize { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0x88 + } + if m.ArrowConversionPlanVersion != 0 { + i = encodeVarintPipeline(dAtA, i, uint64(m.ArrowConversionPlanVersion)) + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0x80 + } + if len(m.ArrowSchemaFingerprint) > 0 { + i -= len(m.ArrowSchemaFingerprint) + copy(dAtA[i:], m.ArrowSchemaFingerprint) + i = encodeVarintPipeline(dAtA, i, uint64(len(m.ArrowSchemaFingerprint))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xfa + } + if len(m.ArrowRecordBatchShards) > 0 { + for iNdEx := len(m.ArrowRecordBatchShards) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.ArrowRecordBatchShards[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPipeline(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xf2 + } + } + if len(m.ArrowObjectIdentities) > 0 { + for iNdEx := len(m.ArrowObjectIdentities) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.ArrowObjectIdentities[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPipeline(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xea + } + } + if m.ArrowExecutionScope != 0 { + i = encodeVarintPipeline(dAtA, i, uint64(m.ArrowExecutionScope)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xe0 + } if m.ParquetWholeFileFanout { i-- if m.ParquetWholeFileFanout { @@ -13834,6 +14195,133 @@ func (m *ODKUForeignKeyCheck) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *ArrowObjectIdentity) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ArrowObjectIdentity) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ArrowObjectIdentity) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.LastModifiedUnixNano != 0 { + i = encodeVarintPipeline(dAtA, i, uint64(m.LastModifiedUnixNano)) + i-- + dAtA[i] = 0x28 + } + if m.Size != 0 { + i = encodeVarintPipeline(dAtA, i, uint64(m.Size)) + i-- + dAtA[i] = 0x20 + } + if len(m.Etag) > 0 { + i -= len(m.Etag) + copy(dAtA[i:], m.Etag) + i = encodeVarintPipeline(dAtA, i, uint64(len(m.Etag))) + i-- + dAtA[i] = 0x1a + } + if len(m.VersionId) > 0 { + i -= len(m.VersionId) + copy(dAtA[i:], m.VersionId) + i = encodeVarintPipeline(dAtA, i, uint64(len(m.VersionId))) + i-- + dAtA[i] = 0x12 + } + if m.FileIndex != 0 { + i = encodeVarintPipeline(dAtA, i, uint64(m.FileIndex)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *ArrowRecordBatchShard) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ArrowRecordBatchShard) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ArrowRecordBatchShard) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.EstimatedWireBytes != 0 { + i = encodeVarintPipeline(dAtA, i, uint64(m.EstimatedWireBytes)) + i-- + dAtA[i] = 0x30 + } + if m.EstimatedRows != 0 { + i = encodeVarintPipeline(dAtA, i, uint64(m.EstimatedRows)) + i-- + dAtA[i] = 0x28 + } + if len(m.RequiredDictionaryBlockIndices) > 0 { + dAtA168 := make([]byte, len(m.RequiredDictionaryBlockIndices)*10) + var j167 int + for _, num1 := range m.RequiredDictionaryBlockIndices { + num := uint64(num1) + for num >= 1<<7 { + dAtA168[j167] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j167++ + } + dAtA168[j167] = uint8(num) + j167++ + } + i -= j167 + copy(dAtA[i:], dAtA168[:j167]) + i = encodeVarintPipeline(dAtA, i, uint64(j167)) + i-- + dAtA[i] = 0x22 + } + if m.RecordBatchEnd != 0 { + i = encodeVarintPipeline(dAtA, i, uint64(m.RecordBatchEnd)) + i-- + dAtA[i] = 0x18 + } + if m.RecordBatchStart != 0 { + i = encodeVarintPipeline(dAtA, i, uint64(m.RecordBatchStart)) + i-- + dAtA[i] = 0x10 + } + if m.FileIndex != 0 { + i = encodeVarintPipeline(dAtA, i, uint64(m.FileIndex)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func encodeVarintPipeline(dAtA []byte, offset int, v uint64) int { offset -= sovPipeline(v) base := offset @@ -15647,6 +16135,34 @@ func (m *ExternalScan) ProtoSize() (n int) { if m.ParquetWholeFileFanout { n += 3 } + if m.ArrowExecutionScope != 0 { + n += 2 + sovPipeline(uint64(m.ArrowExecutionScope)) + } + if len(m.ArrowObjectIdentities) > 0 { + for _, e := range m.ArrowObjectIdentities { + l = e.ProtoSize() + n += 2 + l + sovPipeline(uint64(l)) + } + } + if len(m.ArrowRecordBatchShards) > 0 { + for _, e := range m.ArrowRecordBatchShards { + l = e.ProtoSize() + n += 2 + l + sovPipeline(uint64(l)) + } + } + l = len(m.ArrowSchemaFingerprint) + if l > 0 { + n += 2 + l + sovPipeline(uint64(l)) + } + if m.ArrowConversionPlanVersion != 0 { + n += 2 + sovPipeline(uint64(m.ArrowConversionPlanVersion)) + } + if m.ArrowForceMaterialize { + n += 3 + } + if m.ArrowDistributedExecution { + n += 3 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -16591,6 +17107,69 @@ func (m *ODKUForeignKeyCheck) ProtoSize() (n int) { return n } +func (m *ArrowObjectIdentity) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.FileIndex != 0 { + n += 1 + sovPipeline(uint64(m.FileIndex)) + } + l = len(m.VersionId) + if l > 0 { + n += 1 + l + sovPipeline(uint64(l)) + } + l = len(m.Etag) + if l > 0 { + n += 1 + l + sovPipeline(uint64(l)) + } + if m.Size != 0 { + n += 1 + sovPipeline(uint64(m.Size)) + } + if m.LastModifiedUnixNano != 0 { + n += 1 + sovPipeline(uint64(m.LastModifiedUnixNano)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ArrowRecordBatchShard) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.FileIndex != 0 { + n += 1 + sovPipeline(uint64(m.FileIndex)) + } + if m.RecordBatchStart != 0 { + n += 1 + sovPipeline(uint64(m.RecordBatchStart)) + } + if m.RecordBatchEnd != 0 { + n += 1 + sovPipeline(uint64(m.RecordBatchEnd)) + } + if len(m.RequiredDictionaryBlockIndices) > 0 { + l = 0 + for _, e := range m.RequiredDictionaryBlockIndices { + l += sovPipeline(uint64(e)) + } + n += 1 + sovPipeline(uint64(l)) + l + } + if m.EstimatedRows != 0 { + n += 1 + sovPipeline(uint64(m.EstimatedRows)) + } + if m.EstimatedWireBytes != 0 { + n += 1 + sovPipeline(uint64(m.EstimatedWireBytes)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovPipeline(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -28896,60 +29475,28 @@ func (m *ExternalScan) Unmarshal(dAtA []byte) error { } } m.ParquetWholeFileFanout = bool(v != 0) - default: - iNdEx = preIndex - skippy, err := skipPipeline(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthPipeline - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *TableScan) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPipeline - } - if iNdEx >= l { - return io.ErrUnexpectedEOF + case 28: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ArrowExecutionScope", wireType) } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break + m.ArrowExecutionScope = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ArrowExecutionScope |= ArrowExecutionScope(b&0x7F) << shift + if b < 0x80 { + break + } } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: TableScan: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: TableScan: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: + case 29: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Types", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ArrowObjectIdentities", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -28976,14 +29523,14 @@ func (m *TableScan) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Types = append(m.Types, plan.Type{}) - if err := m.Types[len(m.Types)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.ArrowObjectIdentities = append(m.ArrowObjectIdentities, &ArrowObjectIdentity{}) + if err := m.ArrowObjectIdentities[len(m.ArrowObjectIdentities)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - case 2: + case 30: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field FilterExprs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ArrowRecordBatchShards", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -29010,11 +29557,104 @@ func (m *TableScan) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.FilterExprs = append(m.FilterExprs, &plan.Expr{}) - if err := m.FilterExprs[len(m.FilterExprs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.ArrowRecordBatchShards = append(m.ArrowRecordBatchShards, &ArrowRecordBatchShard{}) + if err := m.ArrowRecordBatchShards[len(m.ArrowRecordBatchShards)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex + case 31: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ArrowSchemaFingerprint", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthPipeline + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthPipeline + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ArrowSchemaFingerprint = append(m.ArrowSchemaFingerprint[:0], dAtA[iNdEx:postIndex]...) + if m.ArrowSchemaFingerprint == nil { + m.ArrowSchemaFingerprint = []byte{} + } + iNdEx = postIndex + case 32: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ArrowConversionPlanVersion", wireType) + } + m.ArrowConversionPlanVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ArrowConversionPlanVersion |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 33: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ArrowForceMaterialize", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ArrowForceMaterialize = bool(v != 0) + case 34: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ArrowDistributedExecution", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ArrowDistributedExecution = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPipeline(dAtA[iNdEx:]) @@ -29037,7 +29677,7 @@ func (m *TableScan) Unmarshal(dAtA []byte) error { } return nil } -func (m *ValueScan) Unmarshal(dAtA []byte) error { +func (m *TableScan) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -29060,17 +29700,17 @@ func (m *ValueScan) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: ValueScan: wiretype end group for non-group") + return fmt.Errorf("proto: TableScan: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: ValueScan: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: TableScan: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field BatchBlock", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Types", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPipeline @@ -29080,23 +29720,142 @@ func (m *ValueScan) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPipeline } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthPipeline } if postIndex > l { return io.ErrUnexpectedEOF } - m.BatchBlock = string(dAtA[iNdEx:postIndex]) + m.Types = append(m.Types, plan.Type{}) + if err := m.Types[len(m.Types)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FilterExprs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPipeline + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPipeline + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FilterExprs = append(m.FilterExprs, &plan.Expr{}) + if err := m.FilterExprs[len(m.FilterExprs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPipeline(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPipeline + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ValueScan) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ValueScan: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ValueScan: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BatchBlock", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPipeline + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPipeline + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BatchBlock = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex @@ -35638,6 +36397,400 @@ func (m *ODKUForeignKeyCheck) Unmarshal(dAtA []byte) error { } return nil } +func (m *ArrowObjectIdentity) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ArrowObjectIdentity: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ArrowObjectIdentity: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FileIndex", wireType) + } + m.FileIndex = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FileIndex |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VersionId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPipeline + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPipeline + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.VersionId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Etag", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPipeline + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPipeline + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Etag = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType) + } + m.Size = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LastModifiedUnixNano", wireType) + } + m.LastModifiedUnixNano = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LastModifiedUnixNano |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPipeline(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPipeline + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ArrowRecordBatchShard) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ArrowRecordBatchShard: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ArrowRecordBatchShard: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FileIndex", wireType) + } + m.FileIndex = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FileIndex |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RecordBatchStart", wireType) + } + m.RecordBatchStart = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.RecordBatchStart |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RecordBatchEnd", wireType) + } + m.RecordBatchEnd = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.RecordBatchEnd |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType == 0 { + var v int32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RequiredDictionaryBlockIndices = append(m.RequiredDictionaryBlockIndices, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPipeline + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPipeline + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.RequiredDictionaryBlockIndices) == 0 { + m.RequiredDictionaryBlockIndices = make([]int32, 0, elementCount) + } + for iNdEx < postIndex { + var v int32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RequiredDictionaryBlockIndices = append(m.RequiredDictionaryBlockIndices, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field RequiredDictionaryBlockIndices", wireType) + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EstimatedRows", wireType) + } + m.EstimatedRows = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EstimatedRows |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EstimatedWireBytes", wireType) + } + m.EstimatedWireBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EstimatedWireBytes |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPipeline(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPipeline + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPipeline(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/pkg/queryservice/client/query_client_test.go b/pkg/queryservice/client/query_client_test.go index f6c4bc5dbb326..af5261b0721db 100644 --- a/pkg/queryservice/client/query_client_test.go +++ b/pkg/queryservice/client/query_client_test.go @@ -34,8 +34,8 @@ func TestMongoDBClientRetireRequiresProtocolVersion5(t *testing.T) { assert.Equal(t, defines.MORPCVersion5, methodVersions[query.CmdMethod_MongoDBClientRetire]) } -func TestRefreshSessionAuthRequiresProtocolVersion54(t *testing.T) { - assert.Equal(t, defines.MORPCVersion55, defines.MORPCLatestVersion) +func TestRefreshSessionAuthRequiresCurrentProtocolVersion(t *testing.T) { + assert.Equal(t, defines.MORPCVersion56, defines.MORPCLatestVersion) assert.Equal(t, defines.MORPCVersion54, methodVersions[query.CmdMethod_RefreshSessionAuth]) } diff --git a/pkg/sql/colexec/external/arrowio/file.go b/pkg/sql/colexec/external/arrowio/file.go new file mode 100644 index 0000000000000..4e08d32b2a7cc --- /dev/null +++ b/pkg/sql/colexec/external/arrowio/file.go @@ -0,0 +1,555 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowio + +import ( + "context" + "encoding/binary" + "io" + "math" + "sort" + "sync/atomic" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/arrowipc" + "github.com/matrixorigin/matrixone/pkg/container/arrowipc/ipcflatbuf" + "github.com/matrixorigin/matrixone/pkg/fileservice" +) + +const ( + ipcContinuationToken = uint32(0xffffffff) + ipcBlockSize = 24 +) + +type fileBlock struct { + offset int64 + metadata int64 + body int64 +} + +func openFile( + ctx context.Context, + rangeReader fileservice.LeasedRangeReader, + path string, + size int64, + admission fileservice.RangeReadAdmission, + options Options, +) (Reader, error) { + recordBlocks, dictionaryBlocks, err := readFooterBlocks( + ctx, rangeReader, path, size, admission, options, + ) + if err != nil { + return nil, err + } + blocks, err := mergeFileBlocks(recordBlocks, dictionaryBlocks) + if err != nil { + return nil, moerr.NewInvalidInputf(ctx, "invalid Arrow IPC File block ordering: %v", err) + } + if options.FileShard != nil { + blocks, err = selectFileShardBlocks(recordBlocks, dictionaryBlocks, *options.FileShard) + if err != nil { + return nil, moerr.NewInvalidInputf(ctx, "invalid Arrow IPC File shard: %v", err) + } + } + + metadataReader := &rangeReadAtSeeker{ + ctx: ctx, reader: rangeReader, path: path, size: size, admission: admission, + } + // Arrow-Go's FileReader does not currently release its dictionary memo from + // Close. Isolate the footer/schema probe in its own admission allocator and + // release that probe as soon as the immutable schema has been extracted. + // The actual range-message reader below replays dictionaries under its own + // normal ref-counted lifetime. + probeAllocator := newAdmissionAllocator(ctx, admission) + defer probeAllocator.releaseAll() + fileReader, err := ipc.NewFileReader( + metadataReader, + ipc.WithAllocator(probeAllocator), + ipc.WithMetadataSizeLimit(options.MaxMetadataBytes), + ipc.WithBodySizeLimit(options.MaxBodyBytes), + ipc.WithEnsureNativeEndian(true), + ) + if err != nil { + return nil, moerr.NewInvalidInputf(ctx, "invalid Arrow IPC File footer: %v", err) + } + schema := fileReader.Schema() + records := fileReader.NumRecords() + _ = fileReader.Close() + if schema == nil || records != len(recordBlocks) { + return nil, moerr.NewInvalidInput(ctx, "Arrow IPC File footer record count is inconsistent") + } + + messageReader, err := newRangeMessageReader( + ctx, rangeReader, path, admission, schema, blocks, options, + ) + if err != nil { + return nil, err + } + reader, err := ipc.NewReaderFromMessageReader( + messageReader, + ipc.WithAllocator(options.Allocator), + ipc.WithMetadataSizeLimit(options.MaxMetadataBytes), + ipc.WithBodySizeLimit(options.MaxBodyBytes), + ipc.WithEnsureNativeEndian(true), + ) + if err != nil { + messageReader.Release() + return nil, moerr.NewInvalidInputf(ctx, "invalid Arrow IPC File schema: %v", err) + } + return &ipcRecordReader{reader: reader, rangeMessageReader: messageReader}, nil +} + +// FileShard is one independently decodable contiguous record-batch interval. +// RequiredDictionaryBlockIndices are indices in the footer dictionary vector. +type FileShard struct { + RecordBatchStart int32 + RecordBatchEnd int32 + RequiredDictionaryBlockIndices []int32 +} + +func selectFileShardBlocks( + records []fileBlock, + dictionaries []fileBlock, + shard FileShard, +) ([]fileBlock, error) { + start, end := int(shard.RecordBatchStart), int(shard.RecordBatchEnd) + if start < 0 || start >= end || end > len(records) { + return nil, moerr.NewInvalidInputNoCtxf("record interval [%d,%d) is outside [0,%d)", start, end, len(records)) + } + lastRecordOffset := records[end-1].offset + expectedDictionaries := make([]int32, 0, len(dictionaries)) + for index, block := range dictionaries { + if block.offset < lastRecordOffset { + expectedDictionaries = append(expectedDictionaries, int32(index)) + } + } + if len(shard.RequiredDictionaryBlockIndices) != len(expectedDictionaries) { + return nil, moerr.NewInvalidInputNoCtxf("dictionary closure has %d blocks, expected %d", + len(shard.RequiredDictionaryBlockIndices), len(expectedDictionaries)) + } + selectedDictionaries := make([]fileBlock, 0, len(expectedDictionaries)) + for index, expected := range expectedDictionaries { + if shard.RequiredDictionaryBlockIndices[index] != expected { + return nil, moerr.NewInvalidInputNoCtxf("dictionary closure index %d is %d, expected %d", + index, shard.RequiredDictionaryBlockIndices[index], expected) + } + selectedDictionaries = append(selectedDictionaries, dictionaries[expected]) + } + return mergeFileBlocks(records[start:end], selectedDictionaries) +} + +func readFooterBlocks( + ctx context.Context, + reader fileservice.LeasedRangeReader, + path string, + size int64, + admission fileservice.RangeReadAdmission, + options Options, +) (records []fileBlock, dictionaries []fileBlock, err error) { + defer func() { + if recovered := recover(); recovered != nil { + records = nil + dictionaries = nil + err = moerr.NewInvalidInputf(ctx, "invalid Arrow IPC File footer: %v", recovered) + } + }() + tailSize := int64(4 + len(ipc.Magic)) + if size <= int64(2*len(ipc.Magic))+4 { + return nil, nil, moerr.NewInvalidInputf(ctx, "Arrow IPC File is too small: %d", size) + } + tail, err := reader.ReadRangeLease(ctx, path, size-tailSize, tailSize, admission) + if err != nil { + return nil, nil, err + } + tailBytes := tail.Bytes() + if len(tailBytes) != int(tailSize) || string(tailBytes[4:]) != string(ipc.Magic) { + tail.Release() + return nil, nil, moerr.NewInvalidInput(ctx, "Arrow IPC File closing magic is invalid") + } + footerLength := int64(binary.LittleEndian.Uint32(tailBytes[:4])) + tail.Release() + if footerLength < 4 || footerLength > options.MaxMetadataBytes || footerLength > size-tailSize-int64(len(ipc.Magic)) { + return nil, nil, moerr.NewInvalidInputf(ctx, "Arrow IPC File footer length %d is invalid", footerLength) + } + footerStart := size - tailSize - footerLength + footer, err := reader.ReadRangeLease(ctx, path, footerStart, footerLength, admission) + if err != nil { + return nil, nil, err + } + defer footer.Release() + footerBytes := footer.Bytes() + if len(footerBytes) < 4 { + return nil, nil, moerr.NewInvalidInput(ctx, "Arrow IPC File footer is truncated") + } + root := binary.LittleEndian.Uint32(footerBytes) + if uint64(root) >= uint64(len(footerBytes)) { + return nil, nil, moerr.NewInvalidInput(ctx, "Arrow IPC File footer root is out of bounds") + } + footerMetadata := ipcflatbuf.GetRootAsFooter(footerBytes) + if err := arrowipc.ValidateSchemaMetadata( + ctx, footerMetadata.Schema(nil), len(footerBytes), + ); err != nil { + return nil, nil, err + } + dictionaryCount := footerMetadata.DictionariesLength() + recordCount := footerMetadata.RecordBatchesLength() + if dictionaryCount < 0 || recordCount < 0 || + uint64(dictionaryCount) > uint64(len(footerBytes))/ipcBlockSize || + uint64(recordCount) > uint64(len(footerBytes))/ipcBlockSize { + return nil, nil, moerr.NewInvalidInput(ctx, "Arrow IPC File block vectors are out of bounds") + } + dictionaries, err = flatbufferBlocks( + dictionaryCount, footerStart, options, footerMetadata.Dictionaries, + ) + if err != nil { + return nil, nil, moerr.NewInvalidInputf(ctx, "invalid Arrow IPC dictionary blocks: %v", err) + } + records, err = flatbufferBlocks( + recordCount, footerStart, options, footerMetadata.RecordBatches, + ) + if err != nil { + return nil, nil, moerr.NewInvalidInputf(ctx, "invalid Arrow IPC record blocks: %v", err) + } + return records, dictionaries, nil +} + +func mergeFileBlocks(records, dictionaries []fileBlock) ([]fileBlock, error) { + blocks := make([]fileBlock, 0, len(records)+len(dictionaries)) + blocks = append(blocks, records...) + blocks = append(blocks, dictionaries...) + sort.Slice(blocks, func(i, j int) bool { return blocks[i].offset < blocks[j].offset }) + previousEnd := int64(len(ipc.Magic)) + for i, block := range blocks { + if block.offset < previousEnd { + return nil, moerr.NewInvalidInputNoCtxf("block %d overlaps the previous block", i) + } + previousEnd = block.offset + block.metadata + block.body + } + return blocks, nil +} + +func flatbufferBlocks( + count int, + footerStart int64, + options Options, + read func(*ipcflatbuf.Block, int) bool, +) ([]fileBlock, error) { + if count == 0 { + return nil, nil + } + blocks := make([]fileBlock, count) + previousEnd := int64(len(ipc.Magic)) + for i := 0; i < count; i++ { + var metadataBlock ipcflatbuf.Block + if !read(&metadataBlock, i) { + return nil, moerr.NewInvalidInputNoCtxf("block %d is missing", i) + } + block := fileBlock{ + offset: metadataBlock.Offset(), + metadata: int64(metadataBlock.MetadataLength()), + body: metadataBlock.BodyLength(), + } + if block.offset < previousEnd || block.offset%8 != 0 || block.metadata < 4 || block.metadata%8 != 0 || + block.body < 0 || block.body%8 != 0 || block.metadata > options.MaxMetadataBytes || + block.body > options.MaxBodyBytes || block.metadata > math.MaxInt64-block.body || + block.offset > footerStart || block.metadata+block.body > footerStart-block.offset { + return nil, moerr.NewInvalidInputNoCtxf("block %d has invalid offset or length", i) + } + previousEnd = block.offset + block.metadata + block.body + blocks[i] = block + } + return blocks, nil +} + +type rangeReadAtSeeker struct { + ctx context.Context + reader fileservice.LeasedRangeReader + path string + size int64 + offset int64 + admission fileservice.RangeReadAdmission +} + +func (r *rangeReadAtSeeker) Read(p []byte) (int, error) { + n, err := r.ReadAt(p, r.offset) + r.offset += int64(n) + return n, err +} + +func (r *rangeReadAtSeeker) ReadAt(p []byte, offset int64) (int, error) { + if len(p) == 0 { + return 0, nil + } + if offset < 0 || offset > r.size || int64(len(p)) > r.size-offset { + return 0, io.EOF + } + lease, err := r.reader.ReadRangeLease(r.ctx, r.path, offset, int64(len(p)), r.admission) + if err != nil { + return 0, err + } + n := copy(p, lease.Bytes()) + lease.Release() + if n != len(p) { + return n, io.ErrUnexpectedEOF + } + return n, nil +} + +func (r *rangeReadAtSeeker) Seek(offset int64, whence int) (int64, error) { + next := offset + switch whence { + case io.SeekStart: + case io.SeekCurrent: + next = r.offset + offset + case io.SeekEnd: + next = r.size + offset + default: + return 0, moerr.NewInvalidInputNoCtx("invalid Arrow IPC seek whence") + } + if next < 0 || next > r.size { + return 0, moerr.NewInvalidInputNoCtx("invalid Arrow IPC seek offset") + } + r.offset = next + return next, nil +} + +type rangeMessageReader struct { + refs atomic.Int64 + ctx context.Context + reader fileservice.LeasedRangeReader + path string + admission fileservice.RangeReadAdmission + maxDecodedRecordBytes int64 + blocks []fileBlock + next int + current *ipc.Message + currentAllocator *rangeLeaseAllocator + generation uint64 + schema *ipc.Message +} + +func newRangeMessageReader( + ctx context.Context, + reader fileservice.LeasedRangeReader, + path string, + admission fileservice.RangeReadAdmission, + schema *arrow.Schema, + blocks []fileBlock, + options Options, +) (*rangeMessageReader, error) { + payload := ipc.GetSchemaPayload(schema, options.Allocator) + defer payload.Release() + meta := payload.Meta() + if meta == nil { + return nil, moerr.NewInternalErrorNoCtx("Arrow schema payload metadata is missing") + } + body := memory.NewBufferBytes(nil) + schemaMessage := ipc.NewMessage(meta, body) + meta.Release() + body.Release() + readerValue := &rangeMessageReader{ + ctx: ctx, reader: reader, path: path, admission: admission, + maxDecodedRecordBytes: options.MaxDecodedRecordBytes, + blocks: blocks, next: -1, schema: schemaMessage, + } + readerValue.refs.Store(1) + return readerValue, nil +} + +func (r *rangeMessageReader) Retain() { + if r == nil { + panic("retain released Arrow range message reader") + } + for { + refs := r.refs.Load() + if refs <= 0 { + panic("retain released Arrow range message reader") + } + if r.refs.CompareAndSwap(refs, refs+1) { + return + } + } +} + +func (r *rangeMessageReader) Release() { + if r == nil { + return + } + refs := r.refs.Add(-1) + if refs < 0 { + panic("Arrow range message reader release underflow") + } + if refs != 0 { + return + } + r.releaseCurrent() + if r.schema != nil { + r.schema.Release() + r.schema = nil + } +} + +// checkpoint identifies the last range generation published to Arrow-Go. +// A failed decoder call may abort only a newer current generation. +func (r *rangeMessageReader) checkpoint() uint64 { + if r == nil { + return 0 + } + return r.generation +} + +func (r *rangeMessageReader) releaseCurrent() { + if r == nil { + return + } + r.currentAllocator = nil + if r.current != nil { + r.current.Release() + r.current = nil + } +} + +// abortAfter releases the current message and forcibly terminates its leased +// range when Arrow-Go abandoned an intermediate owner during a failed decode. +// rangeLeaseAllocator makes a later ref-count cleanup idempotent. +func (r *rangeMessageReader) abortAfter(checkpoint uint64) { + if r == nil || r.generation <= checkpoint || r.currentAllocator == nil { + return + } + allocator := r.currentAllocator + r.currentAllocator = nil + if r.current != nil { + r.current.Release() + r.current = nil + } + allocator.release() +} + +func (r *rangeMessageReader) Message() (message *ipc.Message, err error) { + defer func() { + if recovered := recover(); recovered != nil { + message = nil + err = moerr.NewInvalidInputf(r.ctx, "invalid Arrow IPC File message: %v", recovered) + } + }() + // Message invalidates the preceding message even when the next read fails. + // Any published record has retained its own buffer references by this point. + r.releaseCurrent() + if err := r.ctx.Err(); err != nil { + return nil, err + } + if r.next == -1 { + r.current = r.schema + r.schema = nil + r.next = 0 + return r.current, nil + } + if r.next >= len(r.blocks) { + return nil, io.EOF + } + block := r.blocks[r.next] + r.next++ + lease, err := r.reader.ReadRangeLease( + r.ctx, r.path, block.offset, block.metadata+block.body, r.admission, + ) + if err != nil { + return nil, err + } + var allocator *rangeLeaseAllocator + message, allocator, err = messageFromRangeLease(r.ctx, lease, block, r.maxDecodedRecordBytes) + if err != nil { + lease.Release() + return nil, err + } + r.current = message + r.currentAllocator = allocator + r.generation++ + return message, nil +} + +type rangeLeaseAllocator struct { + lease fileservice.RangeLease + released atomic.Bool +} + +func (a *rangeLeaseAllocator) Allocate(int) []byte { + panic("range lease allocator cannot allocate") +} +func (a *rangeLeaseAllocator) Reallocate(int, []byte) []byte { + panic("range lease allocator cannot reallocate") +} +func (a *rangeLeaseAllocator) Free([]byte) { + a.release() +} + +func (a *rangeLeaseAllocator) release() { + // Keep lease immutable: forced cleanup can race Arrow-Go's late Free, and + // the successful CAS is the sole owner allowed to release it. + if a != nil && a.released.CompareAndSwap(false, true) && a.lease != nil { + a.lease.Release() + } +} + +func messageFromRangeLease( + ctx context.Context, + lease fileservice.RangeLease, + block fileBlock, + maxDecodedRecordBytes int64, +) (message *ipc.Message, allocator *rangeLeaseAllocator, err error) { + defer func() { + if recovered := recover(); recovered != nil { + if allocator != nil { + allocator.release() + } else { + lease.Release() + } + panic(recovered) + } + }() + bytes := lease.Bytes() + if int64(len(bytes)) != block.metadata+block.body || len(bytes) < 4 { + return nil, nil, moerr.NewInvalidInputNoCtx("Arrow IPC File block range is truncated") + } + metadataBytes := bytes[:int(block.metadata)] + prefix := 0 + switch binary.LittleEndian.Uint32(metadataBytes[:4]) { + case 0: + case ipcContinuationToken: + prefix = 8 + default: + prefix = 4 + } + if int(block.metadata) < prefix+4 { + return nil, nil, moerr.NewInvalidInputNoCtx("Arrow IPC File metadata prefix is invalid") + } + if _, err := inspectIPCMessageMetadata( + ctx, metadataBytes[prefix:], block.body, block.body, + bytes[int(block.metadata):], true, maxDecodedRecordBytes, + ); err != nil { + return nil, nil, err + } + allocator = &rangeLeaseAllocator{lease: lease} + owner := memory.NewBufferWithAllocator(bytes, allocator) + meta := memory.SliceBuffer(owner, prefix, int(block.metadata)-prefix) + body := memory.SliceBuffer(owner, int(block.metadata), int(block.body)) + owner.Release() + message = ipc.NewMessage(meta, body) + meta.Release() + body.Release() + return message, allocator, nil +} diff --git a/pkg/sql/colexec/external/arrowio/file_planner.go b/pkg/sql/colexec/external/arrowio/file_planner.go new file mode 100644 index 0000000000000..bd02a939ded96 --- /dev/null +++ b/pkg/sql/colexec/external/arrowio/file_planner.go @@ -0,0 +1,320 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowio + +import ( + "context" + "encoding/binary" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/arrowipc" + "github.com/matrixorigin/matrixone/pkg/fileservice" +) + +const ( + messageHeaderDictionaryBatch = byte(2) + messageHeaderRecordBatch = byte(3) +) + +// RecordBatchInfo is the footer-stable planning metadata for one record block. +type RecordBatchInfo struct { + Index int32 + Rows int64 + WireBytes int64 +} + +// DictionaryBlockInfo describes one dictionary epoch transition in footer +// order. A non-delta block establishes an ID exactly once; subsequent blocks +// for that ID must be deltas. +type DictionaryBlockInfo struct { + Index int32 + ID int64 + IsDelta bool + Rows int64 + WireBytes int64 +} + +// FilePlan is an immutable, bounded description of an IPC File. Schema is the +// official Arrow-Go decoded schema; payload bodies are not read by planning. +type FilePlan struct { + Schema *arrow.Schema + RecordBatches []RecordBatchInfo + Dictionaries []DictionaryBlockInfo + recordBlocks []fileBlock + dictBlocks []fileBlock +} + +// Shard returns a self-contained decoder interval and its conservative +// dictionary closure. Replaying every dictionary transition preceding the last +// selected record is safe even when projection-specific dependencies are not +// provable. +func (p *FilePlan) Shard(start, end int) (FileShard, int64, int64, error) { + if p == nil || p.Schema == nil || len(p.recordBlocks) != len(p.RecordBatches) || + len(p.dictBlocks) != len(p.Dictionaries) { + return FileShard{}, 0, 0, moerr.NewInvalidInputNoCtx("incomplete Arrow IPC File plan") + } + if start < 0 || start >= end || end > len(p.RecordBatches) { + return FileShard{}, 0, 0, moerr.NewInvalidInputNoCtx("invalid Arrow IPC File shard interval") + } + shard := FileShard{RecordBatchStart: int32(start), RecordBatchEnd: int32(end)} + var rows, wireBytes int64 + for _, record := range p.RecordBatches[start:end] { + if record.Rows < 0 || record.WireBytes < 0 { + return FileShard{}, 0, 0, moerr.NewInvalidInputNoCtx("Arrow shard estimates cannot be negative") + } + if record.Rows > 0 && rows > maxInt64-record.Rows { + return FileShard{}, 0, 0, moerr.NewInvalidInputNoCtx("Arrow shard row count overflows") + } + if record.WireBytes > 0 && wireBytes > maxInt64-record.WireBytes { + return FileShard{}, 0, 0, moerr.NewInvalidInputNoCtx("Arrow shard wire size overflows") + } + rows += record.Rows + wireBytes += record.WireBytes + } + lastRecordOffset := p.recordBlocks[end-1].offset + for index, dictionary := range p.dictBlocks { + if dictionary.offset >= lastRecordOffset { + continue + } + shard.RequiredDictionaryBlockIndices = append( + shard.RequiredDictionaryBlockIndices, int32(index), + ) + if dictionary.metadata+dictionary.body > maxInt64-wireBytes { + return FileShard{}, 0, 0, moerr.NewInvalidInputNoCtx("Arrow shard wire size overflows") + } + wireBytes += dictionary.metadata + dictionary.body + } + return shard, rows, wireBytes, nil +} + +const maxInt64 = int64(^uint64(0) >> 1) + +// InspectFile reads only the bounded footer, schema metadata, and block +// metadata. It validates dictionary base/delta ordering before any shard can +// be published. +func InspectFile( + ctx context.Context, + fs fileservice.FileService, + path string, + size int64, + admission fileservice.RangeReadAdmission, + options Options, +) (_ *FilePlan, retErr error) { + options, err := normalizeOptions(options) + if err != nil { + return nil, err + } + if fs == nil || path == "" || size < 0 || admission == nil { + return nil, moerr.NewInvalidInput(ctx, "invalid Arrow IPC File planning source") + } + if options.FileShard != nil { + return nil, moerr.NewInvalidInput(ctx, "Arrow IPC File planning cannot consume a shard") + } + rangeReader := fileservice.NewLeasedRangeReader(fs) + if options.ExpectedIdentity != nil { + expected := *options.ExpectedIdentity + if err := expected.Validate(); err != nil { + return nil, err + } + if expected.Size != size { + return nil, moerr.NewInvalidInputf(ctx, + "Arrow object identity size %d does not match planned size %d", expected.Size, size) + } + conditional, ok := rangeReader.(fileservice.ConditionalLeasedRangeReader) + if !ok { + return nil, moerr.NewNotSupported(ctx, "conditional Arrow object reads") + } + rangeReader = fixedIdentityRangeReader{reader: conditional, expected: expected} + } + ownedAllocator := newAdmissionAllocator(ctx, admission) + defer ownedAllocator.releaseAll() + options.Allocator = ownedAllocator + defer func() { + if recovered := recover(); recovered != nil { + if allocationErr, matched := recoveredAllocationError(recovered); matched { + retErr = allocationErr + return + } + retErr = moerr.NewInvalidInputf(ctx, "invalid Arrow IPC File planning metadata: %v", recovered) + } + }() + + records, dictionaries, err := readFooterBlocks(ctx, rangeReader, path, size, admission, options) + if err != nil { + return nil, err + } + if _, err = mergeFileBlocks(records, dictionaries); err != nil { + return nil, moerr.NewInvalidInputf(ctx, "invalid Arrow IPC File block ordering: %v", err) + } + schema, err := inspectFileSchema(ctx, rangeReader, path, size, admission, options, len(records)) + if err != nil { + return nil, err + } + plan := &FilePlan{ + Schema: schema, RecordBatches: make([]RecordBatchInfo, len(records)), + Dictionaries: make([]DictionaryBlockInfo, len(dictionaries)), + recordBlocks: records, dictBlocks: dictionaries, + } + baseSeen := make(map[int64]struct{}, len(dictionaries)) + for index, block := range dictionaries { + metadata, err := inspectFileBlockMetadata(ctx, rangeReader, path, block, admission) + if err != nil { + return nil, err + } + if metadata.headerType != messageHeaderDictionaryBatch { + return nil, moerr.NewInvalidInputf(ctx, + "Arrow footer dictionary block %d contains message type %d", index, metadata.headerType) + } + if err := acceptDictionaryTransition(baseSeen, metadata.dictionaryID, metadata.isDelta); err != nil { + return nil, err + } + plan.Dictionaries[index] = DictionaryBlockInfo{ + Index: int32(index), ID: metadata.dictionaryID, IsDelta: metadata.isDelta, + Rows: metadata.rows, WireBytes: block.metadata + block.body, + } + } + for index, block := range records { + metadata, err := inspectFileBlockMetadata(ctx, rangeReader, path, block, admission) + if err != nil { + return nil, err + } + if metadata.headerType != messageHeaderRecordBatch { + return nil, moerr.NewInvalidInputf(ctx, + "Arrow footer record block %d contains message type %d", index, metadata.headerType) + } + plan.RecordBatches[index] = RecordBatchInfo{ + Index: int32(index), Rows: metadata.rows, WireBytes: block.metadata + block.body, + } + } + return plan, nil +} + +func acceptDictionaryTransition(baseSeen map[int64]struct{}, id int64, isDelta bool) error { + if isDelta { + if _, ok := baseSeen[id]; !ok { + return moerr.NewInvalidInputNoCtxf("Arrow dictionary %d delta precedes its base", id) + } + return nil + } + if _, exists := baseSeen[id]; exists { + return moerr.NewInvalidInputNoCtxf("Arrow dictionary %d has a replacement base", id) + } + baseSeen[id] = struct{}{} + return nil +} + +func inspectFileSchema( + ctx context.Context, + rangeReader fileservice.LeasedRangeReader, + path string, + size int64, + admission fileservice.RangeReadAdmission, + options Options, + expectedRecords int, +) (*arrow.Schema, error) { + reader, err := ipc.NewFileReader( + &rangeReadAtSeeker{ctx: ctx, reader: rangeReader, path: path, size: size, admission: admission}, + ipc.WithAllocator(options.Allocator), + ipc.WithMetadataSizeLimit(options.MaxMetadataBytes), + ipc.WithBodySizeLimit(options.MaxBodyBytes), + ipc.WithEnsureNativeEndian(true), + ) + if err != nil { + return nil, moerr.NewInvalidInputf(ctx, "invalid Arrow IPC File footer: %v", err) + } + schema := reader.Schema() + records := reader.NumRecords() + _ = reader.Close() + if schema == nil || records != expectedRecords { + return nil, moerr.NewInvalidInput(ctx, "Arrow IPC File footer record count is inconsistent") + } + return schema, nil +} + +type inspectedBlockMetadata struct { + headerType byte + rows int64 + dictionaryID int64 + isDelta bool + bodyBytes int64 +} + +func inspectFileBlockMetadata( + ctx context.Context, + reader fileservice.LeasedRangeReader, + path string, + block fileBlock, + admission fileservice.RangeReadAdmission, +) (inspectedBlockMetadata, error) { + lease, err := reader.ReadRangeLease(ctx, path, block.offset, block.metadata, admission) + if err != nil { + return inspectedBlockMetadata{}, err + } + defer lease.Release() + return inspectFileBlockMetadataBytes(ctx, lease.Bytes(), block) +} + +func inspectFileBlockMetadataBytes( + ctx context.Context, + data []byte, + block fileBlock, +) (inspectedBlockMetadata, error) { + if int64(len(data)) != block.metadata || len(data) < 4 { + return inspectedBlockMetadata{}, moerr.NewInvalidInput(ctx, "Arrow IPC block metadata is truncated") + } + prefix := 4 + if len(data) >= 8 && binary.LittleEndian.Uint32(data) == ipcContinuationToken { + prefix = 8 + } else if binary.LittleEndian.Uint32(data) == 0 { + prefix = 0 + } + if len(data)-prefix < 4 { + return inspectedBlockMetadata{}, moerr.NewInvalidInput(ctx, "Arrow IPC block metadata prefix is invalid") + } + return inspectIPCMessageMetadata( + ctx, data[prefix:], block.body, block.body, nil, false, DefaultMaxDecodedRecordBytes, + ) +} + +// inspectIPCMessageMetadata adapts the shared transport-neutral validator to +// the File planner's private metadata shape. +func inspectIPCMessageMetadata( + ctx context.Context, + payload []byte, + maxBodyBytes int64, + bodyEnvelopeBytes int64, + body []byte, + validateBody bool, + maxDecodedRecordBytes int64, +) (inspectedBlockMetadata, error) { + info, err := arrowipc.InspectMessage(ctx, payload, arrowipc.ValidationOptions{ + MaxMetadataBytes: DefaultMaxMetadataBytes, + MaxBodyBytes: maxBodyBytes, + BodyEnvelopeBytes: bodyEnvelopeBytes, + Body: body, + ValidateBody: validateBody, + MaxDecodedRecordBytes: maxDecodedRecordBytes, + }) + if err != nil { + return inspectedBlockMetadata{}, err + } + return inspectedBlockMetadata{ + headerType: info.HeaderType, rows: info.Rows, + dictionaryID: info.DictionaryID, isDelta: info.IsDelta, + bodyBytes: info.BodyBytes, + }, nil +} diff --git a/pkg/sql/colexec/external/arrowio/reader.go b/pkg/sql/colexec/external/arrowio/reader.go new file mode 100644 index 0000000000000..8287a44f50239 --- /dev/null +++ b/pkg/sql/colexec/external/arrowio/reader.go @@ -0,0 +1,552 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package arrowio provides bounded Arrow IPC File and Stream readers over +// MatrixOne FileService. It deliberately has no Flight/Flight SQL dependency. +package arrowio + +import ( + "context" + "errors" + "io" + "math" + "sync" + "unsafe" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/arrowipc" + "github.com/matrixorigin/matrixone/pkg/fileservice" +) + +const ( + DefaultMaxMetadataBytes = arrowipc.DefaultMaxMetadataBytes + DefaultMaxBodyBytes int64 = 256 << 20 + DefaultMaxDecodedRecordBytes int64 = 256 << 20 +) + +type Container uint8 + +const ( + ContainerAuto Container = iota + ContainerFile + ContainerStream +) + +type Options struct { + MaxMetadataBytes int64 + MaxBodyBytes int64 + MaxDecodedRecordBytes int64 + Allocator memory.Allocator + ExpectedIdentity *fileservice.ObjectIdentity + FileShard *FileShard +} + +type Reader interface { + Schema() *arrow.Schema + Next() bool + RecordBatch() arrow.RecordBatch + Err() error + Close() error +} + +type ipcRecordReader struct { + reader *ipc.Reader + close func() error + allocator *admissionAllocator + rangeMessageReader *rangeMessageReader + err error +} + +func (r *ipcRecordReader) Schema() *arrow.Schema { + if r == nil || r.reader == nil { + return nil + } + return r.reader.Schema() +} + +func (r *ipcRecordReader) Next() (ok bool) { + if r == nil || r.reader == nil || r.err != nil { + return false + } + var checkpoint uint64 + if r.allocator != nil { + checkpoint = r.allocator.checkpoint() + } + var rangeCheckpoint uint64 + if r.rangeMessageReader != nil { + rangeCheckpoint = r.rangeMessageReader.checkpoint() + } + defer func() { + if recovered := recover(); recovered != nil { + if allocationErr, matched := recoveredAllocationError(recovered); matched { + r.err = allocationErr + } else { + r.err = moerr.NewInvalidInputNoCtxf("invalid Arrow IPC record batch: %v", recovered) + } + ok = false + } + if !ok && r.allocator != nil { + // Arrow-Go can recover a panic internally after allocating a body + // buffer but before installing an ArrayData owner. Only allocations + // created by this failed Next call are unowned. Older allocations may + // still be retained by already-published record batches or MO vectors. + r.allocator.releaseAfter(checkpoint) + } + if !ok && r.rangeMessageReader != nil { + // An invalid uncompressed array can panic after Arrow-Go has + // retained ArrayData but before it publishes the array or record. + // Arrow-Go recovers that panic internally, so its ordinary message + // release cannot close the unmatched retain. Abort only a range + // first published by this failed Next call; older ranges may still + // back record batches or MO vectors retained by the caller. + r.rangeMessageReader.abortAfter(rangeCheckpoint) + } + }() + return r.reader.Next() +} + +func (r *ipcRecordReader) RecordBatch() arrow.RecordBatch { + if r == nil || r.reader == nil { + return nil + } + return r.reader.RecordBatch() +} + +func (r *ipcRecordReader) Err() error { + if r == nil { + return nil + } + if r.reader == nil { + return r.err + } + return errors.Join(r.err, r.reader.Err()) +} + +func (r *ipcRecordReader) Close() error { + if r == nil { + return nil + } + if r.reader != nil { + r.reader.Release() + r.reader = nil + } + r.rangeMessageReader = nil + // Do not sweep the allocator here. Arrow buffers retained by a caller are + // allowed to outlive the reader and will free themselves through their + // allocator. Failed Next generations are swept at the Next boundary. + r.allocator = nil + if r.close != nil { + close := r.close + r.close = nil + return close() + } + return nil +} + +func normalizeOptions(options Options) (Options, error) { + if options.MaxMetadataBytes == 0 { + options.MaxMetadataBytes = DefaultMaxMetadataBytes + } + if options.MaxMetadataBytes > DefaultMaxMetadataBytes { + options.MaxMetadataBytes = DefaultMaxMetadataBytes + } + if options.MaxBodyBytes == 0 { + options.MaxBodyBytes = DefaultMaxBodyBytes + } + if options.MaxDecodedRecordBytes == 0 { + options.MaxDecodedRecordBytes = DefaultMaxDecodedRecordBytes + } + if options.MaxMetadataBytes < 4 || options.MaxMetadataBytes > int64(math.MaxInt) || + options.MaxBodyBytes <= 0 || options.MaxBodyBytes > int64(math.MaxInt) || + options.MaxDecodedRecordBytes <= 0 || options.MaxDecodedRecordBytes > int64(math.MaxInt) { + return options, moerr.NewInvalidInputNoCtx("invalid Arrow IPC size limits") + } + return options, nil +} + +// Open selects IPC File or Stream. Auto probes only the final Arrow file magic; +// arbitrary binary input is still validated by the selected official decoder. +func Open( + ctx context.Context, + fs fileservice.FileService, + path string, + size int64, + container Container, + admission fileservice.RangeReadAdmission, + options Options, +) (_ Reader, retErr error) { + options, err := normalizeOptions(options) + if err != nil { + return nil, err + } + if fs == nil || path == "" || size < 0 || admission == nil { + return nil, moerr.NewInvalidInput(ctx, "invalid Arrow IPC source") + } + rangeReader := fileservice.NewLeasedRangeReader(fs) + if options.ExpectedIdentity != nil { + expected := *options.ExpectedIdentity + if err := expected.Validate(); err != nil { + return nil, err + } + if expected.Size != size { + return nil, moerr.NewInvalidInputf(ctx, + "Arrow object identity size %d does not match planned size %d", expected.Size, size) + } + conditional, ok := rangeReader.(fileservice.ConditionalLeasedRangeReader) + if !ok { + return nil, moerr.NewNotSupported(ctx, "conditional Arrow object reads") + } + rangeReader = fixedIdentityRangeReader{reader: conditional, expected: expected} + } + var ownedAllocator *admissionAllocator + if options.Allocator == nil { + ownedAllocator = newAdmissionAllocator(ctx, admission) + options.Allocator = ownedAllocator + } + defer func() { + if recovered := recover(); recovered != nil { + if allocationErr, matched := recoveredAllocationError(recovered); matched { + if ownedAllocator != nil { + ownedAllocator.releaseAll() + } + retErr = allocationErr + return + } + if ownedAllocator != nil { + ownedAllocator.releaseAll() + } + retErr = moerr.NewInvalidInputf(ctx, "invalid Arrow IPC input: %v", recovered) + } + if retErr != nil && ownedAllocator != nil { + ownedAllocator.releaseAll() + } + }() + if container == ContainerAuto { + container, err = detectContainer(ctx, rangeReader, path, size, admission) + if err != nil { + return nil, err + } + } + var result Reader + switch container { + case ContainerFile: + result, err = openFile(ctx, rangeReader, path, size, admission, options) + case ContainerStream: + if options.FileShard != nil { + return nil, moerr.NewInvalidInput(ctx, "Arrow IPC Stream cannot use a record-batch shard") + } + result, err = openStream(ctx, fs, path, options) + default: + return nil, moerr.NewInvalidInputf(ctx, "invalid Arrow IPC container %d", container) + } + if err != nil { + return nil, err + } + if ownedAllocator != nil { + ipcReader, ok := result.(*ipcRecordReader) + if !ok { + _ = result.Close() + return nil, moerr.NewInternalErrorNoCtx("Arrow IPC facade returned an unmanaged reader") + } + ipcReader.allocator = ownedAllocator + } + return result, nil +} + +// DetectContainer performs the same identity-locked tail probe as Open +// without decoding schema or record data. +func DetectContainer( + ctx context.Context, + fs fileservice.FileService, + path string, + size int64, + admission fileservice.RangeReadAdmission, + options Options, +) (Container, error) { + options, err := normalizeOptions(options) + if err != nil { + return 0, err + } + if fs == nil || path == "" || size < 0 || admission == nil { + return 0, moerr.NewInvalidInput(ctx, "invalid Arrow IPC container probe") + } + rangeReader := fileservice.NewLeasedRangeReader(fs) + if options.ExpectedIdentity != nil { + expected := *options.ExpectedIdentity + if err := expected.Validate(); err != nil { + return 0, err + } + if expected.Size != size { + return 0, moerr.NewInvalidInputf(ctx, + "Arrow object identity size %d does not match planned size %d", expected.Size, size) + } + conditional, ok := rangeReader.(fileservice.ConditionalLeasedRangeReader) + if !ok { + return 0, moerr.NewNotSupported(ctx, "conditional Arrow object reads") + } + rangeReader = fixedIdentityRangeReader{reader: conditional, expected: expected} + } + return detectContainer(ctx, rangeReader, path, size, admission) +} + +const arrowAllocatorAlignment = 64 + +type allocationPanic struct{ err error } + +func recoveredAllocationError(value any) (error, bool) { + p, ok := value.(allocationPanic) + if !ok { + return nil, false + } + return p.err, true +} + +// admissionAllocator makes Arrow-Go decoded/metadata buffers participate in +// the same pre-allocation admission protocol as leased FileService ranges. +// Arrow's Allocator API cannot return errors, so a private panic is recovered +// exactly at the facade's Open/Next boundaries and converted back to an error. +type admissionAllocator struct { + ctx context.Context + admission fileservice.RangeReadAdmission + base memory.Allocator + mu sync.Mutex + nextID uint64 + allocated map[uintptr]admissionAllocation +} + +type admissionAllocation struct { + buffer []byte + lease fileservice.CapacityLease + id uint64 +} + +func newAdmissionAllocator(ctx context.Context, admission fileservice.RangeReadAdmission) *admissionAllocator { + return &admissionAllocator{ + ctx: ctx, admission: admission, base: memory.DefaultAllocator, + allocated: make(map[uintptr]admissionAllocation), + } +} + +func (a *admissionAllocator) Allocate(size int) []byte { + if size <= 0 { + return nil + } + if size > int(^uint(0)>>1)-arrowAllocatorAlignment { + panic(allocationPanic{err: moerr.NewInvalidInputNoCtx("Arrow allocation size overflows")}) + } + upper := int64(size) + arrowAllocatorAlignment + reservation, err := a.admission.Reserve(a.ctx, upper) + if err != nil { + panic(allocationPanic{err: err}) + } + var buffer []byte + func() { + defer func() { + if recovered := recover(); recovered != nil { + reservation.Abort() + panic(recovered) + } + }() + buffer = a.base.Allocate(size) + }() + actualCapacity := int64(cap(buffer)) + if actualCapacity <= 0 || actualCapacity > upper { + reservation.Abort() + a.base.Free(buffer) + panic("Arrow allocator exceeded its reserved capacity") + } + lease, err := reservation.Commit(actualCapacity) + if err != nil { + reservation.Abort() + a.base.Free(buffer) + panic(allocationPanic{err: err}) + } + key := arrowBufferKey(buffer) + a.mu.Lock() + if _, exists := a.allocated[key]; exists { + a.mu.Unlock() + lease.Release() + a.base.Free(buffer) + panic("Arrow allocator returned duplicate live backing") + } + a.nextID++ + a.allocated[key] = admissionAllocation{buffer: buffer, lease: lease, id: a.nextID} + a.mu.Unlock() + return buffer +} + +func (a *admissionAllocator) Reallocate(size int, buffer []byte) []byte { + if size <= cap(buffer) { + return buffer[:size] + } + replacement := a.Allocate(size) + copy(replacement, buffer) + a.Free(buffer) + return replacement +} + +func (a *admissionAllocator) Free(buffer []byte) { + key := arrowBufferKey(buffer) + if key == 0 { + return + } + a.mu.Lock() + allocation, exists := a.allocated[key] + delete(a.allocated, key) + a.mu.Unlock() + if !exists { + // releaseAll owns abandoned allocations on decoder failure. Arrow-Go + // cleanup may still invoke Free while unwinding; it must not double-free. + return + } + a.base.Free(allocation.buffer) + if allocation.lease != nil { + allocation.lease.Release() + } +} + +func arrowBufferKey(buffer []byte) uintptr { + if cap(buffer) == 0 { + return 0 + } + return uintptr(unsafe.Pointer(unsafe.SliceData(buffer[:1]))) +} + +func (a *admissionAllocator) releaseAll() { + a.releaseAfter(0) +} + +// checkpoint identifies an allocation generation boundary. It is used to +// reclaim only buffers abandoned by one failed decoder call without touching +// older buffers that may still be retained by published zero-copy views. +func (a *admissionAllocator) checkpoint() uint64 { + a.mu.Lock() + defer a.mu.Unlock() + return a.nextID +} + +func (a *admissionAllocator) releaseAfter(checkpoint uint64) { + a.mu.Lock() + allocated := make([]admissionAllocation, 0) + for key, allocation := range a.allocated { + if allocation.id > checkpoint { + allocated = append(allocated, allocation) + delete(a.allocated, key) + } + } + a.mu.Unlock() + for _, allocation := range allocated { + a.base.Free(allocation.buffer) + if allocation.lease != nil { + allocation.lease.Release() + } + } +} + +var _ memory.Allocator = (*admissionAllocator)(nil) + +func detectContainer( + ctx context.Context, + reader fileservice.LeasedRangeReader, + path string, + size int64, + admission fileservice.RangeReadAdmission, +) (Container, error) { + if size >= int64(len(ipc.Magic)) && admission != nil { + lease, err := reader.ReadRangeLease( + ctx, path, size-int64(len(ipc.Magic)), int64(len(ipc.Magic)), admission, + ) + if err != nil { + return 0, err + } + isFile := string(lease.Bytes()) == string(ipc.Magic) + lease.Release() + if isFile { + return ContainerFile, nil + } + } + return ContainerStream, nil +} + +type fixedIdentityRangeReader struct { + reader fileservice.ConditionalLeasedRangeReader + expected fileservice.ObjectIdentity +} + +func (r fixedIdentityRangeReader) ReadRangeLease( + ctx context.Context, + path string, + offset, size int64, + admission fileservice.RangeReadAdmission, +) (fileservice.RangeLease, error) { + return r.reader.ReadRangeLeaseWithIdentity(ctx, path, offset, size, r.expected, admission) +} + +func openStream( + ctx context.Context, + fs fileservice.FileService, + path string, + options Options, +) (Reader, error) { + var stream io.ReadCloser + if options.ExpectedIdentity != nil { + identityFS, ok := fs.(fileservice.ObjectIdentityFileService) + if !ok { + return nil, moerr.NewNotSupported(ctx, "conditional Arrow stream reads") + } + var err error + stream, err = identityFS.OpenReadWithIdentity(ctx, path, 0, -1, *options.ExpectedIdentity) + if err != nil { + return nil, err + } + } else { + vector := &fileservice.IOVector{ + FilePath: path, + Policy: fileservice.SkipAllCache, + Entries: []fileservice.IOEntry{{ + Offset: 0, Size: -1, ReadCloserForRead: &stream, + }}, + } + if err := fs.Read(ctx, vector); err != nil { + vector.ReleaseReadResultOnError() + return nil, err + } + } + if stream == nil { + return nil, moerr.NewInvalidInput(ctx, "Arrow IPC Stream reader is missing") + } + readerOptions := []ipc.Option{ + ipc.WithAllocator(options.Allocator), + ipc.WithMetadataSizeLimit(options.MaxMetadataBytes), + ipc.WithBodySizeLimit(options.MaxBodyBytes), + ipc.WithEnsureNativeEndian(true), + } + // Arrow-Go's stream messageReader limits the on-wire body only. The facade + // reader additionally validates buffer descriptors and the declared total + // decompressed size before Arrow-Go can allocate decode buffers. + messageReader := newStreamMessageReader(ctx, stream, options) + reader, err := ipc.NewReaderFromMessageReader( + messageReader, + readerOptions..., + ) + if err != nil { + messageReader.Release() + stream.Close() + return nil, moerr.NewInvalidInputf(ctx, "invalid Arrow IPC Stream: %v", err) + } + return &ipcRecordReader{reader: reader, close: stream.Close}, nil +} diff --git a/pkg/sql/colexec/external/arrowio/reader_test.go b/pkg/sql/colexec/external/arrowio/reader_test.go new file mode 100644 index 0000000000000..f4cab0106dd07 --- /dev/null +++ b/pkg/sql/colexec/external/arrowio/reader_test.go @@ -0,0 +1,1160 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowio + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "io" + "sync" + "sync/atomic" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + flatbuffers "github.com/google/flatbuffers/go" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/stretchr/testify/require" +) + +type identityMemoryFS struct { + *fileservice.MemoryFS + identity atomic.Pointer[fileservice.ObjectIdentity] +} + +func newIdentityMemoryFS(fs *fileservice.MemoryFS, identity fileservice.ObjectIdentity) *identityMemoryFS { + result := &identityMemoryFS{MemoryFS: fs} + result.identity.Store(&identity) + return result +} + +func (f *identityMemoryFS) StatFileIdentity(ctx context.Context, _ string) (fileservice.ObjectIdentity, error) { + if err := ctx.Err(); err != nil { + return fileservice.ObjectIdentity{}, err + } + return *f.identity.Load(), nil +} + +func (f *identityMemoryFS) OpenReadWithIdentity( + ctx context.Context, + path string, + offset, size int64, + expected fileservice.ObjectIdentity, +) (io.ReadCloser, error) { + if current := *f.identity.Load(); current != expected { + return nil, fileservice.ErrObjectChanged + } + var stream io.ReadCloser + vector := &fileservice.IOVector{ + FilePath: path, Policy: fileservice.SkipAllCache, + Entries: []fileservice.IOEntry{{Offset: offset, Size: size, ReadCloserForRead: &stream}}, + } + if err := f.Read(ctx, vector); err != nil { + vector.ReleaseReadResultOnError() + return nil, err + } + return stream, nil +} + +type testAdmission struct { + reserved atomic.Int64 + pending atomic.Int64 + released atomic.Int64 + active atomic.Int64 + max atomic.Int64 + reject error +} + +func (a *testAdmission) Reserve(_ context.Context, upper int64) (fileservice.CapacityReservation, error) { + if a.reject != nil { + return nil, a.reject + } + a.reserved.Add(upper) + a.pending.Add(upper) + for { + current := a.max.Load() + if upper <= current || a.max.CompareAndSwap(current, upper) { + break + } + } + return &testReservation{admission: a, upper: upper}, nil +} + +type testReservation struct { + admission *testAdmission + upper int64 + done atomic.Bool +} + +func (r *testReservation) Commit(actual int64) (fileservice.CapacityLease, error) { + if actual < 0 || actual > r.upper || !r.done.CompareAndSwap(false, true) { + return nil, errors.New("invalid reservation commit") + } + r.admission.pending.Add(-r.upper) + r.admission.active.Add(actual) + return &testCapacityLease{admission: r.admission, capacity: actual}, nil +} + +func (r *testReservation) Abort() { + if r.done.CompareAndSwap(false, true) { + r.admission.pending.Add(-r.upper) + } +} + +type testCapacityLease struct { + admission *testAdmission + capacity int64 + released atomic.Bool +} + +type testRangeLease struct { + releases atomic.Int64 +} + +func (l *testRangeLease) Bytes() []byte { return nil } +func (l *testRangeLease) Capacity() int64 { return 0 } +func (l *testRangeLease) Release() { l.releases.Add(1) } + +type panickingAllocator struct{} + +func (panickingAllocator) Allocate(int) []byte { panic("injected allocation panic") } +func (panickingAllocator) Reallocate(int, []byte) []byte { panic("injected reallocation panic") } +func (panickingAllocator) Free([]byte) {} + +func (l *testCapacityLease) Release() { + if l.released.CompareAndSwap(false, true) { + l.admission.active.Add(-l.capacity) + l.admission.released.Add(l.capacity) + } +} + +func TestIPCFileRangeReader(t *testing.T) { + fileBytes, expected := makeIPC(t, ContainerFile) + fs := writeMemoryFile(t, "arrow-file", fileBytes) + admission := new(testAdmission) + reader, err := Open( + context.Background(), fs, "arrow-file", int64(len(fileBytes)), ContainerAuto, admission, Options{}, + ) + require.NoError(t, err) + require.True(t, reader.Schema().Equal(expected[0].Schema())) + + for i := range expected { + require.True(t, reader.Next()) + require.True(t, array.RecordEqual(expected[i], reader.RecordBatch())) + require.Greater(t, admission.active.Load(), int64(0), "current record range must remain pinned") + } + require.False(t, reader.Next()) + require.NoError(t, reader.Err()) + require.NoError(t, reader.Close()) + require.Zero(t, admission.active.Load()) + require.Greater(t, admission.reserved.Load(), int64(0)) + require.Less(t, admission.max.Load(), int64(len(fileBytes)), "file path must not pin the whole object") + releaseRecords(expected) +} + +func TestAdmissionAllocatorFailureAndForcedCleanup(t *testing.T) { + t.Run("forced cleanup frees backing and capacity exactly once", func(t *testing.T) { + admission := new(testAdmission) + checked := memory.NewCheckedAllocator(memory.NewGoAllocator()) + allocator := newAdmissionAllocator(context.Background(), admission) + allocator.base = checked + buffer := allocator.Allocate(127) + require.NotEmpty(t, buffer) + require.Positive(t, admission.active.Load()) + require.Zero(t, admission.pending.Load()) + + allocator.releaseAll() + require.Zero(t, admission.active.Load()) + require.Zero(t, admission.pending.Load()) + allocator.Free(buffer) + require.Zero(t, admission.active.Load(), "late Arrow cleanup must be idempotent") + checked.AssertSize(t, 0) + }) + + t.Run("failed generation cleanup preserves older live backing", func(t *testing.T) { + admission := new(testAdmission) + checked := memory.NewCheckedAllocator(memory.NewGoAllocator()) + allocator := newAdmissionAllocator(context.Background(), admission) + allocator.base = checked + older := allocator.Allocate(64) + checkpoint := allocator.checkpoint() + abandoned := allocator.Allocate(128) + require.Equal(t, int64(cap(older)+cap(abandoned)), admission.active.Load()) + + allocator.releaseAfter(checkpoint) + require.Equal(t, int64(cap(older)), admission.active.Load()) + allocator.Free(abandoned) + require.Equal(t, int64(cap(older)), admission.active.Load(), "late cleanup must not release an older generation") + allocator.Free(older) + require.Zero(t, admission.active.Load()) + checked.AssertSize(t, 0) + }) + + t.Run("base allocation panic aborts reservation", func(t *testing.T) { + admission := new(testAdmission) + allocator := newAdmissionAllocator(context.Background(), admission) + allocator.base = panickingAllocator{} + require.PanicsWithValue(t, "injected allocation panic", func() { + allocator.Allocate(64) + }) + require.Zero(t, admission.pending.Load()) + require.Zero(t, admission.active.Load()) + }) +} + +func TestRangeLeaseAllocatorConcurrentForcedAndLateRelease(t *testing.T) { + lease := new(testRangeLease) + allocator := &rangeLeaseAllocator{lease: lease} + + const releasers = 32 + start := make(chan struct{}) + var wait sync.WaitGroup + wait.Add(releasers) + for i := 0; i < releasers; i++ { + go func(force bool) { + defer wait.Done() + <-start + if force { + allocator.release() + } else { + allocator.Free(nil) + } + }(i%2 == 0) + } + close(start) + wait.Wait() + require.Equal(t, int64(1), lease.releases.Load()) +} + +func TestRangeMessageReaderRetainCannotResurrectTerminalOwner(t *testing.T) { + reader := new(rangeMessageReader) + reader.refs.Store(1) + reader.Retain() + require.Equal(t, int64(2), reader.refs.Load()) + reader.Release() + reader.Release() + require.Zero(t, reader.refs.Load()) + require.Panics(t, reader.Retain) + require.Zero(t, reader.refs.Load()) +} + +func TestStreamMessageReaderRetainCannotResurrectTerminalOwner(t *testing.T) { + reader := new(streamMessageReader) + reader.refs.Store(1) + reader.Retain() + require.Equal(t, int64(2), reader.refs.Load()) + reader.Release() + reader.Release() + require.Zero(t, reader.refs.Load()) + require.Panics(t, reader.Retain) + require.Zero(t, reader.refs.Load()) +} + +func TestIPCFileRecordOutlivesReaderViaArrayData(t *testing.T) { + fileBytes, expected := makeIPC(t, ContainerFile) + fs := writeMemoryFile(t, "arrow-lifetime", fileBytes) + admission := new(testAdmission) + reader, err := Open(context.Background(), fs, "arrow-lifetime", int64(len(fileBytes)), ContainerFile, admission, Options{}) + require.NoError(t, err) + require.True(t, reader.Next()) + record := reader.RecordBatch() + record.Retain() + require.NoError(t, reader.Close()) + require.Greater(t, admission.active.Load(), int64(0)) + require.True(t, array.RecordEqual(expected[0], record)) + record.Release() + require.Zero(t, admission.active.Load()) + releaseRecords(expected) +} + +func TestFailedIPCFileGenerationPreservesOlderRetainedRecord(t *testing.T) { + payload, expected := makeIPC(t, ContainerFile) + defer releaseRecords(expected) + + inspectionFS := writeMemoryFile(t, "arrow-generation-inspect", payload) + inspectionAdmission := new(testAdmission) + options, err := normalizeOptions(Options{Allocator: memory.NewGoAllocator()}) + require.NoError(t, err) + records, _, err := readFooterBlocks( + context.Background(), fileservice.NewLeasedRangeReader(inspectionFS), + "arrow-generation-inspect", int64(len(payload)), inspectionAdmission, options, + ) + require.NoError(t, err) + require.Len(t, records, 2) + require.Zero(t, inspectionAdmission.active.Load()) + + // The second record's String buffers are validity, offsets, and values. + // Keep every descriptor in bounds while making the last logical offset + // exceed the values buffer, so Arrow-Go fails only after ArrayData retain. + malformed := append([]byte(nil), payload...) + second := records[1] + record := firstIPCRecordTable(t, malformed, second) + buffersOffset := flatbuffers.UOffsetT(record.Offset(8)) + require.NotZero(t, buffersOffset) + require.GreaterOrEqual(t, record.VectorLen(buffersOffset), 5) + buffers := record.Vector(buffersOffset) + offsetsDescriptor := buffers + flatbuffers.UOffsetT(3*16) + valuesDescriptor := buffers + flatbuffers.UOffsetT(4*16) + offsetsOffset := record.GetInt64(offsetsDescriptor) + offsetsLength := record.GetInt64(offsetsDescriptor + 8) + valuesLength := record.GetInt64(valuesDescriptor + 8) + require.GreaterOrEqual(t, offsetsLength, int64(12)) + require.GreaterOrEqual(t, valuesLength, int64(0)) + bodyStart := int(second.offset + second.metadata) + lastOffset := bodyStart + int(offsetsOffset) + int(offsetsLength) - 4 + require.LessOrEqual(t, lastOffset+4, len(malformed)) + binary.LittleEndian.PutUint32(malformed[lastOffset:], uint32(valuesLength+1)) + + fs := writeMemoryFile(t, "arrow-generation", malformed) + admission := new(testAdmission) + reader, err := Open( + context.Background(), fs, "arrow-generation", int64(len(malformed)), + ContainerFile, admission, Options{}, + ) + require.NoError(t, err) + require.True(t, reader.Next()) + retained := reader.RecordBatch() + retained.Retain() + olderActive := admission.active.Load() + require.Positive(t, olderActive) + + require.False(t, reader.Next()) + require.Error(t, reader.Err()) + require.Zero(t, admission.pending.Load()) + require.Equal(t, olderActive, admission.active.Load(), "only the retained older generation may remain") + require.NoError(t, reader.Close()) + require.Equal(t, olderActive, admission.active.Load()) + require.True(t, array.RecordEqual(expected[0], retained)) + retained.Release() + require.Zero(t, admission.active.Load()) +} + +func TestMalformedIPCRecordMetadataReleasesRangeLease(t *testing.T) { + original, expected := makeIPC(t, ContainerFile) + releaseRecords(expected) + inspectionFS := writeMemoryFile(t, "arrow-inspect", original) + inspectionAdmission := new(testAdmission) + records, _, err := readFooterBlocks( + context.Background(), fileservice.NewLeasedRangeReader(inspectionFS), "arrow-inspect", + int64(len(original)), inspectionAdmission, Options{ + MaxMetadataBytes: DefaultMaxMetadataBytes, + MaxBodyBytes: DefaultMaxBodyBytes, + Allocator: memory.NewGoAllocator(), + }, + ) + require.NoError(t, err) + require.NotEmpty(t, records) + require.Zero(t, inspectionAdmission.active.Load()) + + block := records[0] + for _, test := range []struct { + name string + errorText string + corrupt func(*testing.T, flatbuffers.Table) + }{ + { + name: "negative-row-count", + errorText: "invalid row count", + corrupt: func(t *testing.T, record flatbuffers.Table) { + lengthOffset := flatbuffers.UOffsetT(record.Offset(4)) + require.NotZero(t, lengthOffset) + binary.LittleEndian.PutUint64(record.Bytes[lengthOffset+record.Pos:], ^uint64(0)) + }, + }, + { + name: "null-count-exceeds-node-length", + errorText: "invalid length", + corrupt: func(t *testing.T, record flatbuffers.Table) { + nodesOffset := flatbuffers.UOffsetT(record.Offset(6)) + require.NotZero(t, nodesOffset) + nodes := record.Vector(nodesOffset) + require.Positive(t, record.VectorLen(nodesOffset)) + length := binary.LittleEndian.Uint64(record.Bytes[nodes:]) + binary.LittleEndian.PutUint64(record.Bytes[nodes+8:], length+1) + }, + }, + { + name: "buffer-exceeds-message-body", + errorText: "exceeds message body", + corrupt: func(t *testing.T, record flatbuffers.Table) { + buffersOffset := flatbuffers.UOffsetT(record.Offset(8)) + require.NotZero(t, buffersOffset) + buffers := record.Vector(buffersOffset) + require.Positive(t, record.VectorLen(buffersOffset)) + binary.LittleEndian.PutUint64(record.Bytes[buffers+8:], uint64(block.body+1)) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + fileBytes := append([]byte(nil), original...) + recordTable := firstIPCRecordTable(t, fileBytes, block) + test.corrupt(t, recordTable) + + path := "arrow-malformed-" + test.name + fs := writeMemoryFile(t, path, fileBytes) + admission := new(testAdmission) + reader, err := Open( + context.Background(), fs, path, int64(len(fileBytes)), + ContainerFile, admission, Options{}, + ) + require.NoError(t, err) + require.False(t, reader.Next()) + require.ErrorContains(t, reader.Err(), test.errorText) + require.NoError(t, reader.Close()) + require.Zero(t, admission.pending.Load()) + require.Zero(t, admission.active.Load()) + }) + } +} + +func firstIPCRecordTable(t *testing.T, fileBytes []byte, block fileBlock) flatbuffers.Table { + t.Helper() + metadata := fileBytes[int(block.offset):int(block.offset+block.metadata)] + prefix := 4 + if binary.LittleEndian.Uint32(metadata) == ipcContinuationToken { + prefix = 8 + } + payload := metadata[prefix:] + messageTable := flatbuffers.Table{Bytes: payload, Pos: flatbuffers.GetUOffsetT(payload)} + headerOffset := flatbuffers.UOffsetT(messageTable.Offset(8)) + require.NotZero(t, headerOffset) + var recordTable flatbuffers.Table + messageTable.Union(&recordTable, headerOffset) + return recordTable +} + +func TestIPCFilePlanningAndIndependentRecordShard(t *testing.T) { + payload, expected := makeIPC(t, ContainerFile) + defer releaseRecords(expected) + fs := writeMemoryFile(t, "arrow-shard", payload) + admission := new(testAdmission) + plan, err := InspectFile( + context.Background(), fs, "arrow-shard", int64(len(payload)), admission, Options{}, + ) + require.NoError(t, err) + require.True(t, plan.Schema.Equal(expected[0].Schema())) + require.Equal(t, []RecordBatchInfo{ + {Index: 0, Rows: 2, WireBytes: plan.RecordBatches[0].WireBytes}, + {Index: 1, Rows: 2, WireBytes: plan.RecordBatches[1].WireBytes}, + }, plan.RecordBatches) + require.Empty(t, plan.Dictionaries) + require.Zero(t, admission.active.Load()) + + shard, rows, wireBytes, err := plan.Shard(1, 2) + require.NoError(t, err) + require.Equal(t, int64(2), rows) + require.Equal(t, plan.RecordBatches[1].WireBytes, wireBytes) + reader, err := Open( + context.Background(), fs, "arrow-shard", int64(len(payload)), ContainerFile, + admission, Options{FileShard: &shard}, + ) + require.NoError(t, err) + require.True(t, reader.Next()) + require.True(t, array.RecordEqual(expected[1], reader.RecordBatch())) + require.False(t, reader.Next()) + require.NoError(t, reader.Err()) + require.NoError(t, reader.Close()) + require.Zero(t, admission.active.Load()) + + shard.RequiredDictionaryBlockIndices = []int32{0} + _, err = Open( + context.Background(), fs, "arrow-shard", int64(len(payload)), ContainerFile, + admission, Options{FileShard: &shard}, + ) + require.ErrorContains(t, err, "dictionary closure") +} + +func TestFilePlanShardRejectsIncompletePlan(t *testing.T) { + plan := &FilePlan{RecordBatches: []RecordBatchInfo{{Index: 0, Rows: 1}}} + var ( + shard FileShard + rows int64 + wireSize int64 + err error + ) + require.NotPanics(t, func() { + shard, rows, wireSize, err = plan.Shard(0, 1) + }) + require.Empty(t, shard) + require.Zero(t, rows) + require.Zero(t, wireSize) + require.ErrorContains(t, err, "incomplete") +} + +func TestFilePlanShardRejectsNegativeEstimates(t *testing.T) { + plan := &FilePlan{ + Schema: arrow.NewSchema([]arrow.Field{{Name: "value", Type: arrow.PrimitiveTypes.Int64}}, nil), + RecordBatches: []RecordBatchInfo{{Index: 0, Rows: -1, WireBytes: -1}}, + recordBlocks: []fileBlock{{offset: 8, metadata: 8, body: 8}}, + } + var ( + shard FileShard + rows int64 + wireSize int64 + err error + ) + require.NotPanics(t, func() { + shard, rows, wireSize, err = plan.Shard(0, 1) + }) + require.Empty(t, shard) + require.Zero(t, rows) + require.Zero(t, wireSize) + require.ErrorContains(t, err, "negative") +} + +func TestIPCFilePlanningDictionaryClosure(t *testing.T) { + payload, expected := makeDictionaryIPC(t, ContainerFile, false) + defer releaseRecords(expected) + fs := writeMemoryFile(t, "arrow-dictionary-shard", payload) + admission := new(testAdmission) + plan, err := InspectFile( + context.Background(), fs, "arrow-dictionary-shard", int64(len(payload)), admission, Options{}, + ) + require.NoError(t, err) + require.Len(t, plan.Dictionaries, 1) + require.False(t, plan.Dictionaries[0].IsDelta) + shard, rows, _, err := plan.Shard(1, 2) + require.NoError(t, err) + require.Equal(t, int64(1), rows) + require.Equal(t, []int32{0}, shard.RequiredDictionaryBlockIndices) + + reader, err := Open( + context.Background(), fs, "arrow-dictionary-shard", int64(len(payload)), ContainerFile, + admission, Options{FileShard: &shard}, + ) + require.NoError(t, err) + require.True(t, reader.Next()) + require.True(t, array.RecordEqual(expected[1], reader.RecordBatch())) + require.False(t, reader.Next()) + require.NoError(t, reader.Err()) + require.NoError(t, reader.Close()) + require.Zero(t, admission.active.Load()) +} + +func TestDictionaryTransitionStateMachine(t *testing.T) { + seen := make(map[int64]struct{}) + require.ErrorContains(t, acceptDictionaryTransition(seen, 7, true), "delta precedes") + require.NoError(t, acceptDictionaryTransition(seen, 7, false)) + require.NoError(t, acceptDictionaryTransition(seen, 7, true)) + require.ErrorContains(t, acceptDictionaryTransition(seen, 7, false), "replacement base") + require.NoError(t, acceptDictionaryTransition(seen, 8, false)) +} + +func TestIPCStreamReaderAndContainerValidation(t *testing.T) { + streamBytes, expected := makeIPC(t, ContainerStream) + fs := writeMemoryFile(t, "arrow-stream", streamBytes) + reader, err := Open(context.Background(), fs, "arrow-stream", int64(len(streamBytes)), ContainerAuto, new(testAdmission), Options{}) + require.NoError(t, err) + for i := range expected { + require.True(t, reader.Next()) + require.True(t, array.RecordEqual(expected[i], reader.RecordBatch())) + } + require.False(t, reader.Next()) + require.NoError(t, reader.Err()) + require.NoError(t, reader.Close()) + releaseRecords(expected) + + bad := writeMemoryFile(t, "bad-arrow", []byte("not arrow")) + _, err = Open(context.Background(), bad, "bad-arrow", 9, ContainerAuto, new(testAdmission), Options{}) + require.Error(t, err) +} + +func TestIPCStreamRequiresEOSMarker(t *testing.T) { + payload, expected := makeIPC(t, ContainerStream) + defer releaseRecords(expected) + require.GreaterOrEqual(t, len(payload), 4) + truncated := payload[:len(payload)-4] + fs := writeMemoryFile(t, "arrow-stream-missing-eos", truncated) + reader, err := Open( + context.Background(), fs, "arrow-stream-missing-eos", int64(len(truncated)), ContainerStream, + new(testAdmission), Options{}, + ) + require.NoError(t, err) + for index := range expected { + require.True(t, reader.Next(), "record %d: %v", index, reader.Err()) + require.True(t, array.RecordEqual(expected[index], reader.RecordBatch())) + } + require.False(t, reader.Next()) + require.Error(t, reader.Err(), "a stream without its EOS marker is truncated") + require.NoError(t, reader.Close()) +} + +func TestIPCFileAndStreamCompression(t *testing.T) { + for _, test := range []struct { + name string + container Container + codec ipc.Option + }{ + {name: "file lz4", container: ContainerFile, codec: ipc.WithLZ4()}, + {name: "file zstd", container: ContainerFile, codec: ipc.WithZstd()}, + {name: "stream lz4", container: ContainerStream, codec: ipc.WithLZ4()}, + {name: "stream zstd", container: ContainerStream, codec: ipc.WithZstd()}, + } { + t.Run(test.name, func(t *testing.T) { + payload, expected := makeIPCWithOptions(t, test.container, test.codec) + defer releaseRecords(expected) + fs := writeMemoryFile(t, "arrow-compressed", payload) + admission := new(testAdmission) + options := Options{MaxDecodedRecordBytes: 1024} + if test.container == ContainerFile { + plan, err := InspectFile( + context.Background(), fs, "arrow-compressed", int64(len(payload)), admission, options, + ) + require.NoError(t, err) + require.Len(t, plan.RecordBatches, len(expected)) + require.Zero(t, admission.active.Load()) + } + + reader, err := Open( + context.Background(), fs, "arrow-compressed", int64(len(payload)), test.container, admission, options, + ) + require.NoError(t, err) + for index := range expected { + require.True(t, reader.Next(), "record %d: %v", index, reader.Err()) + require.True(t, array.RecordEqual(expected[index], reader.RecordBatch())) + } + require.False(t, reader.Next()) + require.NoError(t, reader.Err()) + require.NoError(t, reader.Close()) + require.Zero(t, admission.active.Load()) + }) + } +} + +func TestIPCCompressionMetadataRejectedBeforeDecodeAllocation(t *testing.T) { + for _, container := range []Container{ContainerFile, ContainerStream} { + for _, test := range []struct { + name string + errorText string + corrupt func(*testing.T, []byte, ipcRecordLocation) + }{ + { + name: "decoded-size-exceeds-limit", + errorText: "decoded record body exceeds limit", + corrupt: func(t *testing.T, payload []byte, location ipcRecordLocation) { + bufferOffset, _, _ := firstNonEmptyIPCBuffer(t, location.record) + binary.LittleEndian.PutUint64( + payload[location.bodyStart+int(bufferOffset):], uint64(1025), + ) + }, + }, + { + name: "negative-decoded-size", + errorText: "invalid decoded size", + corrupt: func(t *testing.T, payload []byte, location ipcRecordLocation) { + bufferOffset, _, _ := firstNonEmptyIPCBuffer(t, location.record) + binary.LittleEndian.PutUint64( + payload[location.bodyStart+int(bufferOffset):], ^uint64(1), + ) + }, + }, + { + name: "missing-decoded-size-prefix", + errorText: "shorter than its decoded-size prefix", + corrupt: func(t *testing.T, _ []byte, location ipcRecordLocation) { + _, _, descriptor := firstNonEmptyIPCBuffer(t, location.record) + binary.LittleEndian.PutUint64(location.record.Bytes[descriptor+8:], uint64(7)) + }, + }, + { + name: "unsupported-codec", + errorText: "compression codec 127 is unsupported", + corrupt: func(t *testing.T, _ []byte, location ipcRecordLocation) { + compressionOffset := flatbuffers.UOffsetT(location.record.Offset(10)) + require.NotZero(t, compressionOffset) + compressionPosition := location.record.Indirect(compressionOffset + location.record.Pos) + compression := flatbuffers.Table{Bytes: location.record.Bytes, Pos: compressionPosition} + codecOffset := flatbuffers.UOffsetT(compression.Offset(4)) + require.NotZero(t, codecOffset, "ZSTD codec must be materialized in the flatbuffer") + compression.Bytes[codecOffset+compression.Pos] = 127 + }, + }, + } { + t.Run(containerName(container)+"/"+test.name, func(t *testing.T) { + payload, expected := makeIPCWithOptions(t, container, ipc.WithZstd()) + releaseRecords(expected) + location := locateFirstIPCRecord(t, payload, container) + test.corrupt(t, payload, location) + + path := "arrow-compression-metadata-" + containerName(container) + "-" + test.name + fs := writeMemoryFile(t, path, payload) + admission := new(testAdmission) + reader, err := Open( + context.Background(), fs, path, int64(len(payload)), container, admission, + Options{MaxDecodedRecordBytes: 1024}, + ) + require.NoError(t, err) + require.False(t, reader.Next()) + require.ErrorContains(t, reader.Err(), test.errorText) + require.NoError(t, reader.Close()) + require.Zero(t, admission.pending.Load()) + require.Zero(t, admission.active.Load()) + }) + } + } +} + +type ipcRecordLocation struct { + record flatbuffers.Table + bodyStart int +} + +func locateFirstIPCRecord(t *testing.T, payload []byte, container Container) ipcRecordLocation { + t.Helper() + if container == ContainerFile { + fs := writeMemoryFile(t, "arrow-compression-location", payload) + admission := new(testAdmission) + options, err := normalizeOptions(Options{Allocator: memory.NewGoAllocator()}) + require.NoError(t, err) + records, _, err := readFooterBlocks( + context.Background(), fileservice.NewLeasedRangeReader(fs), + "arrow-compression-location", int64(len(payload)), admission, options, + ) + require.NoError(t, err) + require.NotEmpty(t, records) + require.Zero(t, admission.active.Load()) + return ipcRecordLocation{ + record: firstIPCRecordTable(t, payload, records[0]), + bodyStart: int(records[0].offset + records[0].metadata), + } + } + + for cursor := 0; cursor+4 <= len(payload); { + metadataLength := binary.LittleEndian.Uint32(payload[cursor:]) + cursor += 4 + if metadataLength == ipcContinuationToken { + require.LessOrEqual(t, cursor+4, len(payload)) + metadataLength = binary.LittleEndian.Uint32(payload[cursor:]) + cursor += 4 + } + if metadataLength == 0 { + break + } + require.GreaterOrEqual(t, metadataLength, uint32(4)) + require.LessOrEqual(t, uint64(cursor)+uint64(metadataLength), uint64(len(payload))) + metadata := payload[cursor : cursor+int(metadataLength)] + message := flatbuffers.Table{Bytes: metadata, Pos: flatbuffers.GetUOffsetT(metadata)} + headerTypeOffset := flatbuffers.UOffsetT(message.Offset(6)) + require.NotZero(t, headerTypeOffset) + headerType := message.GetByte(headerTypeOffset + message.Pos) + bodyLengthOffset := flatbuffers.UOffsetT(message.Offset(10)) + var bodyLength int64 + if bodyLengthOffset != 0 { + bodyLength = message.GetInt64(bodyLengthOffset + message.Pos) + } + require.GreaterOrEqual(t, bodyLength, int64(0)) + bodyStart := cursor + int(metadataLength) + require.LessOrEqual(t, uint64(bodyStart)+uint64(bodyLength), uint64(len(payload))) + if headerType == byte(ipc.MessageRecordBatch) { + headerOffset := flatbuffers.UOffsetT(message.Offset(8)) + require.NotZero(t, headerOffset) + var record flatbuffers.Table + message.Union(&record, headerOffset) + return ipcRecordLocation{record: record, bodyStart: bodyStart} + } + cursor = bodyStart + int(bodyLength) + } + t.Fatal("Arrow IPC Stream has no record batch") + return ipcRecordLocation{} +} + +func firstNonEmptyIPCBuffer(t *testing.T, record flatbuffers.Table) (int64, int64, flatbuffers.UOffsetT) { + t.Helper() + buffersOffset := flatbuffers.UOffsetT(record.Offset(8)) + require.NotZero(t, buffersOffset) + buffers := record.Vector(buffersOffset) + for index := 0; index < record.VectorLen(buffersOffset); index++ { + descriptor := buffers + flatbuffers.UOffsetT(index*16) + offset := record.GetInt64(descriptor) + length := record.GetInt64(descriptor + 8) + if length > 0 { + return offset, length, descriptor + } + } + t.Fatal("Arrow record has no non-empty body buffer") + return 0, 0, 0 +} + +func TestCompressedIPCRecordOutlivesReaderAndOwnsAllocation(t *testing.T) { + for _, test := range []struct { + name string + container Container + codec ipc.Option + }{ + {name: "file-lz4", container: ContainerFile, codec: ipc.WithLZ4()}, + {name: "file-zstd", container: ContainerFile, codec: ipc.WithZstd()}, + {name: "stream-lz4", container: ContainerStream, codec: ipc.WithLZ4()}, + {name: "stream-zstd", container: ContainerStream, codec: ipc.WithZstd()}, + } { + t.Run(test.name, func(t *testing.T) { + payload, expected := makeIPCWithOptions(t, test.container, test.codec) + defer releaseRecords(expected) + path := "arrow-compressed-lifetime-" + test.name + fs := writeMemoryFile(t, path, payload) + admission := new(testAdmission) + reader, err := Open( + context.Background(), fs, path, int64(len(payload)), test.container, admission, Options{}, + ) + require.NoError(t, err) + require.True(t, reader.Next()) + record := reader.RecordBatch() + record.Retain() + require.NoError(t, reader.Close()) + require.Positive(t, admission.active.Load(), "retained record must keep decoded or source buffers admitted") + require.True(t, array.RecordEqual(expected[0], record)) + record.Release() + require.Zero(t, admission.pending.Load()) + require.Zero(t, admission.active.Load()) + }) + } +} + +func TestIPCLocalDiskFileAndStream(t *testing.T) { + for _, container := range []Container{ContainerFile, ContainerStream} { + t.Run(containerName(container), func(t *testing.T) { + payload, expected := makeIPC(t, container) + defer releaseRecords(expected) + fs, err := fileservice.NewLocalFS2( + context.Background(), "arrow-disk", t.TempDir(), fileservice.DisabledCacheConfig, nil, + ) + require.NoError(t, err) + t.Cleanup(func() { fs.Close(context.Background()) }) + path := "arrow-disk:input.arrow" + require.NoError(t, fs.Write(context.Background(), fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{Offset: 0, Size: int64(len(payload)), Data: payload}}, + })) + + admission := new(testAdmission) + reader, err := Open( + context.Background(), fs, path, int64(len(payload)), ContainerAuto, admission, Options{}, + ) + require.NoError(t, err) + for index := range expected { + require.True(t, reader.Next(), "record %d: %v", index, reader.Err()) + require.True(t, array.RecordEqual(expected[index], reader.RecordBatch())) + } + require.False(t, reader.Next()) + require.NoError(t, reader.Err()) + require.NoError(t, reader.Close()) + require.Zero(t, admission.pending.Load()) + require.Zero(t, admission.active.Load()) + }) + } +} + +func containerName(container Container) string { + switch container { + case ContainerFile: + return "file" + case ContainerStream: + return "stream" + default: + return "unknown" + } +} + +func TestIPCFileLimitsCancellationAndAdmission(t *testing.T) { + fileBytes, expected := makeIPC(t, ContainerFile) + defer releaseRecords(expected) + fs := writeMemoryFile(t, "arrow-limits", fileBytes) + _, err := Open(context.Background(), fs, "arrow-limits", int64(len(fileBytes)), ContainerFile, new(testAdmission), Options{MaxMetadataBytes: 4}) + require.Error(t, err) + _, err = Open(context.Background(), fs, "arrow-limits", int64(len(fileBytes)), ContainerFile, new(testAdmission), Options{MaxDecodedRecordBytes: -1}) + require.ErrorContains(t, err, "invalid Arrow IPC size limits") + + reject := errors.New("range quota exceeded") + _, err = Open(context.Background(), fs, "arrow-limits", int64(len(fileBytes)), ContainerFile, &testAdmission{reject: reject}, Options{}) + require.ErrorIs(t, err, reject) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = Open(ctx, fs, "arrow-limits", int64(len(fileBytes)), ContainerFile, new(testAdmission), Options{}) + require.ErrorIs(t, err, context.Canceled) +} + +func TestNormalizeOptionsCapsMetadataLimit(t *testing.T) { + options, err := normalizeOptions(Options{ + MaxMetadataBytes: DefaultMaxMetadataBytes * 2, + }) + require.NoError(t, err) + require.Equal(t, int64(DefaultMaxMetadataBytes), options.MaxMetadataBytes) +} + +func TestIPCBodyLimitForFileAndStream(t *testing.T) { + for _, container := range []Container{ContainerFile, ContainerStream} { + t.Run(containerName(container), func(t *testing.T) { + payload, expected := makeIPC(t, container) + defer releaseRecords(expected) + fs := writeMemoryFile(t, "arrow-body-limit", payload) + admission := new(testAdmission) + reader, err := Open( + context.Background(), fs, "arrow-body-limit", int64(len(payload)), container, + admission, Options{MaxBodyBytes: 1}, + ) + if err == nil { + require.False(t, reader.Next()) + require.Error(t, reader.Err()) + require.NoError(t, reader.Close()) + } else { + require.Nil(t, reader) + } + require.Zero(t, admission.pending.Load()) + require.Zero(t, admission.active.Load()) + }) + } +} + +func TestIPCDictionaryReplayForFileAndStreamDelta(t *testing.T) { + for _, test := range []struct { + name string + container Container + delta bool + }{ + {name: "file base dictionary", container: ContainerFile}, + {name: "stream dictionary delta", container: ContainerStream, delta: true}, + } { + t.Run(test.name, func(t *testing.T) { + payload, expected := makeDictionaryIPC(t, test.container, test.delta) + defer releaseRecords(expected) + fs := writeMemoryFile(t, "arrow-dictionary", payload) + admission := new(testAdmission) + reader, err := Open( + context.Background(), fs, "arrow-dictionary", int64(len(payload)), test.container, admission, Options{}, + ) + require.NoError(t, err) + for index := range expected { + require.True(t, reader.Next(), "record %d: %v", index, reader.Err()) + require.True(t, array.RecordEqual(expected[index], reader.RecordBatch())) + } + require.False(t, reader.Next()) + require.NoError(t, reader.Err()) + require.NoError(t, reader.Close()) + require.Zero(t, admission.active.Load()) + }) + } +} + +func TestIPCConditionalIdentityPreventsMixedFileVersions(t *testing.T) { + payload, expectedRecords := makeIPC(t, ContainerFile) + defer releaseRecords(expectedRecords) + base := writeMemoryFile(t, "arrow-identity", payload) + planned := fileservice.ObjectIdentity{ETag: "etag-v1", Size: int64(len(payload))} + fs := newIdentityMemoryFS(base, planned) + reader, err := Open( + context.Background(), fs, "arrow-identity", int64(len(payload)), ContainerFile, + new(testAdmission), Options{ExpectedIdentity: &planned}, + ) + require.NoError(t, err) + + changed := fileservice.ObjectIdentity{ETag: "etag-v2", Size: int64(len(payload))} + fs.identity.Store(&changed) + require.False(t, reader.Next()) + require.ErrorIs(t, reader.Err(), fileservice.ErrObjectChanged) + require.NoError(t, reader.Close()) +} + +func TestIPCConditionalIdentitySupportsSingleStreamGET(t *testing.T) { + payload, expectedRecords := makeIPC(t, ContainerStream) + defer releaseRecords(expectedRecords) + base := writeMemoryFile(t, "arrow-stream-identity", payload) + planned := fileservice.ObjectIdentity{VersionID: "version-1", Size: int64(len(payload))} + fs := newIdentityMemoryFS(base, planned) + reader, err := Open( + context.Background(), fs, "arrow-stream-identity", int64(len(payload)), ContainerStream, + new(testAdmission), Options{ExpectedIdentity: &planned}, + ) + require.NoError(t, err) + for index := range expectedRecords { + require.True(t, reader.Next()) + require.True(t, array.RecordEqual(expectedRecords[index], reader.RecordBatch())) + } + require.False(t, reader.Next()) + require.NoError(t, reader.Err()) + require.NoError(t, reader.Close()) +} + +func makeIPC(t testing.TB, container Container) ([]byte, []arrow.RecordBatch) { + return makeIPCWithOptions(t, container) +} + +func makeIPCWithOptions(t testing.TB, container Container, options ...ipc.Option) ([]byte, []arrow.RecordBatch) { + t.Helper() + alloc := memory.NewGoAllocator() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "name", Type: arrow.BinaryTypes.String, Nullable: true}, + }, nil) + records := make([]arrow.RecordBatch, 0, 2) + for batchIndex := 0; batchIndex < 2; batchIndex++ { + builder := array.NewRecordBuilder(alloc, schema) + builder.Field(0).(*array.Int64Builder).AppendValues( + []int64{int64(batchIndex*2 + 1), int64(batchIndex*2 + 2)}, nil, + ) + builder.Field(1).(*array.StringBuilder).AppendValues( + []string{"a payload longer than twenty three bytes", "short"}, []bool{true, batchIndex == 0}, + ) + records = append(records, builder.NewRecordBatch()) + builder.Release() + } + var output bytes.Buffer + writerOptions := append([]ipc.Option{ + ipc.WithSchema(schema), + ipc.WithAllocator(alloc), + }, options...) + if container == ContainerFile { + writer, err := ipc.NewFileWriter(&output, writerOptions...) + require.NoError(t, err) + for _, record := range records { + require.NoError(t, writer.Write(record)) + } + require.NoError(t, writer.Close()) + } else { + writer := ipc.NewWriter(&output, writerOptions...) + for _, record := range records { + require.NoError(t, writer.Write(record)) + } + require.NoError(t, writer.Close()) + } + return output.Bytes(), records +} + +func makeDictionaryIPC(t testing.TB, container Container, delta bool) ([]byte, []arrow.RecordBatch) { + t.Helper() + alloc := memory.NewGoAllocator() + dictionaryType := &arrow.DictionaryType{ + IndexType: arrow.PrimitiveTypes.Int8, + ValueType: arrow.BinaryTypes.String, + } + schema := arrow.NewSchema([]arrow.Field{{Name: "name", Type: dictionaryType}}, nil) + valueSets := [][]string{{"alpha"}, {"alpha"}} + indices := []int8{0, 0} + if delta { + valueSets[1] = []string{"alpha", "beta"} + indices[1] = 1 + } + records := make([]arrow.RecordBatch, 0, len(valueSets)) + for recordIndex, valueSet := range valueSets { + indexBuilder := array.NewInt8Builder(alloc) + indexBuilder.Append(indices[recordIndex]) + indexArray := indexBuilder.NewArray() + indexBuilder.Release() + valueBuilder := array.NewStringBuilder(alloc) + valueBuilder.AppendValues(valueSet, nil) + valueArray := valueBuilder.NewArray() + valueBuilder.Release() + dictionary := array.NewDictionaryArray(dictionaryType, indexArray, valueArray) + records = append(records, array.NewRecordBatch(schema, []arrow.Array{dictionary}, 1)) + dictionary.Release() + indexArray.Release() + valueArray.Release() + } + + var output bytes.Buffer + if container == ContainerFile { + writer, err := ipc.NewFileWriter(&output, ipc.WithSchema(schema), ipc.WithAllocator(alloc)) + require.NoError(t, err) + for _, record := range records { + require.NoError(t, writer.Write(record)) + } + require.NoError(t, writer.Close()) + } else { + writer := ipc.NewWriter( + &output, + ipc.WithSchema(schema), + ipc.WithAllocator(alloc), + ipc.WithDictionaryDeltas(delta), + ) + for _, record := range records { + require.NoError(t, writer.Write(record)) + } + require.NoError(t, writer.Close()) + } + return output.Bytes(), records +} + +func writeMemoryFile(t testing.TB, path string, data []byte) *fileservice.MemoryFS { + t.Helper() + fs, err := fileservice.NewMemoryFS("arrow-test", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + require.NoError(t, fs.Write(context.Background(), fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{Offset: 0, Size: int64(len(data)), Data: data}}, + })) + return fs +} + +func releaseRecords(records []arrow.RecordBatch) { + for _, record := range records { + record.Release() + } +} + +func FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak(f *testing.F) { + filePayload, fileRecords := makeIPC(f, ContainerFile) + releaseRecords(fileRecords) + streamPayload, streamRecords := makeIPC(f, ContainerStream) + releaseRecords(streamRecords) + f.Add(append([]byte(nil), filePayload...)) + f.Add(append([]byte(nil), streamPayload...)) + f.Add([]byte("not arrow")) + f.Add([]byte{}) + if len(filePayload) > 16 { + f.Add(append([]byte(nil), filePayload[len(filePayload)-16:]...)) + } + + f.Fuzz(func(t *testing.T, payload []byte) { + if len(payload) > 2<<20 { + return + } + // Keep concurrent fuzz workers below the process memory ceiling. The + // production default is protected by statement/global admission, whereas + // this intentionally simple test admission accepts every reservation. + options := Options{ + MaxMetadataBytes: 1 << 20, + MaxBodyBytes: 2 << 20, + MaxDecodedRecordBytes: 2 << 20, + } + fs := writeMemoryFile(t, "arrow-fuzz", append([]byte(nil), payload...)) + admission := new(testAdmission) + plan, _ := InspectFile( + context.Background(), fs, "arrow-fuzz", int64(len(payload)), admission, options, + ) + if plan != nil { + require.NotNil(t, plan.Schema) + } + require.Zero(t, admission.active.Load()) + + reader, openErr := Open( + context.Background(), fs, "arrow-fuzz", int64(len(payload)), ContainerAuto, admission, options, + ) + var readErr error + if reader != nil { + for records := 0; records < 1024 && reader.Next(); records++ { + require.NotNil(t, reader.RecordBatch()) + } + readErr = reader.Err() + require.NoError(t, reader.Close()) + } + require.Zero(t, admission.active.Load(), + "reader=%T openErr=%v readErr=%v reserved=%d released=%d pending=%d max=%d", + reader, openErr, readErr, admission.reserved.Load(), admission.released.Load(), + admission.pending.Load(), admission.max.Load()) + }) +} diff --git a/pkg/sql/colexec/external/arrowio/schema_metadata_test.go b/pkg/sql/colexec/external/arrowio/schema_metadata_test.go new file mode 100644 index 0000000000000..6350d5a86f421 --- /dev/null +++ b/pkg/sql/colexec/external/arrowio/schema_metadata_test.go @@ -0,0 +1,362 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowio + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "strings" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/ipc" + flatbuffers "github.com/google/flatbuffers/go" + "github.com/matrixorigin/matrixone/pkg/container/arrowipc" + "github.com/matrixorigin/matrixone/pkg/container/arrowipc/ipcflatbuf" + "github.com/stretchr/testify/require" +) + +const ( + maxArrowSchemaFields = arrowipc.MaxSchemaFields + maxArrowSchemaDepth = arrowipc.MaxSchemaDepth + maxArrowSchemaMetadataEntries = arrowipc.MaxSchemaMetadataEntries + maxArrowSchemaFeatures = arrowipc.MaxSchemaFeatures + maxArrowUnionTypeIDsPerField = arrowipc.MaxUnionTypeIDsPerField +) + +func TestIPCSchemaVectorCountRejectedBeforeArrowGoForFileAndStream(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{Name: "value", Type: arrow.PrimitiveTypes.Int64}}, nil) + for _, container := range []Container{ContainerFile, ContainerStream} { + t.Run(containerName(container), func(t *testing.T) { + payload := makeEmptyIPC(t, container, schema) + schemaTable := ipcSchemaTable(t, payload, container) + setFlatbufferVectorLength( + t, schemaTable, 6, uint32(maxArrowSchemaFields+1), + ) + + admission := new(testAdmission) + reader, err := Open( + context.Background(), writeMemoryFile(t, "arrow-schema-vector", payload), + "arrow-schema-vector", int64(len(payload)), container, admission, Options{}, + ) + if reader != nil { + require.NoError(t, reader.Close()) + } + require.ErrorContains(t, err, "schema field vector count") + require.Zero(t, admission.pending.Load()) + require.Zero(t, admission.active.Load()) + }) + } +} + +func TestIPCSchemaFieldAndDepthLimits(t *testing.T) { + t.Run("total field boundary", func(t *testing.T) { + fields := make([]arrow.Field, maxArrowSchemaFields) + for index := range fields { + fields[index] = arrow.Field{Name: "f", Type: arrow.PrimitiveTypes.Int8} + } + assertEmptyIPCSchemaOpen(t, arrow.NewSchema(fields, nil)) + + fields = append(fields, arrow.Field{Name: "overflow", Type: arrow.PrimitiveTypes.Int8}) + assertEmptyIPCSchemaError(t, arrow.NewSchema(fields, nil), "schema field count exceeds limit 4096") + }) + + t.Run("nesting depth boundary", func(t *testing.T) { + assertEmptyIPCSchemaOpen(t, nestedListSchema(maxArrowSchemaDepth)) + assertEmptyIPCSchemaError( + t, nestedListSchema(maxArrowSchemaDepth+1), "schema nesting depth 65 exceeds limit 64", + ) + }) +} + +func TestIPCSchemaMetadataAndUnionLimits(t *testing.T) { + t.Run("custom metadata total boundary", func(t *testing.T) { + schemaMetadata := repeatedArrowMetadata(maxArrowSchemaMetadataEntries / 2) + fieldMetadata := repeatedArrowMetadata(maxArrowSchemaMetadataEntries / 2) + field := arrow.Field{Name: "value", Type: arrow.PrimitiveTypes.Int64, Metadata: fieldMetadata} + assertEmptyIPCSchemaOpen(t, arrow.NewSchema([]arrow.Field{field}, &schemaMetadata)) + + fieldMetadata = repeatedArrowMetadata(maxArrowSchemaMetadataEntries/2 + 1) + field.Metadata = fieldMetadata + assertEmptyIPCSchemaError( + t, arrow.NewSchema([]arrow.Field{field}, &schemaMetadata), + "schema custom metadata entry count exceeds limit 4096", + ) + }) + + t.Run("timestamp timezone alias amplification", func(t *testing.T) { + metadata := makeFlatbufferSchemaWithSharedTimezone(t, 128, strings.Repeat("x", 4096)) + var schema ipcflatbuf.Schema + schema.Init(metadata, flatbuffers.GetUOffsetT(metadata)) + require.ErrorContains(t, + arrowipc.ValidateSchemaMetadata(context.Background(), &schema, len(metadata)), + "schema decoded string bytes exceed metadata size", + ) + }) + + t.Run("union type id boundary", func(t *testing.T) { + children := make([]arrow.Field, maxArrowUnionTypeIDsPerField) + codes := make([]arrow.UnionTypeCode, maxArrowUnionTypeIDsPerField) + for index := range children { + children[index] = arrow.Field{ + Name: fmt.Sprintf("member_%d", index), Type: arrow.PrimitiveTypes.Int8, + } + codes[index] = arrow.UnionTypeCode(index) + } + schema := arrow.NewSchema([]arrow.Field{{ + Name: "union", Type: arrow.DenseUnionOf(children, codes), + }}, nil) + assertEmptyIPCSchemaOpen(t, schema) + + payload := makeEmptyIPC(t, ContainerStream, schema) + schemaTable := ipcSchemaTable(t, payload, ContainerStream) + fieldTable := firstFlatbufferSchemaField(t, schemaTable) + unionTable := flatbufferFieldType(t, fieldTable) + setFlatbufferVectorLength( + t, unionTable, 6, uint32(maxArrowUnionTypeIDsPerField+1), + ) + admission := new(testAdmission) + reader, err := Open( + context.Background(), writeMemoryFile(t, "arrow-union-vector", payload), + "arrow-union-vector", int64(len(payload)), ContainerStream, admission, Options{}, + ) + if reader != nil { + require.NoError(t, reader.Close()) + } + require.ErrorContains(t, err, "union type ID count 129 exceeds per-field limit 128") + require.Zero(t, admission.pending.Load()) + require.Zero(t, admission.active.Load()) + }) + + t.Run("duplicate union type ids", func(t *testing.T) { + schema := arrow.NewSchema([]arrow.Field{{ + Name: "union", + Type: arrow.DenseUnionOf([]arrow.Field{ + {Name: "first", Type: arrow.PrimitiveTypes.Int8}, + {Name: "second", Type: arrow.PrimitiveTypes.Int16}, + }, []arrow.UnionTypeCode{1, 1}), + }}, nil) + payload := makeEmptyIPC(t, ContainerStream, schema) + admission := new(testAdmission) + reader, err := Open( + context.Background(), writeMemoryFile(t, "arrow-duplicate-union-id", payload), + "arrow-duplicate-union-id", int64(len(payload)), ContainerStream, admission, Options{}, + ) + if reader != nil { + require.NoError(t, reader.Close()) + } + require.ErrorContains(t, err, "duplicate union type ID") + }) + + t.Run("feature boundary", func(t *testing.T) { + metadata := makeFlatbufferSchemaWithFeatures(t, maxArrowSchemaFeatures) + var schema ipcflatbuf.Schema + schema.Init(metadata, flatbuffers.GetUOffsetT(metadata)) + require.NoError(t, arrowipc.ValidateSchemaMetadata(context.Background(), &schema, len(metadata))) + + metadata = makeFlatbufferSchemaWithFeatures(t, maxArrowSchemaFeatures+1) + schema.Init(metadata, flatbuffers.GetUOffsetT(metadata)) + require.ErrorContains(t, + arrowipc.ValidateSchemaMetadata(context.Background(), &schema, len(metadata)), + "schema feature count 65 exceeds limit 64", + ) + }) +} + +func assertEmptyIPCSchemaOpen(t *testing.T, schema *arrow.Schema) { + t.Helper() + payload := makeEmptyIPC(t, ContainerStream, schema) + admission := new(testAdmission) + reader, err := Open( + context.Background(), writeMemoryFile(t, "arrow-schema-valid", payload), + "arrow-schema-valid", int64(len(payload)), ContainerStream, admission, Options{}, + ) + require.NoError(t, err) + require.NotNil(t, reader.Schema()) + require.False(t, reader.Next()) + require.NoError(t, reader.Err()) + require.NoError(t, reader.Close()) + require.Zero(t, admission.pending.Load()) + require.Zero(t, admission.active.Load()) +} + +func assertEmptyIPCSchemaError(t *testing.T, schema *arrow.Schema, errorText string) { + t.Helper() + payload := makeEmptyIPC(t, ContainerStream, schema) + admission := new(testAdmission) + reader, err := Open( + context.Background(), writeMemoryFile(t, "arrow-schema-invalid", payload), + "arrow-schema-invalid", int64(len(payload)), ContainerStream, admission, Options{}, + ) + if reader != nil { + require.NoError(t, reader.Close()) + } + require.ErrorContains(t, err, errorText) + require.Zero(t, admission.pending.Load()) + require.Zero(t, admission.active.Load()) +} + +func makeEmptyIPC(t testing.TB, container Container, schema *arrow.Schema) []byte { + t.Helper() + var output bytes.Buffer + if container == ContainerFile { + writer, err := ipc.NewFileWriter(&output, ipc.WithSchema(schema)) + require.NoError(t, err) + require.NoError(t, writer.Close()) + } else { + writer := ipc.NewWriter(&output, ipc.WithSchema(schema)) + require.NoError(t, writer.Close()) + } + return output.Bytes() +} + +func nestedListSchema(depth int) *arrow.Schema { + var dataType arrow.DataType = arrow.PrimitiveTypes.Int8 + for current := 1; current < depth; current++ { + dataType = arrow.ListOf(dataType) + } + return arrow.NewSchema([]arrow.Field{{Name: "nested", Type: dataType}}, nil) +} + +func repeatedArrowMetadata(count int) arrow.Metadata { + keys := make([]string, count) + values := make([]string, count) + for index := range keys { + keys[index] = "k" + } + return arrow.NewMetadata(keys, values) +} + +func ipcSchemaTable(t testing.TB, payload []byte, container Container) flatbuffers.Table { + t.Helper() + if container == ContainerFile { + tailStart := len(payload) - 4 - len(ipc.Magic) + require.GreaterOrEqual(t, tailStart, 0) + footerLength := int(binary.LittleEndian.Uint32(payload[tailStart:])) + footerStart := tailStart - footerLength + require.GreaterOrEqual(t, footerStart, len(ipc.Magic)) + footerBytes := payload[footerStart:tailStart] + footer := flatbuffers.Table{Bytes: footerBytes, Pos: flatbuffers.GetUOffsetT(footerBytes)} + schemaOffset := flatbuffers.UOffsetT(footer.Offset(6)) + require.NotZero(t, schemaOffset) + return flatbuffers.Table{ + Bytes: footerBytes, + Pos: footer.Indirect(schemaOffset + footer.Pos), + } + } + + cursor := 0 + require.GreaterOrEqual(t, len(payload), 4) + metadataLength := binary.LittleEndian.Uint32(payload[cursor:]) + cursor += 4 + if metadataLength == ipcContinuationToken { + require.GreaterOrEqual(t, len(payload)-cursor, 4) + metadataLength = binary.LittleEndian.Uint32(payload[cursor:]) + cursor += 4 + } + require.GreaterOrEqual(t, len(payload)-cursor, int(metadataLength)) + metadata := payload[cursor : cursor+int(metadataLength)] + message := flatbuffers.Table{Bytes: metadata, Pos: flatbuffers.GetUOffsetT(metadata)} + headerTypeOffset := flatbuffers.UOffsetT(message.Offset(6)) + require.NotZero(t, headerTypeOffset) + require.Equal(t, byte(ipc.MessageSchema), message.GetByte(headerTypeOffset+message.Pos)) + headerOffset := flatbuffers.UOffsetT(message.Offset(8)) + require.NotZero(t, headerOffset) + var schema flatbuffers.Table + message.Union(&schema, headerOffset) + return schema +} + +func firstFlatbufferSchemaField(t testing.TB, schema flatbuffers.Table) flatbuffers.Table { + t.Helper() + fieldsOffset := flatbuffers.UOffsetT(schema.Offset(6)) + require.NotZero(t, fieldsOffset) + require.Positive(t, schema.VectorLen(fieldsOffset)) + position := schema.Vector(fieldsOffset) + return flatbuffers.Table{Bytes: schema.Bytes, Pos: schema.Indirect(position)} +} + +func flatbufferFieldType(t testing.TB, field flatbuffers.Table) flatbuffers.Table { + t.Helper() + typeOffset := flatbuffers.UOffsetT(field.Offset(10)) + require.NotZero(t, typeOffset) + var dataType flatbuffers.Table + field.Union(&dataType, typeOffset) + return dataType +} + +func setFlatbufferVectorLength( + t testing.TB, + table flatbuffers.Table, + vtableOffset flatbuffers.VOffsetT, + count uint32, +) { + t.Helper() + offset := flatbuffers.UOffsetT(table.Offset(vtableOffset)) + require.NotZero(t, offset) + vectorOffset := offset + table.Pos + require.GreaterOrEqual(t, len(table.Bytes)-int(vectorOffset), 4) + vector := vectorOffset + flatbuffers.GetUOffsetT(table.Bytes[vectorOffset:]) + require.GreaterOrEqual(t, len(table.Bytes)-int(vector), 4) + binary.LittleEndian.PutUint32(table.Bytes[vector:], count) +} + +func makeFlatbufferSchemaWithFeatures(t testing.TB, count int) []byte { + t.Helper() + builder := flatbuffers.NewBuilder(128 + count*8) + builder.StartVector(8, count, 8) + for index := count - 1; index >= 0; index-- { + builder.PrependInt64(int64(index)) + } + features := builder.EndVector(count) + builder.StartObject(4) + builder.PrependUOffsetTSlot(3, features, 0) + schema := builder.EndObject() + builder.Finish(schema) + return append([]byte(nil), builder.FinishedBytes()...) +} + +func makeFlatbufferSchemaWithSharedTimezone( + t testing.TB, + fieldCount int, + timezone string, +) []byte { + t.Helper() + builder := flatbuffers.NewBuilder(128 + fieldCount*32 + len(timezone)) + timezoneOffset := builder.CreateString(timezone) + fields := make([]flatbuffers.UOffsetT, fieldCount) + for index := range fields { + builder.StartObject(2) + builder.PrependUOffsetTSlot(1, timezoneOffset, 0) + timestamp := builder.EndObject() + + builder.StartObject(7) + builder.PrependByteSlot(2, byte(ipcflatbuf.TypeTimestamp), 0) + builder.PrependUOffsetTSlot(3, timestamp, 0) + fields[index] = builder.EndObject() + } + builder.StartVector(4, len(fields), 4) + for index := len(fields) - 1; index >= 0; index-- { + builder.PrependUOffsetT(fields[index]) + } + fieldVector := builder.EndVector(len(fields)) + builder.StartObject(4) + builder.PrependUOffsetTSlot(1, fieldVector, 0) + schema := builder.EndObject() + builder.Finish(schema) + return append([]byte(nil), builder.FinishedBytes()...) +} diff --git a/pkg/sql/colexec/external/arrowio/stream.go b/pkg/sql/colexec/external/arrowio/stream.go new file mode 100644 index 0000000000000..7fa0c133e7b5b --- /dev/null +++ b/pkg/sql/colexec/external/arrowio/stream.go @@ -0,0 +1,163 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowio + +import ( + "context" + "encoding/binary" + "io" + "math" + "sync/atomic" + + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// streamMessageReader is the trust boundary for IPC Stream messages. Arrow-Go +// validates wire metadata/body lengths, but its decompressor trusts each +// buffer's 8-byte decoded-size prefix. Inspecting the complete message here +// keeps that size from reaching an allocator before policy has accepted it. +type streamMessageReader struct { + refs atomic.Int64 + ctx context.Context + stream io.Reader + allocator memory.Allocator + options Options + current *ipc.Message + header [4]byte +} + +func newStreamMessageReader( + ctx context.Context, + stream io.Reader, + options Options, +) *streamMessageReader { + reader := &streamMessageReader{ + ctx: ctx, stream: stream, allocator: options.Allocator, options: options, + } + reader.refs.Store(1) + return reader +} + +func (r *streamMessageReader) Retain() { + if r == nil { + panic("retain released Arrow stream message reader") + } + for { + refs := r.refs.Load() + if refs <= 0 { + panic("retain released Arrow stream message reader") + } + if r.refs.CompareAndSwap(refs, refs+1) { + return + } + } +} + +func (r *streamMessageReader) Release() { + if r == nil { + return + } + refs := r.refs.Add(-1) + if refs < 0 { + panic("Arrow stream message reader release underflow") + } + if refs == 0 && r.current != nil { + r.current.Release() + r.current = nil + } +} + +// Message returns a message that remains valid until Message is called again. +func (r *streamMessageReader) Message() (message *ipc.Message, retErr error) { + defer func() { + if recovered := recover(); recovered != nil { + message = nil + if allocationErr, matched := recoveredAllocationError(recovered); matched { + retErr = allocationErr + return + } + retErr = moerr.NewInvalidInputf(r.ctx, "invalid Arrow IPC Stream message: %v", recovered) + } + }() + if err := r.ctx.Err(); err != nil { + return nil, err + } + + metadataLength, err := r.readMetadataLength() + if err != nil { + return nil, err + } + metadata := memory.NewResizableBuffer(r.allocator) + defer metadata.Release() + metadata.Resize(metadataLength) + if _, err := io.ReadFull(r.stream, metadata.Bytes()); err != nil { + return nil, moerr.NewInvalidInputf(r.ctx, "could not read Arrow IPC Stream metadata: %v", err) + } + + inspected, err := inspectIPCMessageMetadata( + r.ctx, metadata.Bytes(), r.options.MaxBodyBytes, -1, nil, false, + r.options.MaxDecodedRecordBytes, + ) + if err != nil { + return nil, err + } + body := memory.NewResizableBuffer(r.allocator) + defer body.Release() + body.Resize(int(inspected.bodyBytes)) + if _, err := io.ReadFull(r.stream, body.Bytes()); err != nil { + return nil, moerr.NewInvalidInputf(r.ctx, "could not read Arrow IPC Stream body: %v", err) + } + if _, err := inspectIPCMessageMetadata( + r.ctx, metadata.Bytes(), r.options.MaxBodyBytes, inspected.bodyBytes, body.Bytes(), true, + r.options.MaxDecodedRecordBytes, + ); err != nil { + return nil, err + } + + if r.current != nil { + r.current.Release() + r.current = nil + } + r.current = ipc.NewMessage(metadata, body) + return r.current, nil +} + +func (r *streamMessageReader) readMetadataLength() (int, error) { + if _, err := io.ReadFull(r.stream, r.header[:]); err != nil { + return 0, err + } + prefix := binary.LittleEndian.Uint32(r.header[:]) + if prefix == 0 { + return 0, io.EOF + } + if prefix == ipcContinuationToken { + if _, err := io.ReadFull(r.stream, r.header[:]); err != nil { + return 0, moerr.NewInvalidInputf(r.ctx, "could not read Arrow IPC Stream message length: %v", err) + } + prefix = binary.LittleEndian.Uint32(r.header[:]) + if prefix == 0 { + return 0, io.EOF + } + } + if prefix < 4 || uint64(prefix) > uint64(math.MaxInt) || int64(prefix) > r.options.MaxMetadataBytes { + return 0, moerr.NewInvalidInputf(r.ctx, + "Arrow IPC Stream metadata length %d exceeds limit %d", prefix, r.options.MaxMetadataBytes) + } + return int(prefix), nil +} + +var _ ipc.MessageReader = (*streamMessageReader)(nil) diff --git a/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/0764053921689d5c b/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/0764053921689d5c new file mode 100644 index 0000000000000..1dca92a39bfb6 --- /dev/null +++ b/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/0764053921689d5c @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("ARROW1\x00\x00\xff\xff\xff\xff\xb8\x00\x00\x00\x10\x00\x00\x00\x00\x00\n\x00\f\x00\n\x00\t\x00\x04\x00\n\x00\x00\x00\x10\x00\x00\x00\x00\x01\x04\x00\b\x00\b\x00\x00\x00\x04\x00\b\x00\x00\x00\x04\x00\x00\x00\x02\x00\x00\x00T\x00\x00\x00\x14\x00\x00\x00\x10\x00\x14\x00\x10\x00\x0f\x00\x0e\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x14\x00\x00\x00\x00\x00\x05\x01\x10\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00name\x00\x00\x00\x00\x10\x00\x14\x00\x10\x00\x00\x00\x0f\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x02\x1c\x00\x00\x00\x00\x00\x00\x00\b\x00\f\x00\b\x00\a\x00\b\x00\x00\x00\x00\x00\x00\x01@\x00\x00\x00\x02\x00\x00\x00id\x00\x00\xff\xff\xff\xff\xc8\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\f\x00\x16\x00\x14\x00\x13\x00\f\x00\x04\x00\f\x00\x00\x00P\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x03\x04\x00\n\x00\x18\x00\f\x00\b\x00\x04\x00\n\x00\x00\x00\x14\x00\x00\x00h\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\f\x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00(\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00a payload longer than twenty three bytesshort\x00\x00\x00\xff\xff\xff\xff\xc8\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\f\x00\x16\x00\x14\x00\x13\x00\f\x00\x04\x00\f\x00\x00\x00X\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x03\x04\x00\n\x00\x18\x00\f\x00\b\x00\x04\x00\n\x00\x00\x00\x14\x00\x00\x00h\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x00\b\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\f\x00\x00\x00\x00\x00\x00\x00(\x00\x00\x00\x00\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00(\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00a payload longer than twenty three bytesshort\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\f\x00\x14\x00\x12\x00\f\x00\b\x00\x04\x00\f\x00\x00\x00\x10\x00\x00\x00D\x00\x00\x00P\x00\x00\x00\x00\x00\x04\x00\x02\x00\x00\x00\xc8\x00\x00\x00\x00\x00\x00\x00\xd0\x00\x00\x00\x00\x00\x00\x00P\x00\x00\x00\x00\x00\x00\x00\xe8\x01\x00\x00\x00\x00\x00\x00\xd0\x00\x00\x00\x00\x00\x00\x00X\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b\x00\b\x00\x00\x00\x04\x00\b\x00\x00\x00\x04\x00\x00\x00\x02\x00\x00\x00T\x00\x00\x00\x14\x00\x00\x00\x10\x00\x14\x00\x10\x00\x0f\x00\x0e\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x14\x00\x00\x00\x00\x00\x05\x01\x10\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00name\x00\x00\x00\x00\x10\x00\x14\x00\x10\x00\x00\x00\x0f\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x02\x1c\x00\x00\x00\x00\x00\x00\x00\b\x00\f\x00\b\x00\a\x00\b\x00\x00\x00\x00\x00\x00\x01@\x00\x00\x00\x02\x00\x00\x00id\x00\x00\x00\x01\x00\x00ARROW1") diff --git a/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/25977ddbfaf8976e b/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/25977ddbfaf8976e new file mode 100644 index 0000000000000..f88de75739e36 --- /dev/null +++ b/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/25977ddbfaf8976e @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\xff\xff\xff\xff\xb8\x00\x00\x00\x10\x00\x00\x00\x00\x00\n\x00\f\x00\n\x00\t\x00\x04\x00\n\x00\x00\x00\x10\x00\x00\x00\x00\x01\x04\x00\b\x00\b\x00\x00\x00\x04\x00\b\x00\x00\x00\x04\x00\x00\x00\x02\x00\x00T\x00\x00\x00\x14\x00\x00\x00\x10\x00\x14\x00\x10\x00\x0f\x00\x0e\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x14\x00\x00\x00\x00\x00\x05\x01\x10\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00name\x00\x00\x00\x00\x10\x00\x14\x00\x10\x00\x00\x00\x0f\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x02\x1c\x00\x00\x00\x00\x00\x00\x00\b\x00\f\x00\b\x00\a\x00\b\x00\x00\x00\x00\x00\x00\x01@\x00\x00\x00\x02\x00\x00\x00id\x00\x00\xff\xff\xff\xff\xc8\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\f\x00\x16\x00\x14\x00\x13\x00\f\x00\x04\x00\f\x00\x00\x00P\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x03\x04\x00\n\x00\x18\x00\f\x00\b\x00\x04\x00\n\x00\x00\x00\x14\x00\x00\x00h\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\f\x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00") diff --git a/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/5a364ce90e14ce87 b/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/5a364ce90e14ce87 new file mode 100644 index 0000000000000..a54b0e6d680c2 --- /dev/null +++ b/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/5a364ce90e14ce87 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("ARROW1\x00\x00\xff\xff\xff\xff\xb8\x00\x00\x00\x10\x00\x00\x00\x00\x00\n\x00\f\x00\n\x00\t\x00\x04\x00\n\x00\x00\x00\x10\x00\x00\x00\x00\x01\x04\x00\b\x00\b\x00\x00\x00\x04\x00\b\x00\x00\x00\x04\x00\x00\x00\x02\x00\x00\x00T\x00\x00\x00\x14\x00\x00\x00\x10\x00\x14\x00\x10\x00\x0f\x00\x0e\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x14\x00\x00\x00\x00\x00\x05\x01\x10\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00name\x00\x00\x00\x00\x10\x00\x14\x00\x10\x00\x00\x00\x0f\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x02\x1c\x00\x00\x00\x00\x00\x00\x00\b\x00\f\x00\b\x00\a\x00\b\x00\x00\x00\x00\x00\x00\x01@\x00\x00\x00\x02\x00\x00\x00id\x00\x00\xff\xff\xff\xff\xc8\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\f\x00\x16\x00\x14\x00\x13\x00\f\x00\x04\x00\f\x00\x00\x00P\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x03\x04\x00\n\x00\x18\x00\f\x00\b\x00\x04\x00\n\x00\x00\x00\x14\x00\x00\x00h\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\f\x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00(\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00a payload longer than twenty three bytesshort\x00\x00\x00\xff\xff\xff\xff\xc8\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\f\x00\x16\x00\x14\x00\x13\x00\f\x00\x04\x00\f\x00\x00\x00X\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x03\x04\x00\n\x00\x18\x00\f\x00\b\x00\x04\x00\n\x00\x00\x00\x14\x00\x00\x00h\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\f\x00\x00\x00\x00\x00\x00\x00(\x00\x00\x00\x00\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00(\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00a payload longer than twenty three bytesshort\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\x10\x00\x00\x00\f\x00\x14\x00\x12\x00\f\x00\b\x00\x04\x00\f\x00\x00\x00\x10\x00\x00\x00D\x00\x00\x00P\x00\x00\x00\x00\x00\x04\x00\x02\x00\x00\x00\xc8\x00\x00\x00\x00\x00\x00\x00\xd0\x00\x00\x00\x00\x00\x00\x00P\x00\x00\x00\x00\x00\x00\x00\xe8\x01\x00\x00\x00\x00\x00\x00\xd0\x00\x00\x00\x00\x00\x00\x00X\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\b\x00\b\x00\x00\x00\x04\x00\b\x00\x00\x00\x04\x00\x00\x00\x02\x00\x00\x00T\x00\x00\x00\x14\x00\x00\x00\x10\x00\x14\x00\x10\x00\x0f\x00\x0e\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x14\x00\x00\x00\x00\x00\x05\x01\x10\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00name\x00\x00\x00\x00\x10\x00\x14\x00\x10\x00\x00\x00\x0f\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x02\x1c\x00\x00\x00\x00\x00\x00\x00\b\x00\f\x00\b\x00\a\x00\b\x00\x00\x00\x00\x00\x00\x01@\x00\x00\x00\x02\x00\x00\x00id\x00\x00\x00\x01\x00\x00ARROW1") diff --git a/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/b254cb47a7ac7bd1 b/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/b254cb47a7ac7bd1 new file mode 100644 index 0000000000000..7e01c0db4db1c --- /dev/null +++ b/pkg/sql/colexec/external/arrowio/testdata/fuzz/FuzzArrowIPCPlanningAndOpenNeverPanicOrLeak/b254cb47a7ac7bd1 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\xff\xff\xff\xff\xb8\x00\x00\x00\x10\x00\x00\x00\x00\x00\n\x00\f\x00\n\x00\t\x00\x04\x00\n\x00\x00\x00\x10\x00\x00\x00\x00\x01\x04\x00\b\x00\b\x00\x00\x00\x04\x00\b\x00\x00\x00\x04\x00\x00\x00\x02\x00\x00T\x00\x00\x00\x14\x00\x00\x00\x10\x00\x14\x00\x10\x00\x0f\x00\x0e\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x14\x00\x00\x00\x00\x00\x05\x01\x10\x00\x00\x00\x00\x00\x00\x00\x04\x00\x04\x00\x04\x00\x00\x00\x04\x00\x00\x00name\x00\x00\x00\x00\x10\x00\x14\x00\x10\x00\x00\x00\x0f\x00\b\x00\x00\x00\x04\x00\x10\x00\x00\x00\x10\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x02\x1c\x00\x00\x00\x00\x00\x00\x00\b\x00\f\x00\b\x00\a\x00\b\x00\x00\x00\x00\x00\x00\x01@\x00\x00\x00\x02\x00\x00\x00id\x00\x00\xff\xff\xff\xff\xc8\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\f\x00\x16\x00\x14\x00\x13\x00\f\x00\x04\x00\f\x00\x00\x00P\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x03\x04\x00\n\x00\x18\x00\f\x00\b\x00\x04\x00\n\x00\x00\x00\x14\x00\x00\x00h\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\f\x00\x00\x00\x00\x00\x00\x00 \x00\x00\x00\x00\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00(\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00a payload longer than twenty three bytesshort\x00\x00\x00\xff\xff\xff\xff\xc8\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00\f\x00\x16\x00\x14\x00\x13\x00\f\x00\x04\x00\f\x00\x00\x00X\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x03\x04\x00\n\x00\x18\x00\f\x00\b\x00\x04\x00\n\x00\x00\x00\x14\x00\x00\x00h\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x18\x00\x00\x00\x00\x00\x00\x00\f\x00\x00\x00\x00\x00\x00\x00(\x00\x00\x00\x00\x00\x00\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00") diff --git a/pkg/sql/colexec/external/external.go b/pkg/sql/colexec/external/external.go index eeaff59da6cfa..c1d9e3f0274b0 100644 --- a/pkg/sql/colexec/external/external.go +++ b/pkg/sql/colexec/external/external.go @@ -40,6 +40,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/geo" "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/pb/pipeline" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/sql/crt" @@ -81,9 +82,15 @@ func (external *External) Prepare(proc *process.Process) error { } param := external.Es + if param == nil { + return moerr.NewInvalidInput(proc.Ctx, "external parameter is missing") + } if err := validateParquetWholeFileFanoutProtocol(proc, param); err != nil { return err } + if param.Fileparam == nil { + return moerr.NewInvalidInput(proc.Ctx, "external file parameter is missing") + } if proc.GetLim().MaxMsgSize == 0 { param.maxBatchSize = uint64(morpc.GetMessageSize()) } else { @@ -115,12 +122,36 @@ func (external *External) Prepare(proc *process.Process) error { param.Extern.FileService = proc.Base.FileService } } + if param.Extern.FileService == nil { + // Decoded remote parameters carry path/configuration but not the local + // FileService interface. Install the executing CN's service before the + // rollout gate so aliases are classified against that worker's backend. + param.Extern.FileService = proc.Base.FileService + } if param.ForeignScan != nil && param.ForeignScan.Kind == foreignScanKindESQL { param.ESQLTemporalUTC = true } if !loadFormatIsValid(param.Extern) { return moerr.NewNYIf(proc.Ctx, "load format '%s'", param.Extern.Format) } + if param.Extern.Format == tree.ARROW && + (param.Extern.ExternType != int32(plan.ExternType_LOAD) || + param.ArrowExecutionScope != pipeline.ArrowExecutionScope_ArrowLoadData) { + return moerr.NewNotSupported(proc.Ctx, "Arrow format is supported only by LOAD DATA") + } + if param.Extern.Format == tree.ARROW { + // A remote External is reconstructed on the executing CN. The compile + // gate on the coordinator is therefore not sufficient: each worker must + // enforce its own rollout configuration before it opens the source. + settings, err := plan2.RequireArrowLoadEnabled(proc, param.Extern) + if err != nil { + return err + } + if param.ArrowDistributedExecution && !settings.DistributedEnabled { + return moerr.NewNotSupported(proc.Ctx, + "distributed Arrow LOAD is disabled by configuration") + } + } if param.Extern.ExternType == int32(plan.ExternType_LOAD) && (param.Extern.Parallel || param.Extern.ParallelLoadRequested) { param.LoadEmptyNumericAsZero = true @@ -138,6 +169,12 @@ func (external *External) Prepare(proc *process.Process) error { } param.Ctx = proc.Ctx param.addParquetProfile(icebergParquetProfileStats(param)) + // Validate the physical output mapping before constructing a reader. A + // failed mapping must not leave a reader or batch behind for the caller to + // clean up after Prepare returns an error. + if err := validateExternalOutputAttrs(proc.Ctx, param.Attrs, param.Cols); err != nil { + return err + } // Filter public preprocessing if param.Filter == nil { @@ -183,6 +220,12 @@ func (external *External) Prepare(proc *process.Process) error { external.reader = NewZonemapReader(param, proc) case param.Extern.Format == tree.PARQUET: external.reader = NewParquetReader(param, proc) + case param.Extern.Format == tree.ARROW: + reader, err := NewArrowReader(param, proc, external.allocationAccount) + if err != nil { + return err + } + external.reader = reader default: r, err := NewCsvReader(param, proc) if err != nil { @@ -211,9 +254,12 @@ func (external *External) Prepare(proc *process.Process) error { if param.Extern.Format == tree.PARQUET { flag = false } - //alloc space for vector - for i := range param.Attrs { - typ := makeType(¶m.Cols[i].Typ, flag) + // Allocate output vectors in Attrs order, but resolve their physical + // types through ColIndex. Generated or hidden columns can be omitted + // from Attrs, so output position and table-column position differ. + for i, attr := range param.Attrs { + colIndex := int(attr.ColIndex) + typ := makeType(¶m.Cols[colIndex].Typ, flag) external.ctr.buf.Vecs[i] = vector.NewOffHeapVecWithType(typ) } } @@ -247,6 +293,16 @@ func validateParquetWholeFileFanoutProtocol(proc *process.Process, param *Extern return nil } +func validateExternalOutputAttrs(ctx context.Context, attrs []plan.ExternAttr, cols []*plan.ColDef) error { + for _, attr := range attrs { + colIndex := int(attr.ColIndex) + if colIndex < 0 || colIndex >= len(cols) || cols[colIndex] == nil { + return moerr.NewInvalidInputf(ctx, "external output column index %d is invalid", attr.ColIndex) + } + } + return nil +} + func (external *External) checkLoadLockTableBinds(proc *process.Process) error { param := external.Es if param == nil || @@ -331,6 +387,9 @@ func (external *External) Call(proc *process.Process) (vm.CallResult, error) { external.reader.Close() external.fileOpened = false param.Fileparam.End = true + if external.ctr.buf != nil { + external.ctr.buf.CleanOnlyData() + } return result, err } if external.ctr.buf != nil && external.ctr.buf.RowCount() > 0 { @@ -343,7 +402,14 @@ func (external *External) Call(proc *process.Process) (vm.CallResult, error) { } if fileFinished { - external.reader.Close() + if err := external.reader.Close(); err != nil { + external.fileOpened = false + param.Fileparam.End = true + if external.ctr.buf != nil { + external.ctr.buf.CleanOnlyData() + } + return result, err + } external.finishCurrentFile(param) } @@ -2051,7 +2117,7 @@ func parseLoadDataYear(field csvparser.Field) (types.MoYear, error) { func loadFormatIsValid(param *tree.ExternParam) bool { switch param.Format { - case tree.JSONLINE, tree.CSV, tree.PARQUET: + case tree.JSONLINE, tree.CSV, tree.PARQUET, tree.ARROW: return true } return false diff --git a/pkg/sql/colexec/external/external_test.go b/pkg/sql/colexec/external/external_test.go index a2396ce4a10db..54d7f1474cae3 100644 --- a/pkg/sql/colexec/external/external_test.go +++ b/pkg/sql/colexec/external/external_test.go @@ -19,6 +19,7 @@ import ( "compress/zlib" "context" "encoding/json" + "errors" "io" "os" "path/filepath" @@ -407,6 +408,135 @@ func TestPrepareSetsLoadEmptyNumericAsZeroForParallelRequestedLoad(t *testing.T) } } +func TestPrepareAllocatesArrowVectorsByPhysicalColumnIndex(t *testing.T) { + proc := newArrowLoadTestProc(t) + defer proc.Free() + + fs, err := fileservice.NewMemoryFS("arrow-prepare", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + param := externalArrowParam(fs, "arrow-prepare:unused.arrow", 0, tree.ARROW_CONTAINER_FILE) + param.Attrs = []plan.ExternAttr{ + {ColName: "payload", ColIndex: 0}, + {ColName: "id", ColIndex: 2}, + } + param.Cols = []*plan.ColDef{ + {Name: "payload", Typ: plan.Type{Id: int32(types.T_varchar), Width: 100}}, + {Name: "generated", Typ: plan.Type{Id: int32(types.T_varchar), Width: 100}}, + {Name: "id", Typ: plan.Type{Id: int32(types.T_int64)}}, + } + + registry, err := mpool.NewAllocationAccountRegistry(1, 128) + require.NoError(t, err) + account, err := registry.Open(64 << 20) + require.NoError(t, err) + arg := NewArgument().WithEs(param) + require.NoError(t, arg.SetAllocationAccount(account)) + require.NoError(t, arg.Prepare(proc)) + + // Attrs is the physical-column projection. The second output slot maps to + // Cols[2], even though Cols[1] is a generated column omitted from Attrs. + require.Equal(t, types.T_int64, arg.ctr.buf.Vecs[1].GetType().Oid) + + arg.Free(proc, false, nil) + require.NoError(t, arg.ClearAllocationAccount(account)) + arg.Release() + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) +} + +type closeErrorExternalReader struct{ err error } + +func (r *closeErrorExternalReader) Open(*ExternalParam, *process.Process) (bool, error) { + return false, nil +} + +func (r *closeErrorExternalReader) ReadBatch(context.Context, *batch.Batch, *process.Process, process.Analyzer) (bool, error) { + return true, nil +} + +func (r *closeErrorExternalReader) Close() error { return r.err } + +func TestExternalCallPropagatesTerminalReaderCloseError(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + param := &ExternalParam{ + ExParamConst: ExParamConst{Extern: &tree.ExternParam{ExParamConst: tree.ExParamConst{ScanType: tree.INLINE}}}, + ExParam: ExParam{Fileparam: &ExFileparam{}}, + } + external := &External{Es: param, reader: &closeErrorExternalReader{err: errors.New("deferred stream close failed")}, fileOpened: true} + external.OpAnalyzer = process.NewAnalyzer(0, true, true, "external-close-test") + + _, err := external.Call(proc) + require.ErrorContains(t, err, "deferred stream close failed") + require.False(t, external.fileOpened) + require.True(t, param.Fileparam.End) +} + +func TestExternalPrepareRejectsMissingFileParam(t *testing.T) { + proc := testutil.NewProcess(t) + defer proc.Free() + + param := &ExternalParam{ + ExParamConst: ExParamConst{ + FileList: []string{"unused.csv"}, + Extern: &tree.ExternParam{ + ExParamConst: tree.ExParamConst{ + Format: tree.CSV, + Tail: &tree.TailParameter{}, + }, + ExParam: tree.ExParam{ExternType: int32(plan.ExternType_LOAD)}, + }, + }, + } + arg := &External{Es: param} + + var err error + require.NotPanics(t, func() { + err = arg.Prepare(proc) + }) + require.Error(t, err) + require.ErrorContains(t, err, "external file parameter is missing") +} + +func TestExternalPrepareRejectsInvalidAttrIndexWithoutPartialState(t *testing.T) { + proc := newArrowLoadTestProc(t) + defer proc.Free() + + fs, err := fileservice.NewMemoryFS("arrow-prepare-invalid-attr", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + param := externalArrowParam(fs, "arrow-prepare-invalid-attr:unused.arrow", 0, tree.ARROW_CONTAINER_FILE) + param.Attrs = []plan.ExternAttr{ + {ColName: "id", ColIndex: 0}, + {ColName: "missing", ColIndex: 2}, + } + param.Cols = []*plan.ColDef{ + {Name: "id", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "name", Typ: plan.Type{Id: int32(types.T_varchar), Width: 100}}, + } + + registry, err := mpool.NewAllocationAccountRegistry(1, 128) + require.NoError(t, err) + account, err := registry.Open(64 << 20) + require.NoError(t, err) + arg := NewArgument().WithEs(param) + defer func() { + arg.Free(proc, true, err) + require.NoError(t, arg.ClearAllocationAccount(account)) + arg.Release() + account.Seal() + _, finalizeErr := registry.Finalize(account) + require.NoError(t, finalizeErr) + }() + require.NoError(t, arg.SetAllocationAccount(account)) + + err = arg.Prepare(proc) + require.Error(t, err) + require.ErrorContains(t, err, "external output column index 2 is invalid") + require.Nil(t, arg.reader) + require.Nil(t, arg.ctr.buf) +} + func TestGetColDataParallelLoadNullNumericRemainsNull(t *testing.T) { proc := testutil.NewProcess(t) defer proc.Free() diff --git a/pkg/sql/colexec/external/reader_arrow.go b/pkg/sql/colexec/external/reader_arrow.go new file mode 100644 index 0000000000000..4ca8fcc04f4b6 --- /dev/null +++ b/pkg/sql/colexec/external/reader_arrow.go @@ -0,0 +1,743 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package external + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "sync" + "sync/atomic" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/arrowbridge" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/pb/pipeline" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/external/arrowio" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" + plan2 "github.com/matrixorigin/matrixone/pkg/sql/plan" + metric "github.com/matrixorigin/matrixone/pkg/util/metric/v2" + "github.com/matrixorigin/matrixone/pkg/vm/process" +) + +const ( + arrowRangeAllocationSite mpool.AllocationSite = 1 + arrowVectorDataSite mpool.AllocationSite = 3 + arrowVectorAreaSite mpool.AllocationSite = 4 + arrowVectorNullsSite mpool.AllocationSite = 5 + arrowVectorGroupingSite mpool.AllocationSite = 6 + arrowMaxOutputRows = 100_000 +) + +var arrowPinnedMetricState struct { + sync.Mutex + current int64 + peak int64 +} + +// These wrappers observe the same capacity reservation that owns the backing; +// they do not introduce a second quota or release path. Only a successful +// Commit changes the gauge, and the returned lease's idempotent Release removes +// that exact capacity after the FileService/Arrow backing is no longer live. +type meteredArrowRangeAdmission struct { + inner fileservice.RangeReadAdmission +} + +type meteredArrowCapacityReservation struct { + inner fileservice.CapacityReservation +} + +type meteredArrowCapacityLease struct { + inner fileservice.CapacityLease + capacity int64 + released atomic.Bool +} + +func (a meteredArrowRangeAdmission) Reserve( + ctx context.Context, + upperBound int64, +) (fileservice.CapacityReservation, error) { + reservation, err := a.inner.Reserve(ctx, upperBound) + if err != nil { + return nil, err + } + return meteredArrowCapacityReservation{inner: reservation}, nil +} + +func (r meteredArrowCapacityReservation) Commit( + actualCapacity int64, +) (fileservice.CapacityLease, error) { + lease, err := r.inner.Commit(actualCapacity) + if err != nil { + return nil, err + } + adjustArrowPinnedBytes(actualCapacity) + return &meteredArrowCapacityLease{inner: lease, capacity: actualCapacity}, nil +} + +func (r meteredArrowCapacityReservation) Abort() { + r.inner.Abort() +} + +func (l *meteredArrowCapacityLease) Release() { + if l == nil || !l.released.CompareAndSwap(false, true) { + return + } + defer adjustArrowPinnedBytes(-l.capacity) + l.inner.Release() +} + +func adjustArrowPinnedBytes(delta int64) { + arrowPinnedMetricState.Lock() + defer arrowPinnedMetricState.Unlock() + arrowPinnedMetricState.current += delta + // Capacity leases are idempotent, so underflow would indicate a metric-only + // bookkeeping defect. Keep the exported gauge valid while lifecycle tests + // assert that every reader returns to its starting value. + if arrowPinnedMetricState.current < 0 { + arrowPinnedMetricState.current = 0 + } + metric.ArrowLoadPinnedBytesGauge.Set(float64(arrowPinnedMetricState.current)) + if arrowPinnedMetricState.current > arrowPinnedMetricState.peak { + arrowPinnedMetricState.peak = arrowPinnedMetricState.current + metric.ArrowLoadPinnedBytesHighWaterGauge.Set(float64(arrowPinnedMetricState.peak)) + } +} + +func observeArrowPhase(start time.Time, phase string, err error) { + outcome := "success" + if err != nil { + outcome = "error" + } + metric.ArrowLoadPhaseDurationHistogram.WithLabelValues(phase, outcome).Observe(time.Since(start).Seconds()) +} + +func observeArrowConvertStats(stats arrowbridge.ConvertStats) { + metric.ArrowLoadPayloadBytesCounter.WithLabelValues("eligible").Add(float64(stats.EligiblePayloadBytes)) + metric.ArrowLoadPayloadBytesCounter.WithLabelValues("borrowed").Add(float64(stats.BorrowedPayloadBytes)) + metric.ArrowLoadPayloadBytesCounter.WithLabelValues("retained_capacity").Add(float64(stats.RetainedCapacityBytes)) + metric.ArrowLoadCopyBytesCounter.WithLabelValues("arrow_to_mo").Add(float64(stats.MaterializedPayloadBytes)) + metric.ArrowLoadConversionColumnCounter.WithLabelValues("borrowed").Add(float64(stats.BorrowedColumns)) + metric.ArrowLoadConversionColumnCounter.WithLabelValues("materialized").Add(float64(stats.MaterializedColumns)) + metric.ArrowLoadFallbackCounter.WithLabelValues("pin_amplification").Add(float64(stats.PinAmplificationFallbacks)) + metric.ArrowLoadFallbackCounter.WithLabelValues("unaligned").Add(float64(stats.UnalignedFallbacks)) +} + +func arrowLoadErrorCategory(err error) string { + var moError *moerr.Error + moCode := uint16(0) + if errors.As(err, &moError) { + moCode = moError.ErrorCode() + } + switch { + case errors.Is(err, context.Canceled) || moCode == moerr.ErrQueryInterrupted: + return "canceled" + case errors.Is(err, context.DeadlineExceeded) || moCode == moerr.ErrQueryTimeout: + return "deadline_exceeded" + case errors.Is(err, fileservice.ErrObjectChanged): + return "object_changed" + case errors.Is(err, mpool.ErrAllocationAccountCapacity) || + moCode == moerr.ErrOOM || moCode == moerr.ErrMPoolCapacity: + return "resource_exhausted" + case moCode == moerr.ErrNotSupported || moCode == moerr.ErrNYI: + return "not_supported" + case moCode == moerr.ErrConstraintViolation: + return "constraint_violation" + case moCode == moerr.ErrInvalidInput || moCode == moerr.ErrOutOfRange: + return "invalid_input" + case moCode == moerr.ErrInternal: + return "internal" + default: + return "io" + } +} + +func observeArrowError(err error) { + if err != nil { + metric.ArrowLoadErrorCounter.WithLabelValues(arrowLoadErrorCategory(err)).Inc() + } +} + +// ArrowReader is the LOAD-only adapter between the format-neutral External +// operator and the canonical Arrow-to-MO bridge. The IPC reader owns the +// current record; borrowed MO vectors retain only the ArrayData/range leases +// they reference and therefore safely outlive the next IPC Next call. +type ArrowReader struct { + param *ExternalParam + reader arrowio.Reader + plan *arrowbridge.Plan + pending arrow.RecordBatch + rowOffset int64 + admission fileservice.RangeReadAdmission + allocation *vector.AllocationAccountSelection + forceMaterialize bool + conversionFingerprint [sha256.Size]byte + fingerprintSet bool +} + +func NewArrowReader( + param *ExternalParam, + _ *process.Process, + account *mpool.AllocationAccount, +) (*ArrowReader, error) { + if param == nil || param.Extern == nil || account == nil { + return nil, moerr.NewInvalidInputNoCtx("Arrow LOAD requires a statement allocation account") + } + if param.ArrowConversionPlanVersion != arrowbridge.ConversionPlanVersion { + return nil, moerr.NewInvalidInputNoCtxf( + "unsupported Arrow conversion plan version %d", param.ArrowConversionPlanVersion, + ) + } + if len(param.ArrowSchemaFingerprint) != 0 && len(param.ArrowSchemaFingerprint) != sha256.Size { + return nil, moerr.NewInvalidInputNoCtxf( + "invalid Arrow schema fingerprint length %d", len(param.ArrowSchemaFingerprint), + ) + } + admission, err := fileservice.NewAllocationAccountRangeAdmission( + account, + mpool.AllocationOwnerExternal, + arrowRangeAllocationSite, + mpool.AllocationCapacityClassDefault, + ) + if err != nil { + return nil, err + } + allocation, err := vector.NewAllocationAccountSelection( + account, + mpool.AllocationOwnerExternal, + arrowVectorDataSite, + arrowVectorAreaSite, + arrowVectorNullsSite, + arrowVectorGroupingSite, + ) + if err != nil { + return nil, err + } + return &ArrowReader{ + param: param, admission: meteredArrowRangeAdmission{inner: admission}, allocation: allocation, + forceMaterialize: param.ArrowForceMaterialize, + }, nil +} + +func (r *ArrowReader) Open(param *ExternalParam, proc *process.Process) (_ bool, retErr error) { + startTime := time.Now() + defer func() { + observeArrowPhase(startTime, "open", retErr) + outcome := "success" + if retErr != nil { + outcome = "error" + observeArrowError(retErr) + } + metric.ArrowLoadObjectCounter.WithLabelValues(outcome).Inc() + }() + if r == nil || param == nil || param.Extern == nil || param.Fileparam == nil || proc == nil { + if param != nil && param.Fileparam == nil { + return false, moerr.NewInvalidInputNoCtx("Arrow reader file parameter is missing") + } + return false, moerr.NewInvalidInputNoCtx("invalid Arrow reader open") + } + if err := r.Close(); err != nil { + return false, err + } + r.param = param + fileIndex := param.Fileparam.FileIndex - 1 + if fileIndex < 0 || fileIndex >= len(param.FileSize) { + return false, moerr.NewInvalidInputf(proc.Ctx, "Arrow file size is missing for file index %d", fileIndex) + } + size := param.FileSize[fileIndex] + if size < 0 { + return false, moerr.NewInvalidInputf(proc.Ctx, "Arrow file size %d is invalid", size) + } + fs, readPath, err := plan2.GetForETLWithType(param.Extern, param.Fileparam.Filepath) + if err != nil { + return false, err + } + container, err := arrowContainer(param.Extern.ArrowContainer) + if err != nil { + return false, err + } + identity, err := arrowObjectIdentity(proc.Ctx, param, fileIndex, fs, readPath, size) + if err != nil { + return false, err + } + fileShard, err := arrowFileShard(proc.Ctx, param, fileIndex, container) + if err != nil { + return false, err + } + reader, err := arrowio.Open( + proc.Ctx, fs, readPath, size, container, r.admission, + arrowio.Options{ExpectedIdentity: identity, FileShard: fileShard}, + ) + if err != nil { + return false, err + } + r.reader = reader + + targets, err := BuildArrowTargets(proc.Ctx, param.Attrs, param.Cols) + if err != nil { + r.Close() + return false, err + } + mode := arrowbridge.MatchByName + if param.Extern.ArrowMatchByPosition { + mode = arrowbridge.MatchByPosition + } + // LOAD uses a deliberately more permissive type policy than exact result + // protocols such as Python UDF. Keep the policy explicit at this boundary. + r.plan, err = arrowbridge.BindLoad(proc.Ctx, reader.Schema(), targets, mode) + if err != nil { + r.Close() + return false, err + } + fileFingerprint := r.plan.Fingerprint() + if len(param.ArrowSchemaFingerprint) != 0 && + !bytes.Equal(param.ArrowSchemaFingerprint, fileFingerprint[:]) { + r.Close() + return false, moerr.NewInvalidInputf(proc.Ctx, + "Arrow schema and conversion contract for file index %d does not match the planned contract", fileIndex) + } + if r.fingerprintSet && r.conversionFingerprint != fileFingerprint { + r.Close() + return false, moerr.NewInvalidInputf(proc.Ctx, + "Arrow schema and conversion contract for file index %d differs from earlier files", fileIndex) + } + r.conversionFingerprint = fileFingerprint + r.fingerprintSet = true + finished, err := r.advanceToNextNonEmptyRecord() + if err != nil { + r.Close() + return false, err + } + if finished { + if err = r.Close(); err != nil { + return false, err + } + } + if fileShard != nil { + metric.ArrowLoadShardCounter.Inc() + } + return finished, nil +} + +// advanceToNextNonEmptyRecord hides legal zero-row IPC record batches from +// External's non-empty output contract. Calling Next also releases the IPC +// reader's previous record; any published borrowed MO vectors hold their own +// ArrayData references and remain valid. +func (r *ArrowReader) advanceToNextNonEmptyRecord() (bool, error) { + startTime := time.Now() + var retErr error + defer func() { observeArrowPhase(startTime, "next_record", retErr) }() + if r == nil || r.reader == nil { + retErr = moerr.NewInternalErrorNoCtx("Arrow reader is not open") + return false, retErr + } + for r.reader.Next() { + record := r.reader.RecordBatch() + if record == nil { + retErr = moerr.NewInvalidInputNoCtx("Arrow reader returned a nil record batch") + return false, retErr + } + if record.NumRows() == 0 { + continue + } + r.pending = record + r.rowOffset = 0 + metric.ArrowLoadRecordCounter.Inc() + return false, nil + } + if err := r.reader.Err(); err != nil { + retErr = err + return false, retErr + } + r.pending = nil + r.rowOffset = 0 + return true, nil +} + +func arrowFileShard( + ctx context.Context, + param *ExternalParam, + fileIndex int, + container arrowio.Container, +) (*arrowio.FileShard, error) { + var planned *pipeline.ArrowRecordBatchShard + for _, shard := range param.ArrowRecordBatchShards { + if shard == nil || int(shard.FileIndex) != fileIndex { + continue + } + if planned != nil { + return nil, moerr.NewInvalidInputf(ctx, + "multiple Arrow record-batch shards target file index %d in one reader", fileIndex) + } + planned = shard + } + if planned == nil { + return nil, nil + } + if container == arrowio.ContainerStream { + return nil, moerr.NewInvalidInput(ctx, "Arrow IPC Stream cannot use record-batch shards") + } + if planned.RecordBatchStart < 0 || planned.RecordBatchStart >= planned.RecordBatchEnd || + planned.EstimatedRows < 0 || planned.EstimatedWireBytes < 0 { + return nil, moerr.NewInvalidInputf(ctx, + "invalid Arrow record-batch shard [%d,%d)", planned.RecordBatchStart, planned.RecordBatchEnd) + } + return &arrowio.FileShard{ + RecordBatchStart: planned.RecordBatchStart, + RecordBatchEnd: planned.RecordBatchEnd, + RequiredDictionaryBlockIndices: append( + []int32(nil), planned.RequiredDictionaryBlockIndices..., + ), + }, nil +} + +func arrowObjectIdentity( + ctx context.Context, + param *ExternalParam, + fileIndex int, + fs fileservice.FileService, + readPath string, + size int64, +) (*fileservice.ObjectIdentity, error) { + var planned *pipeline.ArrowObjectIdentity + for _, identity := range param.ArrowObjectIdentities { + if identity == nil || int(identity.FileIndex) != fileIndex { + continue + } + if planned != nil { + return nil, moerr.NewInvalidInputf(ctx, "duplicate Arrow object identity for file index %d", fileIndex) + } + planned = identity + } + if planned != nil { + identity := &fileservice.ObjectIdentity{ + VersionID: planned.VersionId, + ETag: planned.Etag, + Size: planned.Size, + } + if planned.LastModifiedUnixNano != 0 { + identity.LastModified = time.Unix(0, planned.LastModifiedUnixNano).UTC() + } + if err := identity.Validate(); err != nil { + return nil, err + } + if identity.Size != size { + return nil, errors.Join(fileservice.ErrObjectChanged, + moerr.NewInternalErrorNoCtxf("Arrow object size changed from %d to %d", identity.Size, size)) + } + return identity, nil + } + identityFS, ok := fs.(fileservice.ObjectIdentityFileService) + if !ok { + return nil, nil + } + identity, err := identityFS.StatFileIdentity(ctx, readPath) + if err != nil { + return nil, err + } + if err := identity.Validate(); err != nil { + return nil, err + } + if identity.Size != size { + return nil, errors.Join(fileservice.ErrObjectChanged, + moerr.NewInternalErrorNoCtxf("Arrow object size changed from %d to %d", size, identity.Size)) + } + return &identity, nil +} + +func (r *ArrowReader) ReadBatch( + ctx context.Context, + buf *batch.Batch, + proc *process.Process, + _ process.Analyzer, +) (_ bool, retErr error) { + defer func() { observeArrowError(retErr) }() + if r == nil || r.reader == nil || r.plan == nil || r.pending == nil || buf == nil || proc == nil { + return false, moerr.NewInvalidInput(ctx, "Arrow reader is not open") + } + if err := ctx.Err(); err != nil { + return false, err + } + record := r.pending + start := r.rowOffset + if start == 0 { + // A RecordBatch is immutable. Validate its complete shape and validity + // exactly once before output-window budgeting; later windows only pay + // their own conversion validation. + if err := r.plan.ValidateRecord(ctx, record); err != nil { + return false, err + } + } + maxRows := min(arrowMaxOutputRows, int(record.NumRows()-start)) + rows, err := r.plan.MaxOutputRows(ctx, record, start, maxRows, r.param.maxBatchSize) + if err != nil { + return false, err + } + end := start + int64(rows) + if start < 0 || start >= end { + return false, moerr.NewInvalidInput(ctx, "Arrow record batch has an invalid row window") + } + view := record + if start != 0 || end != record.NumRows() { + view = record.NewSlice(start, end) + defer view.Release() + } + + location := time.UTC + if proc.GetSessionInfo() != nil && proc.GetSessionInfo().TimeZone != nil { + location = proc.GetSessionInfo().TimeZone + } + convertStart := time.Now() + converted, stats, err := r.plan.ConvertValidatedRecordWindow(ctx, view, proc.Mp(), arrowbridge.ConvertOptions{ + Location: location, Allocation: r.allocation, ForceMaterialize: r.forceMaterialize, + }) + observeArrowPhase(convertStart, "convert", err) + if err != nil { + return false, err + } + observeArrowConvertStats(stats) + wireBudgetStart := time.Now() + converted, actualRows, err := fitArrowBatchToWireBudget(ctx, converted, r.param.maxBatchSize, proc.Mp()) + observeArrowPhase(wireBudgetStart, "wire_budget", err) + if err != nil { + converted.Clean(proc.Mp()) + return false, err + } + if actualRows < rows { + rows = actualRows + end = start + int64(rows) + } + fileFinished := false + if end < record.NumRows() { + r.rowOffset = end + } else { + fileFinished, err = r.advanceToNextNonEmptyRecord() + if err != nil { + converted.Clean(proc.Mp()) + return false, err + } + } + publishStart := time.Now() + if err = replaceArrowBatch(buf, converted, proc.Mp()); err != nil { + observeArrowPhase(publishStart, "publish", err) + converted.Clean(proc.Mp()) + return false, err + } + observeArrowPhase(publishStart, "publish", nil) + metric.ArrowLoadBatchCounter.Inc() + metric.ArrowLoadRowCounter.Add(float64(rows)) + return fileFinished, nil +} + +func fitArrowBatchToWireBudget( + ctx context.Context, + converted *batch.Batch, + maxBytes uint64, + mp *mpool.MPool, +) (*batch.Batch, int, error) { + if converted == nil || converted.RowCount() <= 0 { + return converted, 0, moerr.NewInvalidInput(ctx, "invalid converted Arrow batch") + } + if maxBytes == 0 { + return converted, converted.RowCount(), nil + } + size, err := converted.MarshalBinarySize() + if err != nil { + return converted, 0, err + } + if uint64(size) <= maxBytes { + return converted, converted.RowCount(), nil + } + low, high := 1, converted.RowCount()-1 + best := 0 + for low <= high { + if err := ctx.Err(); err != nil { + return converted, 0, err + } + middle := low + (high-low)/2 + window, err := converted.Window(0, middle) + if err != nil { + return converted, 0, err + } + windowSize, sizeErr := window.MarshalBinarySize() + window.Clean(mp) + if sizeErr != nil { + return converted, 0, sizeErr + } + if uint64(windowSize) <= maxBytes { + best = middle + low = middle + 1 + } else { + high = middle - 1 + } + } + if best == 0 { + one, err := converted.Window(0, 1) + if err != nil { + return converted, 0, err + } + oneSize, sizeErr := one.MarshalBinarySize() + one.Clean(mp) + if sizeErr != nil { + return converted, 0, sizeErr + } + return converted, 0, moerr.NewConstraintViolationf( + ctx, "Arrow row canonical wire size %d exceeds batch limit %d", oneSize, maxBytes, + ) + } + stable, err := stableArrowBatchPrefix(converted, best, mp) + if err != nil { + return converted, 0, err + } + converted.Clean(mp) + return stable, best, nil +} + +// stableArrowBatchPrefix keeps the zero-copy contract column-local while +// making a prefix independent of the source batch lifetime. Borrowed columns +// retain their leased payloads; materialized columns copy only selected rows. +func stableArrowBatchPrefix(source *batch.Batch, rows int, mp *mpool.MPool) (_ *batch.Batch, err error) { + if source == nil || mp == nil || rows <= 0 || rows > source.RowCount() { + return nil, moerr.NewInvalidInputNoCtx("invalid Arrow batch prefix") + } + stable := batch.NewOffHeap(append([]string(nil), source.Attrs...)) + stable.Recursive = source.Recursive + stable.ShuffleIDX = source.ShuffleIDX + defer func() { + if err != nil { + stable.Clean(mp) + } + }() + for index, sourceVector := range source.Vecs { + if sourceVector == nil { + return nil, moerr.NewInvalidInputNoCtxf("Arrow source batch column %d is nil", index) + } + var snapshot *vector.Vector + if sourceVector.HasBorrowedBacking() { + snapshot, err = sourceVector.RetainedReadonlyWindowWithMP(0, rows, mp) + } else { + var window *vector.Vector + selection := sourceVector.AllocationAccountSelection() + if selection == nil { + window, err = sourceVector.WindowByLogicalRows(0, rows) + } else { + window, err = sourceVector.WindowByLogicalRowsWithAllocation(0, rows, mp, selection) + } + if err == nil { + snapshot, err = window.Dup(mp) + window.Free(mp) + } + } + if err != nil { + return nil, err + } + stable.SetVector(int32(index), snapshot) + } + stable.SetRowCount(rows) + return stable, nil +} + +func (r *ArrowReader) Close() error { + if r == nil { + return nil + } + r.pending = nil + r.rowOffset = 0 + r.plan = nil + if r.reader == nil { + return nil + } + reader := r.reader + r.reader = nil + return reader.Close() +} + +func arrowContainer(value string) (arrowio.Container, error) { + switch value { + case "", tree.ARROW_CONTAINER_AUTO: + return arrowio.ContainerAuto, nil + case tree.ARROW_CONTAINER_FILE: + return arrowio.ContainerFile, nil + case tree.ARROW_CONTAINER_STREAM: + return arrowio.ContainerStream, nil + default: + return 0, moerr.NewBadConfigNoCtxf("the arrow_container '%s' is not supported", value) + } +} + +// BuildArrowTargets builds the exact table-side conversion contract shared by +// compile-time fingerprinting and execution-time binding. The returned slice +// follows external source-field order because positional Arrow binding consumes +// targets in that order. MOIndex preserves the table/output position so an +// explicit LOAD column list such as (b, a) still writes the converted vectors to +// their physical destinations. +func BuildArrowTargets( + ctx context.Context, + attrs []plan2.ExternAttr, + cols []*plan2.ColDef, +) ([]arrowbridge.TargetColumn, error) { + targets := make([]arrowbridge.TargetColumn, len(attrs)) + seenSourceFields := make([]bool, len(attrs)) + for outputIndex, attr := range attrs { + colIndex := int(attr.ColIndex) + if colIndex < 0 || colIndex >= len(cols) || cols[colIndex] == nil { + return nil, moerr.NewInvalidInputf(ctx, "Arrow target column index %d is invalid", colIndex) + } + sourceIndex := int(attr.ColFieldIndex) + if sourceIndex < 0 || sourceIndex >= len(attrs) { + return nil, moerr.NewInvalidInputf(ctx, "Arrow source field index %d is invalid", sourceIndex) + } + if seenSourceFields[sourceIndex] { + return nil, moerr.NewInvalidInputf(ctx, "Arrow source field index %d is duplicated", sourceIndex) + } + seenSourceFields[sourceIndex] = true + col := cols[colIndex] + targets[sourceIndex] = arrowbridge.TargetColumn{ + Name: col.Name, + Type: makeType(&col.Typ, false), + NotNull: col.Typ.NotNullable, + MOIndex: outputIndex, + AttrName: attr.ColName, + } + } + return targets, nil +} + +func replaceArrowBatch(dst, src *batch.Batch, mp *mpool.MPool) error { + if dst == nil || src == nil || len(dst.Vecs) != len(src.Vecs) { + return moerr.NewInternalErrorNoCtx("Arrow conversion produced an incompatible MatrixOne batch") + } + for i := range src.Vecs { + if src.Vecs[i] == nil { + return moerr.NewInternalErrorNoCtxf("Arrow conversion produced a nil vector at column %d", i) + } + if dst.Vecs[i] != nil { + dst.Vecs[i].Free(mp) + } + dst.SetVector(int32(i), src.Vecs[i]) + src.SetVector(int32(i), nil) + } + dst.SetRowCount(src.RowCount()) + src.Clean(mp) + return nil +} + +var _ ExternalFileReader = (*ArrowReader)(nil) diff --git a/pkg/sql/colexec/external/reader_arrow_test.go b/pkg/sql/colexec/external/reader_arrow_test.go new file mode 100644 index 0000000000000..6c0b8add35f5f --- /dev/null +++ b/pkg/sql/colexec/external/reader_arrow_test.go @@ -0,0 +1,994 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package external + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + "unsafe" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/config" + "github.com/matrixorigin/matrixone/pkg/container/arrowbridge" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/pb/pipeline" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" + "github.com/matrixorigin/matrixone/pkg/testutil" + metric "github.com/matrixorigin/matrixone/pkg/util/metric/v2" + "github.com/matrixorigin/matrixone/pkg/vm" + "github.com/matrixorigin/matrixone/pkg/vm/process" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + promtestutil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func TestExternalArrowLoadFileAndStream(t *testing.T) { + for _, container := range []string{tree.ARROW_CONTAINER_FILE, tree.ARROW_CONTAINER_STREAM} { + t.Run(container, func(t *testing.T) { + objectsBefore := promtestutil.ToFloat64(metric.ArrowLoadObjectCounter.WithLabelValues("success")) + recordsBefore := promtestutil.ToFloat64(metric.ArrowLoadRecordCounter) + batchesBefore := promtestutil.ToFloat64(metric.ArrowLoadBatchCounter) + rowsBefore := promtestutil.ToFloat64(metric.ArrowLoadRowCounter) + eligibleBefore := promtestutil.ToFloat64(metric.ArrowLoadPayloadBytesCounter.WithLabelValues("eligible")) + borrowedBefore := promtestutil.ToFloat64(metric.ArrowLoadPayloadBytesCounter.WithLabelValues("borrowed")) + copyBefore := promtestutil.ToFloat64(metric.ArrowLoadCopyBytesCounter.WithLabelValues("arrow_to_mo")) + pinnedBefore := promtestutil.ToFloat64(metric.ArrowLoadPinnedBytesGauge) + fileBytes := makeExternalArrowIPC(t, container) + fs, err := fileservice.NewMemoryFS("etl", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + path := "etl:load-" + container + ".arrow" + require.NoError(t, fs.Write(context.Background(), fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{Offset: 0, Size: int64(len(fileBytes)), Data: fileBytes}}, + })) + + registry, err := mpool.NewAllocationAccountRegistry(1, 128) + require.NoError(t, err) + account, err := registry.Open(64 << 20) + require.NoError(t, err) + proc := newArrowLoadTestProc(t) + proc.Base.SessionInfo.TimeZone = nil // reader must use its UTC fallback + + arg := NewArgument().WithEs(externalArrowParam(fs, path, int64(len(fileBytes)), container)) + require.NoError(t, arg.SetAllocationAccount(account)) + require.True(t, arg.ActivatesAllocationAccountLifecycle()) + require.NoError(t, arg.Prepare(proc)) + + result, err := arg.Call(proc) + require.NoError(t, err) + require.Equal(t, vm.ExecNext, result.Status) + require.Equal(t, 2, result.Batch.RowCount()) + require.Equal(t, []int64{1, 2}, vector.MustFixedColNoTypeCheck[int64](result.Batch.Vecs[0])) + require.Equal(t, "a payload longer than twenty three bytes", result.Batch.Vecs[1].GetStringAt(0)) + require.True(t, result.Batch.Vecs[0].HasBorrowedBacking()) + require.True(t, result.Batch.Vecs[1].HasBorrowedBacking()) + require.Greater(t, account.Snapshot().Used, uint64(0)) + + result, err = arg.Call(proc) + require.NoError(t, err) + require.Equal(t, 2, result.Batch.RowCount()) + require.Equal(t, []int64{3, 4}, vector.MustFixedColNoTypeCheck[int64](result.Batch.Vecs[0])) + + result, err = arg.Call(proc) + require.NoError(t, err) + require.Equal(t, vm.ExecStop, result.Status) + arg.Free(proc, false, nil) + require.Zero(t, account.Snapshot().Used) + require.NoError(t, arg.ClearAllocationAccount(account)) + arg.Release() + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) + require.Equal(t, objectsBefore+1, promtestutil.ToFloat64(metric.ArrowLoadObjectCounter.WithLabelValues("success"))) + require.Equal(t, recordsBefore+2, promtestutil.ToFloat64(metric.ArrowLoadRecordCounter)) + require.Equal(t, batchesBefore+2, promtestutil.ToFloat64(metric.ArrowLoadBatchCounter)) + require.Equal(t, rowsBefore+4, promtestutil.ToFloat64(metric.ArrowLoadRowCounter)) + require.Greater(t, promtestutil.ToFloat64(metric.ArrowLoadPayloadBytesCounter.WithLabelValues("eligible")), eligibleBefore) + require.Greater(t, promtestutil.ToFloat64(metric.ArrowLoadPayloadBytesCounter.WithLabelValues("borrowed")), borrowedBefore) + require.Greater(t, promtestutil.ToFloat64(metric.ArrowLoadCopyBytesCounter.WithLabelValues("arrow_to_mo")), copyBefore) + require.Equal(t, pinnedBefore, promtestutil.ToFloat64(metric.ArrowLoadPinnedBytesGauge)) + }) + } +} + +func TestExternalArrowForceMaterialize(t *testing.T) { + borrowedBefore := promtestutil.ToFloat64( + metric.ArrowLoadPayloadBytesCounter.WithLabelValues("borrowed"), + ) + copyBefore := promtestutil.ToFloat64( + metric.ArrowLoadCopyBytesCounter.WithLabelValues("arrow_to_mo"), + ) + fileBytes := makeExternalArrowIPC(t, tree.ARROW_CONTAINER_FILE) + fs, err := fileservice.NewMemoryFS("etl", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + path := "etl:force-materialize.arrow" + require.NoError(t, fs.Write(context.Background(), fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{Offset: 0, Size: int64(len(fileBytes)), Data: fileBytes}}, + })) + + registry, err := mpool.NewAllocationAccountRegistry(1, 128) + require.NoError(t, err) + account, err := registry.Open(64 << 20) + require.NoError(t, err) + proc := newArrowLoadTestProc(t) + param := externalArrowParam(fs, path, int64(len(fileBytes)), tree.ARROW_CONTAINER_FILE) + param.ArrowForceMaterialize = true + arg := NewArgument().WithEs(param) + require.NoError(t, arg.SetAllocationAccount(account)) + require.NoError(t, arg.Prepare(proc)) + + result, err := arg.Call(proc) + require.NoError(t, err) + require.Equal(t, vm.ExecNext, result.Status) + require.False(t, result.Batch.Vecs[0].HasBorrowedBacking()) + require.False(t, result.Batch.Vecs[1].HasBorrowedBacking()) + require.Equal(t, borrowedBefore, promtestutil.ToFloat64( + metric.ArrowLoadPayloadBytesCounter.WithLabelValues("borrowed"), + )) + require.Greater(t, promtestutil.ToFloat64( + metric.ArrowLoadCopyBytesCounter.WithLabelValues("arrow_to_mo"), + ), copyBefore) + + arg.Free(proc, false, nil) + require.Zero(t, account.Snapshot().Used) + require.NoError(t, arg.ClearAllocationAccount(account)) + arg.Release() + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) +} + +func TestArrowLoadErrorCategory(t *testing.T) { + tests := []struct { + err error + category string + }{ + {err: context.Canceled, category: "canceled"}, + {err: context.DeadlineExceeded, category: "deadline_exceeded"}, + {err: fmt.Errorf("wrapped: %w", fileservice.ErrObjectChanged), category: "object_changed"}, + {err: mpool.ErrAllocationAccountCapacity, category: "resource_exhausted"}, + {err: moerr.NewNotSupportedNoCtx("type"), category: "not_supported"}, + {err: moerr.NewConstraintViolationNoCtx("value"), category: "constraint_violation"}, + {err: moerr.NewInvalidInputNoCtx("input"), category: "invalid_input"}, + {err: fmt.Errorf("wrapped: %w", moerr.NewInvalidInputNoCtx("input")), category: "invalid_input"}, + {err: moerr.NewInternalErrorNoCtx("state"), category: "internal"}, + {err: errors.New("backend"), category: "io"}, + } + for _, test := range tests { + require.Equal(t, test.category, arrowLoadErrorCategory(test.err)) + } +} + +func TestArrowReaderOpenRejectsMissingFileParam(t *testing.T) { + proc := newArrowLoadTestProc(t) + reader := new(ArrowReader) + _, err := reader.Open(&ExternalParam{ + ExParamConst: ExParamConst{Extern: &tree.ExternParam{}}, + }, proc) + require.ErrorContains(t, err, "file parameter") +} + +type countingArrowCapacityLease struct { + releases atomic.Int64 +} + +func (l *countingArrowCapacityLease) Release() { + l.releases.Add(1) +} + +func TestMeteredArrowCapacityLeaseConcurrentRelease(t *testing.T) { + const capacity = int64(64) + arrowPinnedMetricState.Lock() + start := arrowPinnedMetricState.current + arrowPinnedMetricState.Unlock() + + inner := new(countingArrowCapacityLease) + lease := &meteredArrowCapacityLease{inner: inner, capacity: capacity} + adjustArrowPinnedBytes(capacity) + + var wait sync.WaitGroup + for index := 0; index < 32; index++ { + wait.Add(1) + go func() { + defer wait.Done() + lease.Release() + }() + } + wait.Wait() + + require.Equal(t, int64(1), inner.releases.Load()) + arrowPinnedMetricState.Lock() + require.Equal(t, start, arrowPinnedMetricState.current) + arrowPinnedMetricState.Unlock() + require.Equal(t, float64(start), promtestutil.ToFloat64(metric.ArrowLoadPinnedBytesGauge)) +} + +func TestExternalArrowLoadFromLocalMinIOAndRejectsObjectChange(t *testing.T) { + minioServer := startLocalArrowMinIO(t) + ctx := context.Background() + payload := makeExternalArrowIPC(t, tree.ARROW_CONTAINER_FILE) + upload := func(key string, data []byte) { + t.Helper() + _, err := minioServer.client.PutObject( + ctx, minioServer.bucket, key, bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "application/vnd.apache.arrow.file"}, + ) + require.NoError(t, err) + } + upload("load.arrow", payload) + upload("identity.arrow", payload) + + fs, err := fileservice.NewS3FS( + ctx, + fileservice.ObjectStorageArguments{ + Name: "etl", Endpoint: "http://" + minioServer.endpoint, + Region: "us-east-1", Bucket: minioServer.bucket, + KeyID: minioServer.user, KeySecret: minioServer.password, IsMinio: true, + }, + fileservice.DisabledCacheConfig, nil, true, true, + ) + require.NoError(t, err) + t.Cleanup(func() { fs.Close(context.Background()) }) + + // Exercise the normal LOAD operator all the way through the S3-compatible + // FileService. A File container must use bounded conditional range GETs; + // it must not stage the entire object locally. + registry, err := mpool.NewAllocationAccountRegistry(1, 128) + require.NoError(t, err) + account, err := registry.Open(64 << 20) + require.NoError(t, err) + proc := newArrowLoadTestProc(t) + proc.Ctx.Value(config.ParameterUnitKey).(*config.ParameterUnit).SV.ArrowLoad.S3Enabled = true + path := "etl:load.arrow" + arg := NewArgument().WithEs(externalArrowParam(fs, path, int64(len(payload)), tree.ARROW_CONTAINER_FILE)) + require.NoError(t, arg.SetAllocationAccount(account)) + require.NoError(t, arg.Prepare(proc)) + for _, expected := range [][]int64{{1, 2}, {3, 4}} { + result, err := arg.Call(proc) + require.NoError(t, err) + require.Equal(t, vm.ExecNext, result.Status) + require.Equal(t, expected, vector.MustFixedColNoTypeCheck[int64](result.Batch.Vecs[0])) + } + result, err := arg.Call(proc) + require.NoError(t, err) + require.Equal(t, vm.ExecStop, result.Status) + arg.Free(proc, false, nil) + require.Zero(t, account.Snapshot().Used) + require.NoError(t, arg.ClearAllocationAccount(account)) + arg.Release() + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) + + // Fix the ETag during Open, replace the object, then force the next record + // range read. An unversioned MinIO bucket must fail rather than combine + // blocks from two object generations. + identityPath := "etl:identity.arrow" + identity, err := fs.StatFileIdentity(ctx, identityPath) + require.NoError(t, err) + require.Empty(t, identity.VersionID) + require.NotEmpty(t, identity.ETag) + registry, err = mpool.NewAllocationAccountRegistry(1, 128) + require.NoError(t, err) + account, err = registry.Open(64 << 20) + require.NoError(t, err) + proc = newArrowLoadTestProc(t) + param := externalArrowParam(fs, identityPath, int64(len(payload)), tree.ARROW_CONTAINER_FILE) + param.Fileparam.FileIndex = 1 + param.Fileparam.Filepath = identityPath + reader, err := NewArrowReader(param, proc, account) + require.NoError(t, err) + fileEmpty, err := reader.Open(param, proc) + require.NoError(t, err) + require.False(t, fileEmpty) + + replacement := append([]byte(nil), payload...) + replacement[len(replacement)/2] ^= 0xff + upload("identity.arrow", replacement) + _, err = reader.ReadBatch(ctx, batch.NewWithSize(2), proc, nil) + require.ErrorIs(t, err, fileservice.ErrObjectChanged) + require.NoError(t, reader.Close()) + require.Zero(t, account.Snapshot().Used) + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) +} + +type localArrowMinIO struct { + endpoint string + bucket string + user string + password string + client *minio.Client +} + +func startLocalArrowMinIO(t *testing.T) localArrowMinIO { + t.Helper() + executable, err := exec.LookPath("minio") + if errors.Is(err, exec.ErrNotFound) { + t.Skip("local MinIO binary is not installed") + } + require.NoError(t, err) + + reserveAddress := func() string { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + address := listener.Addr().String() + require.NoError(t, listener.Close()) + return address + } + endpoint, consoleEndpoint := reserveAddress(), reserveAddress() + dataDir := t.TempDir() + logPath := filepath.Join(t.TempDir(), "minio.log") + logFile, err := os.Create(logPath) + require.NoError(t, err) + + const user = "arrowtest" + const password = "arrowtest-secret" + command := exec.Command( + executable, "server", dataDir, + "--address", endpoint, "--console-address", consoleEndpoint, + ) + command.Env = append(os.Environ(), + "MINIO_ROOT_USER="+user, + "MINIO_ROOT_PASSWORD="+password, + ) + command.Stdout = logFile + command.Stderr = logFile + require.NoError(t, command.Start()) + t.Cleanup(func() { + _ = command.Process.Kill() + _, _ = command.Process.Wait() + _ = logFile.Close() + }) + + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(user, password, ""), + Secure: false, + Region: "us-east-1", + }) + require.NoError(t, err) + bucket := "matrixone-arrow-test" + deadline := time.Now().Add(15 * time.Second) + for { + err = client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{Region: "us-east-1"}) + if err == nil { + break + } + if time.Now().After(deadline) { + _ = command.Process.Kill() + _, _ = command.Process.Wait() + _ = logFile.Close() + logBytes, _ := os.ReadFile(logPath) + t.Fatalf("start local MinIO: %v\n%s", err, logBytes) + } + time.Sleep(100 * time.Millisecond) + } + return localArrowMinIO{ + endpoint: endpoint, bucket: bucket, user: user, password: password, client: client, + } +} + +func TestExternalArrowSkipsZeroRowRecordBatches(t *testing.T) { + for _, container := range []string{tree.ARROW_CONTAINER_FILE, tree.ARROW_CONTAINER_STREAM} { + t.Run(container, func(t *testing.T) { + payload := makeExternalArrowIPCWithRows(t, container, [][]int64{ + {}, {1, 2}, {}, {3}, {}, + }) + fs, err := fileservice.NewMemoryFS("etl", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + path := "etl:empty-records-" + container + ".arrow" + require.NoError(t, fs.Write(context.Background(), fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{Offset: 0, Size: int64(len(payload)), Data: payload}}, + })) + + registry, err := mpool.NewAllocationAccountRegistry(1, 128) + require.NoError(t, err) + account, err := registry.Open(64 << 20) + require.NoError(t, err) + proc := newArrowLoadTestProc(t) + arg := NewArgument().WithEs(externalArrowParam(fs, path, int64(len(payload)), container)) + require.NoError(t, arg.SetAllocationAccount(account)) + require.NoError(t, arg.Prepare(proc)) + + for _, expected := range [][]int64{{1, 2}, {3}} { + result, err := arg.Call(proc) + require.NoError(t, err) + require.Equal(t, vm.ExecNext, result.Status) + require.Equal(t, expected, vector.MustFixedColNoTypeCheck[int64](result.Batch.Vecs[0])) + } + result, err := arg.Call(proc) + require.NoError(t, err) + require.Equal(t, vm.ExecStop, result.Status) + + arg.Free(proc, false, nil) + require.Zero(t, account.Snapshot().Used) + require.NoError(t, arg.ClearAllocationAccount(account)) + arg.Release() + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) + }) + } +} + +func TestExternalArrowPrepareFailsClosedBeforeIO(t *testing.T) { + proc := newArrowLoadTestProc(t) + for _, test := range []struct { + name string + externType int32 + scope pipeline.ArrowExecutionScope + }{ + {"missing-scope", int32(plan.ExternType_LOAD), pipeline.ArrowExecutionScope_UnknownArrowExecutionScope}, + {"external-table", int32(plan.ExternType_EXTERNAL_TB), pipeline.ArrowExecutionScope_ArrowLoadData}, + } { + t.Run(test.name, func(t *testing.T) { + param := externalArrowParam(nil, "does-not-exist.arrow", 1, tree.ARROW_CONTAINER_FILE) + param.Extern.ExternType = test.externType + param.ArrowExecutionScope = test.scope + arg := NewArgument().WithEs(param) + err := arg.Prepare(proc) + require.Error(t, err) + require.Contains(t, err.Error(), "supported only by LOAD DATA") + require.Nil(t, arg.reader) + arg.Free(proc, true, err) + arg.Release() + }) + } +} + +func TestExternalArrowPrepareEnforcesWorkerRolloutGate(t *testing.T) { + proc := newArrowLoadTestProc(t) + defer proc.Free() + settings := proc.Ctx.Value(config.ParameterUnitKey).(*config.ParameterUnit).SV + + param := externalArrowParam(nil, "worker-gate.arrow", 1, tree.ARROW_CONTAINER_FILE) + // Fanout scopes deliberately clear the user PARALLEL request after their + // ranges are assigned. The serialized execution signal, not that request, + // is what makes this an executing distributed scope on the worker. + param.Extern.Parallel = false + param.ArrowDistributedExecution = true + arg := NewArgument().WithEs(param) + err := arg.Prepare(proc) + require.ErrorContains(t, err, "distributed Arrow LOAD is disabled") + require.Nil(t, arg.reader) + arg.Release() + + settings.ArrowLoad.DistributedEnabled = true + registry, err := mpool.NewAllocationAccountRegistry(1, 128) + require.NoError(t, err) + account, err := registry.Open(64 << 20) + require.NoError(t, err) + arg = NewArgument().WithEs(param) + require.NoError(t, arg.SetAllocationAccount(account)) + require.NoError(t, arg.Prepare(proc)) + arg.Free(proc, false, nil) + require.NoError(t, arg.ClearAllocationAccount(account)) + arg.Release() + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) + + settings.ArrowLoad.DistributedEnabled = false + param = externalArrowParam(nil, "s3-gate.arrow", 1, tree.ARROW_CONTAINER_FILE) + param.Extern.ScanType = tree.S3 + arg = NewArgument().WithEs(param) + err = arg.Prepare(proc) + require.ErrorContains(t, err, "S3 or stage is disabled") + require.Nil(t, arg.reader) + arg.Release() +} + +func TestExternalArrowAllocationAccountContract(t *testing.T) { + registry, err := mpool.NewAllocationAccountRegistry(2, 2) + require.NoError(t, err) + first, err := registry.Open(1) + require.NoError(t, err) + second, err := registry.Open(1) + require.NoError(t, err) + arg := NewArgument().WithEs(&ExternalParam{ExParamConst: ExParamConst{ + ArrowExecutionScope: pipeline.ArrowExecutionScope_ArrowLoadData, + }}) + require.ErrorIs(t, arg.SetAllocationAccount(nil), mpool.ErrAllocationAccountInvalid) + require.NoError(t, arg.SetAllocationAccount(first)) + require.NoError(t, arg.SetAllocationAccount(first)) + require.ErrorIs(t, arg.SetAllocationAccount(second), mpool.ErrAllocationAccountMismatch) + require.ErrorIs(t, arg.ClearAllocationAccount(second), mpool.ErrAllocationAccountMismatch) + require.NoError(t, arg.ClearAllocationAccount(first)) + arg.Release() + first.Seal() + second.Seal() + _, err = registry.Finalize(first) + require.NoError(t, err) + _, err = registry.Finalize(second) + require.NoError(t, err) +} + +func TestBuildArrowTargetsUsesPhysicalColumnTypeAcrossGeneratedGap(t *testing.T) { + attrs := []plan.ExternAttr{ + {ColName: "payload", ColIndex: 0, ColFieldIndex: 0}, + {ColName: "id", ColIndex: 2, ColFieldIndex: 1}, + } + cols := []*plan.ColDef{ + {Name: "payload", Typ: plan.Type{Id: int32(types.T_varchar), Width: 128}}, + {Name: "generated", Typ: plan.Type{Id: int32(types.T_float64)}, GeneratedCol: &plan.GeneratedCol{}}, + {Name: "id", Typ: plan.Type{Id: int32(types.T_int64), NotNullable: true}}, + } + targets, err := BuildArrowTargets(context.Background(), attrs, cols) + require.NoError(t, err) + require.Len(t, targets, 2) + require.Equal(t, "payload", targets[0].Name) + require.Equal(t, types.T_varchar, targets[0].Type.Oid) + require.Equal(t, int32(128), targets[0].Type.Width) + require.Equal(t, 0, targets[0].MOIndex) + require.Equal(t, "payload", targets[0].AttrName) + require.Equal(t, "id", targets[1].Name) + require.Equal(t, types.T_int64, targets[1].Type.Oid) + require.True(t, targets[1].NotNull) + require.Equal(t, 1, targets[1].MOIndex) + require.Equal(t, "id", targets[1].AttrName) + + _, err = BuildArrowTargets(context.Background(), []plan.ExternAttr{{ColIndex: 3}}, cols) + require.ErrorContains(t, err, "index 3") +} + +func TestBuildArrowTargetsPreservesExplicitExternalFieldOrder(t *testing.T) { + attrs := []plan.ExternAttr{ + {ColName: "a", ColIndex: 0, ColFieldIndex: 1}, + {ColName: "b", ColIndex: 1, ColFieldIndex: 0}, + } + cols := []*plan.ColDef{ + {Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "b", Typ: plan.Type{Id: int32(types.T_int64)}}, + } + targets, err := BuildArrowTargets(context.Background(), attrs, cols) + require.NoError(t, err) + require.Equal(t, "b", targets[0].Name) + require.Equal(t, 1, targets[0].MOIndex) + require.Equal(t, "a", targets[1].Name) + require.Equal(t, 0, targets[1].MOIndex) + + schema := arrow.NewSchema([]arrow.Field{ + {Name: "source_first", Type: arrow.PrimitiveTypes.Int64}, + {Name: "source_second", Type: arrow.PrimitiveTypes.Int64}, + }, nil) + conversion, err := arrowbridge.BindLoad( + context.Background(), schema, targets, arrowbridge.MatchByPosition, + ) + require.NoError(t, err) + allocator := memory.NewGoAllocator() + firstBuilder := array.NewInt64Builder(allocator) + firstBuilder.Append(11) + first := firstBuilder.NewArray() + firstBuilder.Release() + defer first.Release() + secondBuilder := array.NewInt64Builder(allocator) + secondBuilder.Append(22) + second := secondBuilder.NewArray() + secondBuilder.Release() + defer second.Release() + record := array.NewRecordBatch(schema, []arrow.Array{first, second}, 1) + defer record.Release() + mp := mpool.MustNewZero() + converted, _, err := conversion.Convert(context.Background(), record, mp, arrowbridge.ConvertOptions{}) + require.NoError(t, err) + require.Equal(t, []int64{22}, vector.MustFixedColNoTypeCheck[int64](converted.Vecs[0])) + require.Equal(t, []int64{11}, vector.MustFixedColNoTypeCheck[int64](converted.Vecs[1])) + converted.Clean(mp) + require.Zero(t, mp.CurrNB()) + + _, err = BuildArrowTargets(context.Background(), []plan.ExternAttr{ + {ColIndex: 0, ColFieldIndex: 1}, + {ColIndex: 1, ColFieldIndex: 1}, + }, cols) + require.ErrorContains(t, err, "source field index 1 is duplicated") + _, err = BuildArrowTargets(context.Background(), []plan.ExternAttr{ + {ColIndex: 0, ColFieldIndex: 2}, + {ColIndex: 1, ColFieldIndex: 0}, + }, cols) + require.ErrorContains(t, err, "source field index 2 is invalid") +} + +func TestExternalArrowFileRecordBatchShard(t *testing.T) { + payload := makeExternalArrowIPC(t, tree.ARROW_CONTAINER_FILE) + fs, err := fileservice.NewMemoryFS("etl", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + path := "etl:shard.arrow" + require.NoError(t, fs.Write(context.Background(), fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{Offset: 0, Size: int64(len(payload)), Data: payload}}, + })) + registry, err := mpool.NewAllocationAccountRegistry(1, 128) + require.NoError(t, err) + account, err := registry.Open(64 << 20) + require.NoError(t, err) + proc := newArrowLoadTestProc(t) + param := externalArrowParam(fs, path, int64(len(payload)), tree.ARROW_CONTAINER_FILE) + param.ArrowRecordBatchShards = []*pipeline.ArrowRecordBatchShard{{ + FileIndex: 0, RecordBatchStart: 1, RecordBatchEnd: 2, + EstimatedRows: 2, EstimatedWireBytes: int64(len(payload)), + }} + arg := NewArgument().WithEs(param) + require.NoError(t, arg.SetAllocationAccount(account)) + require.NoError(t, arg.Prepare(proc)) + result, err := arg.Call(proc) + require.NoError(t, err) + require.Equal(t, []int64{3, 4}, vector.MustFixedColNoTypeCheck[int64](result.Batch.Vecs[0])) + result, err = arg.Call(proc) + require.NoError(t, err) + require.Equal(t, vm.ExecStop, result.Status) + arg.Free(proc, false, nil) + require.Zero(t, account.Snapshot().Used) + require.NoError(t, arg.ClearAllocationAccount(account)) + arg.Release() + account.Seal() + _, err = registry.Finalize(account) + require.NoError(t, err) +} + +func TestExternalArrowConversionPlanVersionAndFingerprintFailClosed(t *testing.T) { + proc := newArrowLoadTestProc(t) + for _, test := range []struct { + name string + version uint32 + fingerprint []byte + }{ + {name: "unknown-version", version: arrowbridge.ConversionPlanVersion + 1}, + {name: "malformed-fingerprint", version: arrowbridge.ConversionPlanVersion, fingerprint: []byte{1}}, + } { + t.Run(test.name, func(t *testing.T) { + param := externalArrowParam(nil, "unused.arrow", 1, tree.ARROW_CONTAINER_FILE) + param.ArrowConversionPlanVersion = test.version + param.ArrowSchemaFingerprint = test.fingerprint + registry, err := mpool.NewAllocationAccountRegistry(1, 8) + require.NoError(t, err) + account, err := registry.Open(1 << 20) + require.NoError(t, err) + arg := NewArgument().WithEs(param) + require.NoError(t, arg.SetAllocationAccount(account)) + err = arg.Prepare(proc) + require.Error(t, err) + arg.Free(proc, true, err) + require.NoError(t, arg.ClearAllocationAccount(account)) + arg.Release() + account.Seal() + _, finalizeErr := registry.Finalize(account) + require.NoError(t, finalizeErr) + }) + } +} + +func TestExternalArrowRejectsPlannedFingerprintMismatchBeforeRecordRead(t *testing.T) { + payload := makeExternalArrowIPC(t, tree.ARROW_CONTAINER_FILE) + fs, err := fileservice.NewMemoryFS("etl", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + path := "etl:fingerprint.arrow" + require.NoError(t, fs.Write(context.Background(), fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{Offset: 0, Size: int64(len(payload)), Data: payload}}, + })) + registry, err := mpool.NewAllocationAccountRegistry(1, 128) + require.NoError(t, err) + account, err := registry.Open(64 << 20) + require.NoError(t, err) + proc := newArrowLoadTestProc(t) + param := externalArrowParam(fs, path, int64(len(payload)), tree.ARROW_CONTAINER_FILE) + param.ArrowSchemaFingerprint = make([]byte, 32) + arg := NewArgument().WithEs(param) + require.NoError(t, arg.SetAllocationAccount(account)) + require.NoError(t, arg.Prepare(proc)) + _, err = arg.Call(proc) + require.ErrorContains(t, err, "does not match the planned contract") + arg.Free(proc, true, err) + require.NoError(t, arg.ClearAllocationAccount(account)) + arg.Release() + account.Seal() + _, finalizeErr := registry.Finalize(account) + require.NoError(t, finalizeErr) +} + +func TestExternalArrowFailurePathsReleaseAllocationAccount(t *testing.T) { + payload := makeExternalArrowIPC(t, tree.ARROW_CONTAINER_FILE) + for _, test := range []struct { + name string + limit uint64 + cancelRead bool + }{ + {name: "capacity-during-open", limit: 32}, + {name: "cancellation-after-borrowed-batch", limit: 64 << 20, cancelRead: true}, + } { + t.Run(test.name, func(t *testing.T) { + fs, err := fileservice.NewMemoryFS("etl", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + path := "etl:failure-" + test.name + ".arrow" + require.NoError(t, fs.Write(context.Background(), fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{ + Offset: 0, Size: int64(len(payload)), Data: payload, + }}, + })) + registry, err := mpool.NewAllocationAccountRegistry(1, 128) + require.NoError(t, err) + account, err := registry.Open(test.limit) + require.NoError(t, err) + proc := newArrowLoadTestProc(t) + arg := NewArgument().WithEs(externalArrowParam( + fs, path, int64(len(payload)), tree.ARROW_CONTAINER_FILE, + )) + require.NoError(t, arg.SetAllocationAccount(account)) + require.NoError(t, arg.Prepare(proc)) + + if test.cancelRead { + _, err = arg.Call(proc) + require.NoError(t, err) + require.Greater(t, account.Snapshot().Used, uint64(0)) + ctx, cancel := context.WithCancel(proc.Ctx) + proc.Ctx = ctx + cancel() + _, err = arg.Call(proc) + require.ErrorIs(t, err, context.Canceled) + } else { + _, err = arg.Call(proc) + require.Error(t, err) + require.True(t, + errors.Is(err, mpool.ErrAllocationAccountCapacity) || + mpool.IsMPoolCapacityFailure(err), err) + } + + arg.Free(proc, true, err) + require.Zero(t, account.Snapshot().Used) + require.NoError(t, arg.ClearAllocationAccount(account)) + arg.Release() + account.Seal() + _, finalizeErr := registry.Finalize(account) + require.NoError(t, finalizeErr) + }) + } +} + +func TestFitArrowBatchUsesActualCanonicalWireSize(t *testing.T) { + mp := mpool.MustNewZero() + bat := batch.NewOffHeap([]string{"v"}) + bat.Vecs[0] = vector.NewOffHeapVecWithType(types.T_varchar.ToType()) + for _, value := range []string{ + "first payload longer than twenty three bytes", + "second payload longer than twenty three bytes", + "third payload longer than twenty three bytes", + } { + require.NoError(t, vector.AppendBytes(bat.Vecs[0], []byte(value), false, mp)) + } + bat.SetRowCount(3) + twoRows, err := bat.Window(0, 2) + require.NoError(t, err) + twoSize, err := twoRows.MarshalBinarySize() + require.NoError(t, err) + twoRows.Clean(mp) + oneRow, err := bat.Window(0, 1) + require.NoError(t, err) + oneSize, err := oneRow.MarshalBinarySize() + require.NoError(t, err) + oneRow.Clean(mp) + require.Greater(t, twoSize, oneSize) + + fitted, rows, err := fitArrowBatchToWireBudget( + context.Background(), bat, uint64(twoSize-1), mp, + ) + require.NoError(t, err) + require.Equal(t, 1, rows) + actual, err := fitted.MarshalBinarySize() + require.NoError(t, err) + require.LessOrEqual(t, uint64(actual), uint64(twoSize-1)) + require.Equal(t, "first payload longer than twenty three bytes", fitted.Vecs[0].GetStringAt(0)) + fitted.Clean(mp) + require.Zero(t, mp.CurrNB()) +} + +func TestFitArrowBatchRetainsEligibleBorrowedPrefix(t *testing.T) { + mp := mpool.MustNewZero() + data := types.EncodeSlice([]int64{11, 22, 33}) + lease, err := vector.NewRefCountedBufferLease(data, int64(cap(data)), nil) + require.NoError(t, err) + vec, err := vector.NewBorrowedFixedVector(types.T_int64.ToType(), 3, data, lease) + require.NoError(t, err) + lease.Release() + bat := batch.NewOffHeap([]string{"v"}) + bat.Vecs[0] = vec + bat.SetRowCount(3) + + twoRows, err := bat.Window(0, 2) + require.NoError(t, err) + twoSize, err := twoRows.MarshalBinarySize() + require.NoError(t, err) + twoRows.Clean(mp) + oneRow, err := bat.Window(0, 1) + require.NoError(t, err) + oneSize, err := oneRow.MarshalBinarySize() + require.NoError(t, err) + oneRow.Clean(mp) + require.Greater(t, twoSize, oneSize) + + sourcePointer := uintptr(unsafe.Pointer(unsafe.SliceData(data))) + fitted, rows, err := fitArrowBatchToWireBudget( + context.Background(), bat, uint64(twoSize-1), mp, + ) + require.NoError(t, err) + require.Equal(t, 1, rows) + require.True(t, fitted.Vecs[0].HasBorrowedBacking()) + require.Equal(t, sourcePointer, + uintptr(unsafe.Pointer(unsafe.SliceData(fitted.Vecs[0].GetData())))) + require.Equal(t, []int64{11}, vector.MustFixedColNoTypeCheck[int64](fitted.Vecs[0])) + require.NotNil(t, lease.Bytes()) + + fitted.Clean(mp) + require.Nil(t, lease.Bytes()) + require.Zero(t, mp.CurrNB()) +} + +func TestFitArrowBatchRejectsOversizedFirstRow(t *testing.T) { + for _, rows := range []int{1, 2} { + t.Run(fmt.Sprintf("rows-%d", rows), func(t *testing.T) { + mp := mpool.MustNewZero() + bat := batch.NewOffHeap([]string{"v"}) + bat.Vecs[0] = vector.NewOffHeapVecWithType(types.T_varchar.ToType()) + for row := 0; row < rows; row++ { + value := bytes.Repeat([]byte{'x'}, 256) + require.NoError(t, vector.AppendBytes(bat.Vecs[0], value, false, mp)) + } + bat.SetRowCount(rows) + _, fittedRows, err := fitArrowBatchToWireBudget( + context.Background(), bat, 64, mp, + ) + require.ErrorContains(t, err, "exceeds batch limit") + require.Zero(t, fittedRows) + bat.Clean(mp) + require.Zero(t, mp.CurrNB()) + }) + } +} + +func externalArrowParam(fs fileservice.FileService, path string, size int64, container string) *ExternalParam { + return &ExternalParam{ + ExParamConst: ExParamConst{ + ArrowExecutionScope: pipeline.ArrowExecutionScope_ArrowLoadData, + ArrowConversionPlanVersion: arrowbridge.ConversionPlanVersion, + Attrs: []plan.ExternAttr{ + {ColName: "id", ColIndex: 0, ColFieldIndex: 0}, + {ColName: "name", ColIndex: 1, ColFieldIndex: 1}, + }, + Cols: []*plan.ColDef{ + {Name: "id", Typ: plan.Type{Id: int32(types.T_int64), NotNullable: true}}, + {Name: "name", Typ: plan.Type{Id: int32(types.T_varchar), Width: 100}}, + }, + FileList: []string{path}, + FileSize: []int64{size}, + Extern: &tree.ExternParam{ + ExParamConst: tree.ExParamConst{ + ScanType: tree.INFILE, Filepath: path, Format: tree.ARROW, + ArrowContainer: container, Tail: &tree.TailParameter{}, + }, + ExParam: tree.ExParam{ExternType: int32(plan.ExternType_LOAD), FileService: fs}, + }, + Ctx: context.Background(), + }, + ExParam: ExParam{Fileparam: &ExFileparam{}, Filter: &FilterParam{}}, + } +} + +func newArrowLoadTestProc(t *testing.T) *process.Process { + t.Helper() + proc := testutil.NewProc(t) + frontend := &config.FrontendParameters{} + frontend.SetDefaultValues() + // Reader tests exercise an admitted Arrow LOAD. Product defaults are + // fail-closed, so this fixture supplies the explicit local opt-in. + frontend.ArrowLoad.Enabled = true + proc.Ctx = context.WithValue(proc.Ctx, config.ParameterUnitKey, + config.NewParameterUnit(frontend, nil, nil, nil)) + return proc +} + +func makeExternalArrowIPC(t *testing.T, container string) []byte { + t.Helper() + allocator := memory.NewGoAllocator() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64, Nullable: false}, + {Name: "name", Type: arrow.BinaryTypes.String, Nullable: true}, + }, nil) + var output bytes.Buffer + writeRecords := func(write func(arrow.RecordBatch) error) { + for offset := int64(0); offset < 4; offset += 2 { + builder := array.NewRecordBuilder(allocator, schema) + builder.Field(0).(*array.Int64Builder).AppendValues([]int64{offset + 1, offset + 2}, nil) + builder.Field(1).(*array.StringBuilder).AppendValues( + []string{"a payload longer than twenty three bytes", "short"}, []bool{true, offset == 0}, + ) + record := builder.NewRecordBatch() + require.NoError(t, write(record)) + record.Release() + builder.Release() + } + } + if container == tree.ARROW_CONTAINER_FILE { + writer, err := ipc.NewFileWriter(&output, ipc.WithSchema(schema), ipc.WithAllocator(allocator)) + require.NoError(t, err) + writeRecords(writer.Write) + require.NoError(t, writer.Close()) + } else { + writer := ipc.NewWriter(&output, ipc.WithSchema(schema), ipc.WithAllocator(allocator)) + writeRecords(writer.Write) + require.NoError(t, writer.Close()) + } + return append([]byte(nil), output.Bytes()...) +} + +func makeExternalArrowIPCWithRows(t *testing.T, container string, batches [][]int64) []byte { + t.Helper() + allocator := memory.NewGoAllocator() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64, Nullable: false}, + {Name: "name", Type: arrow.BinaryTypes.String, Nullable: true}, + }, nil) + var output bytes.Buffer + writeRecords := func(write func(arrow.RecordBatch) error) { + for _, ids := range batches { + builder := array.NewRecordBuilder(allocator, schema) + builder.Field(0).(*array.Int64Builder).AppendValues(ids, nil) + names := make([]string, len(ids)) + valid := make([]bool, len(ids)) + for row, id := range ids { + names[row] = fmt.Sprintf("row-%d payload longer than twenty three bytes", id) + valid[row] = true + } + builder.Field(1).(*array.StringBuilder).AppendValues(names, valid) + record := builder.NewRecordBatch() + require.NoError(t, write(record)) + record.Release() + builder.Release() + } + } + if container == tree.ARROW_CONTAINER_FILE { + writer, err := ipc.NewFileWriter(&output, ipc.WithSchema(schema), ipc.WithAllocator(allocator)) + require.NoError(t, err) + writeRecords(writer.Write) + require.NoError(t, writer.Close()) + } else { + writer := ipc.NewWriter(&output, ipc.WithSchema(schema), ipc.WithAllocator(allocator)) + writeRecords(writer.Write) + require.NoError(t, writer.Close()) + } + return append([]byte(nil), output.Bytes()...) +} diff --git a/pkg/sql/colexec/external/types.go b/pkg/sql/colexec/external/types.go index a3ab2719b65f3..49abfabf5e102 100644 --- a/pkg/sql/colexec/external/types.go +++ b/pkg/sql/colexec/external/types.go @@ -22,6 +22,7 @@ import ( "github.com/parquet-go/parquet-go" + "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/common/reuse" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -69,6 +70,20 @@ type ExParamConst struct { Idx int ColumnListLen int32 // load ... (col1, col2 , col3), ColumnListLen is 3 CreateSql string + // ArrowExecutionScope is positive authorization emitted only by compile. + // The zero value must fail closed before Arrow I/O. + ArrowExecutionScope pipeline.ArrowExecutionScope + ArrowObjectIdentities []*pipeline.ArrowObjectIdentity + ArrowRecordBatchShards []*pipeline.ArrowRecordBatchShard + ArrowSchemaFingerprint []byte + ArrowConversionPlanVersion uint32 + // ArrowForceMaterialize is the compile-time rollout snapshot propagated to + // every local or remote External scope. It is not a per-batch heuristic. + ArrowForceMaterialize bool + // ArrowDistributedExecution records that this scope was created by Arrow + // fanout. It intentionally differs from Extern.Parallel: shard scopes clear + // the user request after planning but still require worker-side opt-in. + ArrowDistributedExecution bool // letter case: origin Attrs []plan.ExternAttr @@ -203,10 +218,11 @@ type container struct { } type External struct { - ctr container - Es *ExternalParam - reader ExternalFileReader // unified file reader - fileOpened bool // whether a file is currently active + ctr container + Es *ExternalParam + reader ExternalFileReader // unified file reader + fileOpened bool // whether a file is currently active + allocationAccount *mpool.AllocationAccount vm.OperatorBase colexec.Projection @@ -237,6 +253,36 @@ func NewArgument() *External { return reuse.Alloc[External](nil) } +func (external *External) SetAllocationAccount(account *mpool.AllocationAccount) error { + if account == nil || account.Handle() == 0 { + return mpool.ErrAllocationAccountInvalid + } + if external.allocationAccount != nil && external.allocationAccount != account { + return mpool.ErrAllocationAccountMismatch + } + external.allocationAccount = account + return nil +} + +func (external *External) ActivatesAllocationAccountLifecycle() bool { + return external != nil && external.Es != nil && + external.Es.ArrowExecutionScope == pipeline.ArrowExecutionScope_ArrowLoadData +} + +func (external *External) ClearAllocationAccount(account *mpool.AllocationAccount) error { + if external.allocationAccount == nil { + return nil + } + if external.allocationAccount != account { + return mpool.ErrAllocationAccountMismatch + } + if external.reader != nil || external.fileOpened || external.ctr.buf != nil { + return mpool.ErrAllocationAccountInvariant + } + external.allocationAccount = nil + return nil +} + func (param *ExternalParam) addParquetProfile(stats process.ParquetProfileStats) { if param == nil || param.Extern == nil || !strings.EqualFold(param.Extern.Format, tree.PARQUET) || stats.Empty() { return diff --git a/pkg/sql/compile/arrow_scope_test.go b/pkg/sql/compile/arrow_scope_test.go new file mode 100644 index 0000000000000..3aedb99996a1c --- /dev/null +++ b/pkg/sql/compile/arrow_scope_test.go @@ -0,0 +1,430 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compile + +import ( + "bytes" + "context" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/matrixorigin/matrixone/pkg/config" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/pb/pipeline" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/external" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/stretchr/testify/require" +) + +func TestArrowExecutionScopeRequiresPositiveCompileEvidence(t *testing.T) { + node := &plan.Node{ExternScan: &plan.ExternScan{Type: int32(plan.ExternType_LOAD)}} + param := &tree.ExternParam{ExParamConst: tree.ExParamConst{Format: tree.ARROW}} + newCompile := func(query *plan.Query) *Compile { + return &Compile{anal: &AnalyzeModule{qry: query}} + } + + require.Equal(t, pipeline.ArrowExecutionScope_ArrowLoadData, + newCompile(&plan.Query{LoadTag: true, StmtType: plan.Query_INSERT}).arrowExecutionScope(node, param)) + + tests := []struct { + name string + c *Compile + node *plan.Node + param *tree.ExternParam + }{ + {"nil-compile", nil, node, param}, + {"nil-analysis", &Compile{}, node, param}, + {"missing-load-tag", newCompile(&plan.Query{StmtType: plan.Query_INSERT}), node, param}, + {"wrong-statement", newCompile(&plan.Query{LoadTag: true, StmtType: plan.Query_SELECT}), node, param}, + {"wrong-scan", newCompile(&plan.Query{LoadTag: true, StmtType: plan.Query_INSERT}), &plan.Node{ExternScan: &plan.ExternScan{Type: int32(plan.ExternType_EXTERNAL_TB)}}, param}, + {"wrong-format", newCompile(&plan.Query{LoadTag: true, StmtType: plan.Query_INSERT}), node, &tree.ExternParam{ExParamConst: tree.ExParamConst{Format: tree.PARQUET}}}, + {"nil-node", newCompile(&plan.Query{LoadTag: true, StmtType: plan.Query_INSERT}), nil, param}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, pipeline.ArrowExecutionScope_UnknownArrowExecutionScope, + test.c.arrowExecutionScope(test.node, test.param)) + }) + } +} + +func TestArrowLoadRolloutGateFailsClosedAndSerializesWhenDistributedOff(t *testing.T) { + param := &tree.ExternParam{ + ExParamConst: tree.ExParamConst{Format: tree.ARROW, ScanType: tree.INFILE}, + ExParam: tree.ExParam{Parallel: true}, + } + compile := &Compile{proc: testutil.NewProc(t)} + + _, err := compile.requireArrowLoadEnabled(param) + require.ErrorContains(t, err, "configuration is unavailable") + + frontend := &config.FrontendParameters{} + frontend.SetDefaultValues() + compile.proc.Ctx = context.WithValue( + context.Background(), config.ParameterUnitKey, + config.NewParameterUnit(frontend, nil, nil, nil), + ) + _, err = compile.requireArrowLoadEnabled(param) + require.ErrorContains(t, err, "disabled by configuration") + frontend.ArrowLoad.Enabled = true + settings, err := compile.requireArrowLoadEnabled(param) + require.NoError(t, err) + require.True(t, settings.Enabled) + require.False(t, settings.S3Enabled) + require.False(t, settings.DistributedEnabled) + serial := arrowParamForRollout(param, settings) + require.NotSame(t, param, serial) + require.False(t, serial.Parallel) + + frontend.ArrowLoad.Enabled = false + _, err = compile.requireArrowLoadEnabled(param) + require.ErrorContains(t, err, "disabled by configuration") + frontend.ArrowLoad.Enabled = true + + frontend.ArrowLoad.DistributedEnabled = false + settings, err = compile.requireArrowLoadEnabled(param) + require.NoError(t, err) + serial = arrowParamForRollout(param, settings) + require.NotSame(t, param, serial) + require.True(t, param.Parallel, "rollout fallback must not mutate the reusable parser parameter") + require.False(t, serial.Parallel) + + s3 := new(tree.ExternParam) + *s3 = *param + s3.ScanType = tree.S3 + frontend.ArrowLoad.S3Enabled = false + _, err = compile.requireArrowLoadEnabled(s3) + require.ErrorContains(t, err, "S3 or stage") + dynamicMinIO := new(tree.ExternParam) + *dynamicMinIO = *param + dynamicMinIO.Filepath = "minio,localhost:9000,us-east-1,bucket,key,secret,prefix:input.arrow" + _, err = compile.requireArrowLoadEnabled(dynamicMinIO) + require.ErrorContains(t, err, "S3 or stage") + frontend.ArrowLoad.S3Enabled = true + _, err = compile.requireArrowLoadEnabled(s3) + require.NoError(t, err) + _, err = compile.requireArrowLoadEnabled(dynamicMinIO) + require.NoError(t, err) + + frontend.ArrowLoad.DistributedEnabled = true + require.Same(t, param, arrowParamForRollout(param, frontend.ArrowLoad)) + + frontend.ArrowLoad.ForceMaterialize = true + materialized := arrowParamForRollout(param, frontend.ArrowLoad) + require.NotSame(t, param, materialized) + require.False(t, param.ArrowForceMaterialize, + "rollout fallback must not mutate the reusable parser parameter") + require.True(t, materialized.Parallel) + require.True(t, materialized.ArrowForceMaterialize) +} + +func TestArrowCompileRuntimeRemapsFileIndicesPerScope(t *testing.T) { + runtime := &arrowCompileRuntime{ + identitiesByPath: map[string]*pipeline.ArrowObjectIdentity{ + "a.arrow": {FileIndex: 4, Etag: "a", Size: 10}, + "b.arrow": {FileIndex: 9, VersionId: "b", Size: 20}, + }, + shardsByPath: map[string][]*pipeline.ArrowRecordBatchShard{ + "b.arrow": {{FileIndex: 9, RecordBatchStart: 3, RecordBatchEnd: 4, RequiredDictionaryBlockIndices: []int32{1}}}, + }, + conversionPlanVersion: arrowConversionPlanVersion, + } + identities := runtime.identitiesFor([]string{"b.arrow", "a.arrow"}) + require.Equal(t, int32(0), identities[0].FileIndex) + require.Equal(t, "b", identities[0].VersionId) + require.Equal(t, int32(1), identities[1].FileIndex) + require.Equal(t, "a", identities[1].Etag) + + shards := runtime.shardsFor([]string{"b.arrow", "a.arrow"}) + require.Len(t, shards, 1) + require.Equal(t, int32(0), shards[0].FileIndex) + shards[0].RequiredDictionaryBlockIndices[0] = 99 + require.Equal(t, int32(1), runtime.shardsByPath["b.arrow"][0].RequiredDictionaryBlockIndices[0], + "scope metadata must not alias the compile planner owner") +} + +func TestBuildArrowExternalAttrsUsesOnlyBinderProvenSourceColumns(t *testing.T) { + node := &plan.Node{ + ExternScan: &plan.ExternScan{TbColToDataCol: map[string]int32{"payload": 0, "id": 1}}, + TableDef: &plan.TableDef{Cols: []*plan.ColDef{ + {Name: "payload", Typ: plan.Type{Id: int32(types.T_varchar)}}, + {Name: "generated", Typ: plan.Type{Id: int32(types.T_int64)}, GeneratedCol: &plan.GeneratedCol{}}, + {Name: "id", Typ: plan.Type{Id: int32(types.T_int64)}}, + {Name: "hidden", Typ: plan.Type{Id: int32(types.T_int64)}, Hidden: true}, + {Name: "defaulted", Typ: plan.Type{Id: int32(types.T_int64)}}, + }}, + } + require.Equal(t, []plan.ExternAttr{ + {ColName: "payload", ColIndex: 0, ColFieldIndex: 0}, + {ColName: "id", ColIndex: 2, ColFieldIndex: 1}, + }, buildArrowExternalAttrs(node)) +} + +func TestPlanArrowCompileRuntimeBuildsFingerprintAndRecordShards(t *testing.T) { + ctx := context.Background() + fs, err := fileservice.NewMemoryFS("arrow-plan", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + payload := makeCompileArrowFile(t, nil, 4) + path := "arrow-plan:input.arrow" + require.NoError(t, fs.Write(ctx, fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{Offset: 0, Size: int64(len(payload)), Data: payload}}, + })) + node := &plan.Node{ + ExternScan: &plan.ExternScan{Type: int32(plan.ExternType_LOAD), TbColToDataCol: map[string]int32{"id": 0}}, + TableDef: &plan.TableDef{Cols: []*plan.ColDef{{ + Name: "id", Typ: plan.Type{Id: int32(types.T_int64), NotNullable: true}, + }}}, + } + param := &tree.ExternParam{ + ExParamConst: tree.ExParamConst{ + ScanType: tree.INFILE, Format: tree.ARROW, + ArrowContainer: tree.ARROW_CONTAINER_FILE, + }, + ExParam: tree.ExParam{ExternType: int32(plan.ExternType_LOAD), FileService: fs, Parallel: true}, + } + c := &Compile{proc: testutil.NewProc(t), ncpu: 3, addr: "local-cn"} + runtime, err := c.planArrowCompileRuntime(node, param, []string{path}, []int64{int64(len(payload))}) + require.NoError(t, err) + require.Len(t, runtime.schemaFingerprint, 32) + require.Equal(t, arrowConversionPlanVersion, runtime.conversionPlanVersion) + require.Len(t, runtime.shardsByPath[path], 3) + require.Equal(t, int32(0), runtime.shardsByPath[path][0].RecordBatchStart) + require.Equal(t, int32(4), runtime.shardsByPath[path][2].RecordBatchEnd) + var rows int64 + for _, shard := range runtime.shardsByPath[path] { + rows += shard.EstimatedRows + require.Positive(t, shard.EstimatedWireBytes) + } + require.Equal(t, int64(4), rows) +} + +func TestPlanArrowCompileRuntimeBalancesSkewedRecordWireBytes(t *testing.T) { + ctx := context.Background() + fs, err := fileservice.NewMemoryFS("arrow-skew", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + payload := makeCompileArrowVarlenFile(t, []int{64 << 10, 1, 1, 1}) + path := "arrow-skew:input.arrow" + require.NoError(t, fs.Write(ctx, fileservice.IOVector{ + FilePath: path, + Entries: []fileservice.IOEntry{{Offset: 0, Size: int64(len(payload)), Data: payload}}, + })) + node := &plan.Node{ + ExternScan: &plan.ExternScan{Type: int32(plan.ExternType_LOAD), TbColToDataCol: map[string]int32{"payload": 0}}, + TableDef: &plan.TableDef{Cols: []*plan.ColDef{{ + Name: "payload", Typ: plan.Type{Id: int32(types.T_varbinary)}, + }}}, + } + param := &tree.ExternParam{ + ExParamConst: tree.ExParamConst{ + ScanType: tree.INFILE, Format: tree.ARROW, + ArrowContainer: tree.ARROW_CONTAINER_FILE, + }, + ExParam: tree.ExParam{ExternType: int32(plan.ExternType_LOAD), FileService: fs, Parallel: true}, + } + c := &Compile{proc: testutil.NewProc(t), ncpu: 2, addr: "local-cn"} + runtime, err := c.planArrowCompileRuntime(node, param, []string{path}, []int64{int64(len(payload))}) + require.NoError(t, err) + require.Len(t, runtime.shardsByPath[path], 2) + require.Equal(t, int32(0), runtime.shardsByPath[path][0].RecordBatchStart) + require.Equal(t, int32(1), runtime.shardsByPath[path][0].RecordBatchEnd, + "the dominant record should not be grouped with tiny records") + require.Equal(t, int32(1), runtime.shardsByPath[path][1].RecordBatchStart) + require.Equal(t, int32(4), runtime.shardsByPath[path][1].RecordBatchEnd) +} + +func TestPlanArrowCompileRuntimeRejectsCrossObjectSchemaDrift(t *testing.T) { + ctx := context.Background() + fs, err := fileservice.NewMemoryFS("arrow-drift", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + baseMetadata := arrow.NewMetadata([]string{"source"}, []string{"first"}) + changedMetadata := arrow.NewMetadata([]string{"source"}, []string{"second"}) + payloads := [][]byte{ + makeCompileArrowFile(t, &baseMetadata, 1), + makeCompileArrowFile(t, &changedMetadata, 1), + } + paths := []string{"arrow-drift:first.arrow", "arrow-drift:second.arrow"} + sizes := make([]int64, len(paths)) + for index := range paths { + sizes[index] = int64(len(payloads[index])) + require.NoError(t, fs.Write(ctx, fileservice.IOVector{ + FilePath: paths[index], + Entries: []fileservice.IOEntry{{Offset: 0, Size: sizes[index], Data: payloads[index]}}, + })) + } + node := &plan.Node{ + ExternScan: &plan.ExternScan{Type: int32(plan.ExternType_LOAD), TbColToDataCol: map[string]int32{"id": 0}}, + TableDef: &plan.TableDef{Cols: []*plan.ColDef{{ + Name: "id", Typ: plan.Type{Id: int32(types.T_int64), NotNullable: true}, + }}}, + } + param := &tree.ExternParam{ + ExParamConst: tree.ExParamConst{ + ScanType: tree.INFILE, Format: tree.ARROW, ArrowContainer: tree.ARROW_CONTAINER_FILE, + }, + ExParam: tree.ExParam{ExternType: int32(plan.ExternType_LOAD), FileService: fs}, + } + c := &Compile{proc: testutil.NewProc(t), ncpu: 1} + _, err = c.planArrowCompileRuntime(node, param, paths, sizes) + require.ErrorContains(t, err, "differs from earlier objects") +} + +func TestCompileArrowRecordBatchFanoutPublishesOneShardPerScope(t *testing.T) { + const path = "arrow-plan:input.arrow" + node := &plan.Node{ + ExternScan: &plan.ExternScan{Type: int32(plan.ExternType_LOAD), TbColToDataCol: map[string]int32{"id": 0}}, + TableDef: &plan.TableDef{Cols: []*plan.ColDef{{ + Name: "id", Typ: plan.Type{Id: int32(types.T_int64), NotNullable: true}, + }}}, + } + param := &tree.ExternParam{ + ExParamConst: tree.ExParamConst{ + ScanType: tree.INFILE, Format: tree.ARROW, ArrowContainer: tree.ARROW_CONTAINER_FILE, + }, + ExParam: tree.ExParam{ExternType: int32(plan.ExternType_LOAD), Parallel: true}, + } + runtime := &arrowCompileRuntime{ + identitiesByPath: map[string]*pipeline.ArrowObjectIdentity{ + path: {FileIndex: 0, Etag: "etag-v1", Size: 1000}, + }, + shardsByPath: map[string][]*pipeline.ArrowRecordBatchShard{ + path: { + {FileIndex: 0, RecordBatchStart: 0, RecordBatchEnd: 2, EstimatedRows: 20}, + {FileIndex: 0, RecordBatchStart: 2, RecordBatchEnd: 4, EstimatedRows: 20}, + {FileIndex: 0, RecordBatchStart: 4, RecordBatchEnd: 5, EstimatedRows: 10}, + }, + }, + schemaFingerprint: bytes.Repeat([]byte{0x5a}, 32), + conversionPlanVersion: arrowConversionPlanVersion, + } + c := NewMockCompile(t) + c.addr = "local-cn" + c.ncpu = 3 + c.anal = &AnalyzeModule{isFirst: true, qry: &plan.Query{LoadTag: true, StmtType: plan.Query_INSERT}} + + scopes, err := c.compileExternScanArrowRecordBatchFanout( + node, param, path, 1000, true, runtime, + ) + require.NoError(t, err) + require.Len(t, scopes, 3) + require.True(t, param.Parallel, "compile must not mutate the reusable parser parameter") + require.False(t, c.anal.isFirst) + for index, scope := range scopes { + require.True(t, scope.IsLoad) + require.Equal(t, 1, scope.NodeInfo.Mcpu) + op, ok := scope.RootOp.(*external.External) + require.True(t, ok) + require.False(t, op.Es.Extern.Parallel) + require.True(t, op.Es.ArrowDistributedExecution, + "fanout must retain worker-gate intent after clearing the user request") + require.Equal(t, []string{path}, op.Es.FileList) + require.Equal(t, []int64{1000}, op.Es.FileSize) + require.Equal(t, pipeline.ArrowExecutionScope_ArrowLoadData, op.Es.ArrowExecutionScope) + require.Equal(t, runtime.schemaFingerprint, op.Es.ArrowSchemaFingerprint) + require.Equal(t, arrowConversionPlanVersion, op.Es.ArrowConversionPlanVersion) + require.Len(t, op.Es.ArrowObjectIdentities, 1) + require.Equal(t, int32(0), op.Es.ArrowObjectIdentities[0].FileIndex) + require.Len(t, op.Es.ArrowRecordBatchShards, 1) + got := op.Es.ArrowRecordBatchShards[0] + want := runtime.shardsByPath[path][index] + require.Equal(t, int32(0), got.FileIndex) + require.Equal(t, want.RecordBatchStart, got.RecordBatchStart) + require.Equal(t, want.RecordBatchEnd, got.RecordBatchEnd) + require.Equal(t, want.EstimatedRows, got.EstimatedRows) + } +} + +func TestCompileArrowWholeFileFanoutRetainsWorkerGateIntent(t *testing.T) { + node := &plan.Node{ + ExternScan: &plan.ExternScan{Type: int32(plan.ExternType_LOAD), TbColToDataCol: map[string]int32{"id": 0}}, + TableDef: &plan.TableDef{Cols: []*plan.ColDef{{ + Name: "id", Typ: plan.Type{Id: int32(types.T_int64), NotNullable: true}, + }}}, + } + param := &tree.ExternParam{ + ExParamConst: tree.ExParamConst{ScanType: tree.INFILE, Format: tree.ARROW, ArrowContainer: tree.ARROW_CONTAINER_FILE}, + ExParam: tree.ExParam{ExternType: int32(plan.ExternType_LOAD), Parallel: true}, + } + files := []string{"arrow-plan:one.arrow", "arrow-plan:two.arrow"} + runtime := &arrowCompileRuntime{ + identitiesByPath: map[string]*pipeline.ArrowObjectIdentity{ + files[0]: {FileIndex: 0, Etag: "etag-one", Size: 100}, + files[1]: {FileIndex: 1, Etag: "etag-two", Size: 100}, + }, + shardsByPath: map[string][]*pipeline.ArrowRecordBatchShard{}, + } + c := NewMockCompile(t) + c.addr = "local-cn" + c.ncpu = 2 + c.anal = &AnalyzeModule{isFirst: true, qry: &plan.Query{LoadTag: true, StmtType: plan.Query_INSERT}} + + scopes, err := c.compileExternScanWholeFileFanout(node, param, files, []int64{100, 100}, true, false, runtime) + require.NoError(t, err) + require.Len(t, scopes, 2) + for _, scope := range scopes { + op, ok := scope.RootOp.(*external.External) + require.True(t, ok) + require.False(t, op.Es.Extern.Parallel) + require.True(t, op.Es.ArrowDistributedExecution) + } +} + +func makeCompileArrowFile(t *testing.T, metadata *arrow.Metadata, recordCount int) []byte { + t.Helper() + allocator := memory.NewGoAllocator() + schema := arrow.NewSchema([]arrow.Field{{Name: "id", Type: arrow.PrimitiveTypes.Int64}}, metadata) + var output bytes.Buffer + writer, err := ipc.NewFileWriter(&output, ipc.WithSchema(schema), ipc.WithAllocator(allocator)) + require.NoError(t, err) + for index := 0; index < recordCount; index++ { + builder := array.NewInt64Builder(allocator) + builder.Append(int64(index)) + values := builder.NewArray() + record := array.NewRecordBatch(schema, []arrow.Array{values}, 1) + require.NoError(t, writer.Write(record)) + record.Release() + values.Release() + builder.Release() + } + require.NoError(t, writer.Close()) + return append([]byte(nil), output.Bytes()...) +} + +func makeCompileArrowVarlenFile(t *testing.T, sizes []int) []byte { + t.Helper() + allocator := memory.NewGoAllocator() + schema := arrow.NewSchema([]arrow.Field{{Name: "payload", Type: arrow.BinaryTypes.Binary}}, nil) + var output bytes.Buffer + writer, err := ipc.NewFileWriter(&output, ipc.WithSchema(schema), ipc.WithAllocator(allocator)) + require.NoError(t, err) + for index, size := range sizes { + builder := array.NewBinaryBuilder(allocator, arrow.BinaryTypes.Binary) + builder.Append(bytes.Repeat([]byte{byte(index + 1)}, size)) + values := builder.NewArray() + record := array.NewRecordBatch(schema, []arrow.Array{values}, 1) + require.NoError(t, writer.Write(record)) + record.Release() + values.Release() + builder.Release() + } + require.NoError(t, writer.Close()) + return append([]byte(nil), output.Bytes()...) +} diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index 3943ccad03e07..11462896631be 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -15,6 +15,7 @@ package compile import ( + "bytes" "cmp" "context" "encoding/hex" @@ -29,6 +30,7 @@ import ( "strings" "time" + "github.com/apache/arrow-go/v18/arrow" "github.com/google/uuid" "github.com/parquet-go/parquet-go" @@ -39,6 +41,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/system" commonutil "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/config" + "github.com/matrixorigin/matrixone/pkg/container/arrowbridge" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -58,6 +61,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" "github.com/matrixorigin/matrixone/pkg/sql/colexec/dispatch" "github.com/matrixorigin/matrixone/pkg/sql/colexec/external" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/external/arrowio" "github.com/matrixorigin/matrixone/pkg/sql/colexec/fill" "github.com/matrixorigin/matrixone/pkg/sql/colexec/filter" "github.com/matrixorigin/matrixone/pkg/sql/colexec/group" @@ -2399,7 +2403,7 @@ func (c *Compile) getReadWriteParallelFlag(param *tree.ExternParam, fileList []s if !param.Parallel { return false, false } - if param.Format == tree.PARQUET { + if param.Format == tree.PARQUET || param.Format == tree.ARROW { return false, true } if param.Local || crt.GetCompressType(param.CompressType, fileList[0]) != tree.NOCOMPRESS { @@ -2408,6 +2412,50 @@ func (c *Compile) getReadWriteParallelFlag(param *tree.ExternParam, fileList []s return true, true } +func (c *Compile) arrowExecutionScope(node *plan.Node, param *tree.ExternParam) pipeline.ArrowExecutionScope { + if c == nil || param == nil || param.Format != tree.ARROW || node == nil || node.ExternScan == nil || + c.anal == nil || c.anal.qry == nil { + return pipeline.ArrowExecutionScope_UnknownArrowExecutionScope + } + if c.anal.qry.LoadTag && c.anal.qry.StmtType == plan.Query_INSERT && + node.ExternScan.Type == int32(plan.ExternType_LOAD) { + return pipeline.ArrowExecutionScope_ArrowLoadData + } + return pipeline.ArrowExecutionScope_UnknownArrowExecutionScope +} + +func (c *Compile) requireArrowLoadEnabled( + param *tree.ExternParam, +) (config.ArrowLoadParameters, error) { + var proc *process.Process + if c != nil { + proc = c.proc + } + return plan2.RequireArrowLoadEnabled(proc, param) +} + +func arrowParamForRollout( + param *tree.ExternParam, + settings config.ArrowLoadParameters, +) *tree.ExternParam { + if param == nil { + return param + } + // Rollout settings are sampled once while the statement is compiled. A + // private copy prevents a service-level change, or another compile using the + // parser-owned value, from changing policy halfway through one generation. + parallel := param.Parallel && settings.DistributedEnabled + if parallel == param.Parallel && + settings.ForceMaterialize == param.ArrowForceMaterialize { + return param + } + rollout := new(tree.ExternParam) + *rollout = *param + rollout.Parallel = parallel + rollout.ArrowForceMaterialize = settings.ForceMaterialize + return rollout +} + func (c *Compile) getExternalFileListAndSize(node *plan.Node, param *tree.ExternParam) (fileList []string, fileSize []int64, err error) { // Hive partition tables use recursive list-and-filter discovery, not ReadDir. // ReadDir requires glob patterns in filepath; Hive base paths are opaque directories. @@ -2441,7 +2489,8 @@ func (c *Compile) getExternalFileListAndSize(node *plan.Node, param *tree.Extern return nil, nil, err } case int32(plan.ExternType_LOAD): - if param.Format == tree.PARQUET && strings.ContainsAny(strings.TrimSpace(param.Filepath), "*?[") { + if (param.Format == tree.PARQUET || param.Format == tree.ARROW) && + strings.ContainsAny(strings.TrimSpace(param.Filepath), "*?[") { fileList, fileSize, err = plan2.ReadDir(param) if err != nil { return nil, nil, err @@ -2708,6 +2757,17 @@ func (c *Compile) compileExternScanWithPlanNodeID(node *plan.Node, planNodeID in if err != nil { return nil, err } + if param.Format == tree.ARROW && + c.arrowExecutionScope(node, param) != pipeline.ArrowExecutionScope_ArrowLoadData { + return nil, moerr.NewNotSupported(c.proc.Ctx, "Arrow format is supported only by LOAD DATA") + } + if param.Format == tree.ARROW { + settings, gateErr := c.requireArrowLoadEnabled(param) + if gateErr != nil { + return nil, gateErr + } + param = arrowParamForRollout(param, settings) + } strictSqlMode = effectiveExternalStrictMode(c.proc, param, strictSqlMode) if param.ScanType == tree.INLINE { @@ -2738,6 +2798,23 @@ func (c *Compile) compileExternScanWithPlanNodeID(node *plan.Node, planNodeID in ret.Proc = c.proc.NewNoContextChildProc(0) return []*Scope{ret}, nil } + var arrowRuntime *arrowCompileRuntime + if param.Format == tree.ARROW { + arrowRuntime, err = c.planArrowCompileRuntime(node, param, fileList, fileSize) + if err != nil { + return nil, err + } + if len(fileList) == 1 && len(arrowRuntime.shardsByPath[fileList[0]]) > 1 { + return c.compileExternScanArrowRecordBatchFanout( + node, param, fileList[0], fileSize[0], strictSqlMode, arrowRuntime, + ) + } + if param.Parallel && len(fileList) > 1 { + return c.compileExternScanWholeFileFanout( + node, param, fileList, fileSize, strictSqlMode, false, arrowRuntime, + ) + } + } if param.HivePartitioning { return c.compileExternScanHiveFileFanout(node, param, fileList, fileSize, strictSqlMode) @@ -2772,9 +2849,9 @@ func (c *Compile) compileExternScanWithPlanNodeID(node *plan.Node, planNodeID in if readParallel && writeParallel { return c.compileExternScanParallelReadWrite(node, param, fileList, fileSize, strictSqlMode) } else if writeParallel { - return c.compileExternScanParallelWrite(node, param, fileList, fileSize, strictSqlMode) + return c.compileExternScanParallelWrite(node, param, fileList, fileSize, strictSqlMode, arrowRuntime) } else { - return c.compileExternScanSerialReadWrite(node, param, fileList, fileSize, strictSqlMode) + return c.compileExternScanSerialReadWrite(node, param, fileList, fileSize, strictSqlMode, arrowRuntime) } } @@ -2823,7 +2900,7 @@ func (c *Compile) compileDatastreamScan(node *plan.Node, strictSqlMode bool) ([] scope := c.constructScopeForExternal(c.addr, false) currentFirstFlag := c.anal.isFirst - op := constructExternal(node, param, c.proc.Ctx, nil, nil, nil, strictSqlMode) + op := constructExternal(node, param, c.proc.Ctx, nil, nil, nil, strictSqlMode, c.arrowExecutionScope(node, param)) op.SetAnalyzeControl(c.anal.curNodeIdx, currentFirstFlag) scope.setRootOperator(op) c.anal.isFirst = false @@ -2900,7 +2977,7 @@ func (c *Compile) compileForeignScan(node *plan.Node, strictSqlMode bool) ([]*Sc scope := c.constructScopeForExternal(c.addr, false) currentFirstFlag := c.anal.isFirst - op := constructExternal(node, param, c.proc.Ctx, queryList, fileSize, nil, strictSqlMode) + op := constructExternal(node, param, c.proc.Ctx, queryList, fileSize, nil, strictSqlMode, c.arrowExecutionScope(node, param)) op.SetAnalyzeControl(c.anal.curNodeIdx, currentFirstFlag) scope.setRootOperator(op) c.anal.isFirst = false @@ -2947,7 +3024,7 @@ func (c *Compile) compileKafkaScan(node *plan.Node, strictSqlMode bool) ([]*Scop scope := c.constructScopeForExternal(c.addr, false) currentFirstFlag := c.anal.isFirst - op := constructExternal(node, param, c.proc.Ctx, nil, nil, nil, strictSqlMode) + op := constructExternal(node, param, c.proc.Ctx, nil, nil, nil, strictSqlMode, c.arrowExecutionScope(node, param)) op.SetAnalyzeControl(c.anal.curNodeIdx, currentFirstFlag) scope.setRootOperator(op) c.anal.isFirst = false @@ -3246,7 +3323,7 @@ func (c *Compile) getLoadWriteS3ParallelSize(node *plan.Node, cpuNum int) int { func (c *Compile) compileExternValueScan(node *plan.Node, param *tree.ExternParam, strictSqlMode bool) ([]*Scope, error) { s := c.constructScopeForExternal(c.addr, false) currentFirstFlag := c.anal.isFirst - op := constructExternal(node, param, c.proc.Ctx, nil, nil, nil, strictSqlMode) + op := constructExternal(node, param, c.proc.Ctx, nil, nil, nil, strictSqlMode, c.arrowExecutionScope(node, param)) op.SetIdx(c.anal.curNodeIdx) op.SetIsFirst(currentFirstFlag) s.setRootOperator(op) @@ -3255,7 +3332,7 @@ func (c *Compile) compileExternValueScan(node *plan.Node, param *tree.ExternPara } // construct one thread to read the file data, then dispatch to mcpu thread to get the filedata for insert -func (c *Compile) compileExternScanParallelWrite(node *plan.Node, param *tree.ExternParam, fileList []string, fileSize []int64, strictSqlMode bool) ([]*Scope, error) { +func (c *Compile) compileExternScanParallelWrite(node *plan.Node, param *tree.ExternParam, fileList []string, fileSize []int64, strictSqlMode bool, arrowRuntime ...*arrowCompileRuntime) ([]*Scope, error) { loadEmptyNumericAsZero := param.ExternType == int32(plan.ExternType_LOAD) && (param.Parallel || param.ParallelLoadRequested) param.Parallel = false @@ -3267,7 +3344,7 @@ func (c *Compile) compileExternScanParallelWrite(node *plan.Node, param *tree.Ex } scope := c.constructScopeForExternal(c.addr, false) currentFirstFlag := c.anal.isFirst - extern := constructExternal(node, param, c.proc.Ctx, fileList, fileSize, fileOffsetTmp, strictSqlMode) + extern := constructExternal(node, param, c.proc.Ctx, fileList, fileSize, fileOffsetTmp, strictSqlMode, c.arrowExecutionScope(node, param), arrowRuntime...) parallelLoad := true if len(fileList) > 0 && crt.GetCompressType(param.CompressType, fileList[0]) != tree.NOCOMPRESS { parallelLoad = false @@ -3347,6 +3424,361 @@ type parquetRowGroupSegment struct { load int64 } +const ( + arrowConversionPlanVersion = arrowbridge.ConversionPlanVersion + arrowPlanningAccountLimit uint64 = 64 << 20 + arrowPlanningAllocationSlots = 65_536 + arrowMaxPlannedShards = 4_096 +) + +type arrowCompileRuntime struct { + identitiesByPath map[string]*pipeline.ArrowObjectIdentity + shardsByPath map[string][]*pipeline.ArrowRecordBatchShard + schemaFingerprint []byte + conversionPlanVersion uint32 +} + +func (r *arrowCompileRuntime) identitiesFor(fileList []string) []*pipeline.ArrowObjectIdentity { + if r == nil || len(r.identitiesByPath) == 0 { + return nil + } + identities := make([]*pipeline.ArrowObjectIdentity, 0, len(fileList)) + for localIndex, path := range fileList { + identity := r.identitiesByPath[path] + if identity == nil { + continue + } + clone := *identity + clone.FileIndex = int32(localIndex) + identities = append(identities, &clone) + } + return identities +} + +func (r *arrowCompileRuntime) shardsFor(fileList []string) []*pipeline.ArrowRecordBatchShard { + if r == nil || len(r.shardsByPath) == 0 { + return nil + } + var shards []*pipeline.ArrowRecordBatchShard + for localIndex, path := range fileList { + for _, shard := range r.shardsByPath[path] { + if shard == nil { + continue + } + clone := *shard + clone.FileIndex = int32(localIndex) + clone.RequiredDictionaryBlockIndices = append([]int32(nil), shard.RequiredDictionaryBlockIndices...) + shards = append(shards, &clone) + } + } + return shards +} + +func (r *arrowCompileRuntime) forShard( + path string, + shard *pipeline.ArrowRecordBatchShard, +) *arrowCompileRuntime { + if r == nil || shard == nil { + return nil + } + result := &arrowCompileRuntime{ + identitiesByPath: make(map[string]*pipeline.ArrowObjectIdentity, 1), + shardsByPath: make(map[string][]*pipeline.ArrowRecordBatchShard, 1), + schemaFingerprint: append([]byte(nil), r.schemaFingerprint...), + conversionPlanVersion: r.conversionPlanVersion, + } + if identity := r.identitiesByPath[path]; identity != nil { + clone := *identity + result.identitiesByPath[path] = &clone + } + clone := *shard + clone.FileIndex = 0 + clone.RequiredDictionaryBlockIndices = append([]int32(nil), shard.RequiredDictionaryBlockIndices...) + result.shardsByPath[path] = []*pipeline.ArrowRecordBatchShard{&clone} + return result +} + +func (c *Compile) planArrowCompileRuntime( + node *plan.Node, + param *tree.ExternParam, + fileList []string, + fileSize []int64, +) (_ *arrowCompileRuntime, retErr error) { + ctx := c.proc.Ctx + registry, err := mpool.NewAllocationAccountRegistry(1, arrowPlanningAllocationSlots) + if err != nil { + return nil, err + } + account, err := registry.Open(arrowPlanningAccountLimit) + if err != nil { + return nil, err + } + defer func() { + account.Seal() + _, finalizeErr := registry.Finalize(account) + if finalizeErr != nil { + retErr = errors.Join(retErr, finalizeErr) + } + }() + admission, err := fileservice.NewAllocationAccountRangeAdmission( + account, + mpool.AllocationOwnerExternal, + 2, + mpool.AllocationCapacityClassDefault, + ) + if err != nil { + return nil, err + } + runtime := &arrowCompileRuntime{ + identitiesByPath: make(map[string]*pipeline.ArrowObjectIdentity, len(fileList)), + shardsByPath: make(map[string][]*pipeline.ArrowRecordBatchShard), + conversionPlanVersion: arrowConversionPlanVersion, + } + attrs := buildArrowExternalAttrs(node) + targets, err := external.BuildArrowTargets(ctx, attrs, node.TableDef.Cols) + if err != nil { + return nil, err + } + matchMode := arrowbridge.MatchByName + if param.ArrowMatchByPosition { + matchMode = arrowbridge.MatchByPosition + } + container, err := arrowCompileContainer(param.ArrowContainer) + if err != nil { + return nil, err + } + for fileIndex, filePath := range fileList { + if fileIndex >= len(fileSize) || fileSize[fileIndex] < 0 { + return nil, moerr.NewInvalidInputf(ctx, "Arrow file %d has no valid planned size", fileIndex) + } + fs, readPath, err := plan2.GetForETLWithType(param, filePath) + if err != nil { + return nil, err + } + identityFS, ok := fs.(fileservice.ObjectIdentityFileService) + if !ok { + if param.ScanType == tree.S3 { + return nil, moerr.NewNotSupported(ctx, "S3 Arrow LOAD requires versioned or conditional object reads") + } + } else { + identity, err := identityFS.StatFileIdentity(ctx, readPath) + if err != nil { + return nil, err + } + if err := identity.Validate(); err != nil { + return nil, err + } + if identity.Size != fileSize[fileIndex] { + return nil, errors.Join(fileservice.ErrObjectChanged, + moerr.NewInternalErrorNoCtxf("Arrow object %d size changed from %d to %d", + fileIndex, fileSize[fileIndex], identity.Size)) + } + lastModified := int64(0) + if !identity.LastModified.IsZero() { + lastModified = identity.LastModified.UnixNano() + } + runtime.identitiesByPath[filePath] = &pipeline.ArrowObjectIdentity{ + FileIndex: int32(fileIndex), VersionId: identity.VersionID, Etag: identity.ETag, + Size: identity.Size, LastModifiedUnixNano: lastModified, + } + } + var expectedIdentity *fileservice.ObjectIdentity + if planned := runtime.identitiesByPath[filePath]; planned != nil { + expectedIdentity = &fileservice.ObjectIdentity{ + VersionID: planned.VersionId, ETag: planned.Etag, Size: planned.Size, + } + if planned.LastModifiedUnixNano != 0 { + expectedIdentity.LastModified = time.Unix(0, planned.LastModifiedUnixNano).UTC() + } + } + actualContainer := container + if actualContainer == arrowio.ContainerAuto { + actualContainer, err = arrowio.DetectContainer( + ctx, fs, readPath, fileSize[fileIndex], admission, + arrowio.Options{ExpectedIdentity: expectedIdentity}, + ) + if err != nil { + return nil, err + } + } + + var schema *arrow.Schema + var filePlan *arrowio.FilePlan + switch actualContainer { + case arrowio.ContainerFile: + filePlan, err = arrowio.InspectFile( + ctx, fs, readPath, fileSize[fileIndex], admission, + arrowio.Options{ExpectedIdentity: expectedIdentity}, + ) + if err != nil { + return nil, err + } + schema = filePlan.Schema + case arrowio.ContainerStream: + reader, openErr := arrowio.Open( + ctx, fs, readPath, fileSize[fileIndex], actualContainer, admission, + arrowio.Options{ExpectedIdentity: expectedIdentity}, + ) + if openErr != nil { + return nil, openErr + } + schema = reader.Schema() + if closeErr := reader.Close(); closeErr != nil { + return nil, closeErr + } + default: + return nil, moerr.NewInvalidInputf(ctx, "invalid Arrow IPC container %d", actualContainer) + } + // Compile and execution must fingerprint the same explicit LOAD policy; + // exact result protocols use a separate binder and version contract. + conversionPlan, err := arrowbridge.BindLoad(ctx, schema, targets, matchMode) + if err != nil { + return nil, err + } + fingerprint := conversionPlan.Fingerprint() + if len(runtime.schemaFingerprint) == 0 { + runtime.schemaFingerprint = append([]byte(nil), fingerprint[:]...) + } else if !bytes.Equal(runtime.schemaFingerprint, fingerprint[:]) { + return nil, moerr.NewInvalidInputf(ctx, + "Arrow schema and conversion contract for object %d differs from earlier objects", fileIndex) + } + if filePlan != nil && param.Parallel && len(fileList) == 1 && len(filePlan.RecordBatches) > 1 { + desiredShards := len(c.getHiveFileFanoutNodes( + param, min(len(filePlan.RecordBatches), arrowMaxPlannedShards), + )) + if desiredShards > 1 { + runtime.shardsByPath[filePath], err = buildArrowRecordBatchShards( + fileIndex, filePlan, desiredShards, + ) + if err != nil { + return nil, err + } + } + } + } + return runtime, nil +} + +func arrowCompileContainer(value string) (arrowio.Container, error) { + switch value { + case "", tree.ARROW_CONTAINER_AUTO: + return arrowio.ContainerAuto, nil + case tree.ARROW_CONTAINER_FILE: + return arrowio.ContainerFile, nil + case tree.ARROW_CONTAINER_STREAM: + return arrowio.ContainerStream, nil + default: + return 0, moerr.NewInvalidInputNoCtxf("invalid Arrow IPC container %q", value) + } +} + +func buildArrowRecordBatchShards( + fileIndex int, + filePlan *arrowio.FilePlan, + desired int, +) ([]*pipeline.ArrowRecordBatchShard, error) { + if filePlan == nil || desired <= 0 || len(filePlan.RecordBatches) == 0 { + return nil, moerr.NewInvalidInputNoCtx("invalid Arrow record-batch shard plan") + } + desired = min(desired, len(filePlan.RecordBatches), arrowMaxPlannedShards) + shards := make([]*pipeline.ArrowRecordBatchShard, 0, desired) + var remainingWireBytes int64 + for _, record := range filePlan.RecordBatches { + if record.WireBytes < 0 || record.WireBytes > math.MaxInt64-remainingWireBytes { + return nil, moerr.NewInvalidInputNoCtx("Arrow record-batch wire size overflows") + } + remainingWireBytes += record.WireBytes + } + start := 0 + for shardIndex := 0; shardIndex < desired; shardIndex++ { + remainingRecords := len(filePlan.RecordBatches) - start + remainingShards := desired - shardIndex + end := start + 1 + if remainingShards > 1 { + divisor := int64(remainingShards) + targetBytes := remainingWireBytes / divisor + if remainingWireBytes%divisor != 0 { + targetBytes++ + } + var shardBytes int64 + lastAllowedEnd := len(filePlan.RecordBatches) - remainingShards + 1 + for end <= lastAllowedEnd { + shardBytes += filePlan.RecordBatches[end-1].WireBytes + if shardBytes >= targetBytes || end == lastAllowedEnd { + break + } + end++ + } + } else { + end = start + remainingRecords + } + fileShard, rows, wireBytes, err := filePlan.Shard(start, end) + if err != nil { + return nil, err + } + shards = append(shards, &pipeline.ArrowRecordBatchShard{ + FileIndex: int32(fileIndex), RecordBatchStart: fileShard.RecordBatchStart, + RecordBatchEnd: fileShard.RecordBatchEnd, + RequiredDictionaryBlockIndices: append( + []int32(nil), fileShard.RequiredDictionaryBlockIndices..., + ), + EstimatedRows: rows, EstimatedWireBytes: wireBytes, + }) + for _, record := range filePlan.RecordBatches[start:end] { + remainingWireBytes -= record.WireBytes + } + start = end + } + return shards, nil +} + +func (c *Compile) compileExternScanArrowRecordBatchFanout( + node *plan.Node, + param *tree.ExternParam, + filePath string, + fileSize int64, + strictSQLMode bool, + runtime *arrowCompileRuntime, +) ([]*Scope, error) { + shards := runtime.shardsByPath[filePath] + if len(shards) <= 1 { + return nil, moerr.NewInvalidInput(c.proc.Ctx, "Arrow record-batch fanout requires multiple shards") + } + nodes := c.getHiveFileFanoutNodes(param, len(shards)) + if len(nodes) != len(shards) { + return nil, moerr.NewInternalErrorf(c.proc.Ctx, + "Arrow planned %d record-batch shards for %d workers", len(shards), len(nodes)) + } + stageNodes := c.queryWorkerStageNodes() + scopes := make([]*Scope, 0, len(shards)) + currentFirstFlag := c.anal.isFirst + for index, shard := range shards { + shardParam := new(tree.ExternParam) + *shardParam = *param + shardParam.Parallel = false + remote := param.ScanType == tree.S3 && len(stageNodes) > 0 + scope := c.constructScopeForExternalNode(nodes[index], remote) + scope.NodeInfo.Mcpu = 1 + scope.IsLoad = true + op := constructExternal( + node, shardParam, c.proc.Ctx, + []string{filePath}, []int64{fileSize}, makeWholeFileOffsets(1), + strictSQLMode, c.arrowExecutionScope(node, shardParam), + runtime.forShard(filePath, shard), + ) + // A shard must not retain Parallel=true: that flag requests generic + // splitting and would split this already-planned record range again. + // Preserve the distinct execution fact for the receiving CN's rollout + // gate before this scope is serialized over MORPC. + op.Es.ArrowDistributedExecution = true + op.SetAnalyzeControl(c.anal.curNodeIdx, currentFirstFlag) + scope.setRootOperator(op) + scopes = append(scopes, scope) + } + c.anal.isFirst = false + return scopes, nil +} + type icebergDataFileScopeShard struct { node engine.Node fileList []string @@ -3375,13 +3807,13 @@ func (c *Compile) compileExternScanParquetLoadFileFanout(node *plan.Node, param return c.compileExternScanWholeFileFanout(node, param, fileList, fileSize, strictSqlMode, true) } -func (c *Compile) compileExternScanWholeFileFanout(node *plan.Node, param *tree.ExternParam, fileList []string, fileSize []int64, strictSqlMode bool, parquetWholeFileFanout bool) ([]*Scope, error) { +func (c *Compile) compileExternScanWholeFileFanout(node *plan.Node, param *tree.ExternParam, fileList []string, fileSize []int64, strictSqlMode bool, parquetWholeFileFanout bool, arrowRuntime ...*arrowCompileRuntime) ([]*Scope, error) { nodes := c.getHiveFileFanoutNodes(param, len(fileList)) shards := splitHiveFileShards(fileList, fileSize, nodes) if len(shards) <= 1 { serialParam := *param serialParam.Parallel = false - return c.compileExternScanSerialReadWrite(node, &serialParam, fileList, fileSize, strictSqlMode) + return c.compileExternScanSerialReadWrite(node, &serialParam, fileList, fileSize, strictSqlMode, arrowRuntime...) } ss := make([]*Scope, 0, len(shards)) @@ -3405,8 +3837,13 @@ func (c *Compile) compileExternScanWholeFileFanout(node *plan.Node, param *tree. shard.fileList, shard.fileSize, makeWholeFileOffsets(len(shard.fileList)), strictSqlMode, + c.arrowExecutionScope(node, shardParam), + arrowRuntime..., ) op.Es.ParquetWholeFileFanout = parquetWholeFileFanout + // Whole-file Arrow fanout also clears Extern.Parallel above. Keep the + // execution-side authorization signal independent of that user request. + op.Es.ArrowDistributedExecution = len(arrowRuntime) > 0 op.SetAnalyzeControl(c.anal.curNodeIdx, currentFirstFlag) scope.setRootOperator(op) ss = append(ss, scope) @@ -3452,6 +3889,7 @@ func (c *Compile) compileExternScanIcebergShard( shard.fileList, shard.fileSize, makeWholeFileOffsets(len(shard.fileList)), strictSqlMode, + c.arrowExecutionScope(node, param), ) if err := attachIcebergRuntimeToExternal(c.proc.Ctx, op, runtime, shard.dataTasks); err != nil { return nil, err @@ -3499,6 +3937,7 @@ func (c *Compile) compileExternScanParquetRowGroupFanout( shard.fileList, shard.fileSize, makeWholeFileOffsets(len(shard.fileList)), strictSqlMode, + c.arrowExecutionScope(node, shardParam), ) op.Es.ParquetRowGroupShards = shard.rowGroupShards op.SetAnalyzeControl(c.anal.curNodeIdx, currentFirstFlag) @@ -4364,7 +4803,7 @@ func (c *Compile) compileExternScanParallelReadWrite(node *plan.Node, param *tre } logutil.Infof("compileExternScanParallelReadWrite, len of cnList is %d, cn addr is %s, mcpu is %d, filepath is %s, file size is %d", len(stageNodes), stageNodes[i].Addr, scope.NodeInfo.Mcpu, param.ExParamConst.Filepath, param.ExParamConst.FileSize) logutil.Infof("compileExternScanParallelReadWrite, %v\n", fileOffsetTmp) - op := constructExternal(node, param, c.proc.Ctx, fileList, fileSize, fileOffsetTmp, strictSqlMode) + op := constructExternal(node, param, c.proc.Ctx, fileList, fileSize, fileOffsetTmp, strictSqlMode, c.arrowExecutionScope(node, param)) op.SetAnalyzeControl(c.anal.curNodeIdx, currentFirstFlag) scope.setRootOperator(op) pre += count @@ -4377,7 +4816,7 @@ func (c *Compile) compileExternScanParallelReadWrite(node *plan.Node, param *tre return ss, nil } -func (c *Compile) compileExternScanSerialReadWrite(node *plan.Node, param *tree.ExternParam, fileList []string, fileSize []int64, strictSqlMode bool) ([]*Scope, error) { +func (c *Compile) compileExternScanSerialReadWrite(node *plan.Node, param *tree.ExternParam, fileList []string, fileSize []int64, strictSqlMode bool, arrowRuntime ...*arrowCompileRuntime) ([]*Scope, error) { ss := make([]*Scope, 1) ss[0] = c.constructScopeForExternal(c.addr, param.Parallel) @@ -4389,7 +4828,7 @@ func (c *Compile) compileExternScanSerialReadWrite(node *plan.Node, param *tree. fileOffsetTmp[j].Offset = make([]int64, 0) fileOffsetTmp[j].Offset = append(fileOffsetTmp[j].Offset, []int64{param.FileStartOff, -1}...) } - op := constructExternal(node, param, c.proc.Ctx, fileList, fileSize, fileOffsetTmp, strictSqlMode) + op := constructExternal(node, param, c.proc.Ctx, fileList, fileSize, fileOffsetTmp, strictSqlMode, c.arrowExecutionScope(node, param), arrowRuntime...) op.SetAnalyzeControl(c.anal.curNodeIdx, currentFirstFlag) ss[0].setRootOperator(op) c.anal.isFirst = false @@ -7344,6 +7783,19 @@ func supportsDistributedOrderedTop(service string) bool { return ok && protocolVersion >= defines.MORPCVersion53 } +func supportsRemoteArrowLoadPipeline(service string) bool { + rt := moruntime.ServiceRuntime(service) + if rt == nil { + return false + } + version, ok := rt.GetGlobalVariables(moruntime.MOProtocolVersion) + if !ok { + return false + } + protocolVersion, ok := version.(int64) + return ok && protocolVersion >= defines.MORPCVersion56 +} + func (c *Compile) canCompileShuffleGroup(node *plan.Node) bool { return node.Stats.HashmapStats != nil && node.Stats.HashmapStats.Shuffle && diff --git a/pkg/sql/compile/operator.go b/pkg/sql/compile/operator.go index b78aadd5d6425..c1c50d248ac95 100644 --- a/pkg/sql/compile/operator.go +++ b/pkg/sql/compile/operator.go @@ -1386,26 +1386,30 @@ func constructProjection(node *plan.Node) *projection.Projection { return arg } -func constructExternal(node *plan.Node, param *tree.ExternParam, ctx context.Context, fileList []string, FileSize []int64, fileOffset []*pipeline.FileOffset, strictSqlMode bool) *external.External { +func constructExternal(node *plan.Node, param *tree.ExternParam, ctx context.Context, fileList []string, FileSize []int64, fileOffset []*pipeline.FileOffset, strictSqlMode bool, arrowScope pipeline.ArrowExecutionScope, arrowRuntime ...*arrowCompileRuntime) *external.External { attrs := buildExternalAttrs(node) - - return external.NewArgument().WithEs( + if param != nil && param.Format == tree.ARROW { + attrs = buildArrowExternalAttrs(node) + } + op := external.NewArgument().WithEs( &external.ExternalParam{ ExParamConst: external.ExParamConst{ - Attrs: attrs, - Cols: node.TableDef.Cols, - ColumnListLen: externalColumnListLen(node), - Extern: param, - FileOffsetTotal: fileOffset, - CreateSql: node.TableDef.Createsql, - Ctx: ctx, - FileList: fileList, - FileSize: FileSize, - ClusterTable: node.GetClusterTable(), - StrictSqlMode: strictSqlMode, - DatastreamScan: node.ExternScan.GetDatastreamScan(), - ForeignScan: node.ExternScan.GetForeignScan(), - KafkaScan: node.ExternScan.GetKafkaScan(), + ArrowExecutionScope: arrowScope, + ArrowForceMaterialize: param.ArrowForceMaterialize, + Attrs: attrs, + Cols: node.TableDef.Cols, + ColumnListLen: externalColumnListLen(node), + Extern: param, + FileOffsetTotal: fileOffset, + CreateSql: node.TableDef.Createsql, + Ctx: ctx, + FileList: fileList, + FileSize: FileSize, + ClusterTable: node.GetClusterTable(), + StrictSqlMode: strictSqlMode, + DatastreamScan: node.ExternScan.GetDatastreamScan(), + ForeignScan: node.ExternScan.GetForeignScan(), + KafkaScan: node.ExternScan.GetKafkaScan(), LoadEmptyNumericAsZero: param.ExternType == int32(plan.ExternType_LOAD) && (param.Parallel || param.ParallelLoadRequested), }, @@ -1417,6 +1421,39 @@ func constructExternal(node *plan.Node, param *tree.ExternParam, ctx context.Con }, }, ) + if len(arrowRuntime) > 0 && arrowRuntime[0] != nil { + op.Es.ArrowObjectIdentities = arrowRuntime[0].identitiesFor(fileList) + op.Es.ArrowRecordBatchShards = arrowRuntime[0].shardsFor(fileList) + op.Es.ArrowSchemaFingerprint = append([]byte(nil), arrowRuntime[0].schemaFingerprint...) + op.Es.ArrowConversionPlanVersion = arrowRuntime[0].conversionPlanVersion + } + return op +} + +// buildArrowExternalAttrs uses the LOAD binder's positive source-column map. +// Generated/default/hidden target columns remain owned by the ordinary +// projection/insert pipeline and must never be invented by the Arrow decoder. +func buildArrowExternalAttrs(node *plan.Node) []plan.ExternAttr { + if node == nil || node.TableDef == nil || node.ExternScan == nil { + return nil + } + mapping := node.ExternScan.TbColToDataCol + attrs := make([]plan.ExternAttr, 0, len(mapping)) + for i, col := range node.TableDef.Cols { + if col == nil || col.Hidden || col.GeneratedCol != nil { + continue + } + fieldIndex, ok := mapping[col.Name] + if !ok || fieldIndex < 0 { + continue + } + attrs = append(attrs, plan.ExternAttr{ + ColName: col.Name, + ColIndex: int32(i), + ColFieldIndex: fieldIndex, + }) + } + return attrs } func buildExternalAttrs(node *plan.Node) []plan.ExternAttr { diff --git a/pkg/sql/compile/remoterun.go b/pkg/sql/compile/remoterun.go index 5e30cc142f392..7610327c83862 100644 --- a/pkg/sql/compile/remoterun.go +++ b/pkg/sql/compile/remoterun.go @@ -126,6 +126,9 @@ func encodeRemoteScope(s *Scope, proc *process.Process) ([]byte, error) { if err = validateRemoteODKUAffectedRowsPipelineProtocol(proc, p); err != nil { return nil, err } + if err = validateRemoteArrowLoadPipelineProtocol(proc, p); err != nil { + return nil, err + } return p.Marshal() } @@ -223,6 +226,9 @@ func decodeScope(data []byte, proc *process.Process, isRemote bool, eng engine.E if err = validateRemoteDistributedOrderedTopPipelineProtocol(proc, p); err != nil { return nil, err } + if err = validateRemoteArrowLoadPipelineProtocol(proc, p); err != nil { + return nil, err + } } else if err = plan.ValidateStringLiteralFormsInOwner(p); err != nil { return nil, err } @@ -889,6 +895,13 @@ func convertToPipelineInstruction(op vm.Operator, proc *process.Process, ctx *sc case *external.External: in.ExternalScan = &pipeline.ExternalScan{ + ArrowExecutionScope: t.Es.ArrowExecutionScope, + ArrowForceMaterialize: t.Es.ArrowForceMaterialize, + ArrowDistributedExecution: t.Es.ArrowDistributedExecution, + ArrowObjectIdentities: t.Es.ArrowObjectIdentities, + ArrowRecordBatchShards: t.Es.ArrowRecordBatchShards, + ArrowSchemaFingerprint: t.Es.ArrowSchemaFingerprint, + ArrowConversionPlanVersion: t.Es.ArrowConversionPlanVersion, Attrs: t.Es.Attrs, ColumnListLen: t.Es.ColumnListLen, Cols: t.Es.Cols, @@ -1536,6 +1549,13 @@ func convertToVmOperator(opr *pipeline.Instruction, ctx *scopeContext, eng engin op = external.NewArgument().WithEs( &external.ExternalParam{ ExParamConst: external.ExParamConst{ + ArrowExecutionScope: t.ArrowExecutionScope, + ArrowForceMaterialize: t.ArrowForceMaterialize, + ArrowDistributedExecution: t.ArrowDistributedExecution, + ArrowObjectIdentities: t.ArrowObjectIdentities, + ArrowRecordBatchShards: t.ArrowRecordBatchShards, + ArrowSchemaFingerprint: t.ArrowSchemaFingerprint, + ArrowConversionPlanVersion: t.ArrowConversionPlanVersion, Attrs: t.Attrs, ColumnListLen: t.ColumnListLen, FileSize: t.FileSize, @@ -2334,6 +2354,31 @@ func validateRemoteGroupingSetPipelineProtocol( return nil } +// validateRemoteArrowLoadPipelineProtocol prevents receivers from silently +// ignoring Arrow-specific ExternalScan fields during a mixed-version rollout. +func validateRemoteArrowLoadPipelineProtocol(proc *process.Process, p *pipeline.Pipeline) error { + if p == nil { + return nil + } + for _, instruction := range p.InstructionList { + scan := instruction.GetExternalScan() + if scan == nil || scan.ArrowExecutionScope != pipeline.ArrowExecutionScope_ArrowLoadData { + continue + } + if proc == nil || !supportsRemoteArrowLoadPipeline(proc.GetService()) { + return moerr.NewNotSupportedNoCtx( + "Arrow LOAD remote execution requires MORPC protocol version 56", + ) + } + } + for _, child := range p.Children { + if err := validateRemoteArrowLoadPipelineProtocol(proc, child); err != nil { + return err + } + } + return nil +} + func aggregateUsesCollationAwareTextMinMax(agg aggexec.AggFuncExecExpression) bool { if agg.GetAggID() != aggexec.AggIdOfMin && agg.GetAggID() != aggexec.AggIdOfMax { return false diff --git a/pkg/sql/compile/remoterun_test.go b/pkg/sql/compile/remoterun_test.go index 12432ee947aae..e320a3696bd04 100644 --- a/pkg/sql/compile/remoterun_test.go +++ b/pkg/sql/compile/remoterun_test.go @@ -1684,7 +1684,7 @@ func TestGroupingSetRemoteProtocolValidationRecursesAndIgnoresLegacyGrouping(t * if hadPrevious { rt.SetGlobalVariables(moruntime.MOProtocolVersion, previous) } else { - rt.CompareAndDeleteGlobalVariables(moruntime.MOProtocolVersion, defines.MORPCVersion49) + rt.CompareAndDeleteGlobalVariables(moruntime.MOProtocolVersion, defines.MORPCVersion52) } }) rt.SetGlobalVariables(moruntime.MOProtocolVersion, defines.MORPCVersion48) @@ -1705,6 +1705,94 @@ func TestGroupingSetRemoteProtocolValidationRecursesAndIgnoresLegacyGrouping(t * require.NoError(t, validateRemoteGroupingSetPipelineProtocol(proc, nested)) } +func TestArrowLoadRemoteProtocolValidationAtSendAndReceiveBoundaries(t *testing.T) { + proc := testutil.NewProcess(t) + rt := moruntime.ServiceRuntime(proc.GetService()) + previous, hadPrevious := rt.GetGlobalVariables(moruntime.MOProtocolVersion) + t.Cleanup(func() { + if hadPrevious { + rt.SetGlobalVariables(moruntime.MOProtocolVersion, previous) + } else { + rt.CompareAndDeleteGlobalVariables(moruntime.MOProtocolVersion, defines.MORPCVersion56) + } + }) + + scope := &Scope{Proc: proc, RootOp: external.NewArgument().WithEs( + &external.ExternalParam{ + ExParamConst: external.ExParamConst{ + ArrowExecutionScope: pipeline.ArrowExecutionScope_ArrowLoadData, + ArrowDistributedExecution: true, + }, + ExParam: external.ExParam{Fileparam: &external.ExFileparam{}, Filter: &external.FilterParam{}}, + }, + )} + + rt.SetGlobalVariables(moruntime.MOProtocolVersion, defines.MORPCVersion56) + data, err := encodeRemoteScope(scope, proc) + require.NoError(t, err) + rt.SetGlobalVariables(moruntime.MOProtocolVersion, defines.MORPCVersion55) + _, err = encodeRemoteScope(scope, proc) + require.ErrorContains(t, err, "MORPC protocol version 56") + _, err = decodeScope(data, proc, true, nil) + require.ErrorContains(t, err, "MORPC protocol version 56") +} + +func TestExternalScanArrowRuntimeRoundtrip(t *testing.T) { + ctx := &scopeContext{id: 1, root: &scopeContext{}, parent: &scopeContext{}} + proc := &process.Process{Base: &process.BaseProcess{}} + identities := []*pipeline.ArrowObjectIdentity{{ + FileIndex: 1, VersionId: "version-7", Etag: "etag-7", Size: 8192, + LastModifiedUnixNano: 1234, + }} + shards := []*pipeline.ArrowRecordBatchShard{{ + FileIndex: 1, RecordBatchStart: 2, RecordBatchEnd: 5, + RequiredDictionaryBlockIndices: []int32{0, 3}, + EstimatedRows: 100, EstimatedWireBytes: 4096, + }} + fingerprint := []byte("01234567890123456789012345678901") + op := external.NewArgument().WithEs(&external.ExternalParam{ + ExParamConst: external.ExParamConst{ + ArrowExecutionScope: pipeline.ArrowExecutionScope_ArrowLoadData, + ArrowForceMaterialize: true, + ArrowDistributedExecution: true, + ArrowObjectIdentities: identities, + ArrowRecordBatchShards: shards, + ArrowSchemaFingerprint: fingerprint, + ArrowConversionPlanVersion: arrowConversionPlanVersion, + FileList: []string{"s3://bucket/part.arrow"}, + FileSize: []int64{8192}, + FileOffsetTotal: []*pipeline.FileOffset{{Offset: []int64{0, -1}}}, + }, + ExParam: external.ExParam{Fileparam: &external.ExFileparam{}, Filter: &external.FilterParam{}}, + }) + + _, instruction, err := convertToPipelineInstruction(op, proc, ctx, 1) + require.NoError(t, err) + require.Equal(t, pipeline.ArrowExecutionScope_ArrowLoadData, instruction.ExternalScan.ArrowExecutionScope) + require.True(t, instruction.ExternalScan.ArrowForceMaterialize) + require.True(t, instruction.ExternalScan.ArrowDistributedExecution) + require.Equal(t, identities, instruction.ExternalScan.ArrowObjectIdentities) + require.Equal(t, shards, instruction.ExternalScan.ArrowRecordBatchShards) + require.Equal(t, fingerprint, instruction.ExternalScan.ArrowSchemaFingerprint) + require.Equal(t, arrowConversionPlanVersion, instruction.ExternalScan.ArrowConversionPlanVersion) + + wire, err := instruction.Marshal() + require.NoError(t, err) + wireInstruction := new(pipeline.Instruction) + require.NoError(t, wireInstruction.Unmarshal(wire)) + + restored, err := convertToVmOperator(wireInstruction, ctx, nil) + require.NoError(t, err) + restoredExternal := restored.(*external.External) + require.Equal(t, pipeline.ArrowExecutionScope_ArrowLoadData, restoredExternal.Es.ArrowExecutionScope) + require.True(t, restoredExternal.Es.ArrowForceMaterialize) + require.True(t, restoredExternal.Es.ArrowDistributedExecution) + require.Equal(t, identities, restoredExternal.Es.ArrowObjectIdentities) + require.Equal(t, shards, restoredExternal.Es.ArrowRecordBatchShards) + require.Equal(t, fingerprint, restoredExternal.Es.ArrowSchemaFingerprint) + require.Equal(t, arrowConversionPlanVersion, restoredExternal.Es.ArrowConversionPlanVersion) +} + func TestExternalScanIcebergRuntimeRoundtrip(t *testing.T) { ctx := &scopeContext{ id: 1, diff --git a/pkg/sql/compile/scope_test.go b/pkg/sql/compile/scope_test.go index 3bbdad95cbefd..972c55d658a47 100644 --- a/pkg/sql/compile/scope_test.go +++ b/pkg/sql/compile/scope_test.go @@ -1779,6 +1779,7 @@ func TestConstructExternalLegacyPathDoesNotSetIcebergRuntime(t *testing.T) { []int64{128}, makeWholeFileOffsets(1), true, + pipeline.ArrowExecutionScope_UnknownArrowExecutionScope, ) require.Equal(t, int32(plan.ExternType_EXTERNAL_TB), op.Es.Extern.ExternType) diff --git a/pkg/sql/compile/sidecarflight/arrow_ipc.go b/pkg/sql/compile/sidecarflight/arrow_ipc.go index e56de24c1f6ea..27f896f59ce54 100644 --- a/pkg/sql/compile/sidecarflight/arrow_ipc.go +++ b/pkg/sql/compile/sidecarflight/arrow_ipc.go @@ -15,10 +15,14 @@ package sidecarflight import ( + "context" "encoding/binary" "math" + "math/bits" + "unicode/utf8" "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/arrowipc" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -28,7 +32,7 @@ import ( const ( arrowHeaderSchema = byte(1) arrowHeaderRecordBatch = byte(3) - maxArrowMetadataBytes = 1 << 20 + maxArrowMetadataBytes = int(arrowipc.DefaultMaxMetadataBytes) arrowTypeInt = byte(2) arrowTypeFloatingPoint = byte(3) @@ -69,6 +73,20 @@ func ParseSchema(wire []byte, expected []planpb.Type, headings []string) (*Schem if len(expected) == 0 || len(headings) != len(expected) { return nil, internalErrorf("MatrixOne result schema is empty or inconsistent") } + // The shared pass owns hostile FlatBuffers/vector bounds. The code below + // remains the Sirius-specific exact schema and negotiated type policy. + info, err := arrowipc.InspectMessage(context.Background(), wire, arrowipc.ValidationOptions{ + MaxMetadataBytes: int64(maxArrowMetadataBytes), + MaxBodyBytes: 0, + BodyEnvelopeBytes: 0, + MaxDecodedRecordBytes: 1, + }) + if err != nil { + return nil, internalErrorf("Arrow schema message: %w", err) + } + if info.HeaderType != arrowHeaderSchema { + return nil, internalErrorf("Arrow schema message has header type %d", info.HeaderType) + } metadata, err := ipcMetadata(wire) if err != nil { return nil, err @@ -81,13 +99,6 @@ func ParseSchema(wire []byte, expected []planpb.Type, headings []string) (*Schem if err != nil || version != 4 { return nil, internalErrorf("Arrow schema message has unsupported metadata version %d", version) } - headerType, err := message.byteField(1, 0) - if err != nil || headerType != arrowHeaderSchema { - return nil, internalErrorf("Arrow schema message has header type %d", headerType) - } - if bodyLength, bodyErr := message.int64Field(3, 0); bodyErr != nil || bodyLength != 0 { - return nil, internalErrorf("Arrow schema message has an invalid body length") - } schemaTable, ok, err := message.tableField(2) if err != nil { return nil, internalErrorf("Arrow schema message is missing its schema: %w", err) @@ -219,8 +230,17 @@ func parseArrowField(table flatTable, expected planpb.Type) (arrowField, error) if err != nil { return arrowField{}, err } - if (moType != types.T_decimal64 && moType != types.T_decimal128) || field.precision != expected.Width || - field.scale != expected.Scale || field.bitWidth != 128 { + if moType != types.T_decimal64 && moType != types.T_decimal128 { + return arrowField{}, typeMismatch(typeID, moType) + } + maxPrecision := int32(38) + if moType == types.T_decimal64 { + maxPrecision = 18 + } + if field.precision < 1 || field.precision > maxPrecision { + return arrowField{}, internalErrorf("decimal precision %d is outside [1,%d]", field.precision, maxPrecision) + } + if field.precision != expected.Width || field.scale != expected.Scale || field.bitWidth != 128 { return arrowField{}, typeMismatch(typeID, moType) } case arrowTypeDate: @@ -300,6 +320,22 @@ type arrowBuffer struct { length int64 } +type decimal128Magnitude struct { + low uint64 + high uint64 +} + +var decimal128PowersOfTen = func() [39]decimal128Magnitude { + result := [39]decimal128Magnitude{{low: 1}} + for index := 1; index < len(result); index++ { + lowHigh, low := bits.Mul64(result[index-1].low, 10) + highHigh, highLow := bits.Mul64(result[index-1].high, 10) + result[index] = decimal128Magnitude{low: low, high: lowHigh + highLow} + _ = highHigh // 10^38 is below 2^128; later values are not needed. + } + return result +}() + // decodeRecordBatch converts exactly one flat Arrow record batch into MO // vectors. The returned batch owns its memory and must be cleaned by the // synchronous consumer before the next Flight message is requested. @@ -307,9 +343,31 @@ func (s *Schema) decodeRecordBatch(header, body []byte, maxDecodedBytes uint64, if s == nil || mp == nil || maxDecodedBytes == 0 { return nil, internalErrorf("sidecar flight: missing schema or memory pool") } + if maxDecodedBytes > math.MaxInt64 { + return nil, internalErrorf("Arrow record batch decoded-memory budget overflows") + } if len(header) == 0 || len(header) > maxArrowMetadataBytes { return nil, internalErrorf("Arrow record batch metadata exceeds the supported bound") } + // Validate transport-independent structure before this decoder allocates + // MO vectors. Exact field counts and Sirius conversions remain local. + info, inspectErr := arrowipc.InspectMessage(context.Background(), header, arrowipc.ValidationOptions{ + MaxMetadataBytes: int64(maxArrowMetadataBytes), + MaxBodyBytes: int64(len(body)), + BodyEnvelopeBytes: int64(len(body)), + Body: body, + ValidateBody: true, + MaxDecodedRecordBytes: int64(maxDecodedBytes), + }) + if inspectErr != nil { + return nil, internalErrorf("Arrow record batch: %w", inspectErr) + } + if info.HeaderType != arrowHeaderRecordBatch { + return nil, internalErrorf("Arrow message has unsupported header type %d", info.HeaderType) + } + if info.BodyBytes != int64(len(body)) { + return nil, internalErrorf("Arrow record batch body length mismatch") + } metadata, err := ipcMetadata(header) if err != nil { return nil, err @@ -322,14 +380,6 @@ func (s *Schema) decodeRecordBatch(header, body []byte, maxDecodedBytes uint64, if err != nil || version != 4 { return nil, internalErrorf("Arrow record batch has unsupported metadata version %d", version) } - headerType, err := message.byteField(1, 0) - if err != nil || headerType != arrowHeaderRecordBatch { - return nil, internalErrorf("Arrow message has unsupported header type %d", headerType) - } - bodyLength, err := message.int64Field(3, 0) - if err != nil || bodyLength < 0 || bodyLength != int64(len(body)) { - return nil, internalErrorf("Arrow record batch body length mismatch") - } record, ok, err := message.tableField(2) if err != nil { return nil, internalErrorf("Arrow record batch metadata is missing: %w", err) @@ -426,20 +476,16 @@ func decodeColumn(vec *vector.Vector, field arrowField, node arrowNode, buffers return internalErrorf("required field contains nulls") } validity := sliceBuffer(body, buffers[0]) - if node.nullCount == 0 { - if len(validity) != 0 && int64(len(validity)) < bitmapBytes(node.length) { - return internalErrorf("validity buffer is too short") - } - } else if int64(len(validity)) < bitmapBytes(node.length) { + if (node.nullCount != 0 || len(validity) != 0) && int64(len(validity)) < bitmapBytes(node.length) { return internalErrorf("validity buffer is too short") } isNull := func(row int64) bool { return node.nullCount != 0 && validity[row>>3]&(1<>3]&(1< int64(len(data)) { return internalErrorf("UTF8 offsets are invalid") } - if err := vector.AppendBytes(vec, data[start:end], isNull(row), mp); err != nil { + value := data[start:end] + if !isNull(row) && !utf8.Valid(value) { + return internalErrorf("UTF8 value at row %d is invalid", row) + } + if !isNull(row) && field.expected.Width > 0 && int64(utf8.RuneCount(value)) > int64(field.expected.Width) { + return internalErrorf("UTF8 value at row %d exceeds width %d", row, field.expected.Width) + } + if err := vector.AppendBytes(vec, value, isNull(row), mp); err != nil { return err } previous = end @@ -513,10 +566,20 @@ func decodeColumn(vec *vector.Vector, field arrowField, node arrowNode, buffers err = vector.AppendFixed(vec, math.Float64frombits(binary.LittleEndian.Uint64(values[offset:])), null, mp) } case arrowTypeDate: - err = vector.AppendFixed(vec, types.DaysFromUnixEpochToDate(int32(binary.LittleEndian.Uint32(values[offset:]))), null, mp) + date := types.DaysFromUnixEpochToDate(int32(binary.LittleEndian.Uint32(values[offset:]))) + if !null { + year, month, day, _ := date.Calendar(true) + if !types.ValidDate(year, month, day) { + return internalErrorf("date value at row %d is outside MatrixOne range", row) + } + } + err = vector.AppendFixed(vec, date, null, mp) case arrowTypeDecimal: low := binary.LittleEndian.Uint64(values[offset:]) high := binary.LittleEndian.Uint64(values[offset+8:]) + if !null && !decimal128FitsPrecision(low, high, field.precision) { + return internalErrorf("decimal value at row %d exceeds precision %d", row, field.precision) + } if types.T(field.expected.Id) == types.T_decimal64 { signExtension := uint64(0) if low>>63 != 0 { @@ -539,6 +602,21 @@ func decodeColumn(vec *vector.Vector, field arrowField, node arrowNode, buffers return nil } +func decimal128FitsPrecision(low, high uint64, precision int32) bool { + if precision <= 0 || precision >= int32(len(decimal128PowersOfTen)) { + return false + } + if high>>63 != 0 { + low = ^low + 1 + high = ^high + if low == 0 { + high++ + } + } + limit := decimal128PowersOfTen[precision] + return high < limit.high || high == limit.high && low < limit.low +} + func bitmapBytes(rows int64) int64 { result := rows / 8 if rows%8 != 0 { @@ -554,24 +632,11 @@ func sliceBuffer(body []byte, buffer arrowBuffer) []byte { // ipcMetadata accepts both raw Message flatbuffers and stream-framed IPC // metadata (continuation marker plus size, or the legacy size prefix). func ipcMetadata(wire []byte) ([]byte, error) { - if len(wire) < 4 { - return nil, internalErrorf("Arrow IPC metadata is truncated") - } - if binary.LittleEndian.Uint32(wire[:4]) == math.MaxUint32 { - if len(wire) < 8 { - return nil, internalErrorf("Arrow IPC continuation header is truncated") - } - length := uint64(binary.LittleEndian.Uint32(wire[4:8])) - if length == 0 || length > uint64(len(wire)-8) { - return nil, internalErrorf("Arrow IPC metadata length is invalid") - } - return wire[8 : 8+length], nil - } - length := uint64(binary.LittleEndian.Uint32(wire[:4])) - if length != 0 && length == uint64(len(wire)-4) { - return wire[4:], nil + metadata, err := arrowipc.Metadata(context.Background(), wire, int64(maxArrowMetadataBytes)) + if err != nil { + return nil, internalErrorf("%v", err) } - return wire, nil + return metadata, nil } type flatTable struct { diff --git a/pkg/sql/compile/sidecarflight/arrow_ipc_test.go b/pkg/sql/compile/sidecarflight/arrow_ipc_test.go index 7ad3aeea161c0..8bb7477e8abd8 100644 --- a/pkg/sql/compile/sidecarflight/arrow_ipc_test.go +++ b/pkg/sql/compile/sidecarflight/arrow_ipc_test.go @@ -116,7 +116,7 @@ func TestArrowIPCRejectsMalformedAndMismatchedData(t *testing.T) { require.Error(t, err) } _, err = schema.decodeRecordBatch(mustHex(t, fixtureHeaderHex), body, 1, mp) - require.ErrorContains(t, err, "decoded-memory budget") + require.ErrorContains(t, err, "decoded record body exceeds limit") required := *schema required.fields = append([]arrowField(nil), schema.fields...) @@ -125,10 +125,197 @@ func TestArrowIPCRejectsMalformedAndMismatchedData(t *testing.T) { require.ErrorContains(t, err, "required field contains nulls") _, err = schema.decodeRecordBatch(mustHex(t, fixtureHeaderHex), body[:len(body)-1], 1<<20, mp) - require.ErrorContains(t, err, "body length mismatch") + require.ErrorContains(t, err, "exceeds limit") require.Equal(t, int64(0), mp.CurrNB()) } +func TestArrowIPCRejectsDateOutsideMatrixOneRange(t *testing.T) { + schemaWire := mustHex(t, fixtureSchemaHex) + header := mustHex(t, fixtureHeaderHex) + body := mustHex(t, fixtureBodyHex) + expected, headings := fixtureOutputShape() + schema, err := ParseSchema(schemaWire, expected, headings) + require.NoError(t, err) + + metadata, err := ipcMetadata(header) + require.NoError(t, err) + message, err := rootTable(metadata) + require.NoError(t, err) + record, ok, err := message.tableField(2) + require.NoError(t, err) + require.True(t, ok) + bufferBytes, bufferCount, err := record.structVector(2, 16) + require.NoError(t, err) + dateBuffer := 0 + for _, field := range schema.fields[:10] { + dateBuffer += field.bufferCount + } + dateBuffer++ // skip the date validity bitmap + require.Less(t, dateBuffer, bufferCount) + bodyOffset := binary.LittleEndian.Uint64(bufferBytes[dateBuffer*16:]) + bodyLength := binary.LittleEndian.Uint64(bufferBytes[dateBuffer*16+8:]) + require.GreaterOrEqual(t, bodyLength, uint64(4)) + require.Less(t, bodyOffset+bodyLength, uint64(len(body)+1)) + binary.LittleEndian.PutUint32(body[bodyOffset:], uint32(math.MaxInt32)) + + mp := mpool.MustNewZero() + decoded, err := schema.decodeRecordBatch(header, body, 1<<20, mp) + if decoded != nil { + decoded.Clean(mp) + } + require.ErrorContains(t, err, "date") + require.Zero(t, mp.CurrNB()) +} + +func TestArrowIPCRejectsDecimalValueOutsidePrecision(t *testing.T) { + schemaWire := mustHex(t, fixtureSchemaHex) + header := mustHex(t, fixtureHeaderHex) + body := mustHex(t, fixtureBodyHex) + expected, headings := fixtureOutputShape() + schema, err := ParseSchema(schemaWire, expected, headings) + require.NoError(t, err) + + metadata, err := ipcMetadata(header) + require.NoError(t, err) + message, err := rootTable(metadata) + require.NoError(t, err) + record, ok, err := message.tableField(2) + require.NoError(t, err) + require.True(t, ok) + bufferBytes, bufferCount, err := record.structVector(2, 16) + require.NoError(t, err) + decimalBuffer := 0 + for _, field := range schema.fields[:9] { + decimalBuffer += field.bufferCount + } + decimalBuffer++ // skip the decimal128 validity bitmap + require.Less(t, decimalBuffer, bufferCount) + bodyOffset := binary.LittleEndian.Uint64(bufferBytes[decimalBuffer*16:]) + bodyLength := binary.LittleEndian.Uint64(bufferBytes[decimalBuffer*16+8:]) + require.GreaterOrEqual(t, bodyLength, uint64(16)) + binary.LittleEndian.PutUint64(body[bodyOffset:], math.MaxUint64) + binary.LittleEndian.PutUint64(body[bodyOffset+8:], math.MaxInt64) + + mp := mpool.MustNewZero() + decoded, err := schema.decodeRecordBatch(header, body, 1<<20, mp) + if decoded != nil { + decoded.Clean(mp) + } + require.ErrorContains(t, err, "precision") + require.Zero(t, mp.CurrNB()) +} + +func TestArrowSchemaRejectsDecimal64PrecisionOverflow(t *testing.T) { + schemaWire := mustHex(t, fixtureSchemaHex) + expected, headings := fixtureOutputShape() + expected[8].Width = 38 + metadata, err := ipcMetadata(schemaWire) + require.NoError(t, err) + message, err := rootTable(metadata) + require.NoError(t, err) + schemaTable, ok, err := message.tableField(2) + require.NoError(t, err) + require.True(t, ok) + fields, err := schemaTable.tableVector(1) + require.NoError(t, err) + typeTable, ok, err := fields[8].tableField(3) + require.NoError(t, err) + require.True(t, ok) + precisionPosition, ok, err := typeTable.field(0, 4) + require.NoError(t, err) + require.True(t, ok) + binary.LittleEndian.PutUint32(metadata[precisionPosition:], 38) + + _, err = ParseSchema(schemaWire, expected, headings) + require.ErrorContains(t, err, "precision") +} + +func TestArrowIPCRejectsStringValueOutsideExpectedWidth(t *testing.T) { + schemaWire := mustHex(t, fixtureSchemaHex) + header := mustHex(t, fixtureHeaderHex) + body := mustHex(t, fixtureBodyHex) + expected, headings := fixtureOutputShape() + expected[7] = planpb.Type{Id: int32(types.T_varchar), Width: 1} + schema, err := ParseSchema(schemaWire, expected, headings) + require.NoError(t, err) + + mp := mpool.MustNewZero() + decoded, err := schema.decodeRecordBatch(header, body, 1<<20, mp) + if decoded != nil { + decoded.Clean(mp) + } + require.ErrorContains(t, err, "width") + require.Zero(t, mp.CurrNB()) +} + +func TestArrowIPCRejectsValidityBitmapWithZeroDeclaredNulls(t *testing.T) { + schemaWire := mustHex(t, fixtureSchemaHex) + header := mustHex(t, fixtureHeaderHex) + body := mustHex(t, fixtureBodyHex) + expected, headings := fixtureOutputShape() + schema, err := ParseSchema(schemaWire, expected, headings) + require.NoError(t, err) + + metadata, err := ipcMetadata(header) + require.NoError(t, err) + message, err := rootTable(metadata) + require.NoError(t, err) + record, ok, err := message.tableField(2) + require.NoError(t, err) + require.True(t, ok) + nodeBytes, nodeCount, err := record.structVector(1, 16) + require.NoError(t, err) + require.Positive(t, nodeCount) + // The fixture's first validity bitmap marks its second row NULL. Clearing + // the matching field-node count creates inconsistent Arrow metadata. + binary.LittleEndian.PutUint64(nodeBytes[8:], 0) + + mp := mpool.MustNewZero() + decoded, err := schema.decodeRecordBatch(header, body, 1<<20, mp) + if decoded != nil { + decoded.Clean(mp) + } + require.ErrorContains(t, err, "validity bitmap") + require.Zero(t, mp.CurrNB()) +} + +func TestArrowIPCRejectsInvalidUTF8Value(t *testing.T) { + schemaWire := mustHex(t, fixtureSchemaHex) + header := mustHex(t, fixtureHeaderHex) + body := mustHex(t, fixtureBodyHex) + expected, headings := fixtureOutputShape() + schema, err := ParseSchema(schemaWire, expected, headings) + require.NoError(t, err) + + metadata, err := ipcMetadata(header) + require.NoError(t, err) + message, err := rootTable(metadata) + require.NoError(t, err) + record, ok, err := message.tableField(2) + require.NoError(t, err) + require.True(t, ok) + bufferBytes, bufferCount, err := record.structVector(2, 16) + require.NoError(t, err) + stringBuffer := 0 + for _, field := range schema.fields[:7] { + stringBuffer += field.bufferCount + } + stringBuffer += 2 // skip string validity and offsets + require.Less(t, stringBuffer, bufferCount) + bodyOffset := binary.LittleEndian.Uint64(bufferBytes[stringBuffer*16:]) + bodyLength := binary.LittleEndian.Uint64(bufferBytes[stringBuffer*16+8:]) + require.GreaterOrEqual(t, bodyLength, uint64(1)) + body[bodyOffset] = 0xff + + mp := mpool.MustNewZero() + decoded, err := schema.decodeRecordBatch(header, body, 1<<20, mp) + if decoded != nil { + decoded.Clean(mp) + } + require.ErrorContains(t, err, "UTF8") + require.Zero(t, mp.CurrNB()) +} + func TestArrowIPCLowLevelBoundsAndFraming(t *testing.T) { _, err := ipcMetadata(nil) require.ErrorContains(t, err, "truncated") @@ -296,7 +483,7 @@ func TestArrowSchemaRejectsNegotiationMismatches(t *testing.T) { require.True(t, ok) headerMutation[headerPosition] = 0 _, err = ParseSchema(headerMutation, typesOut, headings) - require.ErrorContains(t, err, "header type") + require.ErrorContains(t, err, "header is missing") missingSchema := append([]byte(nil), metadata...) missingMessage, err := rootTable(missingSchema) @@ -305,7 +492,7 @@ func TestArrowSchemaRejectsNegotiationMismatches(t *testing.T) { require.NoError(t, err) binary.LittleEndian.PutUint16(missingSchema[vtable+8:], 0) _, err = ParseSchema(missingSchema, typesOut, headings) - require.ErrorContains(t, err, "missing its schema") + require.ErrorContains(t, err, "schema header is missing") } func validFlatTableData() []byte { diff --git a/pkg/sql/compile/sidecarflight/client.go b/pkg/sql/compile/sidecarflight/client.go index 86caea65a08cf..ba82bf9a7a440 100644 --- a/pkg/sql/compile/sidecarflight/client.go +++ b/pkg/sql/compile/sidecarflight/client.go @@ -347,7 +347,7 @@ func (r *Runtime) Reconcile( queryID []byte, release func(context.Context) error, ) error { - if r == nil || len(queryID) == 0 || release == nil { + if r == nil || len(queryID) != 16 || release == nil { return internalErrorf("sidecar flight: invalid replayed execution") } idempotencyKey := executionIdempotencyKey(accountID, queryID) @@ -385,10 +385,6 @@ func (r *Runtime) remove(execution *Execution) { func (r *Runtime) retainForReconciliation(execution *Execution) bool { r.mu.Lock() if r.stopped { - if r.executions == nil { - r.executions = make(map[*Execution]struct{}) - } - r.executions[execution] = struct{}{} r.mu.Unlock() return false } diff --git a/pkg/sql/compile/sidecarflight/client_test.go b/pkg/sql/compile/sidecarflight/client_test.go index a87454a314fbf..2aa2bd1f8d769 100644 --- a/pkg/sql/compile/sidecarflight/client_test.go +++ b/pkg/sql/compile/sidecarflight/client_test.go @@ -169,6 +169,60 @@ func TestRuntimeAndExecutionRejectInvalidStates(t *testing.T) { require.NoError(t, nilExecution.CleanupAfterRun(nil, nil)) } +func TestRuntimeReconcileRejectsMalformedQueryIdentity(t *testing.T) { + server := &testFlightServer{} + runtime := &Runtime{ + config: Config{CleanupTimeout: time.Second}, + conn: testFlightConnection(t, server), + executions: make(map[*Execution]struct{}), + } + err := runtime.Reconcile(1, []byte{1}, testFlightRelease) + closeErr := runtime.Close(context.Background()) + require.ErrorContains(t, err, "invalid replayed execution") + require.NoError(t, closeErr) +} + +func TestRuntimeReconcileDoesNotRetainAfterClose(t *testing.T) { + server := &testFlightServer{} + runtime := &Runtime{ + config: Config{CleanupTimeout: time.Second}, + conn: testFlightConnection(t, server), + executions: make(map[*Execution]struct{}), + } + require.NoError(t, runtime.Close(context.Background())) + err := runtime.Reconcile(1, make([]byte, 16), testFlightRelease) + require.ErrorContains(t, err, "runtime is stopping") + require.Zero(t, runtimeExecutionCount(runtime)) +} + +func TestExecutionRunRejectsExecutionCancelledBeforeStart(t *testing.T) { + server := &testFlightServer{ + schema: mustHex(t, fixtureSchemaHex), header: mustHex(t, fixtureHeaderHex), body: mustHex(t, fixtureBodyHex), + ticket: make([]byte, ticketBytes), hash: make([]byte, sha256.Size), doGetStarted: make(chan struct{}), + } + runtime := &Runtime{ + config: Config{MaxBatchBytes: 1 << 20, RequestTimeout: time.Minute, CleanupTimeout: time.Second}, + conn: testFlightConnection(t, server), executions: make(map[*Execution]struct{}), + } + copy(runtime.capabilityHash[:], server.hash) + typesOut, headings := fixtureOutputShape() + execution, err := runtime.Prepare( + context.Background(), 1, make([]byte, 16), []byte("plan"), typesOut, headings, + testFlightDeadline(), testFlightRelease, + ) + require.NoError(t, err) + require.NoError(t, execution.CancelAndJoin(context.Background())) + runErr := execution.Run(context.Background(), mpool.MustNewZero(), nil, + func(*batch.Batch, *perfcounter.CounterSet) error { return nil }) + require.ErrorContains(t, runErr, "already claimed or completed") + select { + case <-server.doGetStarted: + t.Fatal("DoGet was started after cancellation completed") + default: + } + require.NoError(t, runtime.Close(context.Background())) +} + func TestPrepareRejectsUnsafeFlightInfo(t *testing.T) { typesOut, headings := fixtureOutputShape() for _, tc := range []struct { diff --git a/pkg/sql/compile/sidecarflight/stream.go b/pkg/sql/compile/sidecarflight/stream.go index 073f8e5658e56..0cf46a3d2bad8 100644 --- a/pkg/sql/compile/sidecarflight/stream.go +++ b/pkg/sql/compile/sidecarflight/stream.go @@ -42,7 +42,7 @@ func (e *Execution) Run( ctx = context.Background() } e.mu.Lock() - if e.started || e.terminal { + if e.started || e.terminal || e.quiesced { e.mu.Unlock() return internalErrorf("sidecar flight: ticket was already claimed or completed") } diff --git a/pkg/sql/parsers/tree/update.go b/pkg/sql/parsers/tree/update.go index 1954a74fd32a2..52d5e049fb781 100644 --- a/pkg/sql/parsers/tree/update.go +++ b/pkg/sql/parsers/tree/update.go @@ -218,6 +218,15 @@ const ( CSV = "csv" JSONLINE = "jsonline" PARQUET = "parquet" + ARROW = "arrow" +) + +// Arrow IPC container kinds. AUTO probes the object content; it never relies +// on a filename suffix. +const ( + ARROW_CONTAINER_AUTO = "auto" + ARROW_CONTAINER_FILE = "file" + ARROW_CONTAINER_STREAM = "stream" ) // if $format is jsonline @@ -240,16 +249,24 @@ type ExternParam struct { } type ExParamConst struct { - ScanType int - FileSize int64 - FileStartOff int64 - Filepath string - CompressType string - Format string - Option []string - Data string - Tail *TailParameter - StageName Identifier + ScanType int + FileSize int64 + FileStartOff int64 + Filepath string + CompressType string + Format string + ArrowContainer string + // ArrowMatchByPosition is planner-derived from an explicit LOAD column + // list. It is inert unless the compile-only Arrow execution scope is set. + ArrowMatchByPosition bool + // ArrowForceMaterialize is a compile-time snapshot of the CN rollout + // setting. Keeping it with the external-scan payload makes local and remote + // scopes use one conversion policy for the whole statement generation. + ArrowForceMaterialize bool + Option []string + Data string + Tail *TailParameter + StageName Identifier HivePartitioning bool HivePartitionCols []string diff --git a/pkg/sql/plan/arrow_load_gate.go b/pkg/sql/plan/arrow_load_gate.go new file mode 100644 index 0000000000000..e4ba5e5b42d38 --- /dev/null +++ b/pkg/sql/plan/arrow_load_gate.go @@ -0,0 +1,97 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "context" + "strings" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + moruntime "github.com/matrixorigin/matrixone/pkg/common/runtime" + "github.com/matrixorigin/matrixone/pkg/config" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" + "github.com/matrixorigin/matrixone/pkg/vm/process" +) + +// RequireArrowLoadEnabled checks the fail-closed LOAD-only Arrow gates. Every +// Arrow source requires an explicit deployment opt-in; S3-backed sources also +// require the corresponding S3 setting. +// Planner callers check before probing source objects, and compile checks again +// before building execution scopes so a rejected source cannot enter the pipeline. +func RequireArrowLoadEnabled( + proc *process.Process, + param *tree.ExternParam, +) (config.ArrowLoadParameters, error) { + if param == nil || param.Format != tree.ARROW { + return config.ArrowLoadParameters{}, nil + } + + ctx := context.Background() + if proc != nil && proc.Ctx != nil { + ctx = proc.Ctx + } + var parameterUnit *config.ParameterUnit + if value := ctx.Value(config.ParameterUnitKey); value != nil { + parameterUnit, _ = value.(*config.ParameterUnit) + } + if parameterUnit == nil && proc != nil { + if runtime := moruntime.ServiceRuntime(proc.GetService()); runtime != nil { + if value, ok := runtime.GetGlobalVariables("parameter-unit"); ok { + parameterUnit, _ = value.(*config.ParameterUnit) + } + } + } + if parameterUnit == nil || parameterUnit.SV == nil { + return config.ArrowLoadParameters{}, moerr.NewNotSupported( + ctx, "Arrow LOAD is disabled because runtime configuration is unavailable", + ) + } + + settings := parameterUnit.SV.ArrowLoad + if !settings.Enabled { + return settings, moerr.NewNotSupported(ctx, "Arrow LOAD is disabled by configuration") + } + if arrowLoadUsesS3(param) && !settings.S3Enabled { + return settings, moerr.NewNotSupported(ctx, "Arrow LOAD from S3 or stage is disabled by configuration") + } + return settings, nil +} + +func arrowLoadUsesS3(param *tree.ExternParam) bool { + if param == nil { + return false + } + if param.ScanType == tree.S3 { + return true + } + if _, ok := param.FileService.(*fileservice.S3FS); ok { + return true + } + parsed, err := fileservice.ParsePath(param.Filepath) + if err != nil { + return false + } + switch strings.ToLower(parsed.Service) { + case "s3", "s3-no-key", "s3-opts", "opts", "options", "minio": + return true + } + // A configured FileService can be named freely (for example, "archive") + // while still being backed by S3. Resolve that service, including SubPath + // wrappers, so the object-storage kill switch cannot be bypassed by an + // alias. An empty service is intentionally local in GetForETL. + return parsed.Service != "" && + fileservice.IsS3BackedFileService(param.FileService, param.Filepath) +} diff --git a/pkg/sql/plan/arrow_load_gate_test.go b/pkg/sql/plan/arrow_load_gate_test.go new file mode 100644 index 0000000000000..7e80858fe55e8 --- /dev/null +++ b/pkg/sql/plan/arrow_load_gate_test.go @@ -0,0 +1,108 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "context" + "testing" + + "github.com/matrixorigin/matrixone/pkg/config" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/vm/process" + "github.com/stretchr/testify/require" +) + +func TestRequireArrowLoadEnabled(t *testing.T) { + local := &tree.ExternParam{ + ExParamConst: tree.ExParamConst{Format: tree.ARROW, ScanType: tree.INFILE}, + } + require.NoError(t, requireArrowLoadGateError(nil, + &tree.ExternParam{ExParamConst: tree.ExParamConst{Format: tree.PARQUET}})) + require.ErrorContains(t, requireArrowLoadGateError(nil, local), "configuration is unavailable") + + proc := testutil.NewProc(t) + frontend := &config.FrontendParameters{} + frontend.SetDefaultValues() + proc.Ctx = context.WithValue( + context.Background(), config.ParameterUnitKey, + config.NewParameterUnit(frontend, nil, nil, nil), + ) + require.ErrorContains(t, requireArrowLoadGateError(proc, local), "disabled by configuration") + frontend.ArrowLoad.Enabled = true + require.NoError(t, requireArrowLoadGateError(proc, local)) + + directS3 := &tree.ExternParam{ + ExParamConst: tree.ExParamConst{Format: tree.ARROW, ScanType: tree.S3}, + } + dynamicMinIO := &tree.ExternParam{ + ExParamConst: tree.ExParamConst{ + Format: tree.ARROW, + ScanType: tree.INFILE, + Filepath: "minio,localhost:9000,us-east-1,bucket,key,secret,prefix:input.arrow", + }, + } + namedS3, err := fileservice.NewS3FS(context.Background(), fileservice.ObjectStorageArguments{ + Name: "archive", Endpoint: "disk", Bucket: t.TempDir(), NoBucketValidation: true, + }, fileservice.DisabledCacheConfig, nil, true, true) + require.NoError(t, err) + t.Cleanup(func() { namedS3.Close(context.Background()) }) + services, err := fileservice.NewFileServices("archive", namedS3) + require.NoError(t, err) + aliasedS3 := &tree.ExternParam{ + ExParamConst: tree.ExParamConst{ + Format: tree.ARROW, ScanType: tree.INFILE, + Filepath: namedS3.Name() + ":input.arrow", + }, + ExParam: tree.ExParam{FileService: services}, + } + directFileServiceS3 := &tree.ExternParam{ + ExParamConst: tree.ExParamConst{ + Format: tree.ARROW, ScanType: tree.INFILE, Filepath: "input.arrow", + }, + ExParam: tree.ExParam{FileService: namedS3}, + } + require.ErrorContains(t, requireArrowLoadGateError(proc, directS3), "S3 or stage") + require.ErrorContains(t, requireArrowLoadGateError(proc, dynamicMinIO), "S3 or stage") + require.ErrorContains(t, requireArrowLoadGateError(proc, aliasedS3), "S3 or stage") + require.ErrorContains(t, requireArrowLoadGateError(proc, directFileServiceS3), "S3 or stage") + + frontend.ArrowLoad.Enabled = false + require.ErrorContains(t, requireArrowLoadGateError(proc, local), "disabled by configuration") + frontend.ArrowLoad.Enabled = true + + frontend.ArrowLoad.S3Enabled = true + require.NoError(t, requireArrowLoadGateError(proc, directS3)) + require.NoError(t, requireArrowLoadGateError(proc, dynamicMinIO)) + require.NoError(t, requireArrowLoadGateError(proc, aliasedS3)) + require.NoError(t, requireArrowLoadGateError(proc, directFileServiceS3)) + + frontend.ArrowLoad.S3Enabled = false + require.ErrorContains(t, requireArrowLoadGateError(proc, directS3), "S3 or stage") + require.ErrorContains(t, requireArrowLoadGateError(proc, dynamicMinIO), "S3 or stage") + require.ErrorContains(t, requireArrowLoadGateError(proc, aliasedS3), "S3 or stage") + require.ErrorContains(t, requireArrowLoadGateError(proc, directFileServiceS3), "S3 or stage") + frontend.ArrowLoad.S3Enabled = true + require.NoError(t, requireArrowLoadGateError(proc, directS3)) + require.NoError(t, requireArrowLoadGateError(proc, dynamicMinIO)) + require.NoError(t, requireArrowLoadGateError(proc, aliasedS3)) + require.NoError(t, requireArrowLoadGateError(proc, directFileServiceS3)) +} + +func requireArrowLoadGateError(proc *process.Process, param *tree.ExternParam) error { + _, err := RequireArrowLoadEnabled(proc, param) + return err +} diff --git a/pkg/sql/plan/bind_load.go b/pkg/sql/plan/bind_load.go index 908d04d8dc4b2..bb10a363257a4 100644 --- a/pkg/sql/plan/bind_load.go +++ b/pkg/sql/plan/bind_load.go @@ -70,7 +70,7 @@ func (builder *QueryBuilder) bindExternalScan( if err := InitNullMap(stmt.Param, ctx); err != nil { return -1, nil, err } - if err := validateLoadParquetOptions(stmt.Param, ctx); err != nil { + if err := validateLoadColumnarOptions(stmt.Param, ctx); err != nil { return -1, nil, err } defaultParquetLoadParallel(stmt.Param, ctx) @@ -161,6 +161,8 @@ func (builder *QueryBuilder) bindExternalScan( } stmt.Param.FileStartOff = offset } + stmt.Param.ArrowMatchByPosition = stmt.Param.Format == tree.ARROW && + stmt.Param.Tail != nil && len(stmt.Param.Tail.ColumnList) > 0 applyLoadParallelAdmission(stmt.Param, offset) stmt.Param.Tail.ColumnList = nil diff --git a/pkg/sql/plan/build_ddl.go b/pkg/sql/plan/build_ddl.go index 43057d7fcefae..910c9e2bde587 100644 --- a/pkg/sql/plan/build_ddl.go +++ b/pkg/sql/plan/build_ddl.go @@ -2533,11 +2533,15 @@ func buildCreateTable( } else if stmt.Param != nil { for i := 0; i < len(stmt.Param.Option); i += 2 { switch strings.ToLower(stmt.Param.Option[i]) { - case "endpoint", "region", "access_key_id", "secret_access_key", "bucket", "filepath", "compression", "format", "jsondata", "provider", "role_arn", "external_id", "hive_partitioning", "hive_partition_columns", ExternalWriteFilePatternKey, CSVCommentKey: + case "endpoint", "region", "access_key_id", "secret_access_key", "bucket", "filepath", "compression", "format", "jsondata", "provider", "role_arn", "external_id", "hive_partitioning", "hive_partition_columns", "arrow_container", ExternalWriteFilePatternKey, CSVCommentKey: default: return nil, moerr.NewBadConfigf(ctx.GetContext(), "the keyword '%s' is not support", strings.ToLower(stmt.Param.Option[i])) } } + if strings.EqualFold(getRawOption(stmt.Param.Option, "format"), tree.ARROW) || + strings.EqualFold(stmt.Param.Format, tree.ARROW) { + return nil, moerr.NewNotSupported(ctx.GetContext(), "Arrow format is supported only by LOAD DATA") + } if err := validateWriteFilePattern(ctx.GetContext(), stmt.Param, createTable.TableDef); err != nil { return nil, err diff --git a/pkg/sql/plan/build_load.go b/pkg/sql/plan/build_load.go index 7674afdb3b7e2..d9540287d77e1 100644 --- a/pkg/sql/plan/build_load.go +++ b/pkg/sql/plan/build_load.go @@ -217,8 +217,14 @@ func IgnoredLines(param *tree.ExternParam, ctx CompilerContext) (offset int64, e return csvReader.Pos(), nil } -func validateLoadParquetOptions(param *tree.ExternParam, ctx CompilerContext) error { - if param == nil || !isLoadParquetFormat(param) { +func validateLoadColumnarOptions(param *tree.ExternParam, ctx CompilerContext) error { + if param == nil { + return nil + } + if isLoadArrowFormat(param) { + return validateLoadArrowOptions(param, ctx) + } + if !isLoadParquetFormat(param) { return nil } if param.Local { @@ -255,6 +261,76 @@ func validateLoadParquetOptions(param *tree.ExternParam, ctx CompilerContext) er return nil } +// validateLoadParquetOptions is kept as the focused Parquet test seam. LOAD's +// production path uses validateLoadColumnarOptions so new columnar formats do +// not accidentally inherit the CSV option surface. +func validateLoadParquetOptions(param *tree.ExternParam, ctx CompilerContext) error { + if param == nil || !isLoadParquetFormat(param) { + return nil + } + return validateLoadColumnarOptions(param, ctx) +} + +func validateLoadArrowOptions(param *tree.ExternParam, ctx CompilerContext) error { + if param.Local { + return moerr.NewNotSupported(ctx.GetContext(), "Arrow format is supported only by non-LOCAL LOAD DATA") + } + if param.ScanType == tree.INLINE { + return moerr.NewNotSupported(ctx.GetContext(), "Arrow format is supported only by file-backed LOAD DATA") + } + if loadOptionExists(param, "compression") || hasExplicitLoadCompression(param.CompressType) { + return moerr.NewBadConfig(ctx.GetContext(), "LOAD DATA with format='arrow' does not support external compression") + } + if loadOptionExists(param, "jsondata") || param.JsonData != "" { + return moerr.NewBadConfig(ctx.GetContext(), "LOAD DATA with format='arrow' does not support jsondata option") + } + if loadOptionExists(param, "hive_partitioning") || loadOptionExists(param, "hive_partition_columns") || + param.HivePartitioning || len(param.HivePartitionCols) > 0 { + return moerr.NewBadConfig(ctx.GetContext(), "LOAD DATA with format='arrow' does not support hive partitioning options") + } + container := param.ArrowContainer + if container == "" { + container = tree.ARROW_CONTAINER_AUTO + } + switch strings.ToLower(container) { + case tree.ARROW_CONTAINER_AUTO, tree.ARROW_CONTAINER_FILE, tree.ARROW_CONTAINER_STREAM: + param.ArrowContainer = strings.ToLower(container) + default: + return moerr.NewBadConfigf(ctx.GetContext(), "the arrow_container '%s' is not supported", container) + } + if param.Tail == nil { + return nil + } + if param.Tail.Fields != nil { + return moerr.NewBadConfig(ctx.GetContext(), "LOAD DATA with format='arrow' does not support FIELDS option") + } + if param.Tail.Lines != nil { + return moerr.NewBadConfig(ctx.GetContext(), "LOAD DATA with format='arrow' does not support LINES option") + } + if param.Tail.IgnoredLines > 0 { + return moerr.NewBadConfig(ctx.GetContext(), "LOAD DATA with format='arrow' does not support IGNORE LINES") + } + if hasLoadUserVariable(param.Tail.ColumnList) { + return moerr.NewNotSupported(ctx.GetContext(), "Arrow LOAD DATA does not support @variables in column list") + } + if len(param.Tail.Assignments) > 0 { + return moerr.NewNotSupported(ctx.GetContext(), "Arrow LOAD DATA does not support SET clause") + } + return nil +} + +func isLoadArrowFormat(param *tree.ExternParam) bool { + if param.Format == tree.ARROW { + return true + } + for i := 0; i+1 < len(param.Option); i += 2 { + if strings.EqualFold(param.Option[i], "format") && strings.EqualFold(param.Option[i+1], tree.ARROW) { + return true + } + } + return false +} + func isLoadParquetFormat(param *tree.ExternParam) bool { if param.Format == tree.PARQUET { return true @@ -368,7 +444,7 @@ func estimateLoadRowsizeFromFirstLine(param *tree.ExternParam, inputSize int64, if param == nil || param.ScanType == tree.INLINE || param.Local || - param.Format == tree.PARQUET || + param.Format == tree.PARQUET || param.Format == tree.ARROW || getCompressType(param, param.Filepath) != tree.NOCOMPRESS || (lineTerminator != "\n" && lineTerminator != "\r\n") || strings.HasPrefix(param.Filepath, "SHARED:/query_result/") { @@ -456,12 +532,16 @@ func clampLoadRowsize(rowSize float64, inputSize int64) float64 { return rowSize } -func loadParquetMayListFiles(param *tree.ExternParam) bool { +func loadColumnarMayListFiles(param *tree.ExternParam) bool { return param != nil && - param.Format == tree.PARQUET && + (param.Format == tree.PARQUET || param.Format == tree.ARROW) && strings.ContainsAny(strings.TrimSpace(param.Filepath), "*?[") } +func loadParquetMayListFiles(param *tree.ExternParam) bool { + return param != nil && param.Format == tree.PARQUET && loadColumnarMayListFiles(param) +} + func totalLoadFileSize(fileSize []int64) int64 { var total int64 for _, size := range fileSize { @@ -499,7 +579,7 @@ func buildLoad(stmt *tree.Load, ctx CompilerContext, isPrepareStmt bool) (*Plan, if err := InitNullMap(stmt.Param, ctx); err != nil { return nil, err } - if err := validateLoadParquetOptions(stmt.Param, ctx); err != nil { + if err := validateLoadColumnarOptions(stmt.Param, ctx); err != nil { return nil, err } defaultParquetLoadParallel(stmt.Param, ctx) @@ -538,6 +618,8 @@ func buildLoad(stmt *tree.Load, ctx CompilerContext, isPrepareStmt bool) (*Plan, } stmt.Param.FileStartOff = offset } + stmt.Param.ArrowMatchByPosition = stmt.Param.Format == tree.ARROW && + stmt.Param.Tail != nil && len(stmt.Param.Tail.ColumnList) > 0 applyLoadParallelAdmission(stmt.Param, offset) stmt.Param.Tail.ColumnList = nil @@ -610,7 +692,7 @@ func buildLoad(stmt *tree.Load, ctx CompilerContext, isPrepareStmt bool) (*Plan, builder.qry.LoadWriteS3 = false } - if stmt.Param.Parallel && noCompress && stmt.Param.Format != tree.PARQUET { + if stmt.Param.Parallel && noCompress && stmt.Param.Format != tree.PARQUET && stmt.Param.Format != tree.ARROW { projectNode.ProjectList = makeCastExpr(stmt, fileName, originTableDef, projectNode) } lastNodeId = builder.appendNode(projectNode, bindCtx) @@ -750,6 +832,9 @@ func applyLoadParallelAdmission(param *tree.ExternParam, offset int64) { } func checkFileExist(param *tree.ExternParam, ctx CompilerContext) (string, error) { + if _, err := RequireArrowLoadEnabled(ctx.GetProcess(), param); err != nil { + return "", err + } if param.ScanType == tree.INLINE { return "", nil } @@ -763,6 +848,11 @@ func checkFileExist(param *tree.ExternParam, ctx CompilerContext) (string, error return "", err } } + // Stage resolution may reveal an S3-backed source. Recheck the sub-gate + // before ReadDir or StatFile can perform object-store I/O. + if _, err := RequireArrowLoadEnabled(ctx.GetProcess(), param); err != nil { + return "", err + } if param.Local { return param.Filepath, nil } @@ -771,7 +861,7 @@ func checkFileExist(param *tree.ExternParam, ctx CompilerContext) (string, error } param.Ctx = ctx.GetContext() - if loadParquetMayListFiles(param) { + if loadColumnarMayListFiles(param) { fileList, fileSize, err := ReadDir(param) param.Ctx = nil if err != nil { diff --git a/pkg/sql/plan/build_load_arrow_test.go b/pkg/sql/plan/build_load_arrow_test.go new file mode 100644 index 0000000000000..12040ea6d3669 --- /dev/null +++ b/pkg/sql/plan/build_load_arrow_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "context" + "testing" + + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" + "github.com/stretchr/testify/require" +) + +func TestValidateLoadArrowOptionsAcceptsFileStreamAndAuto(t *testing.T) { + ctx := parquetLoadTestCtx{ctx: context.Background()} + for _, container := range []string{"", tree.ARROW_CONTAINER_AUTO, tree.ARROW_CONTAINER_FILE, tree.ARROW_CONTAINER_STREAM} { + t.Run("container="+container, func(t *testing.T) { + param := &tree.ExternParam{ExParamConst: tree.ExParamConst{ + Format: tree.ARROW, ArrowContainer: container, Tail: &tree.TailParameter{}, + }} + require.NoError(t, validateLoadColumnarOptions(param, ctx)) + if container == "" { + require.Equal(t, tree.ARROW_CONTAINER_AUTO, param.ArrowContainer) + } + }) + } +} + +func TestValidateLoadArrowOptionsRejectsConflictingSurface(t *testing.T) { + ctx := parquetLoadTestCtx{ctx: context.Background()} + tests := []struct { + name string + edit func(*tree.ExternParam) + text string + }{ + {"local", func(p *tree.ExternParam) { p.Local = true }, "non-LOCAL"}, + {"inline", func(p *tree.ExternParam) { p.ScanType = tree.INLINE }, "file-backed"}, + {"compression-key", func(p *tree.ExternParam) { p.Option = []string{"compression", "auto"} }, "compression"}, + {"compression-state", func(p *tree.ExternParam) { p.CompressType = tree.GZIP }, "compression"}, + {"json", func(p *tree.ExternParam) { p.JsonData = tree.OBJECT }, "jsondata"}, + {"hive", func(p *tree.ExternParam) { p.HivePartitioning = true }, "hive"}, + {"fields", func(p *tree.ExternParam) { p.Tail.Fields = &tree.Fields{} }, "FIELDS"}, + {"lines", func(p *tree.ExternParam) { p.Tail.Lines = &tree.Lines{} }, "LINES"}, + {"ignore", func(p *tree.ExternParam) { p.Tail.IgnoredLines = 1 }, "IGNORE"}, + {"variable", func(p *tree.ExternParam) { p.Tail.ColumnList = []tree.LoadColumn{&tree.VarExpr{Name: "v"}} }, "@variables"}, + {"set", func(p *tree.ExternParam) { p.Tail.Assignments = tree.UpdateExprs{&tree.UpdateExpr{}} }, "SET"}, + {"container", func(p *tree.ExternParam) { p.ArrowContainer = "flight" }, "arrow_container"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + param := &tree.ExternParam{ExParamConst: tree.ExParamConst{ + Format: tree.ARROW, Tail: &tree.TailParameter{}, + }} + test.edit(param) + err := validateLoadColumnarOptions(param, ctx) + require.Error(t, err) + require.Contains(t, err.Error(), test.text) + }) + } +} + +func TestLoadArrowFormatDetectionAndListing(t *testing.T) { + require.True(t, isLoadArrowFormat(&tree.ExternParam{ExParamConst: tree.ExParamConst{Format: tree.ARROW}})) + require.True(t, isLoadArrowFormat(&tree.ExternParam{ExParamConst: tree.ExParamConst{ + Format: tree.CSV, Option: []string{"FORMAT", "ARROW"}, + }})) + require.False(t, isLoadArrowFormat(&tree.ExternParam{ExParamConst: tree.ExParamConst{Format: tree.PARQUET}})) + require.True(t, loadColumnarMayListFiles(&tree.ExternParam{ExParamConst: tree.ExParamConst{ + Format: tree.ARROW, Filepath: "etl:data/*.arrow", + }})) + require.False(t, loadColumnarMayListFiles(&tree.ExternParam{ExParamConst: tree.ExParamConst{ + Format: tree.ARROW, Filepath: "etl:data/one.arrow", + }})) +} + +func TestValidateLoadColumnarOptionsLeavesCSVUntouched(t *testing.T) { + ctx := parquetLoadTestCtx{ctx: context.Background()} + param := &tree.ExternParam{ExParamConst: tree.ExParamConst{ + Format: tree.CSV, Tail: &tree.TailParameter{Fields: &tree.Fields{}}, + }} + require.NoError(t, validateLoadColumnarOptions(param, ctx)) +} diff --git a/pkg/sql/plan/utils.go b/pkg/sql/plan/utils.go index 0aab46aa39352..1417e8519f992 100644 --- a/pkg/sql/plan/utils.go +++ b/pkg/sql/plan/utils.go @@ -2309,10 +2309,16 @@ func InitInfileParam(param *tree.ExternParam) error { param.CompressType = param.Option[i+1] case "format": format := strings.ToLower(param.Option[i+1]) - if format != tree.CSV && format != tree.JSONLINE && format != tree.PARQUET { + if format != tree.CSV && format != tree.JSONLINE && format != tree.PARQUET && format != tree.ARROW { return moerr.NewBadConfigf(param.Ctx, "the format '%s' is not supported", format) } param.Format = format + case "arrow_container": + container, err := normalizeArrowContainer(param.Ctx, param.Option[i+1]) + if err != nil { + return err + } + param.ArrowContainer = container case "jsondata": jsondata := strings.ToLower(param.Option[i+1]) if jsondata != tree.OBJECT && jsondata != tree.ARRAY { @@ -2339,6 +2345,9 @@ func InitInfileParam(param *tree.ExternParam) error { if len(param.Format) == 0 { param.Format = tree.CSV } + if err := validateArrowContainerOption(param); err != nil { + return err + } return nil } @@ -2375,10 +2384,16 @@ func InitS3Param(param *tree.ExternParam) error { param.S3Param.ExternalId = param.Option[i+1] case "format": format := strings.ToLower(param.Option[i+1]) - if format != tree.CSV && format != tree.JSONLINE && format != tree.PARQUET { + if format != tree.CSV && format != tree.JSONLINE && format != tree.PARQUET && format != tree.ARROW { return moerr.NewBadConfigf(param.Ctx, "the format '%s' is not supported", format) } param.Format = format + case "arrow_container": + container, err := normalizeArrowContainer(param.Ctx, param.Option[i+1]) + if err != nil { + return err + } + param.ArrowContainer = container case "jsondata": jsondata := strings.ToLower(param.Option[i+1]) if jsondata != tree.OBJECT && jsondata != tree.ARRAY { @@ -2402,6 +2417,29 @@ func InitS3Param(param *tree.ExternParam) error { if len(param.Format) == 0 { param.Format = tree.CSV } + if err := validateArrowContainerOption(param); err != nil { + return err + } + return nil +} + +func normalizeArrowContainer(ctx context.Context, value string) (string, error) { + value = strings.ToLower(strings.TrimSpace(value)) + switch value { + case tree.ARROW_CONTAINER_AUTO, tree.ARROW_CONTAINER_FILE, tree.ARROW_CONTAINER_STREAM: + return value, nil + default: + return "", moerr.NewBadConfigf(ctx, "the arrow_container '%s' is not supported", value) + } +} + +func validateArrowContainerOption(param *tree.ExternParam) error { + if param.ArrowContainer != "" && param.Format != tree.ARROW { + return moerr.NewBadConfig(param.Ctx, "arrow_container requires format='arrow'") + } + if param.Format == tree.ARROW && param.ArrowContainer == "" { + param.ArrowContainer = tree.ARROW_CONTAINER_AUTO + } return nil } @@ -2524,10 +2562,16 @@ func InitStageS3Param(param *tree.ExternParam, s stage.StageDef) error { continue case "format": format := strings.ToLower(param.Option[i+1]) - if format != tree.CSV && format != tree.JSONLINE && format != tree.PARQUET { + if format != tree.CSV && format != tree.JSONLINE && format != tree.PARQUET && format != tree.ARROW { return moerr.NewBadConfigf(param.Ctx, "the format '%s' is not supported", format) } param.Format = format + case "arrow_container": + container, err := normalizeArrowContainer(param.Ctx, param.Option[i+1]) + if err != nil { + return err + } + param.ArrowContainer = container case "jsondata": jsondata := strings.ToLower(param.Option[i+1]) if jsondata != tree.OBJECT && jsondata != tree.ARRAY { @@ -2552,6 +2596,9 @@ func InitStageS3Param(param *tree.ExternParam, s stage.StageDef) error { if len(param.Format) == 0 { param.Format = tree.CSV } + if err := validateArrowContainerOption(param); err != nil { + return err + } return nil diff --git a/pkg/sql/plan/utils_test.go b/pkg/sql/plan/utils_test.go index 9c80fe90609da..72a2baacf4283 100644 --- a/pkg/sql/plan/utils_test.go +++ b/pkg/sql/plan/utils_test.go @@ -1858,6 +1858,19 @@ func TestInitInfileParam_Plain(t *testing.T) { require.NoError(t, InitInfileParam(param)) assert.Equal(t, "csv", param.Format) assert.Equal(t, "REM", GetCSVComment(param)) + + param = &tree.ExternParam{ExParamConst: tree.ExParamConst{Option: []string{ + "filepath", "/data.arrow", "format", "ArRoW", "arrow_container", "FiLe", + }}} + require.NoError(t, InitInfileParam(param)) + assert.Equal(t, tree.ARROW, param.Format) + assert.Equal(t, tree.ARROW_CONTAINER_FILE, param.ArrowContainer) + + param = &tree.ExternParam{ExParamConst: tree.ExParamConst{Option: []string{ + "filepath", "/data.arrow", "format", "arrow", + }}} + require.NoError(t, InitInfileParam(param)) + assert.Equal(t, tree.ARROW_CONTAINER_AUTO, param.ArrowContainer) } // TestGetCSVComment covers the COMMENT option accessor. @@ -1977,6 +1990,12 @@ func TestInitS3Param_Plain(t *testing.T) { param.Option = []string{"bucket", "b", "jsondata", "array"} require.NoError(t, InitS3Param(param)) assert.Equal(t, "jsonline", param.Format) + + param = &tree.ExternParam{ExParamConst: tree.ExParamConst{Option: []string{ + "bucket", "b", "filepath", "data.arrow", "format", "arrow", "arrow_container", "stream", + }}} + require.NoError(t, InitS3Param(param)) + assert.Equal(t, tree.ARROW_CONTAINER_STREAM, param.ArrowContainer) } func TestInitS3Param_HiveLegacyOption(t *testing.T) { diff --git a/pkg/tests/arrowload/arrow_load_benchmark_test.go b/pkg/tests/arrowload/arrow_load_benchmark_test.go new file mode 100644 index 0000000000000..c3ef57cc2e915 --- /dev/null +++ b/pkg/tests/arrowload/arrow_load_benchmark_test.go @@ -0,0 +1,127 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowload + +import ( + "fmt" + "testing" + + metric "github.com/matrixorigin/matrixone/pkg/util/metric/v2" + promtestutil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func TestArrowLoadForceMaterializeFallback(t *testing.T) { + for _, test := range []struct { + name string + forceMaterialize bool + }{ + {name: "borrow"}, + {name: "materialize", forceMaterialize: true}, + } { + t.Run(test.name, func(t *testing.T) { + c := startArrowLoadClusterWithOptions(t, arrowLoadClusterOptions{ + cnCount: 1, enabled: true, s3Enabled: true, distributedEnabled: true, + forceMaterialize: test.forceMaterialize, + }) + db := openArrowLoadDB(t, c, 0) + mustExec(t, db, "create database if not exists arrow_materialize") + mustExec(t, db, "use arrow_materialize") + mustExec(t, db, "create table load_target(id bigint not null, payload varchar(128) not null)") + path, _ := fixtureLarge(t) + borrowedBefore := promtestutil.ToFloat64( + metric.ArrowLoadPayloadBytesCounter.WithLabelValues("borrowed"), + ) + copiedBefore := promtestutil.ToFloat64( + metric.ArrowLoadCopyBytesCounter.WithLabelValues("arrow_to_mo"), + ) + + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table load_target", path)) + require.Equal(t, int64(largeFixtureRows), queryCount(t, db, "select count(*) from load_target")) + borrowedDelta := promtestutil.ToFloat64( + metric.ArrowLoadPayloadBytesCounter.WithLabelValues("borrowed"), + ) - borrowedBefore + copiedDelta := promtestutil.ToFloat64( + metric.ArrowLoadCopyBytesCounter.WithLabelValues("arrow_to_mo"), + ) - copiedBefore + if test.forceMaterialize { + require.Zero(t, borrowedDelta) + require.Greater(t, copiedDelta, float64(0)) + } else { + require.Greater(t, borrowedDelta, float64(0)) + } + }) + } +} + +// BenchmarkArrowLoadEndToEndMaterializeAB measures the complete SQL LOAD path +// through the MySQL frontend and embedded storage stack. Table truncation is +// outside the timer; parsing, planning, Arrow I/O/conversion, transaction commit, +// and result acknowledgement remain inside it. The metric assertions prevent a +// benchmark run from silently comparing the same ownership policy twice. +func BenchmarkArrowLoadEndToEndMaterializeAB(b *testing.B) { + path, ddl := fixtureLarge(b) + for _, benchmark := range []struct { + name string + forceMaterialize bool + }{ + {name: "borrow"}, + {name: "materialize", forceMaterialize: true}, + } { + b.Run(benchmark.name, func(b *testing.B) { + c := startArrowLoadClusterWithOptions(b, arrowLoadClusterOptions{ + cnCount: 1, enabled: true, s3Enabled: false, distributedEnabled: false, + forceMaterialize: benchmark.forceMaterialize, + }) + db := openArrowLoadDB(b, c, 0) + mustExec(b, db, "create database if not exists arrow_materialize_benchmark") + mustExec(b, db, "use arrow_materialize_benchmark") + mustExec(b, db, fmt.Sprintf("create table load_target(%s)", ddl)) + borrowedBefore := promtestutil.ToFloat64( + metric.ArrowLoadPayloadBytesCounter.WithLabelValues("borrowed"), + ) + copiedBefore := promtestutil.ToFloat64( + metric.ArrowLoadCopyBytesCounter.WithLabelValues("arrow_to_mo"), + ) + loadSQL := fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table load_target", path) + + b.ReportAllocs() + b.ResetTimer() + for range b.N { + b.StopTimer() + mustExec(b, db, "truncate table load_target") + b.StartTimer() + mustExec(b, db, loadSQL) + } + b.StopTimer() + b.ReportMetric(float64(largeFixtureRows*b.N)/b.Elapsed().Seconds(), "rows/s") + require.Equal(b, int64(largeFixtureRows), queryCount(b, db, "select count(*) from load_target")) + borrowedDelta := promtestutil.ToFloat64( + metric.ArrowLoadPayloadBytesCounter.WithLabelValues("borrowed"), + ) - borrowedBefore + copiedDelta := promtestutil.ToFloat64( + metric.ArrowLoadCopyBytesCounter.WithLabelValues("arrow_to_mo"), + ) - copiedBefore + if benchmark.forceMaterialize { + require.Zero(b, borrowedDelta) + require.Greater(b, copiedDelta, float64(0)) + } else { + require.Greater(b, borrowedDelta, float64(0)) + } + }) + } +} diff --git a/pkg/tests/arrowload/arrow_load_minio_test.go b/pkg/tests/arrowload/arrow_load_minio_test.go new file mode 100644 index 0000000000000..0cb1d51fe3d2f --- /dev/null +++ b/pkg/tests/arrowload/arrow_load_minio_test.go @@ -0,0 +1,357 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowload + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/require" +) + +// testArrowLoadLocalMinIO extends the 1-CN public-path fixture across the real +// S3-compatible boundary. A dedicated MinIO process is required here because +// an in-memory object-store fake cannot prove credential parsing, HTTP request +// cancellation, conditional ETag reads, or stage expansion. +func testArrowLoadLocalMinIO(t *testing.T, db *sql.DB) { + server := startArrowLoadMinIO(t) + + originalPath := fixtureIDName(t, t.TempDir(), "original.arrow", containerFile, + [][]idNameRow{{{id: 1, name: "first"}, {id: 2, name: "second"}}}) + replacementPath := fixtureIDName(t, t.TempDir(), "replacement.arrow", containerFile, + [][]idNameRow{{{id: 7, name: "new-first"}, {id: 8, name: "new-second"}}}) + streamPath := fixtureIDName(t, t.TempDir(), "source.stream", containerStream, + [][]idNameRow{{{id: 3, name: "stream-a"}}, {{id: 4, name: "stream-b"}}}) + original := mustReadFile(t, originalPath) + replacement := mustReadFile(t, replacementPath) + stream := mustReadFile(t, streamPath) + + t.Run("DirectFileAndObjectRefresh", func(t *testing.T) { + const key = "direct/source.arrow" + server.put(t, key, original) + mustExec(t, db, "drop table if exists minio_direct_file") + mustExec(t, db, "create table minio_direct_file(id bigint not null, name varchar(50))") + mustExec(t, db, minioLoadSQL(server.endpointURL, server, key, "minio_direct_file", "file", false)) + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from minio_direct_file")) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from minio_direct_file where id=1")) + + // Replacing the same key between statements checks that a warm cache + // cannot make the second LOAD observe the previous object generation. + server.put(t, key, replacement) + mustExec(t, db, "truncate table minio_direct_file") + mustExec(t, db, minioLoadSQL(server.endpointURL, server, key, "minio_direct_file", "file", false)) + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from minio_direct_file")) + require.Equal(t, int64(0), queryCount(t, db, "select count(*) from minio_direct_file where id=1")) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from minio_direct_file where id=7")) + }) + + t.Run("DirectStream", func(t *testing.T) { + const key = "direct/source.stream" + server.put(t, key, stream) + mustExec(t, db, "drop table if exists minio_direct_stream") + mustExec(t, db, "create table minio_direct_stream(id bigint not null, name varchar(50))") + mustExec(t, db, minioLoadSQL(server.endpointURL, server, key, "minio_direct_stream", "stream", false)) + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from minio_direct_stream")) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from minio_direct_stream where id=4 and name='stream-b'")) + }) + + t.Run("S3BackedStage", func(t *testing.T) { + const key = "stage/source.arrow" + server.put(t, key, original) + mustExec(t, db, "drop stage if exists arrow_minio_stage") + t.Cleanup(func() { _, _ = db.Exec("drop stage if exists arrow_minio_stage") }) + mustExec(t, db, fmt.Sprintf( + "create stage arrow_minio_stage URL='s3://%s/stage/' CREDENTIALS={"+ + "'AWS_KEY_ID'='%s','AWS_SECRET_KEY'='%s','AWS_REGION'='us-east-1',"+ + "'PROVIDER'='minio','ENDPOINT'='%s'}", + server.bucket, server.user, server.password, server.endpointURL)) + mustExec(t, db, "drop table if exists minio_stage") + mustExec(t, db, "create table minio_stage(id bigint not null, name varchar(50))") + mustExec(t, db, + "load data infile {'filepath'='stage://arrow_minio_stage/source.arrow','format'='arrow'} into table minio_stage") + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from minio_stage")) + mustExec(t, db, "drop stage arrow_minio_stage") + }) + + t.Run("MultiObjectSuccessAndCorruptRollback", func(t *testing.T) { + part1Path := fixtureIDName(t, t.TempDir(), "part1.arrow", containerFile, + [][]idNameRow{{{id: 11, name: "part-one"}}}) + part2Path := fixtureIDName(t, t.TempDir(), "part2.arrow", containerFile, + [][]idNameRow{{{id: 12, name: "part-two"}}}) + server.put(t, "multi-ok/part1.arrow", mustReadFile(t, part1Path)) + server.put(t, "multi-ok/part2.arrow", mustReadFile(t, part2Path)) + + mustExec(t, db, "drop table if exists minio_multi_ok") + mustExec(t, db, "create table minio_multi_ok(id bigint not null, name varchar(50))") + mustExec(t, db, minioLoadSQL( + server.endpointURL, server, "multi-ok/part*.arrow", "minio_multi_ok", "file", true)) + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from minio_multi_ok")) + require.Equal(t, int64(2), queryCount(t, db, "select count(distinct id) from minio_multi_ok")) + + server.put(t, "multi-bad/01-valid.arrow", original) + server.put(t, "multi-bad/02-corrupt.arrow", []byte("not an Arrow IPC file")) + mustExec(t, db, "drop table if exists minio_multi_bad") + mustExec(t, db, "create table minio_multi_bad(id bigint not null, name varchar(50))") + mustExec(t, db, "insert into minio_multi_bad values (0, 'seed')") + _, err := db.Exec(minioLoadSQL( + server.endpointURL, server, "multi-bad/*.arrow", "minio_multi_bad", "file", true)) + require.Error(t, err) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from minio_multi_bad")) + + // The next statement is the reuse control for planner/reader cleanup. + mustExec(t, db, minioLoadSQL( + server.endpointURL, server, "multi-bad/01-valid.arrow", "minio_multi_bad", "file", false)) + require.Equal(t, int64(3), queryCount(t, db, "select count(*) from minio_multi_bad")) + }) + + t.Run("ObjectChangeFailsClosed", func(t *testing.T) { + const key = "fault/object-change.arrow" + server.put(t, key, original) + mutationDone := make(chan error, 1) + var mutationStarted atomic.Bool + proxyEndpoint := startArrowMinIOProxy(t, server.endpointURL, + func(w http.ResponseWriter, r *http.Request) bool { + if !isConditionalRangeGET(r) || !mutationStarted.CompareAndSwap(false, true) { + return false + } + mutationCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := server.putObject(mutationCtx, key, replacement); err != nil { + mutationDone <- err + http.Error(w, "failed to replace fault-injection object", http.StatusInternalServerError) + return true + } + mutationDone <- nil + return false + }) + + mustExec(t, db, "drop table if exists minio_object_change") + mustExec(t, db, "create table minio_object_change(id bigint not null, name varchar(50))") + mustExec(t, db, "insert into minio_object_change values (0, 'seed')") + _, err := db.Exec(minioLoadSQL(proxyEndpoint, server, key, "minio_object_change", "file", false)) + require.Error(t, err) + require.Contains(t, strings.ToLower(err.Error()), "object changed") + select { + case mutationErr := <-mutationDone: + require.NoError(t, mutationErr) + case <-time.After(5 * time.Second): + t.Fatal("the fault proxy never observed a conditional range GET") + } + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from minio_object_change")) + }) + + t.Run("CanceledRequestReleasesS3Read", func(t *testing.T) { + const key = "fault/cancel.arrow" + server.put(t, key, original) + requestStarted := make(chan struct{}) + requestCanceled := make(chan struct{}) + var blocked atomic.Bool + proxyEndpoint := startArrowMinIOProxy(t, server.endpointURL, + func(w http.ResponseWriter, r *http.Request) bool { + if !isConditionalRangeGET(r) || !blocked.CompareAndSwap(false, true) { + return false + } + close(requestStarted) + select { + case <-r.Context().Done(): + close(requestCanceled) + case <-time.After(30 * time.Second): + http.Error(w, "timed out waiting for request cancellation", http.StatusGatewayTimeout) + } + return true + }) + + mustExec(t, db, "drop table if exists minio_cancel") + mustExec(t, db, "create table minio_cancel(id bigint not null, name varchar(50))") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + errCh := make(chan error, 1) + go func() { + _, err := db.ExecContext(ctx, minioLoadSQL(proxyEndpoint, server, key, "minio_cancel", "file", false)) + errCh <- err + }() + select { + case <-requestStarted: + cancel() + case <-time.After(30 * time.Second): + cancel() + t.Fatal("timed out waiting for the conditional MinIO request") + } + select { + case err := <-errCh: + require.Error(t, err) + case <-time.After(30 * time.Second): + t.Fatal("timed out waiting for the canceled MinIO LOAD") + } + select { + case <-requestCanceled: + case <-time.After(5 * time.Second): + t.Fatal("the in-flight MinIO range request did not observe cancellation") + } + // go-sql-driver/mysql discards the connection after ExecContext is + // canceled. Re-establish session-local database state on the replacement + // connection before checking durable table state. + mustExec(t, db, "use arrow_bvt") + require.Equal(t, int64(0), queryCount(t, db, "select count(*) from minio_cancel")) + }) +} + +type arrowLoadMinIO struct { + endpoint string + endpointURL string + bucket string + user string + password string + client *minio.Client +} + +func startArrowLoadMinIO(t *testing.T) arrowLoadMinIO { + t.Helper() + executable, err := exec.LookPath("minio") + if errors.Is(err, exec.ErrNotFound) { + t.Skip("local MinIO binary is not installed") + } + require.NoError(t, err) + + reserveAddress := func() string { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + address := listener.Addr().String() + require.NoError(t, listener.Close()) + return address + } + endpoint, consoleEndpoint := reserveAddress(), reserveAddress() + logPath := filepath.Join(t.TempDir(), "minio.log") + logFile, err := os.Create(logPath) + require.NoError(t, err) + + const user = "arrowtest" + const password = "arrowtest-secret" + command := exec.Command(executable, "server", t.TempDir(), "--address", endpoint, "--console-address", consoleEndpoint) + command.Env = append(os.Environ(), "MINIO_ROOT_USER="+user, "MINIO_ROOT_PASSWORD="+password) + command.Stdout = logFile + command.Stderr = logFile + require.NoError(t, command.Start()) + t.Cleanup(func() { + _ = command.Process.Kill() + _, _ = command.Process.Wait() + _ = logFile.Close() + }) + + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(user, password, ""), + Region: "us-east-1", + }) + require.NoError(t, err) + bucket := "matrixone-arrow-load" + deadline := time.Now().Add(15 * time.Second) + for { + attemptCtx, cancel := context.WithTimeout(context.Background(), time.Second) + err = client.MakeBucket(attemptCtx, bucket, minio.MakeBucketOptions{Region: "us-east-1"}) + cancel() + if err == nil { + break + } + if time.Now().After(deadline) { + logBytes, _ := os.ReadFile(logPath) + t.Fatalf("start local MinIO: %v\n%s", err, logBytes) + } + time.Sleep(100 * time.Millisecond) + } + return arrowLoadMinIO{ + endpoint: endpoint, endpointURL: "http://" + endpoint, + bucket: bucket, user: user, password: password, client: client, + } +} + +func (m arrowLoadMinIO) put(t *testing.T, key string, payload []byte) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + require.NoError(t, m.putObject(ctx, key, payload)) +} + +func (m arrowLoadMinIO) putObject(ctx context.Context, key string, payload []byte) error { + _, err := m.client.PutObject( + ctx, m.bucket, key, bytes.NewReader(payload), int64(len(payload)), + minio.PutObjectOptions{ContentType: "application/vnd.apache.arrow.file"}, + ) + return err +} + +func mustReadFile(t *testing.T, path string) []byte { + t.Helper() + payload, err := os.ReadFile(path) + require.NoError(t, err) + return payload +} + +func minioLoadSQL(endpoint string, server arrowLoadMinIO, key, table, container string, parallel bool) string { + containerOption := "" + if container != "" { + containerOption = fmt.Sprintf(",'arrow_container'='%s'", container) + } + stmt := fmt.Sprintf( + "load data url s3option {'endpoint'='%s','access_key_id'='%s','secret_access_key'='%s',"+ + "'bucket'='%s','region'='us-east-1','provider'='minio','filepath'='%s','format'='arrow'%s} into table %s", + endpoint, server.user, server.password, server.bucket, key, containerOption, table) + if parallel { + stmt += " parallel 'true'" + } + return stmt +} + +func isConditionalRangeGET(r *http.Request) bool { + return r.Method == http.MethodGet && r.Header.Get("Range") != "" && r.Header.Get("If-Match") != "" +} + +// startArrowMinIOProxy forwards signed path-style S3 requests to MinIO while +// allowing a test to intercept one protocol phase. interceptor returns true +// only when it has produced the complete response itself. +func startArrowMinIOProxy( + t *testing.T, + targetEndpoint string, + interceptor func(http.ResponseWriter, *http.Request) bool, +) string { + t.Helper() + target, err := url.Parse(targetEndpoint) + require.NoError(t, err) + proxy := httputil.NewSingleHostReverseProxy(target) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if interceptor(w, r) { + return + } + proxy.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + return server.URL +} diff --git a/pkg/tests/arrowload/arrow_load_multicn_test.go b/pkg/tests/arrowload/arrow_load_multicn_test.go new file mode 100644 index 0000000000000..17b5eec0410ef --- /dev/null +++ b/pkg/tests/arrowload/arrow_load_multicn_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowload + +import ( + "database/sql" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestArrowLoadMultiCN covers distributed record-batch fan-out through the +// public path. Shutdown/cancellation coverage uses deterministic request and +// cluster-lifecycle fault injection in the dedicated rollout and MinIO tests. +func TestArrowLoadMultiCN(t *testing.T) { + c := startArrowLoadCluster(t, 2, true, false, true) + db := openArrowLoadDB(t, c, 0) + mustExec(t, db, "create database if not exists arrow_multicn") + mustExec(t, db, "use arrow_multicn") + path, ddl := fixtureLarge(t) + + t.Run("DistributedRecordBatchFanout", func(t *testing.T) { testArrowMultiCNFanout(t, db, path, ddl) }) +} + +// testArrowMultiCNFanout loads the "large" multi-record-batch fixture with +// `PARALLEL 'true'` against the 2-CN cluster and checks full row-count and content +// correctness. Shard-routing internals are already unit-tested in +// pkg/sql/compile's arrow_scope_test.go; this test's job is proving the whole thing +// produces correct data on a real multi-CN cluster, not re-deriving that routing. +func testArrowMultiCNFanout(t *testing.T, db *sql.DB, path, ddl string) { + mustExec(t, db, "drop table if exists large_fanout") + mustExec(t, db, fmt.Sprintf("create table large_fanout(%s)", ddl)) + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table large_fanout parallel 'true'", path)) + + require.Equal(t, int64(largeFixtureRows), queryCount(t, db, "select count(*) from large_fanout")) + require.Equal(t, int64(largeFixtureRows), queryCount(t, db, "select count(distinct id) from large_fanout")) + require.Equal(t, int64(0), queryCount(t, db, + fmt.Sprintf("select count(*) from large_fanout where id < 0 or id >= %d", largeFixtureRows))) +} diff --git a/pkg/tests/arrowload/arrow_load_rollout_test.go b/pkg/tests/arrowload/arrow_load_rollout_test.go new file mode 100644 index 0000000000000..79ef5d4ce700c --- /dev/null +++ b/pkg/tests/arrowload/arrow_load_rollout_test.go @@ -0,0 +1,96 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowload + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestArrowLoadRolloutRollbackDrain exercises the operational transition, not +// just static gate values. It starts from an explicitly enabled local-only policy, stops a +// cluster while an Arrow statement is admitted, restarts with every Arrow gate +// disabled, and finally rolls forward with distributed execution disabled. +// Shutdown may finish the admitted transaction or cancel it; either result must +// be atomic and bounded. +func TestArrowLoadRolloutRollbackDrain(t *testing.T) { + c := startArrowLoadCluster(t, 1, true, false, false) + db := openArrowLoadDB(t, c, 0) + mustExec(t, db, "create database if not exists arrow_rollout") + mustExec(t, db, "use arrow_rollout") + path, ddl := fixtureLarge(t) + mustExec(t, db, fmt.Sprintf("create table rollout_drain(%s)", ddl)) + + ctx := context.Background() + conn, err := db.Conn(ctx) + require.NoError(t, err) + _, err = conn.ExecContext(ctx, "use arrow_rollout") + require.NoError(t, err) + var connID int64 + require.NoError(t, conn.QueryRowContext(ctx, "select connection_id()").Scan(&connID)) + loadErrCh := make(chan error, 1) + go func() { + _, execErr := conn.ExecContext(ctx, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table rollout_drain parallel 'true'", path)) + loadErrCh <- execErr + }() + waitUntilStatementRunning(t, db, connID, "load data", 30*time.Second) + require.NoError(t, c.Close()) + _ = conn.Close() + _ = db.Close() + + var loadErr error + select { + case loadErr = <-loadErrCh: + case <-time.After(30 * time.Second): + t.Fatal("timed out waiting for admitted Arrow LOAD during cluster shutdown") + } + + adjustArrowLoadCluster(c, arrowLoadClusterOptions{cnCount: 1}) + require.NoError(t, c.Start()) + rollbackDB := openArrowLoadDB(t, c, 0) + rows := queryCount(t, rollbackDB, "select count(*) from arrow_rollout.rollout_drain") + if loadErr == nil { + require.Equal(t, int64(largeFixtureRows), rows, + "a drained statement must commit the complete fixture") + } else { + require.Zero(t, rows, "a shutdown-canceled statement must commit no rows") + } + + missing := filepath.Join(t.TempDir(), "must-not-be-read.arrow") + _, err = rollbackDB.Exec(fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table arrow_rollout.rollout_drain", missing)) + require.Error(t, err) + require.Contains(t, strings.ToLower(err.Error()), "disabled by configuration") + + require.NoError(t, rollbackDB.Close()) + require.NoError(t, c.Close()) + adjustArrowLoadCluster(c, arrowLoadClusterOptions{ + cnCount: 1, enabled: true, s3Enabled: false, distributedEnabled: false, + }) + require.NoError(t, c.Start()) + rolledForwardDB := openArrowLoadDB(t, c, 0) + mustExec(t, rolledForwardDB, "truncate table arrow_rollout.rollout_drain") + mustExec(t, rolledForwardDB, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table arrow_rollout.rollout_drain parallel 'true'", path)) + require.Equal(t, int64(largeFixtureRows), queryCount(t, rolledForwardDB, + "select count(*) from arrow_rollout.rollout_drain")) +} diff --git a/pkg/tests/arrowload/arrow_load_test.go b/pkg/tests/arrowload/arrow_load_test.go new file mode 100644 index 0000000000000..ae9b32f21b055 --- /dev/null +++ b/pkg/tests/arrowload/arrow_load_test.go @@ -0,0 +1,612 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowload + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/embed" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/util/fault" + "github.com/stretchr/testify/require" +) + +// TestArrowLoadBVT is the embedded public-path BVT suite for `LOAD DATA ... +// format='arrow'` (issue #23684, design doc sections 14 and 18). It runs every +// subtest against one dedicated 1-CN cluster with S3 explicitly enabled and uses +// the real MySQL protocol rather than the internal executor. Local File and Stream +// and explicitly opts in to every Arrow surface it exercises. +// It proves type-matrix correctness, option/DDL rejection, multi-object +// atomicity, explicit transactions, cross-session visibility, and local gate +// behavior. Standard distributed CI, mixed binaries, and real cloud providers +// remain separate release evidence. +func TestArrowLoadBVT(t *testing.T) { + c := startArrowLoadCluster(t, 1, true, true, false) + db := openArrowLoadDB(t, c, 0) + mustExec(t, db, "create database if not exists arrow_bvt") + mustExec(t, db, "use arrow_bvt") + + t.Run("TypeMatrixNumeric", func(t *testing.T) { testArrowTypeMatrixNumeric(t, db) }) + t.Run("TypeMatrixTimestampDict", func(t *testing.T) { testArrowTypeMatrixTimestampDict(t, db) }) + t.Run("TypeMatrixLongBinary", func(t *testing.T) { testArrowTypeMatrixLongBinary(t, db) }) + t.Run("ExplicitColumnOrder", func(t *testing.T) { testArrowExplicitColumnOrder(t, db) }) + t.Run("ArrowContainerSemantics", func(t *testing.T) { testArrowContainerSemantics(t, db) }) + t.Run("NegativeOptionValidation", func(t *testing.T) { testArrowNegativeOptions(t, db) }) + t.Run("DDLRejects", func(t *testing.T) { testArrowDDLRejects(t, db) }) + t.Run("LocalStage", func(t *testing.T) { testArrowLocalStage(t, db) }) + t.Run("MultiObjectSchemaMismatch", func(t *testing.T) { testArrowSchemaMismatchRollback(t, db) }) + t.Run("ConstraintViolationRollback", func(t *testing.T) { testArrowConstraintViolationRollback(t, db) }) + t.Run("CorruptInputRollback", func(t *testing.T) { testArrowCorruptInputRollback(t, db) }) + t.Run("ExplicitTransactionVisibility", func(t *testing.T) { testArrowExplicitTransaction(t, c) }) + t.Run("TwoSessionIsolation", func(t *testing.T) { testArrowTwoSessionIsolation(t, c) }) + t.Run("DifferentialVsInsert", func(t *testing.T) { testArrowDifferentialVsInsert(t, db) }) + t.Run("CommitPhaseFailureRollback", func(t *testing.T) { testArrowCommitPhaseFailureRollback(t, db) }) + t.Run("LocalMinIO", func(t *testing.T) { testArrowLoadLocalMinIO(t, db) }) + + // Restart is deliberately last: it destroys every existing SQL connection + // while preserving the cluster data directory. No later subtest may depend on + // the original CN generation. + t.Run("ClusterRestartPersistence", func(t *testing.T) { testArrowClusterRestart(t, c, db) }) +} + +func testArrowExplicitColumnOrder(t *testing.T, db *sql.DB) { + mustExec(t, db, "drop table if exists explicit_column_order") + mustExec(t, db, "create table explicit_column_order(a bigint, b bigint)") + for _, container := range []string{containerFile, containerStream} { + t.Run(container, func(t *testing.T) { + mustExec(t, db, "truncate table explicit_column_order") + path := fixtureInt64Pair( + t, t.TempDir(), "explicit_"+container+".arrow", container, + []int64{11, 12}, []int64{21, 22}, + ) + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table explicit_column_order (b,a)", + path, + )) + require.Equal(t, int64(2), queryCount(t, db, + "select count(*) from explicit_column_order where (a=21 and b=11) or (a=22 and b=12)")) + }) + } +} + +// testArrowCommitPhaseFailureRollback injects the transaction failure after +// workspace batches have been dumped but before commit becomes visible. That is +// later than reader/conversion failures, so this test closes the statement-level +// atomicity contract at the actual commit boundary and then proves the same +// source can be retried after the injection is removed. +func testArrowCommitPhaseFailureRollback(t *testing.T, db *sql.DB) { + path := fixtureIDName(t, t.TempDir(), "commit-failure.arrow", containerFile, + [][]idNameRow{{{id: 1, name: "loaded"}}}) + mustExec(t, db, "drop table if exists commit_failure_rollback") + mustExec(t, db, "create table commit_failure_rollback(id bigint not null, name varchar(50))") + mustExec(t, db, "insert into commit_failure_rollback values (0, 'seed')") + + fault.Enable() + defer fault.Disable() + removeFailure, err := objectio.SimpleInject(objectio.FJ_CNCommitAfterWorkspaceDumpFailed) + require.NoError(t, err) + defer removeFailure() + + _, err = db.Exec(fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table commit_failure_rollback", path)) + require.ErrorContains(t, err, "injected commit failure after workspace dump") + removeFailure() + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from commit_failure_rollback"), + "a failed commit must not expose any Arrow rows") + + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table commit_failure_rollback", path)) + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from commit_failure_rollback")) +} + +// TestArrowLoadGateDisabled proves the explicit rollback switch fails closed: +// with `cn.frontend.arrow-load.enabled=false`, any Arrow LOAD must be rejected +// before touching the file at all. This checks the client-visible opt-out +// contract, but it does not replace a true mixed-binary-version rehearsal. +func TestArrowLoadGateDisabled(t *testing.T) { + c := startArrowLoadClusterWithDefaults(t, 1) + db := openArrowLoadDB(t, c, 0) + mustExec(t, db, "create database if not exists arrow_gate_off") + mustExec(t, db, "use arrow_gate_off") + mustExec(t, db, "create table t(id bigint not null, amount decimal(18,2), score double, flag bool)") + path := filepath.Join(t.TempDir(), "missing-before-gate.arrow") + + _, err := db.Exec(fmt.Sprintf("load data infile {'filepath'='%s','format'='arrow'} into table t", path)) + require.Error(t, err) + require.Contains(t, strings.ToLower(err.Error()), "disabled by configuration") + require.Equal(t, int64(0), queryCount(t, db, "select count(*) from t")) +} + +// TestArrowLoadGateS3Disabled proves the default S3 sub-gate fails closed before +// any network I/O. Dummy, unreachable credentials are sufficient proof of the +// ordering: the statement must be rejected by configuration rather than +// attempting HeadObject. +func TestArrowLoadGateS3Disabled(t *testing.T) { + c := startArrowLoadCluster(t, 1, true, false, false) + db := openArrowLoadDB(t, c, 0) + mustExec(t, db, "create database if not exists arrow_s3_gate_off") + mustExec(t, db, "use arrow_s3_gate_off") + mustExec(t, db, "create table t(id bigint not null, amount decimal(18,2), score double, flag bool)") + + _, err := db.Exec( + "load data url s3option " + + "{'endpoint'='http://127.0.0.1:1','access_key_id'='dummy','secret_access_key'='dummy'," + + "'bucket'='no-such-bucket','region'='us-east-1'," + + "'filepath'='does-not-matter.arrow','format'='arrow'} into table t") + require.Error(t, err) + require.Contains(t, strings.ToLower(err.Error()), "disabled by configuration") + require.Equal(t, int64(0), queryCount(t, db, "select count(*) from t")) +} + +// TestArrowLoadGateDistributedDisabledSoftFallback proves DistributedEnabled=false +// is a soft fallback (silently serialize), not a hard rejection. +func TestArrowLoadGateDistributedDisabledSoftFallback(t *testing.T) { + c := startArrowLoadCluster(t, 1, true /*enabled*/, true /*s3Enabled*/, false /*distributedEnabled*/) + db := openArrowLoadDB(t, c, 0) + mustExec(t, db, "create database if not exists arrow_distributed_off") + mustExec(t, db, "use arrow_distributed_off") + mustExec(t, db, "create table t(id bigint not null, name varchar(50))") + + dir := t.TempDir() + fixtureIDName(t, dir, "part1.arrow", containerFile, [][]idNameRow{{{id: 1, name: "a"}}}) + fixtureIDName(t, dir, "part2.arrow", containerFile, [][]idNameRow{{{id: 2, name: "b"}}}) + + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table t parallel 'true'", + filepath.Join(dir, "part*.arrow"))) + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from t")) +} + +func testArrowTypeMatrixNumeric(t *testing.T, db *sql.DB) { + mustExec(t, db, "drop table if exists numeric_matrix") + mustExec(t, db, "create table numeric_matrix(id bigint not null, amount decimal(18,2), score double, flag bool)") + rows := []numericRow{ + {id: 1, amount: 100, score: 1.5, flag: true}, + {id: 2, amountNull: true, score: 2.5, flag: false}, + {id: 3, amount: -50000, scoreNull: true, flag: true}, + {id: 4, amount: 0, score: 4.5, flagNull: true}, + } + for _, container := range []string{containerFile, containerStream} { + t.Run(container, func(t *testing.T) { + mustExec(t, db, "truncate table numeric_matrix") + path := fixtureNumeric(t, t.TempDir(), "numeric_"+container+".arrow", container, rows, 2) + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table numeric_matrix", path)) + require.Equal(t, int64(4), queryCount(t, db, "select count(*) from numeric_matrix")) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from numeric_matrix where amount is null")) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from numeric_matrix where score is null")) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from numeric_matrix where flag is null")) + require.Equal(t, int64(1), queryCount(t, db, + "select count(*) from numeric_matrix where id=3 and amount=-500.00")) + require.Equal(t, int64(1), queryCount(t, db, + "select count(*) from numeric_matrix where id=1 and amount=1.00 and flag=true")) + }) + } +} + +func testArrowTypeMatrixTimestampDict(t *testing.T, db *sql.DB) { + mustExec(t, db, "drop table if exists ts_matrix") + mustExec(t, db, "create table ts_matrix(id bigint not null, ts datetime(6), d date, name varchar(50))") + loc := time.UTC + rows := []timestampRow{ + {id: 1, ts: time.Date(2026, 1, 2, 3, 4, 5, 123456000, loc), date: time.Date(2026, 1, 2, 0, 0, 0, 0, loc), name: "alice"}, + {id: 2, tsNull: true, date: time.Date(2026, 1, 3, 0, 0, 0, 0, loc), name: "bob"}, + {id: 3, ts: time.Date(2026, 1, 4, 6, 7, 8, 0, loc), dateNull: true, nameNull: true}, + } + for _, container := range []string{containerFile, containerStream} { + t.Run(container, func(t *testing.T) { + mustExec(t, db, "truncate table ts_matrix") + path := fixtureTimestampDict(t, t.TempDir(), "ts_"+container+".arrow", container, rows, []string{"alice", "bob"}) + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table ts_matrix", path)) + require.Equal(t, int64(3), queryCount(t, db, "select count(*) from ts_matrix")) + require.Equal(t, int64(1), queryCount(t, db, + "select count(*) from ts_matrix where id=1 and ts='2026-01-02 03:04:05.123456' and d='2026-01-02' and name='alice'")) + require.Equal(t, int64(1), queryCount(t, db, + "select count(*) from ts_matrix where id=2 and ts is null and name='bob'")) + require.Equal(t, int64(1), queryCount(t, db, + "select count(*) from ts_matrix where id=3 and d is null and name is null")) + }) + } +} + +func testArrowTypeMatrixLongBinary(t *testing.T, db *sql.DB) { + mustExec(t, db, "drop table if exists bin_matrix") + mustExec(t, db, "drop table if exists fixed_bin_matrix") + mustExec(t, db, "create table bin_matrix(id bigint not null, payload varbinary(200))") + mustExec(t, db, "create table fixed_bin_matrix(id bigint not null, payload binary(200))") + longA := strings.Repeat("A", 40) + longB := strings.Repeat("B", 64) + rows := []binaryRow{ + {id: 1, payload: []byte(longA)}, + {id: 2, payload: []byte(longB)}, + } + for _, container := range []string{containerFile, containerStream} { + t.Run(container, func(t *testing.T) { + mustExec(t, db, "truncate table bin_matrix") + mustExec(t, db, "truncate table fixed_bin_matrix") + path := fixtureLongBinary(t, t.TempDir(), "bin_"+container+".arrow", container, rows) + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table bin_matrix", path)) + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table fixed_bin_matrix", path)) + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from bin_matrix")) + require.Equal(t, int64(1), queryCount(t, db, + fmt.Sprintf("select count(*) from bin_matrix where id=1 and payload='%s'", longA))) + require.Equal(t, int64(1), queryCount(t, db, + fmt.Sprintf("select count(*) from bin_matrix where id=2 and payload='%s'", longB))) + for _, row := range rows { + var stored []byte + require.NoError(t, db.QueryRow( + "select payload from fixed_bin_matrix where id = ?", row.id, + ).Scan(&stored)) + require.Len(t, stored, 200) + require.Equal(t, row.payload, stored[:len(row.payload)]) + require.Equal(t, make([]byte, 200-len(row.payload)), stored[len(row.payload):]) + } + }) + } +} + +func testArrowContainerSemantics(t *testing.T, db *sql.DB) { + mustExec(t, db, "drop table if exists container_semantics") + mustExec(t, db, "create table container_semantics(id bigint not null, amount decimal(18,2), score double, flag bool)") + row := []numericRow{{id: 1, amount: 100, score: 1.5, flag: true}} + filePath := fixtureNumeric(t, t.TempDir(), "container_file.arrow", containerFile, row, 10) + streamPath := fixtureNumeric(t, t.TempDir(), "container_stream.arrow", containerStream, row, 10) + + for _, arrowContainer := range []string{"auto", "file"} { + t.Run("file_as_"+arrowContainer, func(t *testing.T) { + mustExec(t, db, "truncate table container_semantics") + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow','arrow_container'='%s'} into table container_semantics", + filePath, arrowContainer)) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from container_semantics")) + }) + } + for _, arrowContainer := range []string{"auto", "stream"} { + t.Run("stream_as_"+arrowContainer, func(t *testing.T) { + mustExec(t, db, "truncate table container_semantics") + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow','arrow_container'='%s'} into table container_semantics", + streamPath, arrowContainer)) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from container_semantics")) + }) + } + t.Run("file_as_stream_rejected", func(t *testing.T) { + _, err := db.Exec(fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow','arrow_container'='stream'} into table container_semantics", + filePath)) + require.Error(t, err) + }) + t.Run("stream_as_file_rejected", func(t *testing.T) { + _, err := db.Exec(fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow','arrow_container'='file'} into table container_semantics", + streamPath)) + require.Error(t, err) + }) + t.Run("invalid_container_value_rejected", func(t *testing.T) { + _, err := db.Exec(fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow','arrow_container'='flight'} into table container_semantics", + filePath)) + require.Error(t, err) + }) +} + +func testArrowNegativeOptions(t *testing.T, db *sql.DB) { + mustExec(t, db, "drop table if exists arrow_option_reject") + mustExec(t, db, "create table arrow_option_reject(id bigint, name varchar(50))") + path := fixtureIDName(t, t.TempDir(), "option_reject.arrow", containerFile, + [][]idNameRow{{{id: 1, name: "a"}}}) + + cases := []struct { + name string + sql string + }{ + {"compression", fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow','compression'='gzip'} into table arrow_option_reject", path)}, + {"jsondata", fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow','jsondata'='object'} into table arrow_option_reject", path)}, + {"hive_partitioning", fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow','hive_partitioning'='true'} into table arrow_option_reject", path)}, + {"fields_terminated", fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table arrow_option_reject fields terminated by ','", path)}, + {"lines_terminated", fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table arrow_option_reject lines terminated by '\\n'", path)}, + {"ignore_lines", fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table arrow_option_reject ignore 1 lines", path)}, + {"local_infile", fmt.Sprintf( + "load data local infile {'filepath'='%s','format'='arrow'} into table arrow_option_reject", path)}, + {"at_variable_column", fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table arrow_option_reject (id, @name)", path)}, + {"set_assignment", fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table arrow_option_reject set name=nullif(name,'x')", path)}, + {"invalid_arrow_container", fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow','arrow_container'='flight'} into table arrow_option_reject", path)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := db.Exec(tc.sql) + require.Error(t, err, tc.sql) + }) + } +} + +func testArrowDDLRejects(t *testing.T, db *sql.DB) { + arrowPath := fixtureIDName(t, t.TempDir(), "ddl_reject_source.arrow", containerFile, + [][]idNameRow{{{id: 1, name: "a"}}}) + + mustExec(t, db, "drop table if exists arrow_ext_reject") + _, err := db.Exec(fmt.Sprintf( + "create external table arrow_ext_reject(id bigint, name varchar(50)) infile{'filepath'='%s','format'='arrow'}", + arrowPath)) + require.Error(t, err) + + csvDir := t.TempDir() + csvPath := filepath.Join(csvDir, "placeholder.csv") + require.NoError(t, os.WriteFile(csvPath, []byte("1,a\n"), 0o600)) + mustExec(t, db, "drop table if exists arrow_into_external") + mustExec(t, db, fmt.Sprintf( + "create external table arrow_into_external(id bigint, name varchar(50)) infile{'filepath'='%s','format'='csv'} fields terminated by ','", + csvPath)) + _, err = db.Exec(fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table arrow_into_external", arrowPath)) + require.Error(t, err) +} + +func testArrowLocalStage(t *testing.T, db *sql.DB) { + dir := t.TempDir() + path := fixtureNumeric(t, dir, "stage_source.arrow", containerFile, + []numericRow{{id: 1, amount: 100, score: 1.5, flag: true}}, 10) + + mustExec(t, db, "drop stage if exists arrow_local_stage") + mustExec(t, db, fmt.Sprintf("create stage arrow_local_stage URL='file://%s/'", dir)) + mustExec(t, db, "drop table if exists stage_target") + mustExec(t, db, "create table stage_target(id bigint not null, amount decimal(18,2), score double, flag bool)") + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='stage://arrow_local_stage/%s','format'='arrow'} into table stage_target", + filepath.Base(path))) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from stage_target")) + mustExec(t, db, "drop stage arrow_local_stage") +} + +// testArrowSchemaMismatchRollback proves design invariant I2/I6: two objects in one +// LOAD statement whose Arrow schemas disagree (here, `id` is int64 in one file and +// float64 in the other) must fail the whole statement, leaving only the pre-seeded +// row behind, mirroring load_data_parquet.sql's multi-file negative pattern. +func testArrowSchemaMismatchRollback(t *testing.T, db *sql.DB) { + dir := t.TempDir() + fixtureIDName(t, dir, "mismatch_part1.arrow", containerFile, [][]idNameRow{{{id: 1, name: "a"}}}) + fixtureIDNameMismatchedIDType(t, dir, "mismatch_part2.arrow", containerFile, []float64{2}, []string{"b"}) + + mustExec(t, db, "drop table if exists schema_mismatch") + mustExec(t, db, "create table schema_mismatch(id bigint, name varchar(50))") + mustExec(t, db, "insert into schema_mismatch values (0, 'seed')") + + _, err := db.Exec(fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table schema_mismatch parallel 'true'", + filepath.Join(dir, "mismatch_part*.arrow"))) + require.Error(t, err) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from schema_mismatch")) +} + +// testArrowConstraintViolationRollback proves the same all-or-nothing autocommit +// behavior for a MO-side constraint failure discovered only after the Arrow bridge +// has already converted some rows: a NOT NULL violation split across two same-schema +// files, and a separate INT-range overflow, each leave only the pre-seeded row. +func testArrowConstraintViolationRollback(t *testing.T, db *sql.DB) { + t.Run("not_null_violation", func(t *testing.T) { + dir := t.TempDir() + fixtureIDName(t, dir, "notnull_part1.arrow", containerFile, [][]idNameRow{{{id: 1, name: "a"}}}) + fixtureIDName(t, dir, "notnull_part2.arrow", containerFile, [][]idNameRow{{{idNull: true, name: "b"}}}) + + mustExec(t, db, "drop table if exists notnull_violation") + mustExec(t, db, "create table notnull_violation(id bigint not null, name varchar(50))") + mustExec(t, db, "insert into notnull_violation values (0, 'seed')") + + _, err := db.Exec(fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table notnull_violation parallel 'true'", + filepath.Join(dir, "notnull_part*.arrow"))) + require.Error(t, err) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from notnull_violation")) + }) + + t.Run("integer_range_overflow", func(t *testing.T) { + dir := t.TempDir() + fixtureInt64Overflow(t, dir, "overflow.arrow", containerFile, []int64{1, 1 << 40}) + + mustExec(t, db, "drop table if exists overflow_violation") + mustExec(t, db, "create table overflow_violation(v int not null)") + mustExec(t, db, "insert into overflow_violation values (0)") + + _, err := db.Exec(fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table overflow_violation", + filepath.Join(dir, "overflow.arrow"))) + require.Error(t, err) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from overflow_violation")) + }) +} + +// testArrowCorruptInputRollback reaches malformed File and Stream containers +// through the SQL/frontend/transaction path. Each failure must preserve the +// committed seed row, and a subsequent valid LOAD on the same cluster must +// succeed, proving failed reader generations do not poison later statements. +func testArrowCorruptInputRollback(t *testing.T, db *sql.DB) { + t.Run("file_in_multi_object_load", func(t *testing.T) { + dir := t.TempDir() + validPath := fixtureIDName(t, dir, "01-valid.arrow", containerFile, + [][]idNameRow{{{id: 1, name: "valid"}}}) + require.NoError(t, os.WriteFile(filepath.Join(dir, "02-corrupt.arrow"), []byte("not an Arrow IPC file"), 0o600)) + + mustExec(t, db, "drop table if exists corrupt_file_rollback") + mustExec(t, db, "create table corrupt_file_rollback(id bigint not null, name varchar(50))") + mustExec(t, db, "insert into corrupt_file_rollback values (0, 'seed')") + _, err := db.Exec(fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table corrupt_file_rollback parallel 'true'", + filepath.Join(dir, "*.arrow"))) + require.Error(t, err) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from corrupt_file_rollback")) + + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table corrupt_file_rollback", validPath)) + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from corrupt_file_rollback")) + }) + + t.Run("truncated_stream", func(t *testing.T) { + dir := t.TempDir() + validPath := fixtureIDName(t, dir, "valid.stream", containerStream, + [][]idNameRow{{{id: 1, name: "valid"}}}) + payload, err := os.ReadFile(validPath) + require.NoError(t, err) + require.Greater(t, len(payload), 16) + corruptPath := filepath.Join(dir, "truncated.stream") + require.NoError(t, os.WriteFile(corruptPath, payload[:len(payload)/2], 0o600)) + + mustExec(t, db, "drop table if exists corrupt_stream_rollback") + mustExec(t, db, "create table corrupt_stream_rollback(id bigint not null, name varchar(50))") + mustExec(t, db, "insert into corrupt_stream_rollback values (0, 'seed')") + _, err = db.Exec(fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow','arrow_container'='stream'} into table corrupt_stream_rollback", + corruptPath)) + require.Error(t, err) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from corrupt_stream_rollback")) + + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow','arrow_container'='stream'} into table corrupt_stream_rollback", + validPath)) + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from corrupt_stream_rollback")) + }) +} + +// testArrowExplicitTransaction adapts the begin/rollback and start-transaction/commit +// shape from test/distributed/cases/optimistic/atomicity_1.sql to Arrow LOAD, run +// directly as a Go test (no mo-tester, no @bvt:issue wrapper) against a dedicated +// connection so the transaction stays open across statements. +func testArrowExplicitTransaction(t *testing.T, c embed.Cluster) { + db := openArrowLoadDB(t, c, 0) + mustExec(t, db, "use arrow_bvt") + mustExec(t, db, "drop table if exists txn_visibility") + mustExec(t, db, "create table txn_visibility(id bigint not null, name varchar(50))") + path := fixtureIDName(t, t.TempDir(), "txn.arrow", containerFile, [][]idNameRow{{{id: 1, name: "a"}, {id: 2, name: "b"}}}) + + mustExec(t, db, "begin") + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table txn_visibility", path)) + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from txn_visibility")) + mustExec(t, db, "rollback") + require.Equal(t, int64(0), queryCount(t, db, "select count(*) from txn_visibility")) + + mustExec(t, db, "start transaction") + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table txn_visibility", path)) + mustExec(t, db, "commit") + require.Equal(t, int64(2), queryCount(t, db, "select count(*) from txn_visibility")) +} + +// testArrowTwoSessionIsolation proves an uncommitted Arrow LOAD is invisible to a +// concurrent session until commit, using a real second connection and a channel +// (not a sleep) to sequence "A has loaded but not committed" before B observes. +func testArrowTwoSessionIsolation(t *testing.T, c embed.Cluster) { + sessionA := openArrowLoadDB(t, c, 0) + sessionB := openArrowLoadDB(t, c, 0) + mustExec(t, sessionA, "use arrow_bvt") + mustExec(t, sessionB, "use arrow_bvt") + mustExec(t, sessionA, "drop table if exists two_session_isolation") + mustExec(t, sessionA, "create table two_session_isolation(id bigint not null, name varchar(50))") + path := fixtureIDName(t, t.TempDir(), "isolation.arrow", containerFile, [][]idNameRow{{{id: 1, name: "a"}}}) + + loaded := make(chan struct{}) + committed := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + mustExec(t, sessionA, "begin") + mustExec(t, sessionA, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table two_session_isolation", path)) + close(loaded) + <-committed + mustExec(t, sessionA, "commit") + }() + + <-loaded + require.Equal(t, int64(0), queryCount(t, sessionB, "select count(*) from two_session_isolation"), + "uncommitted Arrow LOAD rows must not be visible to another session") + close(committed) + wg.Wait() + require.Equal(t, int64(1), queryCount(t, sessionB, "select count(*) from two_session_isolation"), + "committed Arrow LOAD rows must become visible to another session") +} + +// testArrowDifferentialVsInsert satisfies design section 14.2's differential-testing +// requirement: the same logical rows loaded through the Arrow bridge and through a +// literal INSERT must be indistinguishable. +func testArrowDifferentialVsInsert(t *testing.T, db *sql.DB) { + mustExec(t, db, "drop table if exists differential_arrow") + mustExec(t, db, "drop table if exists differential_insert") + mustExec(t, db, "create table differential_arrow(id bigint not null, amount decimal(18,2), score double, flag bool)") + mustExec(t, db, "create table differential_insert(id bigint not null, amount decimal(18,2), score double, flag bool)") + + rows := []numericRow{ + {id: 1, amount: 100, score: 1.5, flag: true}, + {id: 2, amountNull: true, score: 2.5, flag: false}, + {id: 3, amount: -50000, scoreNull: true, flag: true}, + } + path := fixtureNumeric(t, t.TempDir(), "differential.arrow", containerFile, rows, 10) + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table differential_arrow", path)) + mustExec(t, db, + "insert into differential_insert values (1,1.00,1.5,true), (2,null,2.5,false), (3,-500.00,null,true)") + + require.Equal(t, int64(0), queryCount(t, db, + "select count(*) from (select * from differential_arrow except select * from differential_insert) x")) + require.Equal(t, int64(0), queryCount(t, db, + "select count(*) from (select * from differential_insert except select * from differential_arrow) x")) +} + +// testArrowClusterRestart proves both sides of the restart boundary: rows loaded +// before a full embedded-cluster stop/start remain committed, and the restarted +// CN generation can build a fresh Arrow reader and LOAD the same disk object. +func testArrowClusterRestart(t *testing.T, c embed.Cluster, db *sql.DB) { + path := fixtureIDName(t, t.TempDir(), "restart.arrow", containerFile, + [][]idNameRow{{{id: 1, name: "before-restart"}}}) + mustExec(t, db, "use arrow_bvt") + mustExec(t, db, "drop table if exists restart_persistence") + mustExec(t, db, "create table restart_persistence(id bigint not null, name varchar(50))") + mustExec(t, db, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table restart_persistence", path)) + require.Equal(t, int64(1), queryCount(t, db, "select count(*) from restart_persistence")) + + require.NoError(t, db.Close()) + require.NoError(t, c.Close()) + require.NoError(t, c.Start()) + + restartedDB := openArrowLoadDB(t, c, 0) + mustExec(t, restartedDB, "use arrow_bvt") + require.Equal(t, int64(1), queryCount(t, restartedDB, "select count(*) from restart_persistence"), + "a committed Arrow LOAD must survive a complete local cluster restart") + mustExec(t, restartedDB, "truncate table restart_persistence") + mustExec(t, restartedDB, fmt.Sprintf( + "load data infile {'filepath'='%s','format'='arrow'} into table restart_persistence", path)) + require.Equal(t, int64(1), queryCount(t, restartedDB, "select count(*) from restart_persistence"), + "the restarted CN generation must accept a new Arrow LOAD") +} diff --git a/pkg/tests/arrowload/cluster_test.go b/pkg/tests/arrowload/cluster_test.go new file mode 100644 index 0000000000000..65cb7b9f6c129 --- /dev/null +++ b/pkg/tests/arrowload/cluster_test.go @@ -0,0 +1,158 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrowload + +import ( + "context" + "database/sql" + "fmt" + "strings" + "testing" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/matrixorigin/matrixone/pkg/embed" + "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/stretchr/testify/require" +) + +// Arrow LOAD tests use dedicated, non-shared embedded clusters and close them at +// cleanup. This keeps process-local metrics and lifecycle state out of pkg/embed's +// package-level shared clusters. All Arrow execution tests opt in explicitly. +type arrowLoadClusterOptions struct { + cnCount int + enabled bool + s3Enabled bool + distributedEnabled bool + forceMaterialize bool + useDefaults bool +} + +func startArrowLoadCluster(t testing.TB, cnCount int, enabled, s3Enabled, distributedEnabled bool) embed.Cluster { + t.Helper() + return startArrowLoadClusterWithOptions(t, arrowLoadClusterOptions{ + cnCount: cnCount, enabled: enabled, s3Enabled: s3Enabled, + distributedEnabled: distributedEnabled, + }) +} + +// startArrowLoadClusterWithDefaults deliberately installs no Arrow-specific +// configuration. Tests using it prove the product default rejects Arrow LOAD. +func startArrowLoadClusterWithDefaults(t testing.TB, cnCount int) embed.Cluster { + t.Helper() + return startArrowLoadClusterWithOptions(t, arrowLoadClusterOptions{ + cnCount: cnCount, useDefaults: true, + }) +} + +func startArrowLoadClusterWithOptions(t testing.TB, options arrowLoadClusterOptions) embed.Cluster { + t.Helper() + clusterOptions := []embed.Option{embed.WithCNCount(options.cnCount)} + if !options.useDefaults { + clusterOptions = append(clusterOptions, embed.WithPreStart(func(svc embed.ServiceOperator) { + if svc.ServiceType() != metadata.ServiceType_CN { + return + } + svc.Adjust(func(cfg *embed.ServiceConfig) { + cfg.CN.Frontend.ArrowLoad.Enabled = options.enabled + cfg.CN.Frontend.ArrowLoad.S3Enabled = options.s3Enabled + cfg.CN.Frontend.ArrowLoad.DistributedEnabled = options.distributedEnabled + cfg.CN.Frontend.ArrowLoad.ForceMaterialize = options.forceMaterialize + }) + })) + } + c, err := embed.StartTestCluster(clusterOptions...) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, c.Close()) + }) + return c +} + +// adjustArrowLoadCluster changes only the next CN generation's rollout +// settings. Callers close the current generation before adjustment and restart +// afterward, so an admitted statement always keeps the policy snapshot carried +// in its compiled external-scan payload. +func adjustArrowLoadCluster(c embed.Cluster, options arrowLoadClusterOptions) { + c.ForeachServices(func(svc embed.ServiceOperator) bool { + if svc.ServiceType() == metadata.ServiceType_CN { + svc.Adjust(func(cfg *embed.ServiceConfig) { + cfg.CN.Frontend.ArrowLoad.Enabled = options.enabled + cfg.CN.Frontend.ArrowLoad.S3Enabled = options.s3Enabled + cfg.CN.Frontend.ArrowLoad.DistributedEnabled = options.distributedEnabled + cfg.CN.Frontend.ArrowLoad.ForceMaterialize = options.forceMaterialize + }) + } + return true + }) +} + +// openArrowLoadDB opens a real MySQL-protocol connection (not the internal SQL +// executor) against the given CN, so statements run through the same frontend path +// a real client would use. This is required for KILL QUERY, multi-session isolation, and +// SHOW-PROCESSLIST-style observation to mean anything. +func openArrowLoadDB(t testing.TB, c embed.Cluster, cnIndex int) *sql.DB { + t.Helper() + cn, err := c.GetCNService(cnIndex) + require.NoError(t, err) + port := cn.GetServiceConfig().CN.Frontend.Port + db, err := sql.Open("mysql", fmt.Sprintf("dump:111@tcp(127.0.0.1:%d)/", port)) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, db.PingContext(ctx)) + return db +} + +func mustExec(t testing.TB, db *sql.DB, stmt string, args ...any) { + t.Helper() + _, err := db.Exec(stmt, args...) + require.NoError(t, err, stmt) +} + +func queryCount(t testing.TB, db *sql.DB, query string, args ...any) int64 { + t.Helper() + var n int64 + require.NoError(t, db.QueryRow(query, args...).Scan(&n), query) + return n +} + +// waitUntilStatementRunning polls information_schema.processlist from a second +// connection until the target connection's current statement text contains +// needle. Callers use it only when the test fixture keeps the statement blocked +// at a deterministic lifecycle boundary before taking its next action. +func waitUntilStatementRunning(t testing.TB, observer *sql.DB, connID int64, needle string, deadline time.Duration) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), deadline) + defer cancel() + ticker := time.NewTicker(5 * time.Millisecond) + defer ticker.Stop() + for { + var info sql.NullString + err := observer.QueryRowContext(ctx, + "select info from information_schema.processlist where conn_id = ?", connID, + ).Scan(&info) + if err == nil && info.Valid && strings.Contains(strings.ToLower(info.String), strings.ToLower(needle)) { + return + } + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for connection %d to run a statement containing %q (last info=%q, err=%v)", + connID, needle, info.String, err) + case <-ticker.C: + } + } +} diff --git a/pkg/tests/arrowload/fixtures_test.go b/pkg/tests/arrowload/fixtures_test.go new file mode 100644 index 0000000000000..77cbfaf03ccbb --- /dev/null +++ b/pkg/tests/arrowload/fixtures_test.go @@ -0,0 +1,432 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package arrowload holds release-level, end-to-end BVT coverage for +// `LOAD DATA ... format='arrow'` (issue #23684). Tests run against dedicated, +// non-shared embedded clusters (see cluster_test.go). The main BVTs exercise the +// local-only default configuration, while focused cases explicitly opt into S3 +// or distributed Arrow LOAD without affecting any other package's shared +// embedded cluster. +package arrowload + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/decimal128" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/stretchr/testify/require" +) + +const ( + containerFile = "file" + containerStream = "stream" +) + +// writeArrowFile writes an Arrow IPC payload (File or Stream container) built by +// emit to a real file under dir and returns its absolute path, so it can be +// referenced directly from SQL as {'filepath'='','format'='arrow'}. Callers +// that need several fixture files visible to one glob/pattern LOAD (multi-object +// tests) pass the same dir to each fixture call; single-file tests just pass a +// fresh t.TempDir(). +func writeArrowFile( + t testing.TB, + dir, filename, container string, + schema *arrow.Schema, + emit func(alloc memory.Allocator, write func(arrow.RecordBatch) error), +) string { + t.Helper() + alloc := memory.NewGoAllocator() + var out bytes.Buffer + switch container { + case containerFile: + w, err := ipc.NewFileWriter(&out, ipc.WithSchema(schema), ipc.WithAllocator(alloc)) + require.NoError(t, err) + emit(alloc, w.Write) + require.NoError(t, w.Close()) + case containerStream: + w := ipc.NewWriter(&out, ipc.WithSchema(schema), ipc.WithAllocator(alloc)) + emit(alloc, w.Write) + require.NoError(t, w.Close()) + default: + t.Fatalf("unknown arrow container %q", container) + } + path := filepath.Join(dir, filename) + require.NoError(t, os.WriteFile(path, out.Bytes(), 0o600)) + return path +} + +// --- representative schema 1: numeric/decimal-heavy ------------------------------- + +// numericRow is one logical row of the numeric/decimal-heavy fixture. Target table: +// `id BIGINT NOT NULL, amount DECIMAL(18,2), score DOUBLE, flag BOOL`. +type numericRow struct { + id int64 + amount int64 // raw decimal128 mantissa at scale 2; ignored if amountNull + amountNull bool + score float64 + scoreNull bool + flag bool + flagNull bool +} + +func numericSchema() *arrow.Schema { + return arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64, Nullable: false}, + {Name: "amount", Type: &arrow.Decimal128Type{Precision: 18, Scale: 2}, Nullable: true}, + {Name: "score", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, + {Name: "flag", Type: arrow.FixedWidthTypes.Boolean, Nullable: true}, + }, nil) +} + +// fixtureNumeric writes rows split into batches of at most batchSize rows each, so +// callers can control record-batch fan-out for parallel-shard coverage. +func fixtureNumeric(t *testing.T, dir, filename, container string, rows []numericRow, batchSize int) string { + t.Helper() + schema := numericSchema() + return writeArrowFile(t, dir, filename, container, schema, func(alloc memory.Allocator, write func(arrow.RecordBatch) error) { + for start := 0; start < len(rows); start += batchSize { + end := min(start+batchSize, len(rows)) + batch := rows[start:end] + builder := array.NewRecordBuilder(alloc, schema) + ids := make([]int64, len(batch)) + amounts := make([]decimal128.Num, len(batch)) + amountValid := make([]bool, len(batch)) + scores := make([]float64, len(batch)) + scoreValid := make([]bool, len(batch)) + flags := make([]bool, len(batch)) + flagValid := make([]bool, len(batch)) + for i, row := range batch { + ids[i] = row.id + amounts[i] = decimal128.FromI64(row.amount) + amountValid[i] = !row.amountNull + scores[i] = row.score + scoreValid[i] = !row.scoreNull + flags[i] = row.flag + flagValid[i] = !row.flagNull + } + builder.Field(0).(*array.Int64Builder).AppendValues(ids, nil) + builder.Field(1).(*array.Decimal128Builder).AppendValues(amounts, amountValid) + builder.Field(2).(*array.Float64Builder).AppendValues(scores, scoreValid) + builder.Field(3).(*array.BooleanBuilder).AppendValues(flags, flagValid) + record := builder.NewRecordBatch() + require.NoError(t, write(record)) + record.Release() + builder.Release() + } + }) +} + +// --- representative schema 2: timestamp + short string + dictionary -------------- + +// timestampRow is one logical row of the timestamp/string fixture. Target table: +// `id BIGINT NOT NULL, ts DATETIME(6), d DATE, name VARCHAR(50)`. +type timestampRow struct { + id int64 + ts time.Time + tsNull bool + date time.Time + dateNull bool + name string + nameNull bool +} + +func timestampDictSchema() *arrow.Schema { + return arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64, Nullable: false}, + {Name: "ts", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true}, + {Name: "d", Type: arrow.FixedWidthTypes.Date32, Nullable: true}, + { + Name: "name", + Type: &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.BinaryTypes.String}, + Nullable: true, + }, + }, nil) +} + +func date32FromTime(ts time.Time) arrow.Date32 { + epoch := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC) + days := int32(ts.UTC().Sub(epoch).Hours() / 24) + return arrow.Date32(days) +} + +// fixtureTimestampDict writes a single record batch (dictionary IPC replay is +// simplest with one base dictionary + one record batch, which is all this BVT +// needs. Dictionary base/delta epoch replay itself is already covered by +// pkg/sql/colexec/external/arrowio's own unit/fuzz suite). +func fixtureTimestampDict(t *testing.T, dir, filename, container string, rows []timestampRow, dictValues []string) string { + t.Helper() + schema := timestampDictSchema() + return writeArrowFile(t, dir, filename, container, schema, func(alloc memory.Allocator, write func(arrow.RecordBatch) error) { + idBuilder := array.NewInt64Builder(alloc) + tsBuilder := array.NewTimestampBuilder(alloc, schema.Field(1).Type.(*arrow.TimestampType)) + dateBuilder := array.NewDate32Builder(alloc) + for _, row := range rows { + idBuilder.Append(row.id) + if row.tsNull { + tsBuilder.AppendNull() + } else { + tsBuilder.Append(arrow.Timestamp(row.ts.UTC().UnixMicro())) + } + if row.dateNull { + dateBuilder.AppendNull() + } else { + dateBuilder.Append(date32FromTime(row.date)) + } + } + idArr := idBuilder.NewArray() + tsArr := tsBuilder.NewArray() + dateArr := dateBuilder.NewArray() + idBuilder.Release() + tsBuilder.Release() + dateBuilder.Release() + + dictType := schema.Field(3).Type.(*arrow.DictionaryType) + indexBuilder := array.NewInt8Builder(alloc) + for _, row := range rows { + if row.nameNull { + indexBuilder.AppendNull() + continue + } + idx := int8(-1) + for i, v := range dictValues { + if v == row.name { + idx = int8(i) + break + } + } + require.GreaterOrEqualf(t, idx, int8(0), "name %q not present in dictionary value set", row.name) + indexBuilder.Append(idx) + } + indexArr := indexBuilder.NewArray() + indexBuilder.Release() + valueBuilder := array.NewStringBuilder(alloc) + valueBuilder.AppendValues(dictValues, nil) + valueArr := valueBuilder.NewArray() + valueBuilder.Release() + nameArr := array.NewDictionaryArray(dictType, indexArr, valueArr) + indexArr.Release() + valueArr.Release() + + record := array.NewRecordBatch(schema, []arrow.Array{idArr, tsArr, dateArr, nameArr}, int64(len(rows))) + idArr.Release() + tsArr.Release() + dateArr.Release() + nameArr.Release() + require.NoError(t, write(record)) + record.Release() + }) +} + +// --- representative schema 3: long binary payloads -------------------------------- + +// binaryRow is one logical row of the long-binary fixture. The same source is +// loaded into VARBINARY(200) and BINARY(200) targets so the public-path test +// distinguishes preserved source length from fixed-width zero padding. +type binaryRow struct { + id int64 + payload []byte +} + +func longBinarySchema() *arrow.Schema { + return arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64, Nullable: false}, + {Name: "payload", Type: arrow.BinaryTypes.Binary, Nullable: true}, + }, nil) +} + +func fixtureLongBinary(t *testing.T, dir, filename, container string, rows []binaryRow) string { + t.Helper() + schema := longBinarySchema() + return writeArrowFile(t, dir, filename, container, schema, func(alloc memory.Allocator, write func(arrow.RecordBatch) error) { + builder := array.NewRecordBuilder(alloc, schema) + ids := make([]int64, len(rows)) + payloads := make([][]byte, len(rows)) + for i, row := range rows { + ids[i] = row.id + payloads[i] = row.payload + require.Greaterf(t, len(row.payload), 23, + "long-binary fixture row %d must exceed the 23-byte inline threshold", i) + } + builder.Field(0).(*array.Int64Builder).AppendValues(ids, nil) + builder.Field(1).(*array.BinaryBuilder).AppendValues(payloads, nil) + record := builder.NewRecordBatch() + require.NoError(t, write(record)) + record.Release() + builder.Release() + }) +} + +// --- generic id/name fixture, used for multi-object schema-mismatch and --------- +// --- constraint-violation coverage ------------------------------------------------- + +// idNameRow is one logical row of the generic id/name fixture. Target table: +// `id BIGINT NOT NULL, name VARCHAR(50)`. Both fields are nullable in the Arrow +// schema (id is nullable here even though the MO target column is NOT NULL) so a +// null id value exercises MO's NOT NULL constraint check at conversion/insert time +// rather than a schema-fingerprint mismatch. +type idNameRow struct { + id int64 + idNull bool + name string + nameNull bool +} + +func idNameSchema() *arrow.Schema { + return arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64, Nullable: true}, + {Name: "name", Type: arrow.BinaryTypes.String, Nullable: true}, + }, nil) +} + +func fixtureIDName(t *testing.T, dir, filename, container string, batches [][]idNameRow) string { + t.Helper() + schema := idNameSchema() + return writeArrowFile(t, dir, filename, container, schema, func(alloc memory.Allocator, write func(arrow.RecordBatch) error) { + for _, rows := range batches { + builder := array.NewRecordBuilder(alloc, schema) + ids := make([]int64, len(rows)) + idValid := make([]bool, len(rows)) + names := make([]string, len(rows)) + nameValid := make([]bool, len(rows)) + for i, row := range rows { + ids[i] = row.id + idValid[i] = !row.idNull + names[i] = row.name + nameValid[i] = !row.nameNull + } + builder.Field(0).(*array.Int64Builder).AppendValues(ids, idValid) + builder.Field(1).(*array.StringBuilder).AppendValues(names, nameValid) + record := builder.NewRecordBatch() + require.NoError(t, write(record)) + record.Release() + builder.Release() + } + }) +} + +func fixtureInt64Pair( + t *testing.T, + dir, filename, container string, + first, second []int64, +) string { + t.Helper() + require.Len(t, second, len(first)) + schema := arrow.NewSchema([]arrow.Field{ + {Name: "source_first", Type: arrow.PrimitiveTypes.Int64}, + {Name: "source_second", Type: arrow.PrimitiveTypes.Int64}, + }, nil) + return writeArrowFile(t, dir, filename, container, schema, func( + alloc memory.Allocator, + write func(arrow.RecordBatch) error, + ) { + builder := array.NewRecordBuilder(alloc, schema) + builder.Field(0).(*array.Int64Builder).AppendValues(first, nil) + builder.Field(1).(*array.Int64Builder).AppendValues(second, nil) + record := builder.NewRecordBatch() + require.NoError(t, write(record)) + record.Release() + builder.Release() + }) +} + +// fixtureIDNameMismatchedIDType writes the same logical (id, name) shape but with +// `id` typed as float64 instead of int64, so pairing this file with one produced by +// fixtureIDName in the same multi-object LOAD trips the cross-object schema +// fingerprint check (design invariant I2/I6) rather than any single-file type error. +func fixtureIDNameMismatchedIDType(t *testing.T, dir, filename, container string, ids []float64, names []string) string { + t.Helper() + require.Equal(t, len(ids), len(names)) + schema := arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, + {Name: "name", Type: arrow.BinaryTypes.String, Nullable: true}, + }, nil) + return writeArrowFile(t, dir, filename, container, schema, func(alloc memory.Allocator, write func(arrow.RecordBatch) error) { + builder := array.NewRecordBuilder(alloc, schema) + builder.Field(0).(*array.Float64Builder).AppendValues(ids, nil) + builder.Field(1).(*array.StringBuilder).AppendValues(names, nil) + record := builder.NewRecordBatch() + require.NoError(t, write(record)) + record.Release() + builder.Release() + }) +} + +// --- overflow fixture: a value that cannot widen into a narrower target column --- + +// fixtureInt64Overflow writes a single BIGINT-ish int64 column with one +// out-of-int32-range value, for use against an `INT NOT NULL` target column. +func fixtureInt64Overflow(t *testing.T, dir, filename, container string, values []int64) string { + t.Helper() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "v", Type: arrow.PrimitiveTypes.Int64, Nullable: false}, + }, nil) + return writeArrowFile(t, dir, filename, container, schema, func(alloc memory.Allocator, write func(arrow.RecordBatch) error) { + builder := array.NewRecordBuilder(alloc, schema) + builder.Field(0).(*array.Int64Builder).AppendValues(values, nil) + record := builder.NewRecordBatch() + require.NoError(t, write(record)) + record.Release() + builder.Release() + }) +} + +// --- large fixture: many record batches, for multi-CN parallel fan-out and the --- +// --- KILL-QUERY cancellation test -------------------------------------------------- + +const ( + largeFixtureBatches = 100 + largeFixtureBatchRows = 1000 + largeFixtureRows = largeFixtureBatches * largeFixtureBatchRows +) + +// fixtureLarge writes an IPC File (so record-batch parallel shard fan-out applies) +// with largeFixtureRows rows of `id BIGINT NOT NULL, payload VARCHAR(64) NOT NULL` +// spread across largeFixtureBatches record batches, giving both the multi-CN +// parallel-shard test and the cancel-mid-LOAD test enough real decode/insert work to +// observe or interrupt without relying on a sleep. +func fixtureLarge(t testing.TB) (path string, schemaDDL string) { + t.Helper() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64, Nullable: false}, + {Name: "payload", Type: arrow.BinaryTypes.String, Nullable: false}, + }, nil) + path = writeArrowFile(t, t.TempDir(), "large.arrow", containerFile, schema, func(alloc memory.Allocator, write func(arrow.RecordBatch) error) { + for b := 0; b < largeFixtureBatches; b++ { + builder := array.NewRecordBuilder(alloc, schema) + ids := make([]int64, largeFixtureBatchRows) + payloads := make([]string, largeFixtureBatchRows) + for r := 0; r < largeFixtureBatchRows; r++ { + id := int64(b*largeFixtureBatchRows + r) + ids[r] = id + payloads[r] = fmt.Sprintf("payload-row-%08d-%s", id, strings.Repeat("x", 32)) + } + builder.Field(0).(*array.Int64Builder).AppendValues(ids, nil) + builder.Field(1).(*array.StringBuilder).AppendValues(payloads, nil) + record := builder.NewRecordBatch() + require.NoError(t, write(record)) + record.Release() + builder.Release() + } + }) + return path, "id BIGINT NOT NULL, payload VARCHAR(128) NOT NULL" +} diff --git a/pkg/util/metric/v2/arrow_load.go b/pkg/util/metric/v2/arrow_load.go new file mode 100644 index 0000000000000..e25cf5c15d343 --- /dev/null +++ b/pkg/util/metric/v2/arrow_load.go @@ -0,0 +1,59 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v2 + +import "github.com/prometheus/client_golang/prometheus" + +var ( + ArrowLoadObjectCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "objects_total", + Help: "Arrow object or object-shard open attempts by bounded outcome."}, []string{"outcome"}) + ArrowLoadShardCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "shards_total", + Help: "Arrow record-batch shards opened successfully."}) + ArrowLoadRecordCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "records_total", + Help: "Non-empty Arrow record batches accepted by the External reader."}) + ArrowLoadBatchCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "batches_total", + Help: "MatrixOne batches published by the Arrow External reader."}) + ArrowLoadRowCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "rows_total", + Help: "Rows published by the Arrow External reader."}) + ArrowLoadPayloadBytesCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "payload_bytes_total", + Help: "Arrow bridge payload bytes by bounded eligibility and ownership kind."}, []string{"kind"}) + ArrowLoadCopyBytesCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "copy_bytes_total", + Help: "Bytes copied at an Arrow LOAD data-plane layer."}, []string{"layer"}) + ArrowLoadConversionColumnCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "conversion_columns_total", + Help: "Arrow columns converted by bounded ownership mode."}, []string{"mode"}) + ArrowLoadFallbackCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "fallbacks_total", + Help: "Arrow zero-copy fallbacks by bounded reason."}, []string{"reason"}) + ArrowLoadErrorCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "errors_total", + Help: "Arrow reader failures by stable category without source-identifying labels."}, []string{"category"}) + ArrowLoadPhaseDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "phase_duration_seconds", + Help: "Arrow reader phase duration by bounded phase and outcome.", Buckets: getDurationBuckets()}, []string{"phase", "outcome"}) + ArrowLoadPinnedBytesGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "pinned_bytes", + Help: "Current Arrow FileService range and decoded-buffer capacity held by live leases."}) + ArrowLoadPinnedBytesHighWaterGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "mo", Subsystem: "arrow_load", Name: "pinned_bytes_high_water", + Help: "Process-lifetime high-water mark of Arrow range and decoded-buffer capacity."}) +) diff --git a/pkg/util/metric/v2/arrow_load_test.go b/pkg/util/metric/v2/arrow_load_test.go new file mode 100644 index 0000000000000..32d683f841b23 --- /dev/null +++ b/pkg/util/metric/v2/arrow_load_test.go @@ -0,0 +1,44 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v2 + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" +) + +func TestArrowLoadMetricsRegister(t *testing.T) { + registry := prometheus.NewPedanticRegistry() + collectors := []prometheus.Collector{ + ArrowLoadObjectCounter, + ArrowLoadShardCounter, + ArrowLoadRecordCounter, + ArrowLoadBatchCounter, + ArrowLoadRowCounter, + ArrowLoadPayloadBytesCounter, + ArrowLoadCopyBytesCounter, + ArrowLoadConversionColumnCounter, + ArrowLoadFallbackCounter, + ArrowLoadErrorCounter, + ArrowLoadPhaseDurationHistogram, + ArrowLoadPinnedBytesGauge, + ArrowLoadPinnedBytesHighWaterGauge, + } + for _, collector := range collectors { + require.NoError(t, registry.Register(collector)) + } +} diff --git a/pkg/util/metric/v2/metrics.go b/pkg/util/metric/v2/metrics.go index 3d3212d0b72f6..2e2555fe4e2f5 100644 --- a/pkg/util/metric/v2/metrics.go +++ b/pkg/util/metric/v2/metrics.go @@ -60,6 +60,7 @@ func init() { initCCPRMetrics() initExecutionResourceMetrics() initHashBuildMetrics() + initArrowLoadMetrics() registry.MustRegister(HeartbeatHistogram) registry.MustRegister(HeartbeatFailureCounter) @@ -73,6 +74,22 @@ func init() { registry.MustRegister(StatsUpdateBlockCounter) } +func initArrowLoadMetrics() { + registry.MustRegister(ArrowLoadObjectCounter) + registry.MustRegister(ArrowLoadShardCounter) + registry.MustRegister(ArrowLoadRecordCounter) + registry.MustRegister(ArrowLoadBatchCounter) + registry.MustRegister(ArrowLoadRowCounter) + registry.MustRegister(ArrowLoadPayloadBytesCounter) + registry.MustRegister(ArrowLoadCopyBytesCounter) + registry.MustRegister(ArrowLoadConversionColumnCounter) + registry.MustRegister(ArrowLoadFallbackCounter) + registry.MustRegister(ArrowLoadErrorCounter) + registry.MustRegister(ArrowLoadPhaseDurationHistogram) + registry.MustRegister(ArrowLoadPinnedBytesGauge) + registry.MustRegister(ArrowLoadPinnedBytesHighWaterGauge) +} + func initMemMetrics() { registry.MustRegister(memMPoolAllocatedSizeGauge) registry.MustRegister(MemTotalCrossPoolFreeCounter) diff --git a/proto/pipeline.proto b/proto/pipeline.proto index 5d7088f4eae44..e16025a5e31d2 100644 --- a/proto/pipeline.proto +++ b/proto/pipeline.proto @@ -488,6 +488,13 @@ message IcebergPlanningStats { int64 planning_cache_miss = 11; } +// ArrowExecutionScope is compile-produced positive authorization for Arrow +// ingestion. Its zero value is deliberately fail-closed on every CN. +enum ArrowExecutionScope { + UnknownArrowExecutionScope = 0; + ArrowLoadData = 1; +} + message ExternalScan { option (gogoproto.goproto_stringer) = false; repeated plan.ExternAttr attrs = 1 [(gogoproto.nullable) = false]; @@ -517,6 +524,18 @@ message ExternalScan { plan.ForeignScan foreign_scan = 25; plan.KafkaScan kafka_scan = 26; bool parquet_whole_file_fanout = 27; + ArrowExecutionScope arrow_execution_scope = 28; + repeated ArrowObjectIdentity arrow_object_identities = 29; + repeated ArrowRecordBatchShard arrow_record_batch_shards = 30; + bytes arrow_schema_fingerprint = 31; + uint32 arrow_conversion_plan_version = 32; + // Compile-time policy snapshot. Older CNs ignore this additive field and + // still reject Arrow format before execution, so mixed versions fail closed. + bool arrow_force_materialize = 33; + // This records actual fanout placement independently of the SQL PARALLEL + // request. Fanout scopes clear that request after splitting, but every + // receiving CN must still enforce its own distributed Arrow rollout gate. + bool arrow_distributed_execution = 34; } message TableScan { @@ -826,3 +845,20 @@ message ODKUForeignKeyCheck { repeated int32 col_idx_list = 1; int32 eligibility_result_pos = 2; } + +message ArrowObjectIdentity { + int32 file_index = 1; + string version_id = 2; + string etag = 3; + int64 size = 4; + int64 last_modified_unix_nano = 5; +} + +message ArrowRecordBatchShard { + int32 file_index = 1; + int32 record_batch_start = 2; + int32 record_batch_end = 3; + repeated int32 required_dictionary_block_indices = 4; + int64 estimated_rows = 5; + int64 estimated_wire_bytes = 6; +} diff --git a/test/distributed/cases/load_data/load_data_arrow.result b/test/distributed/cases/load_data/load_data_arrow.result new file mode 100644 index 0000000000000..e0ce97b656b7e --- /dev/null +++ b/test/distributed/cases/load_data/load_data_arrow.result @@ -0,0 +1,10 @@ +drop database if exists arrow_load_bvt; +create database arrow_load_bvt; +use arrow_load_bvt; +create table arrow_gate_off(id bigint, name varchar(50)); +load data infile {'filepath'='$resources/load_data/arrow_file.arrow', 'format'='arrow'} into table arrow_gate_off; +not supported: Arrow LOAD is disabled by configuration +select count(*) from arrow_gate_off; +➤ count(*)[-5,64,0] 𝄀 +0 +drop database arrow_load_bvt; diff --git a/test/distributed/cases/load_data/load_data_arrow.sql b/test/distributed/cases/load_data/load_data_arrow.sql new file mode 100644 index 0000000000000..65d0894a9bcef --- /dev/null +++ b/test/distributed/cases/load_data/load_data_arrow.sql @@ -0,0 +1,14 @@ +-- The shared compose profile is a production-like default configuration. It +-- must not opt in to Arrow LOAD merely because standard BVT runs on it. +-- Opted-in File/Stream/S3 and multi-CN behavior is covered by the dedicated +-- pkg/tests/arrowload cluster suite, whose fixture explicitly configures every +-- participating CN. +drop database if exists arrow_load_bvt; +create database arrow_load_bvt; +use arrow_load_bvt; + +create table arrow_gate_off(id bigint, name varchar(50)); +load data infile {'filepath'='$resources/load_data/arrow_file.arrow', 'format'='arrow'} into table arrow_gate_off; +select count(*) from arrow_gate_off; + +drop database arrow_load_bvt; diff --git a/test/distributed/resources/load_data/arrow_corrupt.arrow b/test/distributed/resources/load_data/arrow_corrupt.arrow new file mode 100644 index 0000000000000..acb16bd11468c Binary files /dev/null and b/test/distributed/resources/load_data/arrow_corrupt.arrow differ diff --git a/test/distributed/resources/load_data/arrow_file.arrow b/test/distributed/resources/load_data/arrow_file.arrow new file mode 100644 index 0000000000000..ac43665b08427 Binary files /dev/null and b/test/distributed/resources/load_data/arrow_file.arrow differ diff --git a/test/distributed/resources/load_data/arrow_mismatch_1.arrow b/test/distributed/resources/load_data/arrow_mismatch_1.arrow new file mode 100644 index 0000000000000..7d2b98743dcda Binary files /dev/null and b/test/distributed/resources/load_data/arrow_mismatch_1.arrow differ diff --git a/test/distributed/resources/load_data/arrow_mismatch_2.arrow b/test/distributed/resources/load_data/arrow_mismatch_2.arrow new file mode 100644 index 0000000000000..cf7a1aaf56c19 Binary files /dev/null and b/test/distributed/resources/load_data/arrow_mismatch_2.arrow differ diff --git a/test/distributed/resources/load_data/arrow_notnull_1.arrow b/test/distributed/resources/load_data/arrow_notnull_1.arrow new file mode 100644 index 0000000000000..e27eb1ff27d1f Binary files /dev/null and b/test/distributed/resources/load_data/arrow_notnull_1.arrow differ diff --git a/test/distributed/resources/load_data/arrow_notnull_2.arrow b/test/distributed/resources/load_data/arrow_notnull_2.arrow new file mode 100644 index 0000000000000..61b75ba367912 Binary files /dev/null and b/test/distributed/resources/load_data/arrow_notnull_2.arrow differ diff --git a/test/distributed/resources/load_data/arrow_part_1.arrow b/test/distributed/resources/load_data/arrow_part_1.arrow new file mode 100644 index 0000000000000..609ae8cdec36b Binary files /dev/null and b/test/distributed/resources/load_data/arrow_part_1.arrow differ diff --git a/test/distributed/resources/load_data/arrow_part_2.arrow b/test/distributed/resources/load_data/arrow_part_2.arrow new file mode 100644 index 0000000000000..97f130d00422e Binary files /dev/null and b/test/distributed/resources/load_data/arrow_part_2.arrow differ diff --git a/test/distributed/resources/load_data/arrow_stream.arrow b/test/distributed/resources/load_data/arrow_stream.arrow new file mode 100644 index 0000000000000..d5db6887353b9 Binary files /dev/null and b/test/distributed/resources/load_data/arrow_stream.arrow differ