From 3803eae39ec96090d170e123787a2c69291718f9 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Thu, 25 Jun 2026 23:35:59 -0700 Subject: [PATCH 01/36] feat(rest): add scan plan endpoint support to REST catalog client When a table is loaded from a REST catalog that advertises the PlanTableScan endpoint, NewScan() now returns a RestTableScanBuilder whose Build() produces a RestTableScan. PlanFiles() on that scan delegates manifest resolution to the server via POST /plan, GET /plan/{id} (with exponential backoff), POST /tasks/{id}, and DELETE /plan/{id} (best-effort cancel), instead of reading manifests locally. - Add RestTable, RestTableScanBuilder, RestTableScan and RestScanContext - Promote DataTableScan::PlanFiles and TableScanBuilder::Build to virtual - Convert RestCatalog::client_ and paths_ to shared_ptr so RestScanContext can share ownership with live scans --- src/iceberg/catalog/rest/CMakeLists.txt | 2 + src/iceberg/catalog/rest/rest_catalog.cc | 16 +- src/iceberg/catalog/rest/rest_catalog.h | 4 +- src/iceberg/catalog/rest/rest_table.cc | 60 +++++ src/iceberg/catalog/rest/rest_table.h | 61 +++++ src/iceberg/catalog/rest/rest_table_scan.cc | 269 ++++++++++++++++++++ src/iceberg/catalog/rest/rest_table_scan.h | 117 +++++++++ src/iceberg/data_table_scan.h | 2 +- src/iceberg/table_scan.h | 2 +- 9 files changed, 528 insertions(+), 5 deletions(-) create mode 100644 src/iceberg/catalog/rest/rest_table.cc create mode 100644 src/iceberg/catalog/rest/rest_table.h create mode 100644 src/iceberg/catalog/rest/rest_table_scan.cc create mode 100644 src/iceberg/catalog/rest/rest_table_scan.h diff --git a/src/iceberg/catalog/rest/CMakeLists.txt b/src/iceberg/catalog/rest/CMakeLists.txt index f64860ff46..0736290bc5 100644 --- a/src/iceberg/catalog/rest/CMakeLists.txt +++ b/src/iceberg/catalog/rest/CMakeLists.txt @@ -34,6 +34,8 @@ set(ICEBERG_REST_SOURCES rest_catalog.cc rest_file_io.cc rest_metrics_reporter.cc + rest_table.cc + rest_table_scan.cc rest_util.cc types.cc) diff --git a/src/iceberg/catalog/rest/rest_catalog.cc b/src/iceberg/catalog/rest/rest_catalog.cc index 0ca89a725a..34e656bca3 100644 --- a/src/iceberg/catalog/rest/rest_catalog.cc +++ b/src/iceberg/catalog/rest/rest_catalog.cc @@ -41,6 +41,7 @@ #include "iceberg/catalog/rest/resource_paths.h" #include "iceberg/catalog/rest/rest_file_io.h" #include "iceberg/catalog/rest/rest_metrics_reporter_internal.h" +#include "iceberg/catalog/rest/rest_table.h" #include "iceberg/catalog/rest/rest_util.h" #include "iceberg/catalog/rest/types.h" #include "iceberg/json_serde_internal.h" @@ -492,7 +493,7 @@ Result> RestCatalog::Make( RestCatalog::RestCatalog(RestCatalogProperties config, std::shared_ptr file_io, std::shared_ptr client, - std::unique_ptr paths, + std::shared_ptr paths, std::unordered_set endpoints, std::unique_ptr auth_manager, std::shared_ptr catalog_session, @@ -972,6 +973,19 @@ Result> RestCatalog::MakeTableFromLoadResult( auto table_catalog = std::make_shared( shared_from_this(), context, identifier, table_config, table_session, table_io); + if (supported_endpoints_.contains(Endpoint::PlanTableScan())) { + RestScanContext rest_ctx{ + .client = client_, + .paths = paths_, + .session = table_session, + .supported_endpoints = supported_endpoints_, + .identifier = identifier, + }; + return RestTable::Make(identifier, std::move(result.metadata), + std::move(result.metadata_location), std::move(table_io), + std::move(table_catalog), std::move(rest_ctx)); + } + return Table::Make(identifier, std::move(result.metadata), std::move(result.metadata_location), std::move(table_io), std::move(table_catalog), RestTableName(name_, identifier), diff --git a/src/iceberg/catalog/rest/rest_catalog.h b/src/iceberg/catalog/rest/rest_catalog.h index 6f65e6d612..83bce6dced 100644 --- a/src/iceberg/catalog/rest/rest_catalog.h +++ b/src/iceberg/catalog/rest/rest_catalog.h @@ -69,7 +69,7 @@ class ICEBERG_REST_EXPORT RestCatalog final class TableScopedCatalog; RestCatalog(RestCatalogProperties config, std::shared_ptr file_io, - std::shared_ptr client, std::unique_ptr paths, + std::shared_ptr client, std::shared_ptr paths, std::unordered_set endpoints, std::unique_ptr auth_manager, std::shared_ptr catalog_session, @@ -201,7 +201,7 @@ class ICEBERG_REST_EXPORT RestCatalog final RestCatalogProperties config_; std::shared_ptr file_io_; std::shared_ptr client_; - std::unique_ptr paths_; + std::shared_ptr paths_; std::string name_; std::unordered_set supported_endpoints_; std::unique_ptr auth_manager_; diff --git a/src/iceberg/catalog/rest/rest_table.cc b/src/iceberg/catalog/rest/rest_table.cc new file mode 100644 index 0000000000..00f8856d15 --- /dev/null +++ b/src/iceberg/catalog/rest/rest_table.cc @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/catalog/rest/rest_table.h" + +#include +#include + +#include "iceberg/catalog/rest/rest_table_scan.h" +#include "iceberg/result.h" +#include "iceberg/table_metadata.h" +#include "iceberg/util/macros.h" + +namespace iceberg::rest { + +RestTable::RestTable(TableIdentifier identifier, std::shared_ptr metadata, + std::string metadata_location, std::shared_ptr io, + std::shared_ptr catalog, RestScanContext rest_context) + : Table(std::move(identifier), std::move(metadata), std::move(metadata_location), + std::move(io), std::move(catalog)), + rest_context_(std::move(rest_context)) {} + +RestTable::~RestTable() = default; + +Result> RestTable::Make(TableIdentifier identifier, + std::shared_ptr metadata, + std::string metadata_location, + std::shared_ptr io, + std::shared_ptr catalog, + RestScanContext rest_context) { + if (metadata == nullptr) { + return InvalidArgument("Metadata cannot be null"); + } + return std::shared_ptr( + new RestTable(std::move(identifier), std::move(metadata), + std::move(metadata_location), std::move(io), std::move(catalog), + std::move(rest_context))); +} + +Result> RestTable::NewScan() const { + return std::make_unique(metadata_, io_, rest_context_); +} + +} // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/rest_table.h b/src/iceberg/catalog/rest/rest_table.h new file mode 100644 index 0000000000..8e9172b098 --- /dev/null +++ b/src/iceberg/catalog/rest/rest_table.h @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "iceberg/catalog/rest/iceberg_rest_export.h" +#include "iceberg/catalog/rest/rest_table_scan.h" +#include "iceberg/result.h" +#include "iceberg/table.h" +#include "iceberg/type_fwd.h" + +/// \file iceberg/catalog/rest/rest_table.h +/// A Table subclass that uses server-side distributed scan planning via the REST catalog. + +namespace iceberg::rest { + +/// \brief A Table whose NewScan() returns a RestTableScanBuilder, delegating +/// PlanFiles() to the REST catalog server's scan planning endpoints. +class ICEBERG_REST_EXPORT RestTable final : public Table { + public: + static Result> Make(TableIdentifier identifier, + std::shared_ptr metadata, + std::string metadata_location, + std::shared_ptr io, + std::shared_ptr catalog, + RestScanContext rest_context); + + ~RestTable() override; + + /// \brief Returns a RestTableScanBuilder that will delegate PlanFiles() to the + /// REST catalog server. + Result> NewScan() const override; + + private: + RestTable(TableIdentifier identifier, std::shared_ptr metadata, + std::string metadata_location, std::shared_ptr io, + std::shared_ptr catalog, RestScanContext rest_context); + + RestScanContext rest_context_; +}; + +} // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc new file mode 100644 index 0000000000..30dcc95dfd --- /dev/null +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/catalog/rest/rest_table_scan.h" + +#include +#include + +#include + +#include "iceberg/catalog/rest/endpoint.h" +#include "iceberg/catalog/rest/error_handlers.h" +#include "iceberg/catalog/rest/http_client.h" +#include "iceberg/catalog/rest/json_serde_internal.h" +#include "iceberg/catalog/rest/resource_paths.h" +#include "iceberg/catalog/rest/types.h" +#include "iceberg/json_serde_internal.h" +#include "iceberg/partition_spec.h" +#include "iceberg/result.h" +#include "iceberg/schema.h" +#include "iceberg/table_metadata.h" +#include "iceberg/util/macros.h" + +namespace iceberg::rest { + +namespace { + +constexpr int64_t kMinSleepMs = 1'000; +constexpr int64_t kMaxSleepMs = 60'000; +constexpr int kMaxRetries = 10; +constexpr int64_t kMaxWaitTimeMs = 5 * 60 * 1'000; + +#define ICEBERG_ENDPOINT_CHECK(endpoints, endpoint) \ + do { \ + if (!endpoints.contains(endpoint)) { \ + return NotSupported("Not supported endpoint: {}", endpoint.ToString()); \ + } \ + } while (0) + +} // namespace + +// RestTableScan + +RestTableScan::RestTableScan(std::shared_ptr metadata, + std::shared_ptr schema, std::shared_ptr io, + internal::TableScanContext context, + RestScanContext rest_context) + : DataTableScan(std::move(metadata), std::move(schema), std::move(io), + std::move(context)), + rest_context_(std::move(rest_context)) {} + +Result> RestTableScan::Make( + std::shared_ptr metadata, std::shared_ptr schema, + std::shared_ptr io, internal::TableScanContext context, + RestScanContext rest_context) { + ICEBERG_PRECHECK(metadata != nullptr, "Table metadata cannot be null"); + ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null"); + ICEBERG_PRECHECK(io != nullptr, "FileIO cannot be null"); + return std::unique_ptr( + new RestTableScan(std::move(metadata), std::move(schema), std::move(io), + std::move(context), std::move(rest_context))); +} + +Result>> RestTableScan::PlanFiles() const { + TableMetadataCache metadata_cache(metadata_.get()); + ICEBERG_ASSIGN_OR_RAISE(auto specs_by_id, metadata_cache.GetPartitionSpecsById()); + + std::string plan_id; + return PlanTableScan(plan_id, specs_by_id); +} + +Result>> RestTableScan::PlanTableScan( + std::string& plan_id, + const std::unordered_map>& specs) const { + ICEBERG_ENDPOINT_CHECK(rest_context_.supported_endpoints, Endpoint::PlanTableScan()); + + // Build request from scan context + PlanTableScanRequest request; + request.select = context_.selected_columns; + request.filter = context_.filter; + request.case_sensitive = context_.case_sensitive; + request.min_rows_requested = context_.min_rows_requested; + + if (context_.from_snapshot_id.has_value() && context_.to_snapshot_id.has_value()) { + request.start_snapshot_id = context_.from_snapshot_id; + request.end_snapshot_id = context_.to_snapshot_id; + } else if (context_.snapshot_id.has_value()) { + request.snapshot_id = context_.snapshot_id; + } + + if (!context_.columns_to_keep_stats.empty()) { + for (int32_t field_id : context_.columns_to_keep_stats) { + ICEBERG_ASSIGN_OR_RAISE(auto name, schema_->FindColumnNameById(field_id)); + if (name.has_value()) { + request.stats_fields.emplace_back(*name); + } + } + } + + ICEBERG_ASSIGN_OR_RAISE(auto path, + rest_context_.paths->Plan(rest_context_.identifier)); + ICEBERG_ASSIGN_OR_RAISE(auto request_json, ToJson(request)); + ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(request_json)); + ICEBERG_ASSIGN_OR_RAISE( + const auto response, + rest_context_.client->Post(path, json_request, /*headers=*/{}, + *PlanErrorHandler::Instance(), *rest_context_.session)); + ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); + ICEBERG_ASSIGN_OR_RAISE(auto result, + PlanTableScanResponseFromJson(json, specs, *schema_)); + ICEBERG_RETURN_UNEXPECTED(result.Validate()); + + plan_id = result.plan_id; + + switch (result.plan_status) { + case PlanStatus::kCompleted: + return ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); + case PlanStatus::kSubmitted: + return FetchPlanningResult(plan_id, specs); + case PlanStatus::kFailed: + return IOError("Scan planning failed: {}", + result.error ? result.error->message : "unknown error"); + case PlanStatus::kCancelled: + return IOError("Scan planning was cancelled for plan_id={}", plan_id); + } + return IOError("Unexpected plan status"); +} + +Result>> RestTableScan::FetchPlanningResult( + const std::string& plan_id, + const std::unordered_map>& specs) const { + ICEBERG_ENDPOINT_CHECK(rest_context_.supported_endpoints, + Endpoint::FetchPlanningResult()); + + ICEBERG_ASSIGN_OR_RAISE(auto path, + rest_context_.paths->Plan(rest_context_.identifier, plan_id)); + + auto delay_ms = kMinSleepMs; + auto start = std::chrono::steady_clock::now(); + + for (int retry = 0; retry <= kMaxRetries; ++retry) { + ICEBERG_ASSIGN_OR_RAISE( + const auto response, + rest_context_.client->Get(path, /*params=*/{}, /*headers=*/{}, + *PlanErrorHandler::Instance(), *rest_context_.session)); + ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); + ICEBERG_ASSIGN_OR_RAISE(auto result, + FetchPlanningResultResponseFromJson(json, specs, *schema_)); + ICEBERG_RETURN_UNEXPECTED(result.Validate()); + + switch (result.plan_status) { + case PlanStatus::kCompleted: + return ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); + case PlanStatus::kSubmitted: { + auto elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + if (elapsed_ms >= kMaxWaitTimeMs) { + CancelPlanning(plan_id); + return IOError( + "Scan planning timed out after {}ms waiting for plan_id={}", elapsed_ms, + plan_id); + } + std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); + delay_ms = std::min(delay_ms * 2, kMaxSleepMs); + continue; + } + case PlanStatus::kFailed: + CancelPlanning(plan_id); + return IOError("Scan planning failed: {}", + result.error ? result.error->message : "unknown error"); + case PlanStatus::kCancelled: + return IOError("Scan planning was cancelled for plan_id={}", plan_id); + } + } + + CancelPlanning(plan_id); + return IOError("Scan planning exceeded max retries ({}) for plan_id={}", + kMaxRetries, plan_id); +} + +Result>> RestTableScan::FetchScanTasks( + const std::string& plan_task, + const std::unordered_map>& specs) const { + ICEBERG_ENDPOINT_CHECK(rest_context_.supported_endpoints, Endpoint::FetchScanTasks()); + + ICEBERG_ASSIGN_OR_RAISE(auto path, + rest_context_.paths->FetchScanTasks(rest_context_.identifier)); + FetchScanTasksRequest request{.planTask = plan_task}; + ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(ToJson(request))); + ICEBERG_ASSIGN_OR_RAISE( + const auto response, + rest_context_.client->Post(path, json_request, /*headers=*/{}, + *PlanTaskErrorHandler::Instance(), + *rest_context_.session)); + ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); + ICEBERG_ASSIGN_OR_RAISE(auto result, + FetchScanTasksResponseFromJson(json, specs, *schema_)); + ICEBERG_RETURN_UNEXPECTED(result.Validate()); + + return ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); +} + +Result>> RestTableScan::ResolveScanTasks( + const std::optional>& plan_tasks, + const std::optional>>& file_scan_tasks, + const std::unordered_map>& specs) const { + std::vector> result; + + if (file_scan_tasks.has_value()) { + result.insert(result.end(), file_scan_tasks->begin(), file_scan_tasks->end()); + } + + if (plan_tasks.has_value()) { + for (const auto& plan_task : *plan_tasks) { + ICEBERG_ASSIGN_OR_RAISE(auto tasks, FetchScanTasks(plan_task, specs)); + result.insert(result.end(), tasks.begin(), tasks.end()); + } + } + + return result; +} + +void RestTableScan::CancelPlanning(const std::string& plan_id) const { + if (plan_id.empty()) return; + if (!rest_context_.supported_endpoints.contains(Endpoint::CancelPlanning())) return; + + auto path = rest_context_.paths->Plan(rest_context_.identifier, plan_id); + if (!path.has_value()) return; + + // Best-effort: ignore errors. + std::ignore = rest_context_.client->Delete(*path, /*params=*/{}, /*headers=*/{}, + *PlanErrorHandler::Instance(), + *rest_context_.session); +} + +// RestTableScanBuilder + +RestTableScanBuilder::RestTableScanBuilder(std::shared_ptr metadata, + std::shared_ptr io, + RestScanContext rest_context) + : DataTableScanBuilder(std::move(metadata), std::move(io)), + rest_context_(std::move(rest_context)) {} + +Result> RestTableScanBuilder::Build() { + ICEBERG_RETURN_UNEXPECTED(CheckErrors()); + ICEBERG_RETURN_UNEXPECTED(context_.Validate()); + ICEBERG_ASSIGN_OR_RAISE(auto schema, ResolveSnapshotSchema()); + return RestTableScan::Make(metadata_, schema.get(), io_, std::move(context_), + rest_context_); +} + +} // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/rest_table_scan.h b/src/iceberg/catalog/rest/rest_table_scan.h new file mode 100644 index 0000000000..4d924c38fe --- /dev/null +++ b/src/iceberg/catalog/rest/rest_table_scan.h @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "iceberg/catalog/rest/endpoint.h" +#include "iceberg/catalog/rest/iceberg_rest_export.h" +#include "iceberg/data_table_scan.h" +#include "iceberg/result.h" +#include "iceberg/table_identifier.h" +#include "iceberg/table_scan.h" +#include "iceberg/type_fwd.h" + +/// \file iceberg/catalog/rest/rest_table_scan.h +/// REST-specific table scan that delegates scan planning to the REST catalog server. + +namespace iceberg::rest { + +class HttpClient; +class ResourcePaths; + +namespace auth { +class AuthSession; +} // namespace auth + +/// \brief HTTP context shared between RestTable and RestTableScan. +struct ICEBERG_REST_EXPORT RestScanContext { + std::shared_ptr client; + std::shared_ptr paths; + std::shared_ptr session; + std::unordered_set supported_endpoints; + TableIdentifier identifier; +}; + +/// \brief A DataTableScan that delegates PlanFiles() to the REST catalog server +/// via the scan planning endpoints (planTableScan / fetchPlanningResult / +/// cancelPlanning / fetchScanTasks). +class ICEBERG_REST_EXPORT RestTableScan : public DataTableScan { + public: + ~RestTableScan() override = default; + + static Result> Make( + std::shared_ptr metadata, std::shared_ptr schema, + std::shared_ptr io, internal::TableScanContext context, + RestScanContext rest_context); + + /// \brief Plans files via the REST scan planning endpoints. + Result>> PlanFiles() const override; + + private: + RestTableScan(std::shared_ptr metadata, std::shared_ptr schema, + std::shared_ptr io, internal::TableScanContext context, + RestScanContext rest_context); + + /// POST /plan → handle COMPLETED / SUBMITTED / FAILED / CANCELLED. + Result>> PlanTableScan( + std::string& plan_id, + const std::unordered_map>& specs) const; + + /// GET /plan/{plan_id} with exponential backoff until COMPLETED. + Result>> FetchPlanningResult( + const std::string& plan_id, + const std::unordered_map>& specs) const; + + /// POST /tasks/{plan_task_id} → fetch FileScanTasks for one opaque plan task token. + Result>> FetchScanTasks( + const std::string& plan_task, + const std::unordered_map>& specs) const; + + /// Flatten plan_tasks (opaque tokens) + file_scan_tasks into a single list. + Result>> ResolveScanTasks( + const std::optional>& plan_tasks, + const std::optional>>& file_scan_tasks, + const std::unordered_map>& specs) const; + + /// DELETE /plan/{plan_id}; best-effort, errors are silently ignored. + void CancelPlanning(const std::string& plan_id) const; + + RestScanContext rest_context_; +}; + +/// \brief Builder that produces a RestTableScan with the REST HTTP context injected. +class ICEBERG_REST_EXPORT RestTableScanBuilder : public DataTableScanBuilder { + public: + RestTableScanBuilder(std::shared_ptr metadata, std::shared_ptr io, + RestScanContext rest_context); + + /// \brief Resolves schema/context via parent logic then creates a RestTableScan. + Result> Build() override; + + private: + RestScanContext rest_context_; +}; + +} // namespace iceberg::rest diff --git a/src/iceberg/data_table_scan.h b/src/iceberg/data_table_scan.h index a6de0020c9..09cf85fd51 100644 --- a/src/iceberg/data_table_scan.h +++ b/src/iceberg/data_table_scan.h @@ -46,7 +46,7 @@ class ICEBERG_EXPORT DataTableScan : public TableScan { /// /// Collects PlanFilesStream() into a vector. /// \return A Result containing scan tasks or an error. - Result>> PlanFiles() const; + virtual Result>> PlanFiles() const; /// \brief Lazily plans scan tasks by resolving manifests and data files on demand. /// diff --git a/src/iceberg/table_scan.h b/src/iceberg/table_scan.h index 6ec29ad9f6..7702f13e63 100644 --- a/src/iceberg/table_scan.h +++ b/src/iceberg/table_scan.h @@ -402,7 +402,7 @@ class ICEBERG_TEMPLATE_CLASS_EXPORT TableScanBuilder : public ErrorCollector { /// \brief Builds and returns a TableScan instance. /// \return A Result containing the TableScan or an error. - Result> Build(); + virtual Result> Build(); protected: TableScanBuilder(std::shared_ptr metadata, std::shared_ptr io, From cf003c8d574d979133063a431950de1bdc7fe8ab Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 13 Sep 2026 11:52:49 -0700 Subject: [PATCH 02/36] Address review comments and add tests for scan plan endpoints - Cancel server-side plan when ResolveScanTasks fails partway through - Propagate use_snapshot_schema from scan context to PlanTableScanRequest: true for UseSnapshot/AsOfTime/tag refs and incremental scans, false for branch refs and default scans - Gate RestTable creation on effective scan-planning-mode config (table config overrides client config, default is client); error if server mode is requested but endpoint is not advertised - Add ScanPlanningMode enum and ScanPlanningModeFrom() parser to RestCatalogProperties - Make HttpClient methods virtual and add HttpResponse::MakeForTesting() to support unit test mocking - Add tests: use_snapshot_schema in table_scan_test, ScanPlanningModeFrom parsing in catalog_properties_test, and RestTableScan HTTP flow tests in rest_table_scan_test --- .../catalog/rest/catalog_properties.cc | 11 + src/iceberg/catalog/rest/catalog_properties.h | 10 + src/iceberg/catalog/rest/http_client.cc | 9 + src/iceberg/catalog/rest/http_client.h | 42 +- src/iceberg/catalog/rest/rest_catalog.cc | 29 +- src/iceberg/catalog/rest/rest_table_scan.cc | 16 +- src/iceberg/table_scan.cc | 3 + src/iceberg/table_scan.h | 1 + src/iceberg/test/CMakeLists.txt | 2 + src/iceberg/test/catalog_properties_test.cc | 81 ++++ src/iceberg/test/rest_table_scan_test.cc | 403 ++++++++++++++++++ src/iceberg/test/table_scan_test.cc | 39 ++ 12 files changed, 622 insertions(+), 24 deletions(-) create mode 100644 src/iceberg/test/catalog_properties_test.cc create mode 100644 src/iceberg/test/rest_table_scan_test.cc diff --git a/src/iceberg/catalog/rest/catalog_properties.cc b/src/iceberg/catalog/rest/catalog_properties.cc index 0e417e6c39..91d72e8a49 100644 --- a/src/iceberg/catalog/rest/catalog_properties.cc +++ b/src/iceberg/catalog/rest/catalog_properties.cc @@ -20,6 +20,7 @@ #include "iceberg/catalog/rest/catalog_properties.h" #include +#include #include #include @@ -61,4 +62,14 @@ Result RestCatalogProperties::SnapshotLoadingMode() const { } } +Result> RestCatalogProperties::ScanPlanningModeFrom( + const std::unordered_map& config) { + auto it = config.find(kScanPlanningMode.key()); + if (it == config.end()) return std::nullopt; + std::string lower = StringUtils::ToLower(it->second); + if (lower == "client") return ScanPlanningMode::kClient; + if (lower == "server") return ScanPlanningMode::kServer; + return InvalidArgument("Invalid scan planning mode: '{}'.", it->second); +} + } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/catalog_properties.h b/src/iceberg/catalog/rest/catalog_properties.h index d1ee0e9c49..63dcd9a95e 100644 --- a/src/iceberg/catalog/rest/catalog_properties.h +++ b/src/iceberg/catalog/rest/catalog_properties.h @@ -34,6 +34,9 @@ namespace iceberg::rest { /// \brief Snapshot loading mode for REST catalog. enum class SnapshotMode : uint8_t { kAll, kRefs }; +/// \brief Scan planning mode for REST catalog. +enum class ScanPlanningMode : uint8_t { kClient, kServer }; + /// \brief Configuration class for a REST Catalog. class ICEBERG_REST_EXPORT RestCatalogProperties : public ConfigBase { @@ -58,6 +61,8 @@ class ICEBERG_REST_EXPORT RestCatalogProperties /// \brief Whether to report metrics to the REST catalog server (default: true). inline static Entry kMetricsReportingEnabled{ "rest-metrics-reporting-enabled", "true"}; + /// \brief The scan planning mode (client or server). + inline static Entry kScanPlanningMode{"scan-planning-mode", "client"}; /// \brief The prefix for HTTP headers. inline static constexpr std::string_view kHeaderPrefix = "header."; @@ -80,6 +85,11 @@ class ICEBERG_REST_EXPORT RestCatalogProperties /// "REFS", or an error if the value is invalid. Parsing is /// case-insensitive to match Java behavior. Result SnapshotLoadingMode() const; + + /// \brief Get the scan planning mode from the given config map, returning + /// std::nullopt if the key is absent. + static Result> ScanPlanningModeFrom( + const std::unordered_map& config); }; } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/http_client.cc b/src/iceberg/catalog/rest/http_client.cc index 6661c50983..ec870b87d3 100644 --- a/src/iceberg/catalog/rest/http_client.cc +++ b/src/iceberg/catalog/rest/http_client.cc @@ -62,6 +62,15 @@ std::unordered_map HttpResponse::headers() const { return impl_->headers(); } +HttpResponse HttpResponse::MakeForTesting(int32_t status_code, std::string body) { + cpr::Response cpr_response; + cpr_response.status_code = status_code; + cpr_response.text = std::move(body); + HttpResponse response; + response.impl_ = std::make_unique(std::move(cpr_response)); + return response; +} + namespace { /// \brief Default error type for unparseable REST responses. diff --git a/src/iceberg/catalog/rest/http_client.h b/src/iceberg/catalog/rest/http_client.h index ea9c10a399..52522425e5 100644 --- a/src/iceberg/catalog/rest/http_client.h +++ b/src/iceberg/catalog/rest/http_client.h @@ -61,6 +61,9 @@ class ICEBERG_REST_EXPORT HttpResponse { /// \brief Get the headers of the response as a map. std::unordered_map headers() const; + /// \brief Create a response for use in unit tests. + static HttpResponse MakeForTesting(int32_t status_code, std::string body); + private: friend class HttpClient; class Impl; @@ -71,7 +74,7 @@ class ICEBERG_REST_EXPORT HttpResponse { class ICEBERG_REST_EXPORT HttpClient { public: explicit HttpClient(std::unordered_map default_headers = {}); - ~HttpClient(); + virtual ~HttpClient(); HttpClient(const HttpClient&) = delete; HttpClient& operator=(const HttpClient&) = delete; @@ -79,36 +82,37 @@ class ICEBERG_REST_EXPORT HttpClient { HttpClient& operator=(HttpClient&&) = delete; /// \brief Sends a GET request. - Result Get(const std::string& path, - const std::unordered_map& params, - const std::unordered_map& headers, - const ErrorHandler& error_handler, auth::AuthSession& session); + virtual Result Get( + const std::string& path, + const std::unordered_map& params, + const std::unordered_map& headers, + const ErrorHandler& error_handler, auth::AuthSession& session); /// \brief Sends a POST request. - Result Post(const std::string& path, const std::string& body, - const std::unordered_map& headers, - const ErrorHandler& error_handler, - auth::AuthSession& session); + virtual Result Post(const std::string& path, const std::string& body, + const std::unordered_map& headers, + const ErrorHandler& error_handler, + auth::AuthSession& session); /// \brief Sends a POST request with form data. - Result PostForm( + virtual Result PostForm( const std::string& path, const std::unordered_map& form_data, const std::unordered_map& headers, const ErrorHandler& error_handler, auth::AuthSession& session); /// \brief Sends a HEAD request. - Result Head(const std::string& path, - const std::unordered_map& headers, - const ErrorHandler& error_handler, - auth::AuthSession& session); + virtual Result Head( + const std::string& path, + const std::unordered_map& headers, + const ErrorHandler& error_handler, auth::AuthSession& session); /// \brief Sends a DELETE request. - Result Delete(const std::string& path, - const std::unordered_map& params, - const std::unordered_map& headers, - const ErrorHandler& error_handler, - auth::AuthSession& session); + virtual Result Delete( + const std::string& path, + const std::unordered_map& params, + const std::unordered_map& headers, + const ErrorHandler& error_handler, auth::AuthSession& session); private: std::unordered_map default_headers_; diff --git a/src/iceberg/catalog/rest/rest_catalog.cc b/src/iceberg/catalog/rest/rest_catalog.cc index 34e656bca3..90865290ff 100644 --- a/src/iceberg/catalog/rest/rest_catalog.cc +++ b/src/iceberg/catalog/rest/rest_catalog.cc @@ -45,7 +45,9 @@ #include "iceberg/catalog/rest/rest_util.h" #include "iceberg/catalog/rest/types.h" #include "iceberg/json_serde_internal.h" +#include "iceberg/logging/log_level.h" #include "iceberg/logging/log_macros.h" +#include "iceberg/logging/logger.h" #include "iceberg/metrics/metrics_reporters.h" #include "iceberg/partition_spec.h" #include "iceberg/result.h" @@ -973,7 +975,32 @@ Result> RestCatalog::MakeTableFromLoadResult( auto table_catalog = std::make_shared( shared_from_this(), context, identifier, table_config, table_session, table_io); - if (supported_endpoints_.contains(Endpoint::PlanTableScan())) { + // Determine effective scan planning mode: table config overrides client config. + ICEBERG_ASSIGN_OR_RAISE(auto client_mode, + RestCatalogProperties::ScanPlanningModeFrom(config_.configs())); + ICEBERG_ASSIGN_OR_RAISE(auto server_mode, + RestCatalogProperties::ScanPlanningModeFrom(table_config)); + + if (client_mode.has_value() && server_mode.has_value() && + *client_mode != *server_mode) { + Log(LogLevel::kWarn, + "Scan planning mode mismatch for table {}: client config={}, server config={}. " + "Server config will take precedence.", + identifier.ToString(), + *client_mode == ScanPlanningMode::kClient ? "client" : "server", + *server_mode == ScanPlanningMode::kClient ? "client" : "server"); + } + + ScanPlanningMode effective_mode = + server_mode.value_or(client_mode.value_or(ScanPlanningMode::kClient)); + + if (effective_mode == ScanPlanningMode::kServer) { + if (!supported_endpoints_.contains(Endpoint::PlanTableScan())) { + return NotSupported( + "Server requires server-side scan planning for table {} but does not support " + "the PlanTableScan endpoint.", + identifier.ToString()); + } RestScanContext rest_ctx{ .client = client_, .paths = paths_, diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 30dcc95dfd..277fbb630e 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -100,8 +100,10 @@ Result>> RestTableScan::PlanTableScan( if (context_.from_snapshot_id.has_value() && context_.to_snapshot_id.has_value()) { request.start_snapshot_id = context_.from_snapshot_id; request.end_snapshot_id = context_.to_snapshot_id; + request.use_snapshot_schema = true; } else if (context_.snapshot_id.has_value()) { request.snapshot_id = context_.snapshot_id; + request.use_snapshot_schema = context_.use_snapshot_schema; } if (!context_.columns_to_keep_stats.empty()) { @@ -129,8 +131,11 @@ Result>> RestTableScan::PlanTableScan( plan_id = result.plan_id; switch (result.plan_status) { - case PlanStatus::kCompleted: - return ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); + case PlanStatus::kCompleted: { + auto tasks = ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); + if (!tasks.has_value()) CancelPlanning(plan_id); + return tasks; + } case PlanStatus::kSubmitted: return FetchPlanningResult(plan_id, specs); case PlanStatus::kFailed: @@ -165,8 +170,11 @@ Result>> RestTableScan::FetchPlanningR ICEBERG_RETURN_UNEXPECTED(result.Validate()); switch (result.plan_status) { - case PlanStatus::kCompleted: - return ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); + case PlanStatus::kCompleted: { + auto tasks = ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); + if (!tasks.has_value()) CancelPlanning(plan_id); + return tasks; + } case PlanStatus::kSubmitted: { auto elapsed_ms = std::chrono::duration_cast( std::chrono::steady_clock::now() - start) diff --git a/src/iceberg/table_scan.cc b/src/iceberg/table_scan.cc index 46f235fbe2..d8bcda903f 100644 --- a/src/iceberg/table_scan.cc +++ b/src/iceberg/table_scan.cc @@ -316,6 +316,7 @@ TableScanBuilder& TableScanBuilder::UseSnapshot(int64_t snap context_.snapshot_id.value()); ICEBERG_BUILDER_ASSIGN_OR_RETURN(std::ignore, metadata_->SnapshotById(snapshot_id)); context_.snapshot_id = snapshot_id; + context_.use_snapshot_schema = true; return *this; } @@ -323,6 +324,7 @@ template TableScanBuilder& TableScanBuilder::UseRef(const std::string& ref) { if (ref == SnapshotRef::kMainBranch) { context_.snapshot_id.reset(); + context_.use_snapshot_schema = false; return *this; } @@ -335,6 +337,7 @@ TableScanBuilder& TableScanBuilder::UseRef(const std::string const int64_t snapshot_id = iter->second->snapshot_id; ICEBERG_BUILDER_ASSIGN_OR_RETURN(std::ignore, metadata_->SnapshotById(snapshot_id)); context_.snapshot_id = snapshot_id; + context_.use_snapshot_schema = (iter->second->type() == SnapshotRefType::kTag); return *this; } diff --git a/src/iceberg/table_scan.h b/src/iceberg/table_scan.h index 7702f13e63..90d71df27c 100644 --- a/src/iceberg/table_scan.h +++ b/src/iceberg/table_scan.h @@ -234,6 +234,7 @@ struct TableScanContext { std::optional to_snapshot_id; std::string branch{}; std::optional min_rows_requested; + bool use_snapshot_schema{false}; OptionalExecutor plan_executor; std::string table_name; std::shared_ptr metrics_reporter; diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 058a9dd3cc..6e51b2e357 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -321,11 +321,13 @@ if(ICEBERG_BUILD_REST) add_rest_iceberg_test(rest_catalog_test SOURCES auth_manager_test.cc + catalog_properties_test.cc error_handlers_test.cc endpoint_test.cc rest_file_io_test.cc rest_json_serde_test.cc rest_metrics_reporter_test.cc + rest_table_scan_test.cc rest_util_test.cc) if(ICEBERG_SIGV4) diff --git a/src/iceberg/test/catalog_properties_test.cc b/src/iceberg/test/catalog_properties_test.cc new file mode 100644 index 0000000000..47abe7682c --- /dev/null +++ b/src/iceberg/test/catalog_properties_test.cc @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/catalog/rest/catalog_properties.h" + +#include + +#include "iceberg/test/matchers.h" + +namespace iceberg::rest { + +TEST(ScanPlanningModeTest, MissingKeyReturnsNullopt) { + std::unordered_map config; + auto result = RestCatalogProperties::ScanPlanningModeFrom(config); + ASSERT_THAT(result, IsOk()); + EXPECT_FALSE(result->has_value()); +} + +TEST(ScanPlanningModeTest, ClientLowercaseReturnsKClient) { + auto result = RestCatalogProperties::ScanPlanningModeFrom( + {{"scan-planning-mode", "client"}}); + ASSERT_THAT(result, IsOk()); + ASSERT_TRUE(result->has_value()); + EXPECT_EQ(**result, ScanPlanningMode::kClient); +} + +TEST(ScanPlanningModeTest, ServerLowercaseReturnsKServer) { + auto result = RestCatalogProperties::ScanPlanningModeFrom( + {{"scan-planning-mode", "server"}}); + ASSERT_THAT(result, IsOk()); + ASSERT_TRUE(result->has_value()); + EXPECT_EQ(**result, ScanPlanningMode::kServer); +} + +TEST(ScanPlanningModeTest, ClientUppercaseReturnsKClient) { + auto result = RestCatalogProperties::ScanPlanningModeFrom( + {{"scan-planning-mode", "CLIENT"}}); + ASSERT_THAT(result, IsOk()); + ASSERT_TRUE(result->has_value()); + EXPECT_EQ(**result, ScanPlanningMode::kClient); +} + +TEST(ScanPlanningModeTest, ServerUppercaseReturnsKServer) { + auto result = RestCatalogProperties::ScanPlanningModeFrom( + {{"scan-planning-mode", "SERVER"}}); + ASSERT_THAT(result, IsOk()); + ASSERT_TRUE(result->has_value()); + EXPECT_EQ(**result, ScanPlanningMode::kServer); +} + +TEST(ScanPlanningModeTest, InvalidValueReturnsError) { + auto result = RestCatalogProperties::ScanPlanningModeFrom( + {{"scan-planning-mode", "invalid"}}); + EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); +} + +TEST(ScanPlanningModeTest, OtherKeysAreIgnored) { + auto result = RestCatalogProperties::ScanPlanningModeFrom( + {{"other-key", "server"}, {"scan-planning-mode", "client"}}); + ASSERT_THAT(result, IsOk()); + ASSERT_TRUE(result->has_value()); + EXPECT_EQ(**result, ScanPlanningMode::kClient); +} + +} // namespace iceberg::rest diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc new file mode 100644 index 0000000000..89eab80872 --- /dev/null +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/catalog/rest/rest_table_scan.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "iceberg/catalog/rest/auth/auth_session.h" +#include "iceberg/catalog/rest/endpoint.h" +#include "iceberg/catalog/rest/error_handlers.h" +#include "iceberg/catalog/rest/http_client.h" +#include "iceberg/catalog/rest/resource_paths.h" +#include "iceberg/catalog/rest/rest_table.h" +#include "iceberg/file_io.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/snapshot.h" +#include "iceberg/table_identifier.h" +#include "iceberg/table_metadata.h" +#include "iceberg/table_scan.h" +#include "iceberg/test/matchers.h" +#include "iceberg/type.h" + +namespace iceberg::rest { + +using ::testing::_; +using ::testing::Return; + +// -------------------------------------------------------------------------- +// Mock HTTP client that overrides the virtual methods of HttpClient. +// The base class constructor creates a cpr::ConnectionPool, which is a +// lightweight allocation (no network connections are opened at construction). +// -------------------------------------------------------------------------- +class MockHttpClient : public HttpClient { + public: + MockHttpClient() : HttpClient({}) {} + + MOCK_METHOD(Result, Get, + (const std::string& path, + (const std::unordered_map&) params, + (const std::unordered_map&) headers, + const ErrorHandler& error_handler, auth::AuthSession& session), + (override)); + + MOCK_METHOD(Result, Post, + (const std::string& path, const std::string& body, + (const std::unordered_map&) headers, + const ErrorHandler& error_handler, auth::AuthSession& session), + (override)); + + MOCK_METHOD(Result, Delete, + (const std::string& path, + (const std::unordered_map&) params, + (const std::unordered_map&) headers, + const ErrorHandler& error_handler, auth::AuthSession& session), + (override)); +}; + +// -------------------------------------------------------------------------- +// Minimal FileIO stub (no real I/O needed for server-side scan planning tests) +// -------------------------------------------------------------------------- +class NoOpFileIO : public FileIO { + public: + Result ReadFile(const std::string&, + std::optional) override { + return IOError("NoOpFileIO"); + } + Status WriteFile(const std::string&, std::string_view) override { return {}; } + Status DeleteFile(const std::string&) override { return {}; } +}; + +// -------------------------------------------------------------------------- +// Test fixture shared by RestTableScan tests. +// -------------------------------------------------------------------------- +class RestTableScanTest : public ::testing::Test { + protected: + void SetUp() override { + schema_ = std::make_shared(std::vector{ + SchemaField::MakeRequired(1, "id", int32()), + SchemaField::MakeRequired(2, "data", string())}); + + auto spec = PartitionSpec::Unpartitioned(); + + constexpr int64_t kSnapshotId = 1000L; + auto snapshot = std::make_shared( + Snapshot{.snapshot_id = kSnapshotId, + .sequence_number = 1L, + .timestamp_ms = TimePointMsFromUnixMs(1609459200000L), + .manifest_list = "/tmp/manifest-list.avro", + .schema_id = schema_->schema_id()}); + + metadata_ = std::make_shared(TableMetadata{ + .format_version = 2, + .table_uuid = "test-uuid", + .location = "/tmp/table", + .last_sequence_number = 1L, + .last_updated_ms = TimePointMsFromUnixMs(1609459200000L), + .last_column_id = 2, + .schemas = {schema_}, + .current_schema_id = schema_->schema_id(), + .partition_specs = {spec}, + .default_spec_id = spec->spec_id(), + .last_partition_id = 999, + .current_snapshot_id = kSnapshotId, + .snapshots = {snapshot}, + .refs = {{"main", + std::make_shared(SnapshotRef{ + .snapshot_id = kSnapshotId, + .retention = SnapshotRef::Branch{}})}}}); + + file_io_ = std::make_shared(); + + mock_client_ = std::make_shared(); + + ICEBERG_UNWRAP_OR_FAIL(paths_, ResourcePaths::Make( + "http://test-server", /*prefix=*/"", + /*namespace_separator=*/"%1F")); + + session_ = auth::AuthSession::MakeDefault(/*headers=*/{}); + + identifier_ = TableIdentifier{Namespace{{"default"}}, "my_table"}; + + all_plan_endpoints_ = {Endpoint::PlanTableScan(), Endpoint::FetchPlanningResult(), + Endpoint::CancelPlanning(), Endpoint::FetchScanTasks()}; + } + + // Pass std::nullopt to get the full set of plan endpoints (default). + // Pass an explicit set (including empty) to use exactly that set. + RestScanContext MakeContext( + std::optional> endpoints = std::nullopt) { + auto effective = + endpoints.has_value() ? std::move(*endpoints) : all_plan_endpoints_; + return RestScanContext{ + .client = mock_client_, + .paths = paths_, + .session = session_, + .supported_endpoints = std::move(effective), + .identifier = identifier_, + }; + } + + Result> MakeScan(RestScanContext ctx) { + return RestTableScan::Make(metadata_, schema_, file_io_, + internal::TableScanContext{}, std::move(ctx)); + } + + std::shared_ptr schema_; + std::shared_ptr metadata_; + std::shared_ptr file_io_; + std::shared_ptr mock_client_; + std::shared_ptr paths_; + std::shared_ptr session_; + TableIdentifier identifier_; + std::unordered_set all_plan_endpoints_; +}; + +// -------------------------------------------------------------------------- +// PlanFiles: server returns COMPLETED immediately, no file scan tasks. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesCompleted) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// PlanFiles: server returns COMPLETED with a non-empty plan-id (still valid). +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesCompletedWithPlanId) { + constexpr std::string_view kResponseBody = + R"({"status":"completed","plan-id":"plan-abc"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// PlanFiles: server returns SUBMITTED → poll returns COMPLETED. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesSubmittedThenCompleted) { + constexpr std::string_view kSubmittedBody = + R"({"status":"submitted","plan-id":"plan-poll-1"})"; + constexpr std::string_view kCompletedBody = R"({"status":"completed"})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kCompletedBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// PlanFiles: server returns FAILED → scan returns IOError. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesFailed) { + constexpr std::string_view kFailedBody = + R"({"status":"failed","error":{"message":"server error","type":"ServerError","code":500}})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFailedBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kIOError)); +} + +// -------------------------------------------------------------------------- +// PlanFiles: PlanTableScan endpoint missing → NotSupported error. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesEndpointNotSupported) { + ICEBERG_UNWRAP_OR_FAIL( + auto scan, MakeScan(MakeContext(std::unordered_set{}))); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); +} + +// -------------------------------------------------------------------------- +// PlanFiles with plan-tasks: server returns COMPLETED with opaque task token, +// then FetchScanTasks is called and returns no file scan tasks. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesWithPlanTasks) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-1","plan-tasks":["tok-1"]})"; + // FetchScanTasksResponse requires at least one of plan-tasks or file-scan-tasks present. + constexpr std::string_view kTasksResponse = R"({"file-scan-tasks":[]})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// Cancel is called when FetchScanTasks fails after a COMPLETED response that +// included plan-tasks. This mirrors the Java cancelPlan-on-close behavior. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, CancelCalledWhenFetchScanTasksFails) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-cancel-1","plan-tasks":["tok-a"]})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(IOError("FetchScanTasks failed"))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kIOError)); +} + +// -------------------------------------------------------------------------- +// Cancel is a no-op when plan_id is empty (server returned no plan-id). +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, CancelIsNoOpWithEmptyPlanId) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-tasks":["tok-b"]})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(IOError("FetchScanTasks failed"))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kIOError)); +} + +// -------------------------------------------------------------------------- +// Cancel is a no-op when CancelPlanning endpoint is not advertised. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, CancelIsNoOpWhenEndpointNotAdvertised) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-2","plan-tasks":["tok-c"]})"; + + std::unordered_set endpoints_without_cancel = { + Endpoint::PlanTableScan(), Endpoint::FetchPlanningResult(), + Endpoint::FetchScanTasks()}; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(IOError("FetchScanTasks failed"))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, + MakeScan(MakeContext(endpoints_without_cancel))); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kIOError)); +} + +// -------------------------------------------------------------------------- +// FetchPlanningResult: FetchPlanningResult endpoint missing → NotSupported. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, FetchPlanningResultEndpointNotSupported) { + constexpr std::string_view kSubmittedBody = + R"({"status":"submitted","plan-id":"plan-3"})"; + + std::unordered_set endpoints_without_fetch = { + Endpoint::PlanTableScan(), Endpoint::CancelPlanning(), + Endpoint::FetchScanTasks()}; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, + MakeScan(MakeContext(endpoints_without_fetch))); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); +} + +// -------------------------------------------------------------------------- +// FetchScanTasks: endpoint missing → NotSupported. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, FetchScanTasksEndpointNotSupported) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-4","plan-tasks":["tok-d"]})"; + + std::unordered_set endpoints_without_tasks = { + Endpoint::PlanTableScan(), Endpoint::FetchPlanningResult(), + Endpoint::CancelPlanning()}; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, + MakeScan(MakeContext(endpoints_without_tasks))); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); +} + +// -------------------------------------------------------------------------- +// use_snapshot_schema: UseSnapshot() sets it to true in the builder context. +// RestTableScanBuilder propagates context from DataTableScanBuilder. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, UseSnapshotPropagatesUseSnapshotSchemaInContext) { + constexpr int64_t kSnapshotId = 1000L; + RestTableScanBuilder builder(metadata_, file_io_, MakeContext(std::nullopt)); + builder.UseSnapshot(kSnapshotId); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + EXPECT_TRUE(scan->context().use_snapshot_schema); +} + +// -------------------------------------------------------------------------- +// use_snapshot_schema: default scan does not set use_snapshot_schema. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, DefaultScanDoesNotSetUseSnapshotSchema) { + RestTableScanBuilder builder(metadata_, file_io_, MakeContext(std::nullopt)); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + EXPECT_FALSE(scan->context().use_snapshot_schema); +} + +// -------------------------------------------------------------------------- +// RestTable::NewScan returns a RestTableScanBuilder (not a plain builder). +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, RestTableNewScanReturnsRestTableScanBuilder) { + ICEBERG_UNWRAP_OR_FAIL( + auto table, + RestTable::Make(identifier_, metadata_, "/tmp/metadata.json", file_io_, + /*catalog=*/nullptr, MakeContext(std::nullopt))); + ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewScan()); + auto* typed = dynamic_cast(builder.get()); + EXPECT_NE(typed, nullptr); +} + +} // namespace iceberg::rest diff --git a/src/iceberg/test/table_scan_test.cc b/src/iceberg/test/table_scan_test.cc index f08440a10a..623e098021 100644 --- a/src/iceberg/test/table_scan_test.cc +++ b/src/iceberg/test/table_scan_test.cc @@ -842,6 +842,45 @@ TEST_P(TableScanTest, SchemaWithSelectedColumnsAndFilter) { } } +// use_snapshot_schema propagation tests: verify the field is set correctly for +// UseSnapshot, UseRef (tag vs branch), and default/incremental scans. +TEST_P(TableScanTest, UseSnapshotSetsTrueUseSnapshotSchema) { + constexpr int64_t kSnapshotId = 1000L; + ICEBERG_UNWRAP_OR_FAIL(auto builder, + DataTableScanBuilder::Make(table_metadata_, file_io_)); + builder->UseSnapshot(kSnapshotId); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); + EXPECT_TRUE(scan->context().use_snapshot_schema); +} + +TEST_P(TableScanTest, UseRefTagSetsTrueUseSnapshotSchema) { + constexpr int64_t kSnapshotId = 1000L; + table_metadata_->refs["v1.0"] = std::make_shared( + SnapshotRef{.snapshot_id = kSnapshotId, .retention = SnapshotRef::Tag{}}); + + ICEBERG_UNWRAP_OR_FAIL(auto builder, + DataTableScanBuilder::Make(table_metadata_, file_io_)); + builder->UseRef("v1.0"); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); + EXPECT_TRUE(scan->context().use_snapshot_schema); +} + +TEST_P(TableScanTest, UseRefBranchSetsFalseUseSnapshotSchema) { + ICEBERG_UNWRAP_OR_FAIL(auto builder, + DataTableScanBuilder::Make(table_metadata_, file_io_)); + builder->UseRef("main"); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); + EXPECT_FALSE(scan->context().use_snapshot_schema); +} + +TEST_P(TableScanTest, DefaultScanHasFalseUseSnapshotSchema) { + ICEBERG_UNWRAP_OR_FAIL(auto builder, + DataTableScanBuilder::Make(table_metadata_, file_io_)); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); + EXPECT_FALSE(scan->context().use_snapshot_schema); +} + + INSTANTIATE_TEST_SUITE_P(TableScanVersions, TableScanTest, testing::Values(1, 2, 3)); } // namespace iceberg From 4e0b00f9c4ae9cc4101268d9276d94083b247853 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 13 Sep 2026 12:53:34 -0700 Subject: [PATCH 03/36] Fix API compatibility after rebase on main Upstream changed Table and DataTableScanBuilder constructors to require full_name/table_name and MetricsReporter parameters. Updated RestTable, RestTableScanBuilder, and their callers accordingly. --- src/iceberg/catalog/rest/rest_catalog.cc | 3 ++- src/iceberg/catalog/rest/rest_table.cc | 14 ++++++++++---- src/iceberg/catalog/rest/rest_table.h | 6 +++++- src/iceberg/catalog/rest/rest_table_scan.cc | 5 ++++- src/iceberg/catalog/rest/rest_table_scan.h | 3 +++ src/iceberg/test/rest_table_scan_test.cc | 9 ++++++--- 6 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/iceberg/catalog/rest/rest_catalog.cc b/src/iceberg/catalog/rest/rest_catalog.cc index 90865290ff..10d2db92bb 100644 --- a/src/iceberg/catalog/rest/rest_catalog.cc +++ b/src/iceberg/catalog/rest/rest_catalog.cc @@ -1010,7 +1010,8 @@ Result> RestCatalog::MakeTableFromLoadResult( }; return RestTable::Make(identifier, std::move(result.metadata), std::move(result.metadata_location), std::move(table_io), - std::move(table_catalog), std::move(rest_ctx)); + std::move(table_catalog), RestTableName(name_, identifier), + reporter, std::move(rest_ctx)); } return Table::Make(identifier, std::move(result.metadata), diff --git a/src/iceberg/catalog/rest/rest_table.cc b/src/iceberg/catalog/rest/rest_table.cc index 00f8856d15..7274444435 100644 --- a/src/iceberg/catalog/rest/rest_table.cc +++ b/src/iceberg/catalog/rest/rest_table.cc @@ -31,9 +31,12 @@ namespace iceberg::rest { RestTable::RestTable(TableIdentifier identifier, std::shared_ptr metadata, std::string metadata_location, std::shared_ptr io, - std::shared_ptr catalog, RestScanContext rest_context) + std::shared_ptr catalog, std::string full_name, + std::shared_ptr reporter, + RestScanContext rest_context) : Table(std::move(identifier), std::move(metadata), std::move(metadata_location), - std::move(io), std::move(catalog)), + std::move(io), std::move(catalog), std::move(full_name), + std::move(reporter)), rest_context_(std::move(rest_context)) {} RestTable::~RestTable() = default; @@ -43,6 +46,8 @@ Result> RestTable::Make(TableIdentifier identifier, std::string metadata_location, std::shared_ptr io, std::shared_ptr catalog, + std::string full_name, + std::shared_ptr reporter, RestScanContext rest_context) { if (metadata == nullptr) { return InvalidArgument("Metadata cannot be null"); @@ -50,11 +55,12 @@ Result> RestTable::Make(TableIdentifier identifier, return std::shared_ptr( new RestTable(std::move(identifier), std::move(metadata), std::move(metadata_location), std::move(io), std::move(catalog), - std::move(rest_context))); + std::move(full_name), std::move(reporter), std::move(rest_context))); } Result> RestTable::NewScan() const { - return std::make_unique(metadata_, io_, rest_context_); + return std::make_unique(metadata_, io_, full_name_, reporter_, + rest_context_); } } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/rest_table.h b/src/iceberg/catalog/rest/rest_table.h index 8e9172b098..54208d26af 100644 --- a/src/iceberg/catalog/rest/rest_table.h +++ b/src/iceberg/catalog/rest/rest_table.h @@ -24,6 +24,7 @@ #include "iceberg/catalog/rest/iceberg_rest_export.h" #include "iceberg/catalog/rest/rest_table_scan.h" +#include "iceberg/metrics/metrics_reporter.h" #include "iceberg/result.h" #include "iceberg/table.h" #include "iceberg/type_fwd.h" @@ -42,6 +43,8 @@ class ICEBERG_REST_EXPORT RestTable final : public Table { std::string metadata_location, std::shared_ptr io, std::shared_ptr catalog, + std::string full_name, + std::shared_ptr reporter, RestScanContext rest_context); ~RestTable() override; @@ -53,7 +56,8 @@ class ICEBERG_REST_EXPORT RestTable final : public Table { private: RestTable(TableIdentifier identifier, std::shared_ptr metadata, std::string metadata_location, std::shared_ptr io, - std::shared_ptr catalog, RestScanContext rest_context); + std::shared_ptr catalog, std::string full_name, + std::shared_ptr reporter, RestScanContext rest_context); RestScanContext rest_context_; }; diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 277fbb630e..7201cfa355 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -262,8 +262,11 @@ void RestTableScan::CancelPlanning(const std::string& plan_id) const { RestTableScanBuilder::RestTableScanBuilder(std::shared_ptr metadata, std::shared_ptr io, + std::string table_name, + std::shared_ptr metrics_reporter, RestScanContext rest_context) - : DataTableScanBuilder(std::move(metadata), std::move(io)), + : DataTableScanBuilder(std::move(metadata), std::move(io), std::move(table_name), + std::move(metrics_reporter)), rest_context_(std::move(rest_context)) {} Result> RestTableScanBuilder::Build() { diff --git a/src/iceberg/catalog/rest/rest_table_scan.h b/src/iceberg/catalog/rest/rest_table_scan.h index 4d924c38fe..2844d48ca3 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.h +++ b/src/iceberg/catalog/rest/rest_table_scan.h @@ -28,6 +28,7 @@ #include "iceberg/catalog/rest/endpoint.h" #include "iceberg/catalog/rest/iceberg_rest_export.h" #include "iceberg/data_table_scan.h" +#include "iceberg/metrics/metrics_reporter.h" #include "iceberg/result.h" #include "iceberg/table_identifier.h" #include "iceberg/table_scan.h" @@ -105,6 +106,8 @@ class ICEBERG_REST_EXPORT RestTableScan : public DataTableScan { class ICEBERG_REST_EXPORT RestTableScanBuilder : public DataTableScanBuilder { public: RestTableScanBuilder(std::shared_ptr metadata, std::shared_ptr io, + std::string table_name, + std::shared_ptr metrics_reporter, RestScanContext rest_context); /// \brief Resolves schema/context via parent logic then creates a RestTableScan. diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 89eab80872..749b12dcb8 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -372,7 +372,8 @@ TEST_F(RestTableScanTest, FetchScanTasksEndpointNotSupported) { // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, UseSnapshotPropagatesUseSnapshotSchemaInContext) { constexpr int64_t kSnapshotId = 1000L; - RestTableScanBuilder builder(metadata_, file_io_, MakeContext(std::nullopt)); + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext(std::nullopt)); builder.UseSnapshot(kSnapshotId); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); EXPECT_TRUE(scan->context().use_snapshot_schema); @@ -382,7 +383,8 @@ TEST_F(RestTableScanTest, UseSnapshotPropagatesUseSnapshotSchemaInContext) { // use_snapshot_schema: default scan does not set use_snapshot_schema. // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, DefaultScanDoesNotSetUseSnapshotSchema) { - RestTableScanBuilder builder(metadata_, file_io_, MakeContext(std::nullopt)); + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext(std::nullopt)); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); EXPECT_FALSE(scan->context().use_snapshot_schema); } @@ -394,7 +396,8 @@ TEST_F(RestTableScanTest, RestTableNewScanReturnsRestTableScanBuilder) { ICEBERG_UNWRAP_OR_FAIL( auto table, RestTable::Make(identifier_, metadata_, "/tmp/metadata.json", file_io_, - /*catalog=*/nullptr, MakeContext(std::nullopt))); + /*catalog=*/nullptr, "test.my_table", nullptr, + MakeContext(std::nullopt))); ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewScan()); auto* typed = dynamic_cast(builder.get()); EXPECT_NE(typed, nullptr); From 0a284275b00722b7c5fa6da875081c376375e4d4 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 13 Sep 2026 14:25:50 -0700 Subject: [PATCH 04/36] feat(rest): support storage credentials in scan planning responses Parse storage-credentials from PlanTableScanResponse, FetchPlanningResultResponse, and FetchScanTasksResponse. When credentials are present, build a scan-scoped FileIO via MakeTableFileIO and expose it through RestTableScan::effective_io() for callers to use when reading the returned scan tasks. --- src/iceberg/catalog/rest/json_serde.cc | 24 +++ src/iceberg/catalog/rest/rest_catalog.cc | 2 + src/iceberg/catalog/rest/rest_table_scan.cc | 58 ++++--- src/iceberg/catalog/rest/rest_table_scan.h | 22 ++- src/iceberg/catalog/rest/types.cc | 1 + src/iceberg/catalog/rest/types.h | 5 +- src/iceberg/test/rest_table_scan_test.cc | 169 +++++++++++++------- src/iceberg/test/table_scan_test.cc | 13 +- 8 files changed, 205 insertions(+), 89 deletions(-) diff --git a/src/iceberg/catalog/rest/json_serde.cc b/src/iceberg/catalog/rest/json_serde.cc index ba348dec3f..c5bf08a808 100644 --- a/src/iceberg/catalog/rest/json_serde.cc +++ b/src/iceberg/catalog/rest/json_serde.cc @@ -292,6 +292,15 @@ Result ScanTaskFieldsToJson( json[kFileScanTasks] = std::move(tasks_json); } + if (!response.storage_credentials.empty()) { + nlohmann::json creds_json = nlohmann::json::array(); + for (const auto& cred : response.storage_credentials) { + ICEBERG_ASSIGN_OR_RAISE(auto entry, StorageCredentialToJson(cred)); + creds_json.push_back(std::move(entry)); + } + json[kStorageCredentials] = std::move(creds_json); + } + return json; } @@ -333,6 +342,21 @@ Status ScanTaskFieldsFromJson( FileScanTasksFromJson(file_scan_tasks_json, response.delete_files, partition_specs_by_id, schema)); } + + // 4. storage_credentials + if (json.contains(kStorageCredentials)) { + ICEBERG_ASSIGN_OR_RAISE(auto creds_json, + GetJsonValue(json, kStorageCredentials)); + if (!creds_json.is_array()) { + return JsonParseError("Cannot parse storage credentials from non-array: {}", + SafeDumpJson(creds_json)); + } + for (const auto& entry : creds_json) { + ICEBERG_ASSIGN_OR_RAISE(auto cred, StorageCredentialFromJson(entry)); + response.storage_credentials.push_back(std::move(cred)); + } + } + return {}; } diff --git a/src/iceberg/catalog/rest/rest_catalog.cc b/src/iceberg/catalog/rest/rest_catalog.cc index 10d2db92bb..384124ce73 100644 --- a/src/iceberg/catalog/rest/rest_catalog.cc +++ b/src/iceberg/catalog/rest/rest_catalog.cc @@ -1007,6 +1007,8 @@ Result> RestCatalog::MakeTableFromLoadResult( .session = table_session, .supported_endpoints = supported_endpoints_, .identifier = identifier, + .catalog_config = config_.configs(), + .table_config = table_config, }; return RestTable::Make(identifier, std::move(result.metadata), std::move(result.metadata_location), std::move(table_io), diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 7201cfa355..3e11c71dc2 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -29,6 +29,7 @@ #include "iceberg/catalog/rest/http_client.h" #include "iceberg/catalog/rest/json_serde_internal.h" #include "iceberg/catalog/rest/resource_paths.h" +#include "iceberg/catalog/rest/rest_file_io.h" #include "iceberg/catalog/rest/types.h" #include "iceberg/json_serde_internal.h" #include "iceberg/partition_spec.h" @@ -115,8 +116,7 @@ Result>> RestTableScan::PlanTableScan( } } - ICEBERG_ASSIGN_OR_RAISE(auto path, - rest_context_.paths->Plan(rest_context_.identifier)); + ICEBERG_ASSIGN_OR_RAISE(auto path, rest_context_.paths->Plan(rest_context_.identifier)); ICEBERG_ASSIGN_OR_RAISE(auto request_json, ToJson(request)); ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(request_json)); ICEBERG_ASSIGN_OR_RAISE( @@ -132,6 +132,7 @@ Result>> RestTableScan::PlanTableScan( switch (result.plan_status) { case PlanStatus::kCompleted: { + ICEBERG_RETURN_UNEXPECTED(ApplyStorageCredentials(result.storage_credentials)); auto tasks = ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); if (!tasks.has_value()) CancelPlanning(plan_id); return tasks; @@ -140,7 +141,7 @@ Result>> RestTableScan::PlanTableScan( return FetchPlanningResult(plan_id, specs); case PlanStatus::kFailed: return IOError("Scan planning failed: {}", - result.error ? result.error->message : "unknown error"); + result.error ? result.error->message : "unknown error"); case PlanStatus::kCancelled: return IOError("Scan planning was cancelled for plan_id={}", plan_id); } @@ -171,6 +172,7 @@ Result>> RestTableScan::FetchPlanningR switch (result.plan_status) { case PlanStatus::kCompleted: { + ICEBERG_RETURN_UNEXPECTED(ApplyStorageCredentials(result.storage_credentials)); auto tasks = ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); if (!tasks.has_value()) CancelPlanning(plan_id); return tasks; @@ -181,9 +183,8 @@ Result>> RestTableScan::FetchPlanningR .count(); if (elapsed_ms >= kMaxWaitTimeMs) { CancelPlanning(plan_id); - return IOError( - "Scan planning timed out after {}ms waiting for plan_id={}", elapsed_ms, - plan_id); + return IOError("Scan planning timed out after {}ms waiting for plan_id={}", + elapsed_ms, plan_id); } std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); delay_ms = std::min(delay_ms * 2, kMaxSleepMs); @@ -192,15 +193,15 @@ Result>> RestTableScan::FetchPlanningR case PlanStatus::kFailed: CancelPlanning(plan_id); return IOError("Scan planning failed: {}", - result.error ? result.error->message : "unknown error"); + result.error ? result.error->message : "unknown error"); case PlanStatus::kCancelled: return IOError("Scan planning was cancelled for plan_id={}", plan_id); } } CancelPlanning(plan_id); - return IOError("Scan planning exceeded max retries ({}) for plan_id={}", - kMaxRetries, plan_id); + return IOError("Scan planning exceeded max retries ({}) for plan_id={}", kMaxRetries, + plan_id); } Result>> RestTableScan::FetchScanTasks( @@ -212,15 +213,15 @@ Result>> RestTableScan::FetchScanTasks rest_context_.paths->FetchScanTasks(rest_context_.identifier)); FetchScanTasksRequest request{.planTask = plan_task}; ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(ToJson(request))); - ICEBERG_ASSIGN_OR_RAISE( - const auto response, - rest_context_.client->Post(path, json_request, /*headers=*/{}, - *PlanTaskErrorHandler::Instance(), - *rest_context_.session)); + ICEBERG_ASSIGN_OR_RAISE(const auto response, + rest_context_.client->Post(path, json_request, /*headers=*/{}, + *PlanTaskErrorHandler::Instance(), + *rest_context_.session)); ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); ICEBERG_ASSIGN_OR_RAISE(auto result, FetchScanTasksResponseFromJson(json, specs, *schema_)); ICEBERG_RETURN_UNEXPECTED(result.Validate()); + ICEBERG_RETURN_UNEXPECTED(ApplyStorageCredentials(result.storage_credentials)); return ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); } @@ -253,18 +254,31 @@ void RestTableScan::CancelPlanning(const std::string& plan_id) const { if (!path.has_value()) return; // Best-effort: ignore errors. - std::ignore = rest_context_.client->Delete(*path, /*params=*/{}, /*headers=*/{}, - *PlanErrorHandler::Instance(), - *rest_context_.session); + std::ignore = + rest_context_.client->Delete(*path, /*params=*/{}, /*headers=*/{}, + *PlanErrorHandler::Instance(), *rest_context_.session); +} + +const std::shared_ptr& RestTableScan::effective_io() const { + return scan_io_ ? scan_io_ : io_; +} + +Status RestTableScan::ApplyStorageCredentials( + const std::vector& credentials) const { + if (credentials.empty()) return {}; + ICEBERG_ASSIGN_OR_RAISE( + auto io, MakeTableFileIO(rest_context_.catalog_config, rest_context_.table_config, + credentials)); + scan_io_ = std::move(io); + return {}; } // RestTableScanBuilder -RestTableScanBuilder::RestTableScanBuilder(std::shared_ptr metadata, - std::shared_ptr io, - std::string table_name, - std::shared_ptr metrics_reporter, - RestScanContext rest_context) +RestTableScanBuilder::RestTableScanBuilder( + std::shared_ptr metadata, std::shared_ptr io, + std::string table_name, std::shared_ptr metrics_reporter, + RestScanContext rest_context) : DataTableScanBuilder(std::move(metadata), std::move(io), std::move(table_name), std::move(metrics_reporter)), rest_context_(std::move(rest_context)) {} diff --git a/src/iceberg/catalog/rest/rest_table_scan.h b/src/iceberg/catalog/rest/rest_table_scan.h index 2844d48ca3..5e3258086f 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.h +++ b/src/iceberg/catalog/rest/rest_table_scan.h @@ -30,6 +30,7 @@ #include "iceberg/data_table_scan.h" #include "iceberg/metrics/metrics_reporter.h" #include "iceberg/result.h" +#include "iceberg/storage_credential.h" #include "iceberg/table_identifier.h" #include "iceberg/table_scan.h" #include "iceberg/type_fwd.h" @@ -53,6 +54,11 @@ struct ICEBERG_REST_EXPORT RestScanContext { std::shared_ptr session; std::unordered_set supported_endpoints; TableIdentifier identifier; + /// Catalog-level config, used with table_config to build a scan-scoped FileIO + /// when the server vends storage credentials in a planning response. + std::unordered_map catalog_config; + /// Table-level config merged with catalog_config for scan-scoped FileIO creation. + std::unordered_map table_config; }; /// \brief A DataTableScan that delegates PlanFiles() to the REST catalog server @@ -70,6 +76,13 @@ class ICEBERG_REST_EXPORT RestTableScan : public DataTableScan { /// \brief Plans files via the REST scan planning endpoints. Result>> PlanFiles() const override; + /// \brief Returns the FileIO to use when reading scan results. + /// + /// If the server vended storage credentials during planning, returns a FileIO + /// initialised with those credentials; otherwise returns the table's FileIO. + /// Must be called after PlanFiles(). + const std::shared_ptr& effective_io() const; + private: RestTableScan(std::shared_ptr metadata, std::shared_ptr schema, std::shared_ptr io, internal::TableScanContext context, @@ -99,14 +112,19 @@ class ICEBERG_REST_EXPORT RestTableScan : public DataTableScan { /// DELETE /plan/{plan_id}; best-effort, errors are silently ignored. void CancelPlanning(const std::string& plan_id) const; + /// Builds a scan-scoped FileIO from vended credentials and caches it in scan_io_. + /// No-op if credentials is empty. + Status ApplyStorageCredentials(const std::vector& credentials) const; + RestScanContext rest_context_; + mutable std::shared_ptr scan_io_; }; /// \brief Builder that produces a RestTableScan with the REST HTTP context injected. class ICEBERG_REST_EXPORT RestTableScanBuilder : public DataTableScanBuilder { public: - RestTableScanBuilder(std::shared_ptr metadata, std::shared_ptr io, - std::string table_name, + RestTableScanBuilder(std::shared_ptr metadata, + std::shared_ptr io, std::string table_name, std::shared_ptr metrics_reporter, RestScanContext rest_context); diff --git a/src/iceberg/catalog/rest/types.cc b/src/iceberg/catalog/rest/types.cc index 84fba9a7ca..ef20f44da7 100644 --- a/src/iceberg/catalog/rest/types.cc +++ b/src/iceberg/catalog/rest/types.cc @@ -210,6 +210,7 @@ bool OptionalSharedPtrVectorEqual( template bool ScanTaskFieldsEqual(const Response& lhs, const Response& rhs) { return lhs.plan_tasks == rhs.plan_tasks && + lhs.storage_credentials == rhs.storage_credentials && SharedPtrVectorEqual(lhs.delete_files, rhs.delete_files) && OptionalSharedPtrVectorEqual(lhs.file_scan_tasks, rhs.file_scan_tasks, FileScanTaskEqual); diff --git a/src/iceberg/catalog/rest/types.h b/src/iceberg/catalog/rest/types.h index a47a8e8515..aa2b57d17b 100644 --- a/src/iceberg/catalog/rest/types.h +++ b/src/iceberg/catalog/rest/types.h @@ -352,7 +352,7 @@ struct ICEBERG_REST_EXPORT PlanTableScanResponse { PlanStatus plan_status = PlanStatus::kCompleted; std::string plan_id; std::optional error; - // TODO(sandeepg): Add storage credentials and bind scan FileIO to them. + std::vector storage_credentials; Status Validate() const; @@ -367,7 +367,7 @@ struct ICEBERG_REST_EXPORT FetchPlanningResultResponse { std::vector> delete_files; PlanStatus plan_status = PlanStatus::kCompleted; std::optional error; - // TODO(sandeepg): Add storage credentials and bind scan FileIO to them. + std::vector storage_credentials; Status Validate() const; @@ -388,6 +388,7 @@ struct ICEBERG_REST_EXPORT FetchScanTasksResponse { std::optional> plan_tasks; std::optional>> file_scan_tasks; std::vector> delete_files; + std::vector storage_credentials; Status Validate() const; diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 749b12dcb8..9c5efd929b 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -61,21 +61,21 @@ class MockHttpClient : public HttpClient { MOCK_METHOD(Result, Get, (const std::string& path, - (const std::unordered_map&) params, - (const std::unordered_map&) headers, + (const std::unordered_map&)params, + (const std::unordered_map&)headers, const ErrorHandler& error_handler, auth::AuthSession& session), (override)); MOCK_METHOD(Result, Post, (const std::string& path, const std::string& body, - (const std::unordered_map&) headers, + (const std::unordered_map&)headers, const ErrorHandler& error_handler, auth::AuthSession& session), (override)); MOCK_METHOD(Result, Delete, (const std::string& path, - (const std::unordered_map&) params, - (const std::unordered_map&) headers, + (const std::unordered_map&)params, + (const std::unordered_map&)headers, const ErrorHandler& error_handler, auth::AuthSession& session), (override)); }; @@ -85,8 +85,7 @@ class MockHttpClient : public HttpClient { // -------------------------------------------------------------------------- class NoOpFileIO : public FileIO { public: - Result ReadFile(const std::string&, - std::optional) override { + Result ReadFile(const std::string&, std::optional) override { return IOError("NoOpFileIO"); } Status WriteFile(const std::string&, std::string_view) override { return {}; } @@ -99,9 +98,9 @@ class NoOpFileIO : public FileIO { class RestTableScanTest : public ::testing::Test { protected: void SetUp() override { - schema_ = std::make_shared(std::vector{ - SchemaField::MakeRequired(1, "id", int32()), - SchemaField::MakeRequired(2, "data", string())}); + schema_ = std::make_shared( + std::vector{SchemaField::MakeRequired(1, "id", int32()), + SchemaField::MakeRequired(2, "data", string())}); auto spec = PartitionSpec::Unpartitioned(); @@ -113,36 +112,35 @@ class RestTableScanTest : public ::testing::Test { .manifest_list = "/tmp/manifest-list.avro", .schema_id = schema_->schema_id()}); - metadata_ = std::make_shared(TableMetadata{ - .format_version = 2, - .table_uuid = "test-uuid", - .location = "/tmp/table", - .last_sequence_number = 1L, - .last_updated_ms = TimePointMsFromUnixMs(1609459200000L), - .last_column_id = 2, - .schemas = {schema_}, - .current_schema_id = schema_->schema_id(), - .partition_specs = {spec}, - .default_spec_id = spec->spec_id(), - .last_partition_id = 999, - .current_snapshot_id = kSnapshotId, - .snapshots = {snapshot}, - .refs = {{"main", - std::make_shared(SnapshotRef{ - .snapshot_id = kSnapshotId, - .retention = SnapshotRef::Branch{}})}}}); + metadata_ = std::make_shared( + TableMetadata{.format_version = 2, + .table_uuid = "test-uuid", + .location = "/tmp/table", + .last_sequence_number = 1L, + .last_updated_ms = TimePointMsFromUnixMs(1609459200000L), + .last_column_id = 2, + .schemas = {schema_}, + .current_schema_id = schema_->schema_id(), + .partition_specs = {spec}, + .default_spec_id = spec->spec_id(), + .last_partition_id = 999, + .current_snapshot_id = kSnapshotId, + .snapshots = {snapshot}, + .refs = {{"main", std::make_shared(SnapshotRef{ + .snapshot_id = kSnapshotId, + .retention = SnapshotRef::Branch{}})}}}); file_io_ = std::make_shared(); mock_client_ = std::make_shared(); - ICEBERG_UNWRAP_OR_FAIL(paths_, ResourcePaths::Make( - "http://test-server", /*prefix=*/"", - /*namespace_separator=*/"%1F")); + ICEBERG_UNWRAP_OR_FAIL(paths_, + ResourcePaths::Make("http://test-server", /*prefix=*/"", + /*namespace_separator=*/"%1F")); session_ = auth::AuthSession::MakeDefault(/*headers=*/{}); - identifier_ = TableIdentifier{Namespace{{"default"}}, "my_table"}; + identifier_ = TableIdentifier{.ns = Namespace{{"default"}}, .name = "my_table"}; all_plan_endpoints_ = {Endpoint::PlanTableScan(), Endpoint::FetchPlanningResult(), Endpoint::CancelPlanning(), Endpoint::FetchScanTasks()}; @@ -152,8 +150,7 @@ class RestTableScanTest : public ::testing::Test { // Pass an explicit set (including empty) to use exactly that set. RestScanContext MakeContext( std::optional> endpoints = std::nullopt) { - auto effective = - endpoints.has_value() ? std::move(*endpoints) : all_plan_endpoints_; + auto effective = endpoints.has_value() ? std::move(*endpoints) : all_plan_endpoints_; return RestScanContext{ .client = mock_client_, .paths = paths_, @@ -164,8 +161,8 @@ class RestTableScanTest : public ::testing::Test { } Result> MakeScan(RestScanContext ctx) { - return RestTableScan::Make(metadata_, schema_, file_io_, - internal::TableScanContext{}, std::move(ctx)); + return RestTableScan::Make(metadata_, schema_, file_io_, internal::TableScanContext{}, + std::move(ctx)); } std::shared_ptr schema_; @@ -241,8 +238,8 @@ TEST_F(RestTableScanTest, PlanFilesFailed) { // PlanFiles: PlanTableScan endpoint missing → NotSupported error. // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, PlanFilesEndpointNotSupported) { - ICEBERG_UNWRAP_OR_FAIL( - auto scan, MakeScan(MakeContext(std::unordered_set{}))); + ICEBERG_UNWRAP_OR_FAIL(auto scan, + MakeScan(MakeContext(std::unordered_set{}))); auto result = scan->PlanFiles(); EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); } @@ -254,7 +251,8 @@ TEST_F(RestTableScanTest, PlanFilesEndpointNotSupported) { TEST_F(RestTableScanTest, PlanFilesWithPlanTasks) { constexpr std::string_view kPlanResponse = R"({"status":"completed","plan-id":"plan-1","plan-tasks":["tok-1"]})"; - // FetchScanTasksResponse requires at least one of plan-tasks or file-scan-tasks present. + // FetchScanTasksResponse requires at least one of plan-tasks or file-scan-tasks + // present. constexpr std::string_view kTasksResponse = R"({"file-scan-tasks":[]})"; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) @@ -318,8 +316,7 @@ TEST_F(RestTableScanTest, CancelIsNoOpWhenEndpointNotAdvertised) { .WillOnce(Return(IOError("FetchScanTasks failed"))); EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); - ICEBERG_UNWRAP_OR_FAIL(auto scan, - MakeScan(MakeContext(endpoints_without_cancel))); + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext(endpoints_without_cancel))); auto result = scan->PlanFiles(); EXPECT_THAT(result, IsError(ErrorKind::kIOError)); } @@ -332,14 +329,12 @@ TEST_F(RestTableScanTest, FetchPlanningResultEndpointNotSupported) { R"({"status":"submitted","plan-id":"plan-3"})"; std::unordered_set endpoints_without_fetch = { - Endpoint::PlanTableScan(), Endpoint::CancelPlanning(), - Endpoint::FetchScanTasks()}; + Endpoint::PlanTableScan(), Endpoint::CancelPlanning(), Endpoint::FetchScanTasks()}; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); - ICEBERG_UNWRAP_OR_FAIL(auto scan, - MakeScan(MakeContext(endpoints_without_fetch))); + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext(endpoints_without_fetch))); auto result = scan->PlanFiles(); EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); } @@ -351,17 +346,16 @@ TEST_F(RestTableScanTest, FetchScanTasksEndpointNotSupported) { constexpr std::string_view kPlanResponse = R"({"status":"completed","plan-id":"plan-4","plan-tasks":["tok-d"]})"; - std::unordered_set endpoints_without_tasks = { - Endpoint::PlanTableScan(), Endpoint::FetchPlanningResult(), - Endpoint::CancelPlanning()}; + std::unordered_set endpoints_without_tasks = {Endpoint::PlanTableScan(), + Endpoint::FetchPlanningResult(), + Endpoint::CancelPlanning()}; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))); EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); - ICEBERG_UNWRAP_OR_FAIL(auto scan, - MakeScan(MakeContext(endpoints_without_tasks))); + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext(endpoints_without_tasks))); auto result = scan->PlanFiles(); EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); } @@ -389,15 +383,82 @@ TEST_F(RestTableScanTest, DefaultScanDoesNotSetUseSnapshotSchema) { EXPECT_FALSE(scan->context().use_snapshot_schema); } +// -------------------------------------------------------------------------- +// Storage credentials in COMPLETED response: effective_io() returns a +// credential-scoped IO, not the original table IO. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, StorageCredentialsInPlanResponseUpdatesEffectiveIO) { + constexpr std::string_view kResponseBody = R"({ + "status": "completed", + "storage-credentials": [ + {"prefix": "s3://bucket/prefix", "config": {"key": "value"}} + ] + })"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); + + auto* rest_scan = dynamic_cast(scan.get()); + ASSERT_NE(rest_scan, nullptr); + // effective_io() must return a credential-scoped IO, not the original file_io_. + EXPECT_NE(rest_scan->effective_io().get(), file_io_.get()); +} + +// -------------------------------------------------------------------------- +// No storage credentials: effective_io() falls back to the table's FileIO. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, NoStorageCredentialsEffectiveIoFallsBackToTableIO) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); + + auto* rest_scan = dynamic_cast(scan.get()); + ASSERT_NE(rest_scan, nullptr); + EXPECT_EQ(rest_scan->effective_io().get(), file_io_.get()); +} + +// -------------------------------------------------------------------------- +// Storage credentials returned in FetchScanTasksResponse also update +// effective_io(). +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, StorageCredentialsInFetchScanTasksResponseUpdatesEffectiveIO) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-cred","plan-tasks":["tok-cred"]})"; + constexpr std::string_view kTasksResponse = R"({ + "file-scan-tasks": [], + "storage-credentials": [ + {"prefix": "s3://bucket/prefix", "config": {"key": "value"}} + ] + })"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); + + auto* rest_scan = dynamic_cast(scan.get()); + ASSERT_NE(rest_scan, nullptr); + EXPECT_NE(rest_scan->effective_io().get(), file_io_.get()); +} + // -------------------------------------------------------------------------- // RestTable::NewScan returns a RestTableScanBuilder (not a plain builder). // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, RestTableNewScanReturnsRestTableScanBuilder) { ICEBERG_UNWRAP_OR_FAIL( - auto table, - RestTable::Make(identifier_, metadata_, "/tmp/metadata.json", file_io_, - /*catalog=*/nullptr, "test.my_table", nullptr, - MakeContext(std::nullopt))); + auto table, RestTable::Make(identifier_, metadata_, "/tmp/metadata.json", file_io_, + /*catalog=*/nullptr, "test.my_table", nullptr, + MakeContext(std::nullopt))); ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewScan()); auto* typed = dynamic_cast(builder.get()); EXPECT_NE(typed, nullptr); diff --git a/src/iceberg/test/table_scan_test.cc b/src/iceberg/test/table_scan_test.cc index 623e098021..1214a05360 100644 --- a/src/iceberg/test/table_scan_test.cc +++ b/src/iceberg/test/table_scan_test.cc @@ -846,8 +846,7 @@ TEST_P(TableScanTest, SchemaWithSelectedColumnsAndFilter) { // UseSnapshot, UseRef (tag vs branch), and default/incremental scans. TEST_P(TableScanTest, UseSnapshotSetsTrueUseSnapshotSchema) { constexpr int64_t kSnapshotId = 1000L; - ICEBERG_UNWRAP_OR_FAIL(auto builder, - DataTableScanBuilder::Make(table_metadata_, file_io_)); + ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); builder->UseSnapshot(kSnapshotId); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); EXPECT_TRUE(scan->context().use_snapshot_schema); @@ -858,29 +857,25 @@ TEST_P(TableScanTest, UseRefTagSetsTrueUseSnapshotSchema) { table_metadata_->refs["v1.0"] = std::make_shared( SnapshotRef{.snapshot_id = kSnapshotId, .retention = SnapshotRef::Tag{}}); - ICEBERG_UNWRAP_OR_FAIL(auto builder, - DataTableScanBuilder::Make(table_metadata_, file_io_)); + ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); builder->UseRef("v1.0"); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); EXPECT_TRUE(scan->context().use_snapshot_schema); } TEST_P(TableScanTest, UseRefBranchSetsFalseUseSnapshotSchema) { - ICEBERG_UNWRAP_OR_FAIL(auto builder, - DataTableScanBuilder::Make(table_metadata_, file_io_)); + ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); builder->UseRef("main"); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); EXPECT_FALSE(scan->context().use_snapshot_schema); } TEST_P(TableScanTest, DefaultScanHasFalseUseSnapshotSchema) { - ICEBERG_UNWRAP_OR_FAIL(auto builder, - DataTableScanBuilder::Make(table_metadata_, file_io_)); + ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); EXPECT_FALSE(scan->context().use_snapshot_schema); } - INSTANTIATE_TEST_SUITE_P(TableScanVersions, TableScanTest, testing::Values(1, 2, 3)); } // namespace iceberg From e40bc52df6b66001e6a03c8da6b3bcb8ae6f7457 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 13 Sep 2026 15:58:17 -0700 Subject: [PATCH 05/36] style: apply clang-format to fix CI pre-commit failures --- src/iceberg/catalog/rest/http_client.h | 14 ++++++-------- src/iceberg/catalog/rest/rest_table.cc | 16 ++++++---------- src/iceberg/catalog/rest/rest_table.h | 13 +++++-------- src/iceberg/test/catalog_properties_test.cc | 20 ++++++++++---------- 4 files changed, 27 insertions(+), 36 deletions(-) diff --git a/src/iceberg/catalog/rest/http_client.h b/src/iceberg/catalog/rest/http_client.h index 52522425e5..d42798faea 100644 --- a/src/iceberg/catalog/rest/http_client.h +++ b/src/iceberg/catalog/rest/http_client.h @@ -83,16 +83,15 @@ class ICEBERG_REST_EXPORT HttpClient { /// \brief Sends a GET request. virtual Result Get( - const std::string& path, - const std::unordered_map& params, + const std::string& path, const std::unordered_map& params, const std::unordered_map& headers, const ErrorHandler& error_handler, auth::AuthSession& session); /// \brief Sends a POST request. - virtual Result Post(const std::string& path, const std::string& body, - const std::unordered_map& headers, - const ErrorHandler& error_handler, - auth::AuthSession& session); + virtual Result Post( + const std::string& path, const std::string& body, + const std::unordered_map& headers, + const ErrorHandler& error_handler, auth::AuthSession& session); /// \brief Sends a POST request with form data. virtual Result PostForm( @@ -109,8 +108,7 @@ class ICEBERG_REST_EXPORT HttpClient { /// \brief Sends a DELETE request. virtual Result Delete( - const std::string& path, - const std::unordered_map& params, + const std::string& path, const std::unordered_map& params, const std::unordered_map& headers, const ErrorHandler& error_handler, auth::AuthSession& session); diff --git a/src/iceberg/catalog/rest/rest_table.cc b/src/iceberg/catalog/rest/rest_table.cc index 7274444435..6fb3306703 100644 --- a/src/iceberg/catalog/rest/rest_table.cc +++ b/src/iceberg/catalog/rest/rest_table.cc @@ -35,20 +35,16 @@ RestTable::RestTable(TableIdentifier identifier, std::shared_ptr std::shared_ptr reporter, RestScanContext rest_context) : Table(std::move(identifier), std::move(metadata), std::move(metadata_location), - std::move(io), std::move(catalog), std::move(full_name), - std::move(reporter)), + std::move(io), std::move(catalog), std::move(full_name), std::move(reporter)), rest_context_(std::move(rest_context)) {} RestTable::~RestTable() = default; -Result> RestTable::Make(TableIdentifier identifier, - std::shared_ptr metadata, - std::string metadata_location, - std::shared_ptr io, - std::shared_ptr catalog, - std::string full_name, - std::shared_ptr reporter, - RestScanContext rest_context) { +Result> RestTable::Make( + TableIdentifier identifier, std::shared_ptr metadata, + std::string metadata_location, std::shared_ptr io, + std::shared_ptr catalog, std::string full_name, + std::shared_ptr reporter, RestScanContext rest_context) { if (metadata == nullptr) { return InvalidArgument("Metadata cannot be null"); } diff --git a/src/iceberg/catalog/rest/rest_table.h b/src/iceberg/catalog/rest/rest_table.h index 54208d26af..fb23ca66b0 100644 --- a/src/iceberg/catalog/rest/rest_table.h +++ b/src/iceberg/catalog/rest/rest_table.h @@ -38,14 +38,11 @@ namespace iceberg::rest { /// PlanFiles() to the REST catalog server's scan planning endpoints. class ICEBERG_REST_EXPORT RestTable final : public Table { public: - static Result> Make(TableIdentifier identifier, - std::shared_ptr metadata, - std::string metadata_location, - std::shared_ptr io, - std::shared_ptr catalog, - std::string full_name, - std::shared_ptr reporter, - RestScanContext rest_context); + static Result> Make( + TableIdentifier identifier, std::shared_ptr metadata, + std::string metadata_location, std::shared_ptr io, + std::shared_ptr catalog, std::string full_name, + std::shared_ptr reporter, RestScanContext rest_context); ~RestTable() override; diff --git a/src/iceberg/test/catalog_properties_test.cc b/src/iceberg/test/catalog_properties_test.cc index 47abe7682c..5f22e60d45 100644 --- a/src/iceberg/test/catalog_properties_test.cc +++ b/src/iceberg/test/catalog_properties_test.cc @@ -33,40 +33,40 @@ TEST(ScanPlanningModeTest, MissingKeyReturnsNullopt) { } TEST(ScanPlanningModeTest, ClientLowercaseReturnsKClient) { - auto result = RestCatalogProperties::ScanPlanningModeFrom( - {{"scan-planning-mode", "client"}}); + auto result = + RestCatalogProperties::ScanPlanningModeFrom({{"scan-planning-mode", "client"}}); ASSERT_THAT(result, IsOk()); ASSERT_TRUE(result->has_value()); EXPECT_EQ(**result, ScanPlanningMode::kClient); } TEST(ScanPlanningModeTest, ServerLowercaseReturnsKServer) { - auto result = RestCatalogProperties::ScanPlanningModeFrom( - {{"scan-planning-mode", "server"}}); + auto result = + RestCatalogProperties::ScanPlanningModeFrom({{"scan-planning-mode", "server"}}); ASSERT_THAT(result, IsOk()); ASSERT_TRUE(result->has_value()); EXPECT_EQ(**result, ScanPlanningMode::kServer); } TEST(ScanPlanningModeTest, ClientUppercaseReturnsKClient) { - auto result = RestCatalogProperties::ScanPlanningModeFrom( - {{"scan-planning-mode", "CLIENT"}}); + auto result = + RestCatalogProperties::ScanPlanningModeFrom({{"scan-planning-mode", "CLIENT"}}); ASSERT_THAT(result, IsOk()); ASSERT_TRUE(result->has_value()); EXPECT_EQ(**result, ScanPlanningMode::kClient); } TEST(ScanPlanningModeTest, ServerUppercaseReturnsKServer) { - auto result = RestCatalogProperties::ScanPlanningModeFrom( - {{"scan-planning-mode", "SERVER"}}); + auto result = + RestCatalogProperties::ScanPlanningModeFrom({{"scan-planning-mode", "SERVER"}}); ASSERT_THAT(result, IsOk()); ASSERT_TRUE(result->has_value()); EXPECT_EQ(**result, ScanPlanningMode::kServer); } TEST(ScanPlanningModeTest, InvalidValueReturnsError) { - auto result = RestCatalogProperties::ScanPlanningModeFrom( - {{"scan-planning-mode", "invalid"}}); + auto result = + RestCatalogProperties::ScanPlanningModeFrom({{"scan-planning-mode", "invalid"}}); EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); } From be13f9167fb54414df7ce7f4482c407c87d7ec52 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 13 Sep 2026 16:01:55 -0700 Subject: [PATCH 06/36] fix: export TableScanContext so Validate() is linkable from iceberg_rest RestTableScanBuilder::Build() calls context_.Validate() across the iceberg_rest/iceberg library boundary. Without ICEBERG_EXPORT on TableScanContext the symbol is hidden in the shared library and the linker fails on arm64. --- src/iceberg/table_scan.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iceberg/table_scan.h b/src/iceberg/table_scan.h index 90d71df27c..4707049769 100644 --- a/src/iceberg/table_scan.h +++ b/src/iceberg/table_scan.h @@ -219,7 +219,7 @@ class ICEBERG_EXPORT DeletedDataFileScanTask : public ChangelogScanTask { namespace internal { // Internal table scan context used by different scan implementations. -struct TableScanContext { +struct ICEBERG_EXPORT TableScanContext { std::optional snapshot_id; std::shared_ptr filter; bool ignore_residuals{false}; From 1d7cf266d9784cc6187f313db1241c54e129be1f Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 13 Sep 2026 16:04:13 -0700 Subject: [PATCH 07/36] fix: explicitly default TableScanBuilder move ctor for Windows DLL export MSVC does not export the implicitly-generated move constructor of a template class instantiation. RestTableScanBuilder (introduced in iceberg_rest) is exported with ICEBERG_REST_EXPORT, so its compiler- generated move constructor must call the base TableScanBuilder move constructor as an imported symbol. Explicitly defaulting it makes it part of the explicit template instantiation and therefore exported. --- src/iceberg/table_scan.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/iceberg/table_scan.h b/src/iceberg/table_scan.h index 4707049769..d7b00779d5 100644 --- a/src/iceberg/table_scan.h +++ b/src/iceberg/table_scan.h @@ -410,6 +410,9 @@ class ICEBERG_TEMPLATE_CLASS_EXPORT TableScanBuilder : public ErrorCollector { std::string table_name, std::shared_ptr metrics_reporter); + TableScanBuilder(TableScanBuilder&&) = default; + TableScanBuilder& operator=(TableScanBuilder&&) = default; + // Return the schema bound to the specified snapshot. Result>> ResolveSnapshotSchema(); Status ResolveColumnStatsSelection(); From 443443e2756b81347e0050ad811f99d76b5b0025 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Mon, 28 Sep 2026 20:20:15 -0700 Subject: [PATCH 08/36] feat(rest): server-side planning for incremental append scans, PlanFilesStream, and virtual io() - Add RestIncrementalAppendScan and RestIncrementalAppendScanBuilder; override RestTable::NewIncrementalAppendScan() to delegate PlanFiles() to the REST server via the same scan planning endpoints. IncrementalChangelogScan is left local: the planTableScan response only carries FileScanTask objects, not the per-snapshot operation metadata needed to reconstruct ChangelogScanTask entries. Refactor shared HTTP planning logic into free functions (ExecuteScanPlan, FetchPlanningResult, FetchScanTasks, ResolveScanTasks, ApplyStorageCredentials, CancelPlanning) used by both scan types. - Make DataTableScan::PlanFilesStream() virtual and implement it in RestTableScan with a lazy RestFileScanTaskStream that drives FetchScanTasks per plan_task token. The base-class PlanFiles() delegates to PlanFilesStream()->ToVector(), so the server planning path is used for both the eager and streaming APIs. - Make TableScan::io() virtual and override it in RestTableScan to return the credential-scoped FileIO when the server vends storage credentials, without requiring a downcast. Remove effective_io(). A shared ScanIoSlot (shared_ptr>) is held by both the scan and the lazy stream so credentials vended by any FetchScanTasks response are visible through io() even after the stream is consumed. Update tests accordingly. --- src/iceberg/catalog/rest/rest_table.cc | 6 + src/iceberg/catalog/rest/rest_table.h | 4 + src/iceberg/catalog/rest/rest_table_scan.cc | 503 ++++++++++++++------ src/iceberg/catalog/rest/rest_table_scan.h | 91 ++-- src/iceberg/data_table_scan.h | 2 +- src/iceberg/table_scan.h | 2 +- src/iceberg/test/rest_table_scan_test.cc | 21 +- 7 files changed, 437 insertions(+), 192 deletions(-) diff --git a/src/iceberg/catalog/rest/rest_table.cc b/src/iceberg/catalog/rest/rest_table.cc index 6fb3306703..dd2b43b960 100644 --- a/src/iceberg/catalog/rest/rest_table.cc +++ b/src/iceberg/catalog/rest/rest_table.cc @@ -59,4 +59,10 @@ Result> RestTable::NewScan() const { rest_context_); } +Result> +RestTable::NewIncrementalAppendScan() const { + return std::make_unique(metadata_, io_, full_name_, + reporter_, rest_context_); +} + } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/rest_table.h b/src/iceberg/catalog/rest/rest_table.h index fb23ca66b0..f47dbe5b35 100644 --- a/src/iceberg/catalog/rest/rest_table.h +++ b/src/iceberg/catalog/rest/rest_table.h @@ -50,6 +50,10 @@ class ICEBERG_REST_EXPORT RestTable final : public Table { /// REST catalog server. Result> NewScan() const override; + /// \brief Returns a RestIncrementalAppendScanBuilder that delegates to the server. + Result> NewIncrementalAppendScan() + const override; + private: RestTable(TableIdentifier identifier, std::shared_ptr metadata, std::string metadata_location, std::shared_ptr io, diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 3e11c71dc2..5a346b03e9 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -35,6 +35,7 @@ #include "iceberg/partition_spec.h" #include "iceberg/result.h" #include "iceberg/schema.h" +#include "iceberg/snapshot.h" #include "iceberg/table_metadata.h" #include "iceberg/util/macros.h" @@ -47,6 +48,12 @@ constexpr int64_t kMaxSleepMs = 60'000; constexpr int kMaxRetries = 10; constexpr int64_t kMaxWaitTimeMs = 5 * 60 * 1'000; +using SpecsById = std::unordered_map>; +// A shared slot that lets the scan and the stream share a single FileIO reference. +// Credentials vended by any lazy FetchScanTasks response are written into *slot, +// making them visible to RestTableScan::io() even after the stream is consumed. +using ScanIoSlot = std::shared_ptr>; + #define ICEBERG_ENDPOINT_CHECK(endpoints, endpoint) \ do { \ if (!endpoints.contains(endpoint)) { \ @@ -54,108 +61,83 @@ constexpr int64_t kMaxWaitTimeMs = 5 * 60 * 1'000; } \ } while (0) -} // namespace - -// RestTableScan - -RestTableScan::RestTableScan(std::shared_ptr metadata, - std::shared_ptr schema, std::shared_ptr io, - internal::TableScanContext context, - RestScanContext rest_context) - : DataTableScan(std::move(metadata), std::move(schema), std::move(io), - std::move(context)), - rest_context_(std::move(rest_context)) {} +// --------------------------------------------------------------------------- +// Shared HTTP scan planning helpers used by all REST scan implementations. +// --------------------------------------------------------------------------- -Result> RestTableScan::Make( - std::shared_ptr metadata, std::shared_ptr schema, - std::shared_ptr io, internal::TableScanContext context, - RestScanContext rest_context) { - ICEBERG_PRECHECK(metadata != nullptr, "Table metadata cannot be null"); - ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null"); - ICEBERG_PRECHECK(io != nullptr, "FileIO cannot be null"); - return std::unique_ptr( - new RestTableScan(std::move(metadata), std::move(schema), std::move(io), - std::move(context), std::move(rest_context))); +Status ApplyStorageCredentials(const RestScanContext& ctx, + const std::vector& credentials, + std::shared_ptr& scan_io) { + if (credentials.empty()) return {}; + ICEBERG_ASSIGN_OR_RAISE( + auto io, MakeTableFileIO(ctx.catalog_config, ctx.table_config, credentials)); + scan_io = std::move(io); + return {}; } -Result>> RestTableScan::PlanFiles() const { - TableMetadataCache metadata_cache(metadata_.get()); - ICEBERG_ASSIGN_OR_RAISE(auto specs_by_id, metadata_cache.GetPartitionSpecsById()); +void CancelPlanning(const RestScanContext& ctx, const std::string& plan_id) { + if (plan_id.empty()) return; + if (!ctx.supported_endpoints.contains(Endpoint::CancelPlanning())) return; + + auto path = ctx.paths->Plan(ctx.identifier, plan_id); + if (!path.has_value()) return; - std::string plan_id; - return PlanTableScan(plan_id, specs_by_id); + std::ignore = ctx.client->Delete(*path, /*params=*/{}, /*headers=*/{}, + *PlanErrorHandler::Instance(), *ctx.session); } -Result>> RestTableScan::PlanTableScan( - std::string& plan_id, - const std::unordered_map>& specs) const { - ICEBERG_ENDPOINT_CHECK(rest_context_.supported_endpoints, Endpoint::PlanTableScan()); +Result>> FetchScanTasks( + const RestScanContext& ctx, const Schema& schema, const std::string& plan_task, + const SpecsById& specs, std::shared_ptr& scan_io); - // Build request from scan context - PlanTableScanRequest request; - request.select = context_.selected_columns; - request.filter = context_.filter; - request.case_sensitive = context_.case_sensitive; - request.min_rows_requested = context_.min_rows_requested; +Result>> ResolveScanTasks( + const RestScanContext& ctx, const Schema& schema, + const std::optional>& plan_tasks, + const std::optional>>& file_scan_tasks, + const SpecsById& specs, std::shared_ptr& scan_io) { + std::vector> result; - if (context_.from_snapshot_id.has_value() && context_.to_snapshot_id.has_value()) { - request.start_snapshot_id = context_.from_snapshot_id; - request.end_snapshot_id = context_.to_snapshot_id; - request.use_snapshot_schema = true; - } else if (context_.snapshot_id.has_value()) { - request.snapshot_id = context_.snapshot_id; - request.use_snapshot_schema = context_.use_snapshot_schema; + if (file_scan_tasks.has_value()) { + result.insert(result.end(), file_scan_tasks->begin(), file_scan_tasks->end()); } - if (!context_.columns_to_keep_stats.empty()) { - for (int32_t field_id : context_.columns_to_keep_stats) { - ICEBERG_ASSIGN_OR_RAISE(auto name, schema_->FindColumnNameById(field_id)); - if (name.has_value()) { - request.stats_fields.emplace_back(*name); - } + if (plan_tasks.has_value()) { + for (const auto& token : *plan_tasks) { + ICEBERG_ASSIGN_OR_RAISE(auto tasks, FetchScanTasks(ctx, schema, token, specs, scan_io)); + result.insert(result.end(), tasks.begin(), tasks.end()); } } - ICEBERG_ASSIGN_OR_RAISE(auto path, rest_context_.paths->Plan(rest_context_.identifier)); - ICEBERG_ASSIGN_OR_RAISE(auto request_json, ToJson(request)); - ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(request_json)); + return result; +} + +Result>> FetchScanTasks( + const RestScanContext& ctx, const Schema& schema, const std::string& plan_task, + const SpecsById& specs, std::shared_ptr& scan_io) { + ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::FetchScanTasks()); + + ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->FetchScanTasks(ctx.identifier)); + FetchScanTasksRequest request{.planTask = plan_task}; + ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(ToJson(request))); ICEBERG_ASSIGN_OR_RAISE( const auto response, - rest_context_.client->Post(path, json_request, /*headers=*/{}, - *PlanErrorHandler::Instance(), *rest_context_.session)); + ctx.client->Post(path, json_request, /*headers=*/{}, + *PlanTaskErrorHandler::Instance(), *ctx.session)); ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); - ICEBERG_ASSIGN_OR_RAISE(auto result, - PlanTableScanResponseFromJson(json, specs, *schema_)); + ICEBERG_ASSIGN_OR_RAISE(auto result, FetchScanTasksResponseFromJson(json, specs, schema)); ICEBERG_RETURN_UNEXPECTED(result.Validate()); + ICEBERG_RETURN_UNEXPECTED(ApplyStorageCredentials(ctx, result.storage_credentials, scan_io)); - plan_id = result.plan_id; - - switch (result.plan_status) { - case PlanStatus::kCompleted: { - ICEBERG_RETURN_UNEXPECTED(ApplyStorageCredentials(result.storage_credentials)); - auto tasks = ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); - if (!tasks.has_value()) CancelPlanning(plan_id); - return tasks; - } - case PlanStatus::kSubmitted: - return FetchPlanningResult(plan_id, specs); - case PlanStatus::kFailed: - return IOError("Scan planning failed: {}", - result.error ? result.error->message : "unknown error"); - case PlanStatus::kCancelled: - return IOError("Scan planning was cancelled for plan_id={}", plan_id); - } - return IOError("Unexpected plan status"); + return ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs, + scan_io); } -Result>> RestTableScan::FetchPlanningResult( - const std::string& plan_id, - const std::unordered_map>& specs) const { - ICEBERG_ENDPOINT_CHECK(rest_context_.supported_endpoints, - Endpoint::FetchPlanningResult()); +Result>> FetchPlanningResult( + const RestScanContext& ctx, const Schema& schema, const std::string& plan_id, + const SpecsById& specs, std::shared_ptr& scan_io) { + ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::FetchPlanningResult()); - ICEBERG_ASSIGN_OR_RAISE(auto path, - rest_context_.paths->Plan(rest_context_.identifier, plan_id)); + ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->Plan(ctx.identifier, plan_id)); auto delay_ms = kMinSleepMs; auto start = std::chrono::steady_clock::now(); @@ -163,18 +145,21 @@ Result>> RestTableScan::FetchPlanningR for (int retry = 0; retry <= kMaxRetries; ++retry) { ICEBERG_ASSIGN_OR_RAISE( const auto response, - rest_context_.client->Get(path, /*params=*/{}, /*headers=*/{}, - *PlanErrorHandler::Instance(), *rest_context_.session)); + ctx.client->Get(path, /*params=*/{}, /*headers=*/{}, + *PlanErrorHandler::Instance(), *ctx.session)); ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); ICEBERG_ASSIGN_OR_RAISE(auto result, - FetchPlanningResultResponseFromJson(json, specs, *schema_)); + FetchPlanningResultResponseFromJson(json, specs, schema)); ICEBERG_RETURN_UNEXPECTED(result.Validate()); switch (result.plan_status) { case PlanStatus::kCompleted: { - ICEBERG_RETURN_UNEXPECTED(ApplyStorageCredentials(result.storage_credentials)); - auto tasks = ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); - if (!tasks.has_value()) CancelPlanning(plan_id); + ICEBERG_RETURN_UNEXPECTED( + ApplyStorageCredentials(ctx, result.storage_credentials, scan_io)); + auto tasks = + ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs, + scan_io); + if (!tasks.has_value()) CancelPlanning(ctx, plan_id); return tasks; } case PlanStatus::kSubmitted: { @@ -182,7 +167,7 @@ Result>> RestTableScan::FetchPlanningR std::chrono::steady_clock::now() - start) .count(); if (elapsed_ms >= kMaxWaitTimeMs) { - CancelPlanning(plan_id); + CancelPlanning(ctx, plan_id); return IOError("Scan planning timed out after {}ms waiting for plan_id={}", elapsed_ms, plan_id); } @@ -191,7 +176,7 @@ Result>> RestTableScan::FetchPlanningR continue; } case PlanStatus::kFailed: - CancelPlanning(plan_id); + CancelPlanning(ctx, plan_id); return IOError("Scan planning failed: {}", result.error ? result.error->message : "unknown error"); case PlanStatus::kCancelled: @@ -199,81 +184,234 @@ Result>> RestTableScan::FetchPlanningR } } - CancelPlanning(plan_id); + CancelPlanning(ctx, plan_id); return IOError("Scan planning exceeded max retries ({}) for plan_id={}", kMaxRetries, plan_id); } -Result>> RestTableScan::FetchScanTasks( - const std::string& plan_task, - const std::unordered_map>& specs) const { - ICEBERG_ENDPOINT_CHECK(rest_context_.supported_endpoints, Endpoint::FetchScanTasks()); +/// A lazy stream that drives FetchScanTasks calls on demand. +/// +/// The eager POST /plan (or poll until COMPLETED) is done before this stream is +/// constructed. The stream then yields the directly-returned file_scan_tasks first, +/// then fetches each plan_task token lazily one at a time as Next() is called. +/// +/// scan_io_slot is shared with the owning RestTableScan so that credentials vended +/// by any FetchScanTasks response are visible through RestTableScan::io() even after +/// the stream has been consumed. +class RestFileScanTaskStream final : public FileScanTaskStream { + public: + RestFileScanTaskStream(RestScanContext ctx, std::shared_ptr schema, + std::string plan_id, + std::vector> initial_tasks, + std::vector plan_task_tokens, SpecsById specs, + ScanIoSlot scan_io_slot) + : ctx_(std::move(ctx)), + schema_(std::move(schema)), + plan_id_(std::move(plan_id)), + buffer_(std::move(initial_tasks)), + plan_task_tokens_(std::move(plan_task_tokens)), + specs_(std::move(specs)), + scan_io_slot_(std::move(scan_io_slot)) {} + + ~RestFileScanTaskStream() override { + if (!consumed_) CancelPlanning(ctx_, plan_id_); + } - ICEBERG_ASSIGN_OR_RAISE(auto path, - rest_context_.paths->FetchScanTasks(rest_context_.identifier)); - FetchScanTasksRequest request{.planTask = plan_task}; - ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(ToJson(request))); - ICEBERG_ASSIGN_OR_RAISE(const auto response, - rest_context_.client->Post(path, json_request, /*headers=*/{}, - *PlanTaskErrorHandler::Instance(), - *rest_context_.session)); + protected: + Result>> NextImpl() override { + while (true) { + if (buffer_pos_ < buffer_.size()) { + return buffer_[buffer_pos_++]; + } + if (token_pos_ >= plan_task_tokens_.size()) { + consumed_ = true; + return std::nullopt; + } + const auto& token = plan_task_tokens_[token_pos_++]; + ICEBERG_ASSIGN_OR_RAISE( + buffer_, FetchScanTasks(ctx_, *schema_, token, specs_, *scan_io_slot_)); + buffer_pos_ = 0; + } + } + + private: + RestScanContext ctx_; + std::shared_ptr schema_; + std::string plan_id_; + std::vector> buffer_; + size_t buffer_pos_ = 0; + std::vector plan_task_tokens_; + size_t token_pos_ = 0; + SpecsById specs_; + ScanIoSlot scan_io_slot_; + bool consumed_ = false; +}; + +/// POST /plan (polling if SUBMITTED), apply credentials, return a lazy stream. +/// +/// scan_io_slot is shared with the caller (RestTableScan) so that credentials +/// vended by FetchScanTasks responses update RestTableScan::io() in place. +Result ExecuteScanPlanStream(const RestScanContext& ctx, + const Schema& schema, + std::shared_ptr schema_ptr, + PlanTableScanRequest request, + const SpecsById& specs, + ScanIoSlot scan_io_slot) { + ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::PlanTableScan()); + + ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->Plan(ctx.identifier)); + ICEBERG_ASSIGN_OR_RAISE(auto request_json, ToJson(request)); + ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(request_json)); + ICEBERG_ASSIGN_OR_RAISE( + const auto response, + ctx.client->Post(path, json_request, /*headers=*/{}, *PlanErrorHandler::Instance(), + *ctx.session)); ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); - ICEBERG_ASSIGN_OR_RAISE(auto result, - FetchScanTasksResponseFromJson(json, specs, *schema_)); + ICEBERG_ASSIGN_OR_RAISE(auto result, PlanTableScanResponseFromJson(json, specs, schema)); ICEBERG_RETURN_UNEXPECTED(result.Validate()); - ICEBERG_RETURN_UNEXPECTED(ApplyStorageCredentials(result.storage_credentials)); - return ResolveScanTasks(result.plan_tasks, result.file_scan_tasks, specs); -} + const std::string plan_id = result.plan_id; -Result>> RestTableScan::ResolveScanTasks( - const std::optional>& plan_tasks, - const std::optional>>& file_scan_tasks, - const std::unordered_map>& specs) const { - std::vector> result; + if (result.plan_status == PlanStatus::kSubmitted) { + // Poll until COMPLETED, eagerly collecting all tasks into the buffer. + std::string mutable_plan_id = plan_id; + ICEBERG_ASSIGN_OR_RAISE( + auto tasks, FetchPlanningResult(ctx, schema, mutable_plan_id, specs, *scan_io_slot)); + return std::make_unique(ctx, schema_ptr, mutable_plan_id, + std::move(tasks), {}, specs, scan_io_slot); + } - if (file_scan_tasks.has_value()) { - result.insert(result.end(), file_scan_tasks->begin(), file_scan_tasks->end()); + if (result.plan_status == PlanStatus::kFailed) { + return IOError("Scan planning failed: {}", + result.error ? result.error->message : "unknown error"); + } + if (result.plan_status == PlanStatus::kCancelled) { + return IOError("Scan planning was cancelled for plan_id={}", plan_id); } - if (plan_tasks.has_value()) { - for (const auto& plan_task : *plan_tasks) { - ICEBERG_ASSIGN_OR_RAISE(auto tasks, FetchScanTasks(plan_task, specs)); - result.insert(result.end(), tasks.begin(), tasks.end()); - } + // kCompleted: apply credentials from the initial response, then build the lazy stream. + ICEBERG_RETURN_UNEXPECTED(ApplyStorageCredentials(ctx, result.storage_credentials, *scan_io_slot)); + + std::vector> initial_tasks; + if (result.file_scan_tasks.has_value()) { + initial_tasks = std::move(*result.file_scan_tasks); + } + std::vector plan_task_tokens; + if (result.plan_tasks.has_value()) { + plan_task_tokens = std::move(*result.plan_tasks); } - return result; + return std::make_unique( + ctx, std::move(schema_ptr), plan_id, std::move(initial_tasks), + std::move(plan_task_tokens), specs, std::move(scan_io_slot)); } -void RestTableScan::CancelPlanning(const std::string& plan_id) const { - if (plan_id.empty()) return; - if (!rest_context_.supported_endpoints.contains(Endpoint::CancelPlanning())) return; +/// Eager batch planning: used by RestIncrementalAppendScan which has no stream path. +Result>> ExecuteScanPlan( + const RestScanContext& ctx, const Schema& schema, PlanTableScanRequest request, + const SpecsById& specs, std::shared_ptr& scan_io) { + ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::PlanTableScan()); - auto path = rest_context_.paths->Plan(rest_context_.identifier, plan_id); - if (!path.has_value()) return; + ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->Plan(ctx.identifier)); + ICEBERG_ASSIGN_OR_RAISE(auto request_json, ToJson(request)); + ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(request_json)); + ICEBERG_ASSIGN_OR_RAISE( + const auto response, + ctx.client->Post(path, json_request, /*headers=*/{}, *PlanErrorHandler::Instance(), + *ctx.session)); + ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); + ICEBERG_ASSIGN_OR_RAISE(auto result, PlanTableScanResponseFromJson(json, specs, schema)); + ICEBERG_RETURN_UNEXPECTED(result.Validate()); + + const std::string plan_id = result.plan_id; + + switch (result.plan_status) { + case PlanStatus::kCompleted: { + ICEBERG_RETURN_UNEXPECTED( + ApplyStorageCredentials(ctx, result.storage_credentials, scan_io)); + auto tasks = + ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs, + scan_io); + if (!tasks.has_value()) CancelPlanning(ctx, plan_id); + return tasks; + } + case PlanStatus::kSubmitted: + return FetchPlanningResult(ctx, schema, plan_id, specs, scan_io); + case PlanStatus::kFailed: + return IOError("Scan planning failed: {}", + result.error ? result.error->message : "unknown error"); + case PlanStatus::kCancelled: + return IOError("Scan planning was cancelled for plan_id={}", plan_id); + } + return IOError("Unexpected plan status"); +} + +} // namespace + +// --------------------------------------------------------------------------- +// RestTableScan +// --------------------------------------------------------------------------- + +RestTableScan::RestTableScan(std::shared_ptr metadata, + std::shared_ptr schema, std::shared_ptr io, + internal::TableScanContext context, + RestScanContext rest_context) + : DataTableScan(std::move(metadata), std::move(schema), std::move(io), + std::move(context)), + rest_context_(std::move(rest_context)), + scan_io_slot_(std::make_shared>()) {} - // Best-effort: ignore errors. - std::ignore = - rest_context_.client->Delete(*path, /*params=*/{}, /*headers=*/{}, - *PlanErrorHandler::Instance(), *rest_context_.session); +Result> RestTableScan::Make( + std::shared_ptr metadata, std::shared_ptr schema, + std::shared_ptr io, internal::TableScanContext context, + RestScanContext rest_context) { + ICEBERG_PRECHECK(metadata != nullptr, "Table metadata cannot be null"); + ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null"); + ICEBERG_PRECHECK(io != nullptr, "FileIO cannot be null"); + return std::unique_ptr( + new RestTableScan(std::move(metadata), std::move(schema), std::move(io), + std::move(context), std::move(rest_context))); } -const std::shared_ptr& RestTableScan::effective_io() const { - return scan_io_ ? scan_io_ : io_; +Result RestTableScan::PlanFilesStream() const { + TableMetadataCache metadata_cache(metadata_.get()); + ICEBERG_ASSIGN_OR_RAISE(auto specs, metadata_cache.GetPartitionSpecsById()); + + PlanTableScanRequest request; + request.select = context_.selected_columns.value_or(std::vector{}); + request.filter = context_.filter; + request.case_sensitive = context_.case_sensitive; + request.min_rows_requested = context_.min_rows_requested; + + if (context_.from_snapshot_id.has_value() && context_.to_snapshot_id.has_value()) { + request.start_snapshot_id = context_.from_snapshot_id; + request.end_snapshot_id = context_.to_snapshot_id; + request.use_snapshot_schema = true; + } else if (context_.snapshot_id.has_value()) { + request.snapshot_id = context_.snapshot_id; + request.use_snapshot_schema = context_.use_snapshot_schema; + } + + if (!context_.columns_to_keep_stats.empty()) { + for (int32_t field_id : context_.columns_to_keep_stats) { + ICEBERG_ASSIGN_OR_RAISE(auto name, schema_->FindColumnNameById(field_id)); + if (name.has_value()) { + request.stats_fields.emplace_back(*name); + } + } + } + + return ExecuteScanPlanStream(rest_context_, *schema_, schema_, std::move(request), specs, + scan_io_slot_); } -Status RestTableScan::ApplyStorageCredentials( - const std::vector& credentials) const { - if (credentials.empty()) return {}; - ICEBERG_ASSIGN_OR_RAISE( - auto io, MakeTableFileIO(rest_context_.catalog_config, rest_context_.table_config, - credentials)); - scan_io_ = std::move(io); - return {}; +const std::shared_ptr& RestTableScan::io() const { + return *scan_io_slot_ ? *scan_io_slot_ : io_; } +// --------------------------------------------------------------------------- // RestTableScanBuilder +// --------------------------------------------------------------------------- RestTableScanBuilder::RestTableScanBuilder( std::shared_ptr metadata, std::shared_ptr io, @@ -291,4 +429,91 @@ Result> RestTableScanBuilder::Build() { rest_context_); } +// --------------------------------------------------------------------------- +// RestIncrementalAppendScan +// --------------------------------------------------------------------------- + +RestIncrementalAppendScan::RestIncrementalAppendScan( + std::shared_ptr metadata, std::shared_ptr schema, + std::shared_ptr io, internal::TableScanContext context, + RestScanContext rest_context) + : IncrementalAppendScan(std::move(metadata), std::move(schema), std::move(io), + std::move(context)), + rest_context_(std::move(rest_context)) {} + +Result> RestIncrementalAppendScan::Make( + std::shared_ptr metadata, std::shared_ptr schema, + std::shared_ptr io, internal::TableScanContext context, + RestScanContext rest_context) { + ICEBERG_PRECHECK(metadata != nullptr, "Table metadata cannot be null"); + ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null"); + ICEBERG_PRECHECK(io != nullptr, "FileIO cannot be null"); + return std::unique_ptr(new RestIncrementalAppendScan( + std::move(metadata), std::move(schema), std::move(io), std::move(context), + std::move(rest_context))); +} + +Result>> RestIncrementalAppendScan::PlanFiles() + const { + TableMetadataCache metadata_cache(metadata_.get()); + ICEBERG_ASSIGN_OR_RAISE(auto specs, metadata_cache.GetPartitionSpecsById()); + + PlanTableScanRequest request; + request.select = context_.selected_columns.value_or(std::vector{}); + request.filter = context_.filter; + request.case_sensitive = context_.case_sensitive; + request.min_rows_requested = context_.min_rows_requested; + + // Resolve end snapshot: use to_snapshot_id if set, else current table snapshot. + if (context_.to_snapshot_id.has_value()) { + request.end_snapshot_id = context_.to_snapshot_id; + } else { + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, metadata_->Snapshot()); + if (!snapshot) return {}; + request.end_snapshot_id = snapshot->snapshot_id; + } + + // Resolve start snapshot (exclusive): respect from_snapshot_id_inclusive. + if (context_.from_snapshot_id.has_value()) { + if (context_.from_snapshot_id_inclusive) { + ICEBERG_ASSIGN_OR_RAISE(auto from_snap, + metadata_->SnapshotById(*context_.from_snapshot_id)); + request.start_snapshot_id = from_snap->parent_snapshot_id; + } else { + request.start_snapshot_id = context_.from_snapshot_id; + } + } + + if (!context_.columns_to_keep_stats.empty()) { + for (int32_t field_id : context_.columns_to_keep_stats) { + ICEBERG_ASSIGN_OR_RAISE(auto name, schema_->FindColumnNameById(field_id)); + if (name.has_value()) { + request.stats_fields.emplace_back(*name); + } + } + } + + return ExecuteScanPlan(rest_context_, *schema_, std::move(request), specs, scan_io_); +} + +// --------------------------------------------------------------------------- +// RestIncrementalAppendScanBuilder +// --------------------------------------------------------------------------- + +RestIncrementalAppendScanBuilder::RestIncrementalAppendScanBuilder( + std::shared_ptr metadata, std::shared_ptr io, + std::string table_name, std::shared_ptr metrics_reporter, + RestScanContext rest_context) + : IncrementalAppendScanBuilder(std::move(metadata), std::move(io), + std::move(table_name), std::move(metrics_reporter)), + rest_context_(std::move(rest_context)) {} + +Result> RestIncrementalAppendScanBuilder::Build() { + ICEBERG_RETURN_UNEXPECTED(CheckErrors()); + ICEBERG_RETURN_UNEXPECTED(context_.Validate()); + ICEBERG_ASSIGN_OR_RAISE(auto schema, ResolveSnapshotSchema()); + return RestIncrementalAppendScan::Make(metadata_, schema.get(), io_, std::move(context_), + rest_context_); +} + } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/rest_table_scan.h b/src/iceberg/catalog/rest/rest_table_scan.h index 5e3258086f..41c4f8d7f1 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.h +++ b/src/iceberg/catalog/rest/rest_table_scan.h @@ -28,6 +28,7 @@ #include "iceberg/catalog/rest/endpoint.h" #include "iceberg/catalog/rest/iceberg_rest_export.h" #include "iceberg/data_table_scan.h" +#include "iceberg/incremental_append_scan.h" #include "iceberg/metrics/metrics_reporter.h" #include "iceberg/result.h" #include "iceberg/storage_credential.h" @@ -36,7 +37,7 @@ #include "iceberg/type_fwd.h" /// \file iceberg/catalog/rest/rest_table_scan.h -/// REST-specific table scan that delegates scan planning to the REST catalog server. +/// REST-specific table scans that delegate scan planning to the REST catalog server. namespace iceberg::rest { @@ -47,7 +48,7 @@ namespace auth { class AuthSession; } // namespace auth -/// \brief HTTP context shared between RestTable and RestTableScan. +/// \brief HTTP context shared between RestTable and REST scan classes. struct ICEBERG_REST_EXPORT RestScanContext { std::shared_ptr client; std::shared_ptr paths; @@ -61,7 +62,7 @@ struct ICEBERG_REST_EXPORT RestScanContext { std::unordered_map table_config; }; -/// \brief A DataTableScan that delegates PlanFiles() to the REST catalog server +/// \brief A DataTableScan that delegates PlanFilesStream() to the REST catalog server /// via the scan planning endpoints (planTableScan / fetchPlanningResult / /// cancelPlanning / fetchScanTasks). class ICEBERG_REST_EXPORT RestTableScan : public DataTableScan { @@ -73,51 +74,24 @@ class ICEBERG_REST_EXPORT RestTableScan : public DataTableScan { std::shared_ptr io, internal::TableScanContext context, RestScanContext rest_context); - /// \brief Plans files via the REST scan planning endpoints. - Result>> PlanFiles() const override; + /// \brief Plans files lazily via the REST scan planning endpoints. + Result PlanFilesStream() const override; - /// \brief Returns the FileIO to use when reading scan results. + /// \brief Returns the effective FileIO for reading scan results. /// /// If the server vended storage credentials during planning, returns a FileIO /// initialised with those credentials; otherwise returns the table's FileIO. - /// Must be called after PlanFiles(). - const std::shared_ptr& effective_io() const; + const std::shared_ptr& io() const override; private: RestTableScan(std::shared_ptr metadata, std::shared_ptr schema, std::shared_ptr io, internal::TableScanContext context, RestScanContext rest_context); - /// POST /plan → handle COMPLETED / SUBMITTED / FAILED / CANCELLED. - Result>> PlanTableScan( - std::string& plan_id, - const std::unordered_map>& specs) const; - - /// GET /plan/{plan_id} with exponential backoff until COMPLETED. - Result>> FetchPlanningResult( - const std::string& plan_id, - const std::unordered_map>& specs) const; - - /// POST /tasks/{plan_task_id} → fetch FileScanTasks for one opaque plan task token. - Result>> FetchScanTasks( - const std::string& plan_task, - const std::unordered_map>& specs) const; - - /// Flatten plan_tasks (opaque tokens) + file_scan_tasks into a single list. - Result>> ResolveScanTasks( - const std::optional>& plan_tasks, - const std::optional>>& file_scan_tasks, - const std::unordered_map>& specs) const; - - /// DELETE /plan/{plan_id}; best-effort, errors are silently ignored. - void CancelPlanning(const std::string& plan_id) const; - - /// Builds a scan-scoped FileIO from vended credentials and caches it in scan_io_. - /// No-op if credentials is empty. - Status ApplyStorageCredentials(const std::vector& credentials) const; - RestScanContext rest_context_; - mutable std::shared_ptr scan_io_; + /// Shared slot so credentials vended by any lazy FetchScanTasks response are + /// visible through io() even after the stream has been consumed. + mutable std::shared_ptr> scan_io_slot_; }; /// \brief Builder that produces a RestTableScan with the REST HTTP context injected. @@ -135,4 +109,47 @@ class ICEBERG_REST_EXPORT RestTableScanBuilder : public DataTableScanBuilder { RestScanContext rest_context_; }; +/// \brief An IncrementalAppendScan that delegates PlanFiles() to the REST catalog server. +/// +/// IncrementalChangelogScan is not delegated because the REST planTableScan response +/// only carries FileScanTask objects; reconstructing ChangelogScanTask entries +/// (AddedRowsScanTask vs DeletedDataFileScanTask) requires per-snapshot operation +/// metadata that the server does not return. Changelog scans always plan locally. +class ICEBERG_REST_EXPORT RestIncrementalAppendScan : public IncrementalAppendScan { + public: + ~RestIncrementalAppendScan() override = default; + + static Result> Make( + std::shared_ptr metadata, std::shared_ptr schema, + std::shared_ptr io, internal::TableScanContext context, + RestScanContext rest_context); + + /// \brief Plans files via the REST scan planning endpoints. + Result>> PlanFiles() const override; + + private: + RestIncrementalAppendScan(std::shared_ptr metadata, + std::shared_ptr schema, std::shared_ptr io, + internal::TableScanContext context, + RestScanContext rest_context); + + RestScanContext rest_context_; + mutable std::shared_ptr scan_io_; +}; + +/// \brief Builder that produces a RestIncrementalAppendScan. +class ICEBERG_REST_EXPORT RestIncrementalAppendScanBuilder + : public IncrementalAppendScanBuilder { + public: + RestIncrementalAppendScanBuilder(std::shared_ptr metadata, + std::shared_ptr io, std::string table_name, + std::shared_ptr metrics_reporter, + RestScanContext rest_context); + + Result> Build() override; + + private: + RestScanContext rest_context_; +}; + } // namespace iceberg::rest diff --git a/src/iceberg/data_table_scan.h b/src/iceberg/data_table_scan.h index 09cf85fd51..e841413893 100644 --- a/src/iceberg/data_table_scan.h +++ b/src/iceberg/data_table_scan.h @@ -54,7 +54,7 @@ class ICEBERG_EXPORT DataTableScan : public TableScan { /// can outlive this scan. An executor configured through PlanWith() is borrowed and /// must remain alive until the stream is destroyed, as later Next() calls may submit /// work to it. - Result PlanFilesStream() const; + virtual Result PlanFilesStream() const; protected: using TableScan::TableScan; diff --git a/src/iceberg/table_scan.h b/src/iceberg/table_scan.h index d7b00779d5..5abb564b2c 100644 --- a/src/iceberg/table_scan.h +++ b/src/iceberg/table_scan.h @@ -442,7 +442,7 @@ class ICEBERG_EXPORT TableScan { const internal::TableScanContext& context() const; /// \brief Returns the file I/O instance used for reading files. - const std::shared_ptr& io() const; + virtual const std::shared_ptr& io() const; /// \brief Returns this scan's filter expression. const std::shared_ptr& filter() const; diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 9c5efd929b..ed9a000b65 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -384,7 +384,7 @@ TEST_F(RestTableScanTest, DefaultScanDoesNotSetUseSnapshotSchema) { } // -------------------------------------------------------------------------- -// Storage credentials in COMPLETED response: effective_io() returns a +// Storage credentials in COMPLETED response: io() returns a // credential-scoped IO, not the original table IO. // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, StorageCredentialsInPlanResponseUpdatesEffectiveIO) { @@ -401,14 +401,12 @@ TEST_F(RestTableScanTest, StorageCredentialsInPlanResponseUpdatesEffectiveIO) { ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); - auto* rest_scan = dynamic_cast(scan.get()); - ASSERT_NE(rest_scan, nullptr); - // effective_io() must return a credential-scoped IO, not the original file_io_. - EXPECT_NE(rest_scan->effective_io().get(), file_io_.get()); + // io() must return a credential-scoped IO without requiring a downcast. + EXPECT_NE(scan->io().get(), file_io_.get()); } // -------------------------------------------------------------------------- -// No storage credentials: effective_io() falls back to the table's FileIO. +// No storage credentials: io() falls back to the table's FileIO. // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, NoStorageCredentialsEffectiveIoFallsBackToTableIO) { constexpr std::string_view kResponseBody = R"({"status":"completed"})"; @@ -419,14 +417,11 @@ TEST_F(RestTableScanTest, NoStorageCredentialsEffectiveIoFallsBackToTableIO) { ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); - auto* rest_scan = dynamic_cast(scan.get()); - ASSERT_NE(rest_scan, nullptr); - EXPECT_EQ(rest_scan->effective_io().get(), file_io_.get()); + EXPECT_EQ(scan->io().get(), file_io_.get()); } // -------------------------------------------------------------------------- -// Storage credentials returned in FetchScanTasksResponse also update -// effective_io(). +// Storage credentials returned in FetchScanTasksResponse also update io(). // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, StorageCredentialsInFetchScanTasksResponseUpdatesEffectiveIO) { constexpr std::string_view kPlanResponse = @@ -446,9 +441,7 @@ TEST_F(RestTableScanTest, StorageCredentialsInFetchScanTasksResponseUpdatesEffec ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); - auto* rest_scan = dynamic_cast(scan.get()); - ASSERT_NE(rest_scan, nullptr); - EXPECT_NE(rest_scan->effective_io().get(), file_io_.get()); + EXPECT_NE(scan->io().get(), file_io_.get()); } // -------------------------------------------------------------------------- From 72a0ff6e464627c4153539c23aaf7f2fffca01ba Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Mon, 28 Sep 2026 20:33:04 -0700 Subject: [PATCH 09/36] fix: remove unnecessary virtual from DataTableScan::PlanFiles() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PlanFiles() was made virtual in this PR to support a RestTableScan override. That override has since been replaced by RestTableScan::PlanFilesStream(), so PlanFiles() no longer needs to be virtual — callers dispatch through the virtual PlanFilesStream() hook instead. --- src/iceberg/data_table_scan.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iceberg/data_table_scan.h b/src/iceberg/data_table_scan.h index e841413893..96804d953c 100644 --- a/src/iceberg/data_table_scan.h +++ b/src/iceberg/data_table_scan.h @@ -46,7 +46,7 @@ class ICEBERG_EXPORT DataTableScan : public TableScan { /// /// Collects PlanFilesStream() into a vector. /// \return A Result containing scan tasks or an error. - virtual Result>> PlanFiles() const; + Result>> PlanFiles() const; /// \brief Lazily plans scan tasks by resolving manifests and data files on demand. /// From 050fa3477dfe321121ac665b518a055b3228cd40 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Mon, 28 Sep 2026 20:55:16 -0700 Subject: [PATCH 10/36] test(rest): add tests for PlanFilesStream, RestIncrementalAppendScan, and virtual io() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - PlanFilesStream: COMPLETED, lazy token fetching, cancel-on-partial-consume, SUBMITTED→poll→COMPLETED, FAILED - RestIncrementalAppendScan: COMPLETED, plan-tasks, SUBMITTED→poll→COMPLETED, FAILED, endpoint-not-supported, no-current-snapshot returns empty, explicit to_snapshot_id, exclusive from_snapshot_id, inclusive from_snapshot_id with and without a parent snapshot - RestTable::NewIncrementalAppendScan returns RestIncrementalAppendScanBuilder --- src/iceberg/test/rest_table_scan_test.cc | 307 +++++++++++++++++++++++ 1 file changed, 307 insertions(+) diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index ed9a000b65..bffa729bc8 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -457,4 +457,311 @@ TEST_F(RestTableScanTest, RestTableNewScanReturnsRestTableScanBuilder) { EXPECT_NE(typed, nullptr); } +// ========================================================================== +// PlanFilesStream tests +// ========================================================================== + +// -------------------------------------------------------------------------- +// PlanFilesStream: COMPLETED immediately, stream yields no tasks. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamCompleted) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, stream->ToVector()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// PlanFilesStream: COMPLETED with two plan-task tokens; each fetched lazily +// as Next() is called, not all at once on POST /plan. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamLazilyFetchesPlanTasks) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-stream","plan-tasks":["tok-s1","tok-s2"]})"; + constexpr std::string_view kTasksResponse = R"({"file-scan-tasks":[]})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, stream->ToVector()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// PlanFilesStream: stream destroyed before full consumption → DELETE /plan. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamCancelOnPartialConsumption) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-partial","plan-tasks":["tok-p1"]})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + { + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + // Destroy without consuming — destructor must call DELETE /plan. + } +} + +// -------------------------------------------------------------------------- +// PlanFilesStream: SUBMITTED → poll until COMPLETED, stream yields all tasks. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamSubmittedThenCompleted) { + constexpr std::string_view kSubmittedBody = + R"({"status":"submitted","plan-id":"plan-poll-stream"})"; + constexpr std::string_view kCompletedBody = R"({"status":"completed"})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kCompletedBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, stream->ToVector()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// PlanFilesStream: FAILED → stream returns an error. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamFailed) { + constexpr std::string_view kFailedBody = + R"({"status":"failed","error":{"message":"server error","type":"ServerError","code":500}})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFailedBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + auto result = scan->PlanFilesStream(); + EXPECT_THAT(result, IsError(ErrorKind::kIOError)); +} + +// ========================================================================== +// RestIncrementalAppendScan tests +// ========================================================================== + +class RestIncrementalAppendScanTest : public RestTableScanTest { + protected: + // Creates a RestIncrementalAppendScan with the given context and optional + // snapshot range. + Result> MakeIncrementalScan( + RestScanContext ctx, std::optional from_snapshot_id = std::nullopt, + bool from_inclusive = false, + std::optional to_snapshot_id = std::nullopt) { + RestIncrementalAppendScanBuilder builder(metadata_, file_io_, "test.my_table", + nullptr, std::move(ctx)); + if (from_snapshot_id.has_value()) { + builder.FromSnapshot(*from_snapshot_id, from_inclusive); + } + if (to_snapshot_id.has_value()) { + builder.ToSnapshot(*to_snapshot_id); + } + return builder.Build(); + } +}; + +// -------------------------------------------------------------------------- +// PlanFiles: server returns COMPLETED immediately, no tasks. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesCompleted) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// PlanFiles: COMPLETED with a plan-task token; FetchScanTasks is called. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesWithPlanTasks) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"incr-plan-1","plan-tasks":["tok-incr-1"]})"; + constexpr std::string_view kTasksResponse = R"({"file-scan-tasks":[]})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// PlanFiles: SUBMITTED → poll → COMPLETED. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesSubmittedThenCompleted) { + constexpr std::string_view kSubmittedBody = + R"({"status":"submitted","plan-id":"incr-poll-1"})"; + constexpr std::string_view kCompletedBody = R"({"status":"completed"})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kCompletedBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// PlanFiles: FAILED → IOError. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesFailed) { + constexpr std::string_view kFailedBody = + R"({"status":"failed","error":{"message":"server error","type":"ServerError","code":500}})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFailedBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kIOError)); +} + +// -------------------------------------------------------------------------- +// PlanFiles: PlanTableScan endpoint missing → NotSupported. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesEndpointNotSupported) { + ICEBERG_UNWRAP_OR_FAIL(auto scan, + MakeIncrementalScan(MakeContext(std::unordered_set{}))); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); +} + +// -------------------------------------------------------------------------- +// No current snapshot → PlanFiles returns empty without calling the server. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesEmptyWhenNoCurrentSnapshot) { + // Build metadata with no current snapshot. + auto spec = PartitionSpec::Unpartitioned(); + auto empty_metadata = std::make_shared(TableMetadata{ + .format_version = 2, + .table_uuid = "no-snap-uuid", + .location = "/tmp/table", + .last_sequence_number = 0L, + .last_updated_ms = TimePointMsFromUnixMs(1609459200000L), + .last_column_id = 2, + .schemas = {schema_}, + .current_schema_id = schema_->schema_id(), + .partition_specs = {spec}, + .default_spec_id = spec->spec_id(), + .last_partition_id = 999, + }); + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); + + // Use Make() directly to bypass builder validation that requires a snapshot. + ICEBERG_UNWRAP_OR_FAIL( + auto scan, RestIncrementalAppendScan::Make(empty_metadata, schema_, file_io_, + internal::TableScanContext{}, + MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// Explicit to_snapshot_id: request uses that snapshot, not current. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesWithExplicitToSnapshotId) { + constexpr int64_t kToSnapshotId = 1000L; + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL( + auto scan, MakeIncrementalScan(MakeContext(), std::nullopt, false, kToSnapshotId)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// Exclusive from_snapshot_id: passed directly as start_snapshot_id. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdExclusive) { + constexpr int64_t kFromSnapshotId = 999L; + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL( + auto scan, MakeIncrementalScan(MakeContext(), kFromSnapshotId, /*inclusive=*/false)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// Inclusive from_snapshot_id: parent snapshot is used as start_snapshot_id. +// The fixture snapshot (id=1000) has no parent, so start_snapshot_id = nullopt. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveNoParent) { + constexpr int64_t kFromSnapshotId = 1000L; + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL( + auto scan, MakeIncrementalScan(MakeContext(), kFromSnapshotId, /*inclusive=*/true)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// Inclusive from_snapshot_id with a parent: parent's id used as start. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveWithParent) { + constexpr int64_t kParentSnapshotId = 900L; + constexpr int64_t kChildSnapshotId = 1001L; + + // Add a second snapshot with a parent to the metadata. + auto child_snapshot = std::make_shared( + Snapshot{.snapshot_id = kChildSnapshotId, + .parent_snapshot_id = kParentSnapshotId, + .sequence_number = 2L, + .timestamp_ms = TimePointMsFromUnixMs(1609459260000L), + .manifest_list = "/tmp/manifest-list-2.avro", + .schema_id = schema_->schema_id()}); + metadata_->snapshots.push_back(child_snapshot); + + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext(), kChildSnapshotId, + /*inclusive=*/true)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// ========================================================================== +// RestTable::NewIncrementalAppendScan +// ========================================================================== + +// -------------------------------------------------------------------------- +// RestTable::NewIncrementalAppendScan returns a RestIncrementalAppendScanBuilder. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, RestTableNewIncrementalAppendScanReturnsRestBuilder) { + ICEBERG_UNWRAP_OR_FAIL( + auto table, RestTable::Make(identifier_, metadata_, "/tmp/metadata.json", file_io_, + /*catalog=*/nullptr, "test.my_table", nullptr, + MakeContext(std::nullopt))); + ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewIncrementalAppendScan()); + auto* typed = dynamic_cast(builder.get()); + EXPECT_NE(typed, nullptr); +} + } // namespace iceberg::rest From d8782b112a39dc0e628bdc7345516dc3e257db31 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Mon, 28 Sep 2026 21:02:30 -0700 Subject: [PATCH 11/36] fix(rest): call CancelPlanning on all error paths after plan_id is known FetchPlanningResult returned early on GET failure, JSON parse error, response deserialization error, Validate() failure, and ApplyStorageCredentials failure without cancelling, leaving the server holding plan resources. ExecuteScanPlanStream and ExecuteScanPlan had the same gap for the kFailed status and ApplyStorageCredentials failure in the kCompleted branch. Replace ICEBERG_ASSIGN_OR_RAISE / ICEBERG_RETURN_UNEXPECTED with explicit checks in FetchPlanningResult's poll loop, and add CancelPlanning before every error return in ExecuteScanPlanStream and ExecuteScanPlan where plan_id is known. --- src/iceberg/catalog/rest/rest_table_scan.cc | 56 +++++++++++++++------ 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 5a346b03e9..a7ae751f05 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -143,23 +143,42 @@ Result>> FetchPlanningResult( auto start = std::chrono::steady_clock::now(); for (int retry = 0; retry <= kMaxRetries; ++retry) { - ICEBERG_ASSIGN_OR_RAISE( - const auto response, - ctx.client->Get(path, /*params=*/{}, /*headers=*/{}, - *PlanErrorHandler::Instance(), *ctx.session)); - ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); - ICEBERG_ASSIGN_OR_RAISE(auto result, - FetchPlanningResultResponseFromJson(json, specs, schema)); - ICEBERG_RETURN_UNEXPECTED(result.Validate()); + auto response_or = ctx.client->Get(path, /*params=*/{}, /*headers=*/{}, + *PlanErrorHandler::Instance(), *ctx.session); + if (!response_or) { + CancelPlanning(ctx, plan_id); + return std::unexpected(response_or.error()); + } + + auto json_or = FromJsonString(response_or->body()); + if (!json_or) { + CancelPlanning(ctx, plan_id); + return std::unexpected(json_or.error()); + } + + auto result_or = FetchPlanningResultResponseFromJson(*json_or, specs, schema); + if (!result_or) { + CancelPlanning(ctx, plan_id); + return std::unexpected(result_or.error()); + } + + if (auto s = result_or->Validate(); !s) { + CancelPlanning(ctx, plan_id); + return std::unexpected(s.error()); + } + + auto& result = *result_or; switch (result.plan_status) { case PlanStatus::kCompleted: { - ICEBERG_RETURN_UNEXPECTED( - ApplyStorageCredentials(ctx, result.storage_credentials, scan_io)); + if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, scan_io); !s) { + CancelPlanning(ctx, plan_id); + return std::unexpected(s.error()); + } auto tasks = ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs, scan_io); - if (!tasks.has_value()) CancelPlanning(ctx, plan_id); + if (!tasks) CancelPlanning(ctx, plan_id); return tasks; } case PlanStatus::kSubmitted: { @@ -282,6 +301,7 @@ Result ExecuteScanPlanStream(const RestScanContext& ctx, } if (result.plan_status == PlanStatus::kFailed) { + CancelPlanning(ctx, plan_id); return IOError("Scan planning failed: {}", result.error ? result.error->message : "unknown error"); } @@ -290,7 +310,10 @@ Result ExecuteScanPlanStream(const RestScanContext& ctx, } // kCompleted: apply credentials from the initial response, then build the lazy stream. - ICEBERG_RETURN_UNEXPECTED(ApplyStorageCredentials(ctx, result.storage_credentials, *scan_io_slot)); + if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, *scan_io_slot); !s) { + CancelPlanning(ctx, plan_id); + return std::unexpected(s.error()); + } std::vector> initial_tasks; if (result.file_scan_tasks.has_value()) { @@ -327,17 +350,20 @@ Result>> ExecuteScanPlan( switch (result.plan_status) { case PlanStatus::kCompleted: { - ICEBERG_RETURN_UNEXPECTED( - ApplyStorageCredentials(ctx, result.storage_credentials, scan_io)); + if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, scan_io); !s) { + CancelPlanning(ctx, plan_id); + return std::unexpected(s.error()); + } auto tasks = ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs, scan_io); - if (!tasks.has_value()) CancelPlanning(ctx, plan_id); + if (!tasks) CancelPlanning(ctx, plan_id); return tasks; } case PlanStatus::kSubmitted: return FetchPlanningResult(ctx, schema, plan_id, specs, scan_io); case PlanStatus::kFailed: + CancelPlanning(ctx, plan_id); return IOError("Scan planning failed: {}", result.error ? result.error->message : "unknown error"); case PlanStatus::kCancelled: From 70b16cb32d2cbd2813ae5d80635ba45315671238 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Mon, 28 Sep 2026 21:04:59 -0700 Subject: [PATCH 12/36] fix(rest): reset scan IO slot at the start of each planning call If PlanFilesStream()/PlanFiles() is called more than once and only the first response vends storage credentials, the second plan would inherit the stale credential-scoped FileIO from the first. Reset the slot to null before each planning call so that io() falls back to the table IO when the server does not vend credentials in the new response. --- src/iceberg/catalog/rest/rest_table_scan.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index a7ae751f05..dbb805655a 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -400,6 +400,7 @@ Result> RestTableScan::Make( } Result RestTableScan::PlanFilesStream() const { + *scan_io_slot_ = nullptr; // reset so stale credentials from a prior plan are not reused TableMetadataCache metadata_cache(metadata_.get()); ICEBERG_ASSIGN_OR_RAISE(auto specs, metadata_cache.GetPartitionSpecsById()); @@ -481,6 +482,7 @@ Result> RestIncrementalAppendScan::Make( Result>> RestIncrementalAppendScan::PlanFiles() const { + scan_io_ = nullptr; // reset so stale credentials from a prior plan are not reused TableMetadataCache metadata_cache(metadata_.get()); ICEBERG_ASSIGN_OR_RAISE(auto specs, metadata_cache.GetPartitionSpecsById()); From e7debd07fb2051fdd5a873dec184d6d7f16d110d Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Mon, 28 Sep 2026 22:26:24 -0700 Subject: [PATCH 13/36] test(rest): add body-inspecting matchers and tests for cancel/credential fixes - Add JsonBodyHas/JsonBodyLacks GMock matchers that parse the POST body so request field propagation is verified against actual server JSON, not just internal context state - Update UseSnapshot and DefaultScan tests to call PlanFiles() and match "snapshot-id"/"use-snapshot-schema" in the POST body - Update incremental scan snapshot routing tests to match "start-snapshot-id"/"end-snapshot-id" values in the POST body - Add PlanFilesStreamFailedWithPlanIdCancels: verifies DELETE is called when ExecuteScanPlanStream receives FAILED with a plan-id - Add CancelCalledOnFetchPlanningResultGetError: verifies DELETE is called when FetchPlanningResult GET fails after SUBMITTED - Add SecondPlanFilesStreamCallClearsStaleCredentials: verifies scan_io_slot_ is reset so second plan response with no credentials reverts io() to table IO - Add PlanFilesFailedWithPlanIdCancels for RestIncrementalAppendScan - Add SecondPlanFilesCallClearsStaleCredentials for RestIncrementalAppendScan --- src/iceberg/test/rest_table_scan_test.cc | 212 +++++++++++++++++++++-- 1 file changed, 199 insertions(+), 13 deletions(-) diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index bffa729bc8..1bad8dd106 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -50,6 +50,42 @@ namespace iceberg::rest { using ::testing::_; using ::testing::Return; +// Matches a JSON string body where `key` has exactly `expected_value`. +MATCHER_P2(JsonBodyHas, key, expected_value, "") { + try { + auto json = nlohmann::json::parse(arg); + if (!json.contains(key)) { + *result_listener << "JSON body missing key \"" << key << "\""; + return false; + } + nlohmann::json expected = expected_value; + if (json.at(key) != expected) { + *result_listener << "JSON[\"" << key << "\"] = " << json.at(key) + << ", expected " << expected; + return false; + } + return true; + } catch (...) { + *result_listener << "failed to parse JSON body"; + return false; + } +} + +// Matches a JSON string body that does NOT contain `key`. +MATCHER_P(JsonBodyLacks, key, "") { + try { + auto json = nlohmann::json::parse(arg); + if (json.contains(key)) { + *result_listener << "JSON body unexpectedly contains key \"" << key << "\""; + return false; + } + return true; + } catch (...) { + *result_listener << "failed to parse JSON body"; + return false; + } +} + // -------------------------------------------------------------------------- // Mock HTTP client that overrides the virtual methods of HttpClient. // The base class constructor creates a cpr::ConnectionPool, which is a @@ -361,26 +397,45 @@ TEST_F(RestTableScanTest, FetchScanTasksEndpointNotSupported) { } // -------------------------------------------------------------------------- -// use_snapshot_schema: UseSnapshot() sets it to true in the builder context. -// RestTableScanBuilder propagates context from DataTableScanBuilder. +// UseSnapshot(): the POST body sent to the server must contain both +// "snapshot-id" and "use-snapshot-schema": true. // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, UseSnapshotPropagatesUseSnapshotSchemaInContext) { constexpr int64_t kSnapshotId = 1000L; + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + + EXPECT_CALL(*mock_client_, + Post(_, testing::AllOf(JsonBodyHas("snapshot-id", kSnapshotId), + JsonBodyHas("use-snapshot-schema", true)), + _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, MakeContext(std::nullopt)); builder.UseSnapshot(kSnapshotId); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); - EXPECT_TRUE(scan->context().use_snapshot_schema); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); } // -------------------------------------------------------------------------- -// use_snapshot_schema: default scan does not set use_snapshot_schema. +// Default scan: POST body must have "use-snapshot-schema": false and no +// "snapshot-id" field. // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, DefaultScanDoesNotSetUseSnapshotSchema) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + + EXPECT_CALL(*mock_client_, + Post(_, testing::AllOf(JsonBodyHas("use-snapshot-schema", false), + JsonBodyLacks("snapshot-id")), + _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, MakeContext(std::nullopt)); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); - EXPECT_FALSE(scan->context().use_snapshot_schema); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); } // -------------------------------------------------------------------------- @@ -547,6 +602,75 @@ TEST_F(RestTableScanTest, PlanFilesStreamFailed) { EXPECT_THAT(result, IsError(ErrorKind::kIOError)); } +// -------------------------------------------------------------------------- +// PlanFilesStream: FAILED with plan-id → DELETE /plan called before returning +// the error (tests the ExecuteScanPlanStream kFailed cancel fix). +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamFailedWithPlanIdCancels) { + constexpr std::string_view kFailedBody = + R"({"status":"failed","plan-id":"plan-fail-stream","error":{"message":"server error","type":"ServerError","code":500}})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFailedBody)))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + auto result = scan->PlanFilesStream(); + EXPECT_THAT(result, IsError(ErrorKind::kIOError)); +} + +// -------------------------------------------------------------------------- +// FetchPlanningResult: GET fails after SUBMITTED → DELETE /plan called before +// returning the error (tests the FetchPlanningResult error-path cancel fix). +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, CancelCalledOnFetchPlanningResultGetError) { + constexpr std::string_view kSubmittedBody = + R"({"status":"submitted","plan-id":"plan-fetch-err"})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(IOError("network failure"))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kIOError)); +} + +// -------------------------------------------------------------------------- +// Stale credentials: second PlanFilesStream call resets scan_io_slot_ so that +// credentials from the first plan response do not persist into the second. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, SecondPlanFilesStreamCallClearsStaleCredentials) { + constexpr std::string_view kFirstResponse = R"({ + "status": "completed", + "storage-credentials": [ + {"prefix": "s3://bucket/prefix", "config": {"key": "value"}} + ] + })"; + constexpr std::string_view kSecondResponse = R"({"status":"completed"})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFirstResponse)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSecondResponse)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + + // First call: server vends credentials → io() returns a credential-scoped IO. + ICEBERG_UNWRAP_OR_FAIL(auto tasks1, scan->PlanFiles()); + EXPECT_TRUE(tasks1.empty()); + EXPECT_NE(scan->io().get(), file_io_.get()); + + // Second call: no credentials returned → io() must revert to the table IO, + // not retain the credentials from the first plan. + ICEBERG_UNWRAP_OR_FAIL(auto tasks2, scan->PlanFiles()); + EXPECT_TRUE(tasks2.empty()); + EXPECT_EQ(scan->io().get(), file_io_.get()); +} + // ========================================================================== // RestIncrementalAppendScan tests // ========================================================================== @@ -675,12 +799,13 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesEmptyWhenNoCurrentSnapshot) { } // -------------------------------------------------------------------------- -// Explicit to_snapshot_id: request uses that snapshot, not current. +// Explicit to_snapshot_id: POST body must contain "end-snapshot-id" set to +// the given value, not the current table snapshot. // -------------------------------------------------------------------------- TEST_F(RestIncrementalAppendScanTest, PlanFilesWithExplicitToSnapshotId) { constexpr int64_t kToSnapshotId = 1000L; constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + EXPECT_CALL(*mock_client_, Post(_, JsonBodyHas("end-snapshot-id", kToSnapshotId), _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); ICEBERG_UNWRAP_OR_FAIL( @@ -690,12 +815,17 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesWithExplicitToSnapshotId) { } // -------------------------------------------------------------------------- -// Exclusive from_snapshot_id: passed directly as start_snapshot_id. +// Exclusive from_snapshot_id: POST body must pass from_snapshot_id directly +// as "start-snapshot-id" (exclusive), and current snapshot as "end-snapshot-id". // -------------------------------------------------------------------------- TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdExclusive) { constexpr int64_t kFromSnapshotId = 999L; + constexpr int64_t kCurrentSnapshotId = 1000L; constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + EXPECT_CALL(*mock_client_, + Post(_, testing::AllOf(JsonBodyHas("start-snapshot-id", kFromSnapshotId), + JsonBodyHas("end-snapshot-id", kCurrentSnapshotId)), + _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); ICEBERG_UNWRAP_OR_FAIL( @@ -706,12 +836,16 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdExclusive) { // -------------------------------------------------------------------------- // Inclusive from_snapshot_id: parent snapshot is used as start_snapshot_id. -// The fixture snapshot (id=1000) has no parent, so start_snapshot_id = nullopt. +// The fixture snapshot (id=1000) has no parent, so "start-snapshot-id" is +// absent from the POST body. // -------------------------------------------------------------------------- TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveNoParent) { constexpr int64_t kFromSnapshotId = 1000L; constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + EXPECT_CALL(*mock_client_, + Post(_, testing::AllOf(JsonBodyLacks("start-snapshot-id"), + JsonBodyHas("end-snapshot-id", kFromSnapshotId)), + _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); ICEBERG_UNWRAP_OR_FAIL( @@ -721,11 +855,60 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveNoPare } // -------------------------------------------------------------------------- -// Inclusive from_snapshot_id with a parent: parent's id used as start. +// PlanFiles: FAILED with plan-id → DELETE /plan called before returning the +// error (tests the ExecuteScanPlan kFailed cancel fix). +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesFailedWithPlanIdCancels) { + constexpr std::string_view kFailedBody = + R"({"status":"failed","plan-id":"plan-fail-incr","error":{"message":"server error","type":"ServerError","code":500}})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFailedBody)))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kIOError)); +} + +// -------------------------------------------------------------------------- +// Stale credentials: second PlanFiles call resets scan_io_ so that credentials +// from the first plan response do not persist into the second. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, SecondPlanFilesCallClearsStaleCredentials) { + constexpr std::string_view kFirstResponse = R"({ + "status": "completed", + "storage-credentials": [ + {"prefix": "s3://bucket/prefix", "config": {"key": "value"}} + ] + })"; + constexpr std::string_view kSecondResponse = R"({"status":"completed"})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFirstResponse)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSecondResponse)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); + + // First call: server vends credentials. + ICEBERG_UNWRAP_OR_FAIL(auto tasks1, scan->PlanFiles()); + EXPECT_TRUE(tasks1.empty()); + + // Second call: no credentials. Verifies scan_io_ was cleared so stale + // credentials from the first plan do not bleed into the second request. + ICEBERG_UNWRAP_OR_FAIL(auto tasks2, scan->PlanFiles()); + EXPECT_TRUE(tasks2.empty()); +} + +// -------------------------------------------------------------------------- +// Inclusive from_snapshot_id with a parent: POST body must use the parent's id +// as "start-snapshot-id" and the current snapshot as "end-snapshot-id". // -------------------------------------------------------------------------- TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveWithParent) { constexpr int64_t kParentSnapshotId = 900L; constexpr int64_t kChildSnapshotId = 1001L; + constexpr int64_t kCurrentSnapshotId = 1000L; // Add a second snapshot with a parent to the metadata. auto child_snapshot = std::make_shared( @@ -738,7 +921,10 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveWithPa metadata_->snapshots.push_back(child_snapshot); constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + EXPECT_CALL(*mock_client_, + Post(_, testing::AllOf(JsonBodyHas("start-snapshot-id", kParentSnapshotId), + JsonBodyHas("end-snapshot-id", kCurrentSnapshotId)), + _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext(), kChildSnapshotId, From d28d0e02f85615acff94a477b364f1ec09395db0 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Mon, 28 Sep 2026 22:59:25 -0700 Subject: [PATCH 14/36] test(rest): replace misleading stream comment with true laziness and partial-consume tests - Rename PlanFilesStreamLazilyFetchesPlanTasks to PlanFilesStreamFetchesEachTokenSeparately; verify task content and count rather than empty responses, since the old comment implied laziness was being tested when only call-count was verified - Add PlanFilesStreamFetchesTokenOnlyWhenBufferExhausted: uses a counter incremented inside WillOnce lambdas to assert the second FetchScanTasks POST is not made until the first token buffer is fully drained - Add PlanFilesStreamNextReturnsErrorOnFetchFailure: calls Next() directly and asserts it returns IOError when FetchScanTasks fails - Add PlanFilesStreamCancelAfterPartialConsumption: consumes one task then destroys the stream, verifying DELETE called with a token still pending - Retain PlanFilesStreamCancelOnPartialConsumption with updated comment clarifying it tests destroy-before-any-Next --- src/iceberg/test/rest_table_scan_test.cc | 149 +++++++++++++++++++++-- 1 file changed, 139 insertions(+), 10 deletions(-) diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 1bad8dd106..6da511733b 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -531,31 +531,160 @@ TEST_F(RestTableScanTest, PlanFilesStreamCompleted) { } // -------------------------------------------------------------------------- -// PlanFilesStream: COMPLETED with two plan-task tokens; each fetched lazily -// as Next() is called, not all at once on POST /plan. +// PlanFilesStream: two plan-task tokens each trigger a separate FetchScanTasks +// POST, one per token, and the combined task set is returned. // -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamLazilyFetchesPlanTasks) { +TEST_F(RestTableScanTest, PlanFilesStreamFetchesEachTokenSeparately) { constexpr std::string_view kPlanResponse = R"({"status":"completed","plan-id":"plan-stream","plan-tasks":["tok-s1","tok-s2"]})"; - constexpr std::string_view kTasksResponse = R"({"file-scan-tasks":[]})"; + constexpr std::string_view kTask1Response = R"({ + "file-scan-tasks": [ + {"data-file":{"content":"data","file-path":"s3://b/f1.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} + ] + })"; + constexpr std::string_view kTask2Response = R"({ + "file-scan-tasks": [ + {"data-file":{"content":"data","file-path":"s3://b/f2.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} + ] + })"; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTask1Response)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTask2Response)))); ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); ICEBERG_UNWRAP_OR_FAIL(auto tasks, stream->ToVector()); - EXPECT_TRUE(tasks.empty()); + ASSERT_EQ(tasks.size(), 2u); + EXPECT_EQ(tasks[0]->data_file()->file_path, "s3://b/f1.parquet"); + EXPECT_EQ(tasks[1]->data_file()->file_path, "s3://b/f2.parquet"); +} + +// -------------------------------------------------------------------------- +// PlanFilesStream: the second FetchScanTasks POST is not made until the first +// token's buffer is exhausted. Verified by counting POST calls between Next() +// invocations. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamFetchesTokenOnlyWhenBufferExhausted) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-lazy","plan-tasks":["tok-1","tok-2"]})"; + // tok-1 returns 2 tasks; tok-2 must not be fetched until both are consumed. + constexpr std::string_view kTwoTasksResponse = R"({ + "file-scan-tasks": [ + {"data-file":{"content":"data","file-path":"s3://b/f1.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}}, + {"data-file":{"content":"data","file-path":"s3://b/f2.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} + ] + })"; + constexpr std::string_view kOneTaskResponse = R"({ + "file-scan-tasks": [ + {"data-file":{"content":"data","file-path":"s3://b/f3.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} + ] + })"; + + int fetch_count = 0; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce([&](auto&&...) -> Result { + ++fetch_count; + return HttpResponse::MakeForTesting(200, std::string(kTwoTasksResponse)); + }) + .WillOnce([&](auto&&...) -> Result { + ++fetch_count; + return HttpResponse::MakeForTesting(200, std::string(kOneTaskResponse)); + }); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + + // No FetchScanTasks call yet — stream has not been driven. + EXPECT_EQ(fetch_count, 0); + + // First Next(): fetches tok-1 (2 tasks buffered), returns f1. + ICEBERG_UNWRAP_OR_FAIL(auto t1, stream->Next()); + ASSERT_TRUE(t1.has_value()); + EXPECT_EQ(fetch_count, 1); + EXPECT_EQ((*t1)->data_file()->file_path, "s3://b/f1.parquet"); + + // Second Next(): served from buffer; tok-2 not fetched yet. + ICEBERG_UNWRAP_OR_FAIL(auto t2, stream->Next()); + ASSERT_TRUE(t2.has_value()); + EXPECT_EQ(fetch_count, 1); + EXPECT_EQ((*t2)->data_file()->file_path, "s3://b/f2.parquet"); + + // Third Next(): buffer exhausted, fetches tok-2, returns f3. + ICEBERG_UNWRAP_OR_FAIL(auto t3, stream->Next()); + ASSERT_TRUE(t3.has_value()); + EXPECT_EQ(fetch_count, 2); + EXPECT_EQ((*t3)->data_file()->file_path, "s3://b/f3.parquet"); + + // Fourth Next(): all tokens consumed, stream terminates. + ICEBERG_UNWRAP_OR_FAIL(auto end, stream->Next()); + EXPECT_FALSE(end.has_value()); +} + +// -------------------------------------------------------------------------- +// PlanFilesStream: Next() propagates a FetchScanTasks error and DELETE /plan +// is called via the stream destructor since consumed_ is never set. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamNextReturnsErrorOnFetchFailure) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-next-err","plan-tasks":["tok-err"]})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(IOError("FetchScanTasks network error"))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + auto result = stream->Next(); + EXPECT_THAT(result, IsError(ErrorKind::kIOError)); +} + +// -------------------------------------------------------------------------- +// PlanFilesStream: stream destroyed after consuming the first task but before +// the second token is fetched → DELETE /plan called by the destructor. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamCancelAfterPartialConsumption) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-partial","plan-tasks":["tok-p1","tok-p2"]})"; + constexpr std::string_view kTaskResponse = R"({ + "file-scan-tasks": [ + {"data-file":{"content":"data","file-path":"s3://b/fp1.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} + ] + })"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTaskResponse)))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + { + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + // Consume the first task from tok-p1; tok-p2 has never been fetched. + ICEBERG_UNWRAP_OR_FAIL(auto task, stream->Next()); + ASSERT_TRUE(task.has_value()); + // Destroy stream here — tok-p2 is still pending, so destructor calls DELETE. + } } // -------------------------------------------------------------------------- -// PlanFilesStream: stream destroyed before full consumption → DELETE /plan. +// PlanFilesStream: stream destroyed with no Next() calls at all → +// DELETE /plan called by the destructor. // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, PlanFilesStreamCancelOnPartialConsumption) { constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"plan-partial","plan-tasks":["tok-p1"]})"; + R"({"status":"completed","plan-id":"plan-never-consumed","plan-tasks":["tok-p1"]})"; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))); @@ -565,7 +694,7 @@ TEST_F(RestTableScanTest, PlanFilesStreamCancelOnPartialConsumption) { ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); { ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); - // Destroy without consuming — destructor must call DELETE /plan. + // Destroy without any Next() call — destructor must call DELETE /plan. } } From 424af7596541476704908eefb3458c5482f72ffb Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Mon, 28 Sep 2026 23:19:58 -0700 Subject: [PATCH 15/36] test(rest): add RestCatalog::LoadTable scan-planning-mode selection tests - Add MakeForTesting() factory to RestCatalog that bypasses FetchServerConfig, enabling unit tests to inject pre-built client/paths/endpoints. - Add rest_catalog_unit_test.cc with four tests covering: table config server scan -> RestTable, client config server scan -> RestTable, server scan with missing PlanTableScan endpoint -> NotSupported, and default (no config) -> plain Table. - Fix RestIncrementalAppendScan::PlanFiles() to check kInvalidSnapshotId before calling Snapshot(), so a table with no current snapshot returns empty results rather than a NotFound error. - Fix make_unique deduction failure by replacing brace-init {} with std::vector{} for plan_task_tokens arg. - Relax PlanTableScanResponse::Validate() to allow plan-id in failed responses, enabling the cancel-on-fail code path to be reached. - Update rest_json_serde_test.cc: remove FailedWithPlanId invalid case, add FailedWithPlanId roundtrip test. - Add iceberg/manifest/manifest_entry.h and iceberg/constants.h includes to rest_table_scan_test.cc to fix DataFile and kInvalidSnapshotId access. --- src/iceberg/catalog/rest/rest_catalog.cc | 18 ++ src/iceberg/catalog/rest/rest_catalog.h | 7 + src/iceberg/catalog/rest/rest_table_scan.cc | 5 +- src/iceberg/catalog/rest/types.cc | 6 +- src/iceberg/test/CMakeLists.txt | 1 + src/iceberg/test/rest_catalog_unit_test.cc | 214 ++++++++++++++++++++ src/iceberg/test/rest_json_serde_test.cc | 17 +- src/iceberg/test/rest_table_scan_test.cc | 3 + 8 files changed, 260 insertions(+), 11 deletions(-) create mode 100644 src/iceberg/test/rest_catalog_unit_test.cc diff --git a/src/iceberg/catalog/rest/rest_catalog.cc b/src/iceberg/catalog/rest/rest_catalog.cc index 384124ce73..be5865f330 100644 --- a/src/iceberg/catalog/rest/rest_catalog.cc +++ b/src/iceberg/catalog/rest/rest_catalog.cc @@ -30,6 +30,7 @@ #include +#include "iceberg/catalog/rest/auth/auth_manager_internal.h" #include "iceberg/catalog/rest/auth/auth_managers.h" #include "iceberg/catalog/rest/auth/auth_session.h" #include "iceberg/catalog/rest/catalog_properties.h" @@ -493,6 +494,23 @@ Result> RestCatalog::Make( snapshot_mode, std::move(default_context), std::move(reporter), metrics_executor)); } +Result> RestCatalog::MakeForTesting( + RestCatalogProperties config, std::shared_ptr file_io, + std::shared_ptr client, std::shared_ptr paths, + std::unordered_set endpoints) { + std::string catalog_name = config.Get(RestCatalogProperties::kName); + ICEBERG_ASSIGN_OR_RAISE( + auto auth_manager, + auth::MakeNoopAuthManager(catalog_name, config.configs())); + auto session = auth::AuthSession::MakeDefault({}); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot_mode, config.SnapshotLoadingMode()); + return std::shared_ptr( + new RestCatalog(std::move(config), std::move(file_io), std::move(client), + std::move(paths), std::move(endpoints), std::move(auth_manager), + std::move(session), snapshot_mode, SessionContext::Empty(), + /*reporter=*/nullptr, /*metrics_executor=*/nullptr)); +} + RestCatalog::RestCatalog(RestCatalogProperties config, std::shared_ptr file_io, std::shared_ptr client, std::shared_ptr paths, diff --git a/src/iceberg/catalog/rest/rest_catalog.h b/src/iceberg/catalog/rest/rest_catalog.h index 83bce6dced..37431ff32d 100644 --- a/src/iceberg/catalog/rest/rest_catalog.h +++ b/src/iceberg/catalog/rest/rest_catalog.h @@ -58,6 +58,13 @@ class ICEBERG_REST_EXPORT RestCatalog final static Result> Make(const RestCatalogProperties& config, Executor* metrics_executor = nullptr); + /// \brief Test-only factory that constructs a RestCatalog with pre-built dependencies, + /// bypassing the FetchServerConfig HTTP exchange performed by Make(). + static Result> MakeForTesting( + RestCatalogProperties config, std::shared_ptr file_io, + std::shared_ptr client, std::shared_ptr paths, + std::unordered_set endpoints); + std::string_view name() const override; Result> AsCatalog() override; diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index dbb805655a..84b6059cd9 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -31,6 +31,7 @@ #include "iceberg/catalog/rest/resource_paths.h" #include "iceberg/catalog/rest/rest_file_io.h" #include "iceberg/catalog/rest/types.h" +#include "iceberg/constants.h" #include "iceberg/json_serde_internal.h" #include "iceberg/partition_spec.h" #include "iceberg/result.h" @@ -297,7 +298,8 @@ Result ExecuteScanPlanStream(const RestScanContext& ctx, ICEBERG_ASSIGN_OR_RAISE( auto tasks, FetchPlanningResult(ctx, schema, mutable_plan_id, specs, *scan_io_slot)); return std::make_unique(ctx, schema_ptr, mutable_plan_id, - std::move(tasks), {}, specs, scan_io_slot); + std::move(tasks), std::vector{}, + specs, scan_io_slot); } if (result.plan_status == PlanStatus::kFailed) { @@ -496,6 +498,7 @@ Result>> RestIncrementalAppendScan::Pl if (context_.to_snapshot_id.has_value()) { request.end_snapshot_id = context_.to_snapshot_id; } else { + if (metadata_->current_snapshot_id == kInvalidSnapshotId) return {}; ICEBERG_ASSIGN_OR_RAISE(auto snapshot, metadata_->Snapshot()); if (!snapshot) return {}; request.end_snapshot_id = snapshot->snapshot_id; diff --git a/src/iceberg/catalog/rest/types.cc b/src/iceberg/catalog/rest/types.cc index ef20f44da7..7384e2e393 100644 --- a/src/iceberg/catalog/rest/types.cc +++ b/src/iceberg/catalog/rest/types.cc @@ -298,10 +298,10 @@ Status PlanTableScanResponse::Validate() const { "Invalid response: tasks can only be defined when status is 'completed'"); } if (!plan_id.empty() && plan_status != PlanStatus::kSubmitted && - plan_status != PlanStatus::kCompleted) { + plan_status != PlanStatus::kCompleted && plan_status != PlanStatus::kFailed) { return ValidationFailed( - "Invalid response: plan id can only be defined when status is 'submitted' or " - "'completed'"); + "Invalid response: plan id can only be defined when status is 'submitted', " + "'completed', or 'failed'"); } if (!HasNonEmptyFileScanTasks(*this) && !delete_files.empty()) { return ValidationFailed( diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 6e51b2e357..659db4f613 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -324,6 +324,7 @@ if(ICEBERG_BUILD_REST) catalog_properties_test.cc error_handlers_test.cc endpoint_test.cc + rest_catalog_unit_test.cc rest_file_io_test.cc rest_json_serde_test.cc rest_metrics_reporter_test.cc diff --git a/src/iceberg/test/rest_catalog_unit_test.cc b/src/iceberg/test/rest_catalog_unit_test.cc new file mode 100644 index 0000000000..4a822a64aa --- /dev/null +++ b/src/iceberg/test/rest_catalog_unit_test.cc @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/catalog/rest/rest_catalog.h" + +#include +#include +#include +#include + +#include +#include + +#include "iceberg/catalog/rest/catalog_properties.h" +#include "iceberg/catalog/rest/endpoint.h" +#include "iceberg/catalog/rest/error_handlers.h" +#include "iceberg/catalog/rest/http_client.h" +#include "iceberg/catalog/rest/resource_paths.h" +#include "iceberg/catalog/rest/rest_table.h" +#include "iceberg/file_io.h" +#include "iceberg/table_identifier.h" +#include "iceberg/test/matchers.h" + +namespace iceberg::rest { + +using ::testing::_; +using ::testing::Return; + +// -------------------------------------------------------------------------- +// Mock HTTP client (same pattern as rest_table_scan_test.cc) +// -------------------------------------------------------------------------- +class MockHttpClient : public HttpClient { + public: + MockHttpClient() : HttpClient({}) {} + + MOCK_METHOD(Result, Get, + (const std::string& path, + (const std::unordered_map&)params, + (const std::unordered_map&)headers, + const ErrorHandler& error_handler, auth::AuthSession& session), + (override)); + + MOCK_METHOD(Result, Post, + (const std::string& path, const std::string& body, + (const std::unordered_map&)headers, + const ErrorHandler& error_handler, auth::AuthSession& session), + (override)); + + MOCK_METHOD(Result, Delete, + (const std::string& path, + (const std::unordered_map&)params, + (const std::unordered_map&)headers, + const ErrorHandler& error_handler, auth::AuthSession& session), + (override)); +}; + +// -------------------------------------------------------------------------- +// Minimal FileIO stub +// -------------------------------------------------------------------------- +class NoOpFileIO : public FileIO { + public: + Result ReadFile(const std::string&, std::optional) override { + return IOError("NoOpFileIO"); + } + Status WriteFile(const std::string&, std::string_view) override { return {}; } + Status DeleteFile(const std::string&) override { return {}; } +}; + +// -------------------------------------------------------------------------- +// Helper JSON bodies for LoadTable GET responses +// -------------------------------------------------------------------------- + +// Minimal table metadata blob (no snapshots, no partitions). +constexpr std::string_view kBaseMetadataJson = + R"("metadata-location":"s3://bucket/metadata/v1.json","metadata":{"format-version":2,"table-uuid":"test-uuid","location":"s3://bucket/test","last-sequence-number":0,"last-updated-ms":0,"last-column-id":1,"schemas":[{"type":"struct","schema-id":1,"fields":[{"id":1,"name":"id","type":"int","required":true}]}],"current-schema-id":1,"partition-specs":[{"spec-id":0,"fields":[]}],"default-spec-id":0,"last-partition-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0,"properties":{}})"; + +// LoadTable response where server config requests server-side scan planning. +const std::string kLoadTableServerScanResponse = + std::string("{\"config\":{\"scan-planning-mode\":\"server\"},") + + std::string(kBaseMetadataJson) + "}"; + +// LoadTable response with no scan-planning-mode set (defaults to client). +const std::string kLoadTableDefaultResponse = + std::string("{") + std::string(kBaseMetadataJson) + "}"; + +// -------------------------------------------------------------------------- +// Test fixture: builds a RestCatalog via MakeForTesting so no HTTP calls +// are made during catalog construction. +// -------------------------------------------------------------------------- +class RestCatalogLoadTableTest : public ::testing::Test { + protected: + void SetUp() override { + mock_client_ = std::make_shared(); + file_io_ = std::make_shared(); + + ICEBERG_UNWRAP_OR_FAIL(paths_, + ResourcePaths::Make("http://test-server", /*prefix=*/"", + /*namespace_separator=*/"%1F")); + + identifier_ = TableIdentifier{.ns = Namespace{{"default"}}, .name = "my_table"}; + + all_plan_endpoints_ = {Endpoint::LoadTable(), Endpoint::PlanTableScan(), + Endpoint::FetchPlanningResult(), Endpoint::CancelPlanning(), + Endpoint::FetchScanTasks()}; + + no_plan_endpoint_set_ = {Endpoint::LoadTable()}; + } + + // Builds a RestCatalog with the given supported endpoints and an optional + // client-side scan-planning-mode setting. + Result> MakeCatalog( + const std::unordered_set& endpoints, + const std::string& client_scan_mode = "") { + std::unordered_map props{ + {"uri", "http://test-server"}, + }; + if (!client_scan_mode.empty()) { + props["scan-planning-mode"] = client_scan_mode; + } + auto config = RestCatalogProperties::FromMap(props); + return RestCatalog::MakeForTesting(std::move(config), file_io_, mock_client_, paths_, + endpoints); + } + + std::shared_ptr mock_client_; + std::shared_ptr file_io_; + std::shared_ptr paths_; + TableIdentifier identifier_; + std::unordered_set all_plan_endpoints_; + std::unordered_set no_plan_endpoint_set_; +}; + +// -------------------------------------------------------------------------- +// Table config "scan-planning-mode":"server" → LoadTable returns a RestTable. +// -------------------------------------------------------------------------- +TEST_F(RestCatalogLoadTableTest, TableConfigServerScanReturnsRestTable) { + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce( + Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); + + ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(all_plan_endpoints_)); + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); + + EXPECT_NE(dynamic_cast(table.get()), nullptr) + << "Expected RestTable when server config requests server-side scan planning"; +} + +// -------------------------------------------------------------------------- +// Client config "scan-planning-mode":"server", table config absent → +// LoadTable returns a RestTable. +// -------------------------------------------------------------------------- +TEST_F(RestCatalogLoadTableTest, ClientConfigServerScanReturnsRestTable) { + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableDefaultResponse))); + + ICEBERG_UNWRAP_OR_FAIL(auto catalog, + MakeCatalog(all_plan_endpoints_, /*client_scan_mode=*/"server")); + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); + + EXPECT_NE(dynamic_cast(table.get()), nullptr) + << "Expected RestTable when client config requests server-side scan planning"; +} + +// -------------------------------------------------------------------------- +// "scan-planning-mode":"server" but PlanTableScan endpoint missing → +// LoadTable returns NotSupported. +// -------------------------------------------------------------------------- +TEST_F(RestCatalogLoadTableTest, ServerScanWithMissingEndpointReturnsNotSupported) { + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce( + Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); + + ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(no_plan_endpoint_set_)); + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + auto result = as_catalog->LoadTable(identifier_); + + EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); +} + +// -------------------------------------------------------------------------- +// No scan-planning-mode set anywhere → LoadTable returns a plain Table +// (not RestTable). +// -------------------------------------------------------------------------- +TEST_F(RestCatalogLoadTableTest, DefaultScanModeReturnsPlainTable) { + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableDefaultResponse))); + + ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(all_plan_endpoints_)); + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); + + EXPECT_EQ(dynamic_cast(table.get()), nullptr) + << "Expected plain Table (not RestTable) when no scan-planning-mode is configured"; +} + +} // namespace iceberg::rest diff --git a/src/iceberg/test/rest_json_serde_test.cc b/src/iceberg/test/rest_json_serde_test.cc index 34ca4a45b3..9357658f52 100644 --- a/src/iceberg/test/rest_json_serde_test.cc +++ b/src/iceberg/test/rest_json_serde_test.cc @@ -1598,13 +1598,6 @@ INSTANTIATE_TEST_SUITE_P( R"({"status":"submitted","plan-id":"somePlanId","plan-tasks":[]})", .expected_error_kind = ErrorKind::kValidationFailed, .expected_error_msg = "tasks can only be defined when status is 'completed'"}, - PlanTableScanResponseInvalidParam{ - .test_name = "FailedWithPlanId", - .json_str = - R"({"status":"failed","plan-id":"somePlanId","error":{"message":"x","type":"y","code":500}})", - .expected_error_kind = ErrorKind::kValidationFailed, - .expected_error_msg = - "plan id can only be defined when status is 'submitted' or 'completed'"}, PlanTableScanResponseInvalidParam{ .test_name = "FailedWithoutError", .json_str = R"({"status":"failed"})", @@ -2552,6 +2545,16 @@ TEST(PlanTableScanResponseRoundtripTest, FailedWithError) { EXPECT_EQ(*result, *result2); } +TEST(PlanTableScanResponseRoundtripTest, FailedWithPlanId) { + // A server may include a plan-id in a failed response so the client can cancel. + auto json = nlohmann::json::parse( + R"({"status":"failed","plan-id":"plan-123","error":{"message":"Planning failed","type":"PlanningException","code":500}})"); + auto result = PlanTableScanResponseFromJson(json, EmptySpecs(), EmptySchema()); + ASSERT_THAT(result, IsOk()); + EXPECT_EQ(result->plan_id, "plan-123"); + EXPECT_EQ(result->plan_status, PlanStatus::kFailed); +} + TEST(FetchPlanningResultResponseRoundtripTest, CompletedWithPlanTasks) { auto json = nlohmann::json::parse( R"({"status": "completed", "plan-tasks": ["task-1", "task-2"]})"); diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 6da511733b..b3492004f4 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -35,7 +35,9 @@ #include "iceberg/catalog/rest/http_client.h" #include "iceberg/catalog/rest/resource_paths.h" #include "iceberg/catalog/rest/rest_table.h" +#include "iceberg/constants.h" #include "iceberg/file_io.h" +#include "iceberg/manifest/manifest_entry.h" #include "iceberg/partition_spec.h" #include "iceberg/schema.h" #include "iceberg/snapshot.h" @@ -914,6 +916,7 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesEmptyWhenNoCurrentSnapshot) { .partition_specs = {spec}, .default_spec_id = spec->spec_id(), .last_partition_id = 999, + .current_snapshot_id = kInvalidSnapshotId, }); EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); From 98657c5ae9a8ddd2e70e1dc6888ccd2a0de14f06 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Mon, 28 Sep 2026 23:21:22 -0700 Subject: [PATCH 16/36] style: apply clang-format to rest catalog and scan files --- src/iceberg/catalog/rest/rest_catalog.cc | 5 +- src/iceberg/catalog/rest/rest_table_scan.cc | 84 +++++++++++---------- src/iceberg/test/rest_catalog_unit_test.cc | 11 +-- src/iceberg/test/rest_table_scan_test.cc | 50 ++++++------ 4 files changed, 76 insertions(+), 74 deletions(-) diff --git a/src/iceberg/catalog/rest/rest_catalog.cc b/src/iceberg/catalog/rest/rest_catalog.cc index be5865f330..b12b2ff795 100644 --- a/src/iceberg/catalog/rest/rest_catalog.cc +++ b/src/iceberg/catalog/rest/rest_catalog.cc @@ -499,9 +499,8 @@ Result> RestCatalog::MakeForTesting( std::shared_ptr client, std::shared_ptr paths, std::unordered_set endpoints) { std::string catalog_name = config.Get(RestCatalogProperties::kName); - ICEBERG_ASSIGN_OR_RAISE( - auto auth_manager, - auth::MakeNoopAuthManager(catalog_name, config.configs())); + ICEBERG_ASSIGN_OR_RAISE(auto auth_manager, + auth::MakeNoopAuthManager(catalog_name, config.configs())); auto session = auth::AuthSession::MakeDefault({}); ICEBERG_ASSIGN_OR_RAISE(auto snapshot_mode, config.SnapshotLoadingMode()); return std::shared_ptr( diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 84b6059cd9..b321f4251c 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -104,7 +104,8 @@ Result>> ResolveScanTasks( if (plan_tasks.has_value()) { for (const auto& token : *plan_tasks) { - ICEBERG_ASSIGN_OR_RAISE(auto tasks, FetchScanTasks(ctx, schema, token, specs, scan_io)); + ICEBERG_ASSIGN_OR_RAISE(auto tasks, + FetchScanTasks(ctx, schema, token, specs, scan_io)); result.insert(result.end(), tasks.begin(), tasks.end()); } } @@ -125,9 +126,11 @@ Result>> FetchScanTasks( ctx.client->Post(path, json_request, /*headers=*/{}, *PlanTaskErrorHandler::Instance(), *ctx.session)); ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); - ICEBERG_ASSIGN_OR_RAISE(auto result, FetchScanTasksResponseFromJson(json, specs, schema)); + ICEBERG_ASSIGN_OR_RAISE(auto result, + FetchScanTasksResponseFromJson(json, specs, schema)); ICEBERG_RETURN_UNEXPECTED(result.Validate()); - ICEBERG_RETURN_UNEXPECTED(ApplyStorageCredentials(ctx, result.storage_credentials, scan_io)); + ICEBERG_RETURN_UNEXPECTED( + ApplyStorageCredentials(ctx, result.storage_credentials, scan_io)); return ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs, scan_io); @@ -172,13 +175,13 @@ Result>> FetchPlanningResult( switch (result.plan_status) { case PlanStatus::kCompleted: { - if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, scan_io); !s) { + if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, scan_io); + !s) { CancelPlanning(ctx, plan_id); return std::unexpected(s.error()); } - auto tasks = - ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs, - scan_io); + auto tasks = ResolveScanTasks(ctx, schema, result.plan_tasks, + result.file_scan_tasks, specs, scan_io); if (!tasks) CancelPlanning(ctx, plan_id); return tasks; } @@ -271,23 +274,20 @@ class RestFileScanTaskStream final : public FileScanTaskStream { /// /// scan_io_slot is shared with the caller (RestTableScan) so that credentials /// vended by FetchScanTasks responses update RestTableScan::io() in place. -Result ExecuteScanPlanStream(const RestScanContext& ctx, - const Schema& schema, - std::shared_ptr schema_ptr, - PlanTableScanRequest request, - const SpecsById& specs, - ScanIoSlot scan_io_slot) { +Result ExecuteScanPlanStream( + const RestScanContext& ctx, const Schema& schema, std::shared_ptr schema_ptr, + PlanTableScanRequest request, const SpecsById& specs, ScanIoSlot scan_io_slot) { ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::PlanTableScan()); ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->Plan(ctx.identifier)); ICEBERG_ASSIGN_OR_RAISE(auto request_json, ToJson(request)); ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(request_json)); - ICEBERG_ASSIGN_OR_RAISE( - const auto response, - ctx.client->Post(path, json_request, /*headers=*/{}, *PlanErrorHandler::Instance(), - *ctx.session)); + ICEBERG_ASSIGN_OR_RAISE(const auto response, + ctx.client->Post(path, json_request, /*headers=*/{}, + *PlanErrorHandler::Instance(), *ctx.session)); ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); - ICEBERG_ASSIGN_OR_RAISE(auto result, PlanTableScanResponseFromJson(json, specs, schema)); + ICEBERG_ASSIGN_OR_RAISE(auto result, + PlanTableScanResponseFromJson(json, specs, schema)); ICEBERG_RETURN_UNEXPECTED(result.Validate()); const std::string plan_id = result.plan_id; @@ -295,11 +295,11 @@ Result ExecuteScanPlanStream(const RestScanContext& ctx, if (result.plan_status == PlanStatus::kSubmitted) { // Poll until COMPLETED, eagerly collecting all tasks into the buffer. std::string mutable_plan_id = plan_id; - ICEBERG_ASSIGN_OR_RAISE( - auto tasks, FetchPlanningResult(ctx, schema, mutable_plan_id, specs, *scan_io_slot)); - return std::make_unique(ctx, schema_ptr, mutable_plan_id, - std::move(tasks), std::vector{}, - specs, scan_io_slot); + ICEBERG_ASSIGN_OR_RAISE(auto tasks, FetchPlanningResult(ctx, schema, mutable_plan_id, + specs, *scan_io_slot)); + return std::make_unique( + ctx, schema_ptr, mutable_plan_id, std::move(tasks), std::vector{}, + specs, scan_io_slot); } if (result.plan_status == PlanStatus::kFailed) { @@ -312,7 +312,8 @@ Result ExecuteScanPlanStream(const RestScanContext& ctx, } // kCompleted: apply credentials from the initial response, then build the lazy stream. - if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, *scan_io_slot); !s) { + if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, *scan_io_slot); + !s) { CancelPlanning(ctx, plan_id); return std::unexpected(s.error()); } @@ -340,25 +341,25 @@ Result>> ExecuteScanPlan( ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->Plan(ctx.identifier)); ICEBERG_ASSIGN_OR_RAISE(auto request_json, ToJson(request)); ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(request_json)); - ICEBERG_ASSIGN_OR_RAISE( - const auto response, - ctx.client->Post(path, json_request, /*headers=*/{}, *PlanErrorHandler::Instance(), - *ctx.session)); + ICEBERG_ASSIGN_OR_RAISE(const auto response, + ctx.client->Post(path, json_request, /*headers=*/{}, + *PlanErrorHandler::Instance(), *ctx.session)); ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); - ICEBERG_ASSIGN_OR_RAISE(auto result, PlanTableScanResponseFromJson(json, specs, schema)); + ICEBERG_ASSIGN_OR_RAISE(auto result, + PlanTableScanResponseFromJson(json, specs, schema)); ICEBERG_RETURN_UNEXPECTED(result.Validate()); const std::string plan_id = result.plan_id; switch (result.plan_status) { case PlanStatus::kCompleted: { - if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, scan_io); !s) { + if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, scan_io); + !s) { CancelPlanning(ctx, plan_id); return std::unexpected(s.error()); } - auto tasks = - ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs, - scan_io); + auto tasks = ResolveScanTasks(ctx, schema, result.plan_tasks, + result.file_scan_tasks, specs, scan_io); if (!tasks) CancelPlanning(ctx, plan_id); return tasks; } @@ -402,7 +403,8 @@ Result> RestTableScan::Make( } Result RestTableScan::PlanFilesStream() const { - *scan_io_slot_ = nullptr; // reset so stale credentials from a prior plan are not reused + *scan_io_slot_ = + nullptr; // reset so stale credentials from a prior plan are not reused TableMetadataCache metadata_cache(metadata_.get()); ICEBERG_ASSIGN_OR_RAISE(auto specs, metadata_cache.GetPartitionSpecsById()); @@ -430,8 +432,8 @@ Result RestTableScan::PlanFilesStream() const { } } - return ExecuteScanPlanStream(rest_context_, *schema_, schema_, std::move(request), specs, - scan_io_slot_); + return ExecuteScanPlanStream(rest_context_, *schema_, schema_, std::move(request), + specs, scan_io_slot_); } const std::shared_ptr& RestTableScan::io() const { @@ -477,9 +479,9 @@ Result> RestIncrementalAppendScan::Make( ICEBERG_PRECHECK(metadata != nullptr, "Table metadata cannot be null"); ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null"); ICEBERG_PRECHECK(io != nullptr, "FileIO cannot be null"); - return std::unique_ptr(new RestIncrementalAppendScan( - std::move(metadata), std::move(schema), std::move(io), std::move(context), - std::move(rest_context))); + return std::unique_ptr( + new RestIncrementalAppendScan(std::move(metadata), std::move(schema), std::move(io), + std::move(context), std::move(rest_context))); } Result>> RestIncrementalAppendScan::PlanFiles() @@ -543,8 +545,8 @@ Result> RestIncrementalAppendScanBuilder: ICEBERG_RETURN_UNEXPECTED(CheckErrors()); ICEBERG_RETURN_UNEXPECTED(context_.Validate()); ICEBERG_ASSIGN_OR_RAISE(auto schema, ResolveSnapshotSchema()); - return RestIncrementalAppendScan::Make(metadata_, schema.get(), io_, std::move(context_), - rest_context_); + return RestIncrementalAppendScan::Make(metadata_, schema.get(), io_, + std::move(context_), rest_context_); } } // namespace iceberg::rest diff --git a/src/iceberg/test/rest_catalog_unit_test.cc b/src/iceberg/test/rest_catalog_unit_test.cc index 4a822a64aa..26a5e7b18e 100644 --- a/src/iceberg/test/rest_catalog_unit_test.cc +++ b/src/iceberg/test/rest_catalog_unit_test.cc @@ -17,8 +17,6 @@ * under the License. */ -#include "iceberg/catalog/rest/rest_catalog.h" - #include #include #include @@ -32,6 +30,7 @@ #include "iceberg/catalog/rest/error_handlers.h" #include "iceberg/catalog/rest/http_client.h" #include "iceberg/catalog/rest/resource_paths.h" +#include "iceberg/catalog/rest/rest_catalog.h" #include "iceberg/catalog/rest/rest_table.h" #include "iceberg/file_io.h" #include "iceberg/table_identifier.h" @@ -115,7 +114,7 @@ class RestCatalogLoadTableTest : public ::testing::Test { identifier_ = TableIdentifier{.ns = Namespace{{"default"}}, .name = "my_table"}; - all_plan_endpoints_ = {Endpoint::LoadTable(), Endpoint::PlanTableScan(), + all_plan_endpoints_ = {Endpoint::LoadTable(), Endpoint::PlanTableScan(), Endpoint::FetchPlanningResult(), Endpoint::CancelPlanning(), Endpoint::FetchScanTasks()}; @@ -151,8 +150,7 @@ class RestCatalogLoadTableTest : public ::testing::Test { // -------------------------------------------------------------------------- TEST_F(RestCatalogLoadTableTest, TableConfigServerScanReturnsRestTable) { EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce( - Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); + .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(all_plan_endpoints_)); ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); @@ -185,8 +183,7 @@ TEST_F(RestCatalogLoadTableTest, ClientConfigServerScanReturnsRestTable) { // -------------------------------------------------------------------------- TEST_F(RestCatalogLoadTableTest, ServerScanWithMissingEndpointReturnsNotSupported) { EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce( - Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); + .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(no_plan_endpoint_set_)); ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index b3492004f4..e3c61e21c2 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -62,8 +62,8 @@ MATCHER_P2(JsonBodyHas, key, expected_value, "") { } nlohmann::json expected = expected_value; if (json.at(key) != expected) { - *result_listener << "JSON[\"" << key << "\"] = " << json.at(key) - << ", expected " << expected; + *result_listener << "JSON[\"" << key << "\"] = " << json.at(key) << ", expected " + << expected; return false; } return true; @@ -407,8 +407,9 @@ TEST_F(RestTableScanTest, UseSnapshotPropagatesUseSnapshotSchemaInContext) { constexpr std::string_view kResponseBody = R"({"status":"completed"})"; EXPECT_CALL(*mock_client_, - Post(_, testing::AllOf(JsonBodyHas("snapshot-id", kSnapshotId), - JsonBodyHas("use-snapshot-schema", true)), + Post(_, + testing::AllOf(JsonBodyHas("snapshot-id", kSnapshotId), + JsonBodyHas("use-snapshot-schema", true)), _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); @@ -428,8 +429,9 @@ TEST_F(RestTableScanTest, DefaultScanDoesNotSetUseSnapshotSchema) { constexpr std::string_view kResponseBody = R"({"status":"completed"})"; EXPECT_CALL(*mock_client_, - Post(_, testing::AllOf(JsonBodyHas("use-snapshot-schema", false), - JsonBodyLacks("snapshot-id")), + Post(_, + testing::AllOf(JsonBodyHas("use-snapshot-schema", false), + JsonBodyLacks("snapshot-id")), _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); @@ -812,8 +814,7 @@ class RestIncrementalAppendScanTest : public RestTableScanTest { // snapshot range. Result> MakeIncrementalScan( RestScanContext ctx, std::optional from_snapshot_id = std::nullopt, - bool from_inclusive = false, - std::optional to_snapshot_id = std::nullopt) { + bool from_inclusive = false, std::optional to_snapshot_id = std::nullopt) { RestIncrementalAppendScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, std::move(ctx)); if (from_snapshot_id.has_value()) { @@ -892,8 +893,8 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesFailed) { // PlanFiles: PlanTableScan endpoint missing → NotSupported. // -------------------------------------------------------------------------- TEST_F(RestIncrementalAppendScanTest, PlanFilesEndpointNotSupported) { - ICEBERG_UNWRAP_OR_FAIL(auto scan, - MakeIncrementalScan(MakeContext(std::unordered_set{}))); + ICEBERG_UNWRAP_OR_FAIL( + auto scan, MakeIncrementalScan(MakeContext(std::unordered_set{}))); auto result = scan->PlanFiles(); EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); } @@ -922,10 +923,9 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesEmptyWhenNoCurrentSnapshot) { EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); // Use Make() directly to bypass builder validation that requires a snapshot. - ICEBERG_UNWRAP_OR_FAIL( - auto scan, RestIncrementalAppendScan::Make(empty_metadata, schema_, file_io_, - internal::TableScanContext{}, - MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto scan, RestIncrementalAppendScan::Make( + empty_metadata, schema_, file_io_, + internal::TableScanContext{}, MakeContext())); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); } @@ -937,7 +937,8 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesEmptyWhenNoCurrentSnapshot) { TEST_F(RestIncrementalAppendScanTest, PlanFilesWithExplicitToSnapshotId) { constexpr int64_t kToSnapshotId = 1000L; constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, Post(_, JsonBodyHas("end-snapshot-id", kToSnapshotId), _, _, _)) + EXPECT_CALL(*mock_client_, + Post(_, JsonBodyHas("end-snapshot-id", kToSnapshotId), _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); ICEBERG_UNWRAP_OR_FAIL( @@ -955,13 +956,14 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdExclusive) { constexpr int64_t kCurrentSnapshotId = 1000L; constexpr std::string_view kResponseBody = R"({"status":"completed"})"; EXPECT_CALL(*mock_client_, - Post(_, testing::AllOf(JsonBodyHas("start-snapshot-id", kFromSnapshotId), - JsonBodyHas("end-snapshot-id", kCurrentSnapshotId)), + Post(_, + testing::AllOf(JsonBodyHas("start-snapshot-id", kFromSnapshotId), + JsonBodyHas("end-snapshot-id", kCurrentSnapshotId)), _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - ICEBERG_UNWRAP_OR_FAIL( - auto scan, MakeIncrementalScan(MakeContext(), kFromSnapshotId, /*inclusive=*/false)); + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext(), kFromSnapshotId, + /*inclusive=*/false)); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); } @@ -975,8 +977,9 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveNoPare constexpr int64_t kFromSnapshotId = 1000L; constexpr std::string_view kResponseBody = R"({"status":"completed"})"; EXPECT_CALL(*mock_client_, - Post(_, testing::AllOf(JsonBodyLacks("start-snapshot-id"), - JsonBodyHas("end-snapshot-id", kFromSnapshotId)), + Post(_, + testing::AllOf(JsonBodyLacks("start-snapshot-id"), + JsonBodyHas("end-snapshot-id", kFromSnapshotId)), _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); @@ -1054,8 +1057,9 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveWithPa constexpr std::string_view kResponseBody = R"({"status":"completed"})"; EXPECT_CALL(*mock_client_, - Post(_, testing::AllOf(JsonBodyHas("start-snapshot-id", kParentSnapshotId), - JsonBodyHas("end-snapshot-id", kCurrentSnapshotId)), + Post(_, + testing::AllOf(JsonBodyHas("start-snapshot-id", kParentSnapshotId), + JsonBodyHas("end-snapshot-id", kCurrentSnapshotId)), _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); From ad31c277bad0047b0e11fce5a743eae9204572a5 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Tue, 29 Sep 2026 23:05:11 -0700 Subject: [PATCH 17/36] style: use raw string literal in rest_catalog_unit_test --- src/iceberg/test/rest_catalog_unit_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iceberg/test/rest_catalog_unit_test.cc b/src/iceberg/test/rest_catalog_unit_test.cc index 26a5e7b18e..b23d92a77f 100644 --- a/src/iceberg/test/rest_catalog_unit_test.cc +++ b/src/iceberg/test/rest_catalog_unit_test.cc @@ -91,7 +91,7 @@ constexpr std::string_view kBaseMetadataJson = // LoadTable response where server config requests server-side scan planning. const std::string kLoadTableServerScanResponse = - std::string("{\"config\":{\"scan-planning-mode\":\"server\"},") + + std::string(R"({"config":{"scan-planning-mode":"server"},)") + std::string(kBaseMetadataJson) + "}"; // LoadTable response with no scan-planning-mode set (defaults to client). From 0ca7b8c38b8a2a6e83178eb2a57898a84eb64dad Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 17:41:36 -0700 Subject: [PATCH 18/36] fix(rest): resolve incremental append scan range via base class Override the two-arg PlanFiles so the snapshot range is resolved by ResolvePlanFiles, which uses the branch head when UseBranch() is set and validates explicit start/end snapshots against it. Return NotSupported when there is no start snapshot, since the REST API requires both start-snapshot-id and end-snapshot-id. --- src/iceberg/catalog/rest/rest_table_scan.cc | 36 +--- src/iceberg/catalog/rest/rest_table_scan.h | 9 +- src/iceberg/test/rest_table_scan_test.cc | 227 ++++++++++++++------ 3 files changed, 185 insertions(+), 87 deletions(-) diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index b321f4251c..3d21dbb505 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -31,12 +31,10 @@ #include "iceberg/catalog/rest/resource_paths.h" #include "iceberg/catalog/rest/rest_file_io.h" #include "iceberg/catalog/rest/types.h" -#include "iceberg/constants.h" #include "iceberg/json_serde_internal.h" #include "iceberg/partition_spec.h" #include "iceberg/result.h" #include "iceberg/schema.h" -#include "iceberg/snapshot.h" #include "iceberg/table_metadata.h" #include "iceberg/util/macros.h" @@ -484,8 +482,14 @@ Result> RestIncrementalAppendScan::Make( std::move(context), std::move(rest_context))); } -Result>> RestIncrementalAppendScan::PlanFiles() - const { +Result>> RestIncrementalAppendScan::PlanFiles( + std::optional from_snapshot_id_exclusive, + int64_t to_snapshot_id_inclusive) const { + // The REST API requires start-snapshot-id for incremental scans. + if (!from_snapshot_id_exclusive.has_value()) { + return NotSupported("REST incremental append scan requires a start snapshot"); + } + scan_io_ = nullptr; // reset so stale credentials from a prior plan are not reused TableMetadataCache metadata_cache(metadata_.get()); ICEBERG_ASSIGN_OR_RAISE(auto specs, metadata_cache.GetPartitionSpecsById()); @@ -495,27 +499,9 @@ Result>> RestIncrementalAppendScan::Pl request.filter = context_.filter; request.case_sensitive = context_.case_sensitive; request.min_rows_requested = context_.min_rows_requested; - - // Resolve end snapshot: use to_snapshot_id if set, else current table snapshot. - if (context_.to_snapshot_id.has_value()) { - request.end_snapshot_id = context_.to_snapshot_id; - } else { - if (metadata_->current_snapshot_id == kInvalidSnapshotId) return {}; - ICEBERG_ASSIGN_OR_RAISE(auto snapshot, metadata_->Snapshot()); - if (!snapshot) return {}; - request.end_snapshot_id = snapshot->snapshot_id; - } - - // Resolve start snapshot (exclusive): respect from_snapshot_id_inclusive. - if (context_.from_snapshot_id.has_value()) { - if (context_.from_snapshot_id_inclusive) { - ICEBERG_ASSIGN_OR_RAISE(auto from_snap, - metadata_->SnapshotById(*context_.from_snapshot_id)); - request.start_snapshot_id = from_snap->parent_snapshot_id; - } else { - request.start_snapshot_id = context_.from_snapshot_id; - } - } + request.start_snapshot_id = from_snapshot_id_exclusive; + request.end_snapshot_id = to_snapshot_id_inclusive; + request.use_snapshot_schema = true; if (!context_.columns_to_keep_stats.empty()) { for (int32_t field_id : context_.columns_to_keep_stats) { diff --git a/src/iceberg/catalog/rest/rest_table_scan.h b/src/iceberg/catalog/rest/rest_table_scan.h index 41c4f8d7f1..0c05d619e7 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.h +++ b/src/iceberg/catalog/rest/rest_table_scan.h @@ -19,7 +19,9 @@ #pragma once +#include #include +#include #include #include #include @@ -124,8 +126,13 @@ class ICEBERG_REST_EXPORT RestIncrementalAppendScan : public IncrementalAppendSc std::shared_ptr io, internal::TableScanContext context, RestScanContext rest_context); + using IncrementalAppendScan::PlanFiles; + + protected: /// \brief Plans files via the REST scan planning endpoints. - Result>> PlanFiles() const override; + Result>> PlanFiles( + std::optional from_snapshot_id_exclusive, + int64_t to_snapshot_id_inclusive) const override; private: RestIncrementalAppendScan(std::shared_ptr metadata, diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index e3c61e21c2..6ea4744443 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -807,16 +807,52 @@ TEST_F(RestTableScanTest, SecondPlanFilesStreamCallClearsStaleCredentials) { // ========================================================================== // RestIncrementalAppendScan tests // ========================================================================== - +// Lineage: 100 <- 200 <- 300 (main), and 200 <- 400 (branch "audit"). class RestIncrementalAppendScanTest : public RestTableScanTest { protected: - // Creates a RestIncrementalAppendScan with the given context and optional - // snapshot range. + static constexpr int64_t kRootId = 100L; + static constexpr int64_t kMiddleId = 200L; + static constexpr int64_t kMainHeadId = 300L; + static constexpr int64_t kBranchHeadId = 400L; + static constexpr std::string_view kBranch = "audit"; + + void SetUp() override { + RestTableScanTest::SetUp(); + + auto make_snapshot = [&](int64_t id, std::optional parent_id, + int64_t sequence_number) { + return std::make_shared( + Snapshot{.snapshot_id = id, + .parent_snapshot_id = parent_id, + .sequence_number = sequence_number, + .timestamp_ms = TimePointMsFromUnixMs(1609459200000L + id), + .manifest_list = "/tmp/manifest-list.avro", + .schema_id = schema_->schema_id()}); + }; + auto make_branch = [](int64_t id) { + return std::make_shared( + SnapshotRef{.snapshot_id = id, .retention = SnapshotRef::Branch{}}); + }; + + metadata_->snapshots = {make_snapshot(kRootId, std::nullopt, 1L), + make_snapshot(kMiddleId, kRootId, 2L), + make_snapshot(kMainHeadId, kMiddleId, 3L), + make_snapshot(kBranchHeadId, kMiddleId, 4L)}; + metadata_->last_sequence_number = 4L; + metadata_->current_snapshot_id = kMainHeadId; + metadata_->refs = {{"main", make_branch(kMainHeadId)}, + {std::string(kBranch), make_branch(kBranchHeadId)}}; + } + + RestIncrementalAppendScanBuilder MakeBuilder(RestScanContext ctx) { + return RestIncrementalAppendScanBuilder(metadata_, file_io_, "test.my_table", nullptr, + std::move(ctx)); + } + Result> MakeIncrementalScan( - RestScanContext ctx, std::optional from_snapshot_id = std::nullopt, + RestScanContext ctx, std::optional from_snapshot_id = kRootId, bool from_inclusive = false, std::optional to_snapshot_id = std::nullopt) { - RestIncrementalAppendScanBuilder builder(metadata_, file_io_, "test.my_table", - nullptr, std::move(ctx)); + auto builder = MakeBuilder(std::move(ctx)); if (from_snapshot_id.has_value()) { builder.FromSnapshot(*from_snapshot_id, from_inclusive); } @@ -931,64 +967,166 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesEmptyWhenNoCurrentSnapshot) { } // -------------------------------------------------------------------------- -// Explicit to_snapshot_id: POST body must contain "end-snapshot-id" set to -// the given value, not the current table snapshot. +// POST body carries the resolved snapshot range. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesSendsResolvedRange) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, + Post(_, + testing::AllOf(JsonBodyHas("start-snapshot-id", kRootId), + JsonBodyHas("end-snapshot-id", kMainHeadId), + JsonBodyHas("use-snapshot-schema", true)), + _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// Explicit to_snapshot_id is sent as end-snapshot-id. // -------------------------------------------------------------------------- TEST_F(RestIncrementalAppendScanTest, PlanFilesWithExplicitToSnapshotId) { - constexpr int64_t kToSnapshotId = 1000L; constexpr std::string_view kResponseBody = R"({"status":"completed"})"; EXPECT_CALL(*mock_client_, - Post(_, JsonBodyHas("end-snapshot-id", kToSnapshotId), _, _, _)) + Post(_, + testing::AllOf(JsonBodyHas("start-snapshot-id", kRootId), + JsonBodyHas("end-snapshot-id", kMiddleId)), + _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - ICEBERG_UNWRAP_OR_FAIL( - auto scan, MakeIncrementalScan(MakeContext(), std::nullopt, false, kToSnapshotId)); + ICEBERG_UNWRAP_OR_FAIL(auto scan, + MakeIncrementalScan(MakeContext(), kRootId, + /*from_inclusive=*/false, kMiddleId)); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); } // -------------------------------------------------------------------------- -// Exclusive from_snapshot_id: POST body must pass from_snapshot_id directly -// as "start-snapshot-id" (exclusive), and current snapshot as "end-snapshot-id". +// Exclusive from_snapshot_id is sent as start-snapshot-id. // -------------------------------------------------------------------------- TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdExclusive) { - constexpr int64_t kFromSnapshotId = 999L; - constexpr int64_t kCurrentSnapshotId = 1000L; constexpr std::string_view kResponseBody = R"({"status":"completed"})"; EXPECT_CALL(*mock_client_, Post(_, - testing::AllOf(JsonBodyHas("start-snapshot-id", kFromSnapshotId), - JsonBodyHas("end-snapshot-id", kCurrentSnapshotId)), + testing::AllOf(JsonBodyHas("start-snapshot-id", kMiddleId), + JsonBodyHas("end-snapshot-id", kMainHeadId)), _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext(), kFromSnapshotId, - /*inclusive=*/false)); + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext(), kMiddleId, + /*from_inclusive=*/false)); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); } // -------------------------------------------------------------------------- -// Inclusive from_snapshot_id: parent snapshot is used as start_snapshot_id. -// The fixture snapshot (id=1000) has no parent, so "start-snapshot-id" is -// absent from the POST body. +// Inclusive from_snapshot_id sends its parent as start-snapshot-id. // -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveNoParent) { - constexpr int64_t kFromSnapshotId = 1000L; +TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveWithParent) { constexpr std::string_view kResponseBody = R"({"status":"completed"})"; EXPECT_CALL(*mock_client_, Post(_, - testing::AllOf(JsonBodyLacks("start-snapshot-id"), - JsonBodyHas("end-snapshot-id", kFromSnapshotId)), + testing::AllOf(JsonBodyHas("start-snapshot-id", kRootId), + JsonBodyHas("end-snapshot-id", kMainHeadId)), _, _, _)) .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext(), kMiddleId, + /*from_inclusive=*/true)); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// Inclusive from_snapshot_id on the root snapshot → NotSupported. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveNoParent) { + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); + ICEBERG_UNWRAP_OR_FAIL( - auto scan, MakeIncrementalScan(MakeContext(), kFromSnapshotId, /*inclusive=*/true)); + auto scan, MakeIncrementalScan(MakeContext(), kRootId, /*from_inclusive=*/true)); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); +} + +// -------------------------------------------------------------------------- +// No from_snapshot_id → NotSupported. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesWithoutFromSnapshotIdNotSupported) { + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); + + ICEBERG_UNWRAP_OR_FAIL( + auto scan, MakeIncrementalScan(MakeContext(), /*from_snapshot_id=*/std::nullopt)); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); +} + +// -------------------------------------------------------------------------- +// UseBranch() without to_snapshot_id sends the branch head as end-snapshot-id. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, UseBranchWithoutToSnapshotUsesBranchHead) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, + Post(_, + testing::AllOf(JsonBodyHas("start-snapshot-id", kRootId), + JsonBodyHas("end-snapshot-id", kBranchHeadId)), + _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + auto builder = MakeBuilder(MakeContext()); + builder.FromSnapshot(kRootId).UseBranch(std::string(kBranch)); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// UseBranch() with to_snapshot_id on the branch. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, UseBranchWithToSnapshotOnBranch) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, + Post(_, + testing::AllOf(JsonBodyHas("start-snapshot-id", kRootId), + JsonBodyHas("end-snapshot-id", kMiddleId)), + _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + auto builder = MakeBuilder(MakeContext()); + builder.FromSnapshot(kRootId).ToSnapshot(kMiddleId).UseBranch(std::string(kBranch)); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); } +// -------------------------------------------------------------------------- +// UseBranch() with to_snapshot_id not on the branch → ValidationFailed. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, UseBranchRejectsToSnapshotNotOnBranch) { + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); + + auto builder = MakeBuilder(MakeContext()); + builder.FromSnapshot(kRootId).ToSnapshot(kMainHeadId).UseBranch(std::string(kBranch)); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("not a valid snapshot on the current branch")); +} + +// -------------------------------------------------------------------------- +// from_snapshot_id not an ancestor of the end snapshot → ValidationFailed. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, RejectsFromSnapshotNotAncestorOfEnd) { + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext(), kBranchHeadId, + /*from_inclusive=*/false)); + auto result = scan->PlanFiles(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); +} + // -------------------------------------------------------------------------- // PlanFiles: FAILED with plan-id → DELETE /plan called before returning the // error (tests the ExecuteScanPlan kFailed cancel fix). @@ -1036,39 +1174,6 @@ TEST_F(RestIncrementalAppendScanTest, SecondPlanFilesCallClearsStaleCredentials) EXPECT_TRUE(tasks2.empty()); } -// -------------------------------------------------------------------------- -// Inclusive from_snapshot_id with a parent: POST body must use the parent's id -// as "start-snapshot-id" and the current snapshot as "end-snapshot-id". -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveWithParent) { - constexpr int64_t kParentSnapshotId = 900L; - constexpr int64_t kChildSnapshotId = 1001L; - constexpr int64_t kCurrentSnapshotId = 1000L; - - // Add a second snapshot with a parent to the metadata. - auto child_snapshot = std::make_shared( - Snapshot{.snapshot_id = kChildSnapshotId, - .parent_snapshot_id = kParentSnapshotId, - .sequence_number = 2L, - .timestamp_ms = TimePointMsFromUnixMs(1609459260000L), - .manifest_list = "/tmp/manifest-list-2.avro", - .schema_id = schema_->schema_id()}); - metadata_->snapshots.push_back(child_snapshot); - - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, - Post(_, - testing::AllOf(JsonBodyHas("start-snapshot-id", kParentSnapshotId), - JsonBodyHas("end-snapshot-id", kCurrentSnapshotId)), - _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext(), kChildSnapshotId, - /*inclusive=*/true)); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - // ========================================================================== // RestTable::NewIncrementalAppendScan // ========================================================================== From 24fb9da2287fcccb832f5d0376bf90b6d7a2aed1 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 17:44:36 -0700 Subject: [PATCH 19/36] fix(rest): expose vended credentials via RestIncrementalAppendScan::io() Override io() so callers read incremental scan tasks with the FileIO built from storage credentials returned during planning, falling back to the table's FileIO when none are returned. --- src/iceberg/catalog/rest/rest_table_scan.cc | 4 ++ src/iceberg/catalog/rest/rest_table_scan.h | 4 ++ src/iceberg/test/rest_table_scan_test.cc | 59 +++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 3d21dbb505..36d871d3cf 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -515,6 +515,10 @@ Result>> RestIncrementalAppendScan::Pl return ExecuteScanPlan(rest_context_, *schema_, std::move(request), specs, scan_io_); } +const std::shared_ptr& RestIncrementalAppendScan::io() const { + return scan_io_ ? scan_io_ : io_; +} + // --------------------------------------------------------------------------- // RestIncrementalAppendScanBuilder // --------------------------------------------------------------------------- diff --git a/src/iceberg/catalog/rest/rest_table_scan.h b/src/iceberg/catalog/rest/rest_table_scan.h index 0c05d619e7..d5e2e5c7b0 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.h +++ b/src/iceberg/catalog/rest/rest_table_scan.h @@ -128,6 +128,10 @@ class ICEBERG_REST_EXPORT RestIncrementalAppendScan : public IncrementalAppendSc using IncrementalAppendScan::PlanFiles; + /// \brief Returns the FileIO with vended credentials if the server returned any + /// during planning; otherwise returns the table's FileIO. + const std::shared_ptr& io() const override; + protected: /// \brief Plans files via the REST scan planning endpoints. Result>> PlanFiles( diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 6ea4744443..3f92c56b58 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -1167,11 +1167,70 @@ TEST_F(RestIncrementalAppendScanTest, SecondPlanFilesCallClearsStaleCredentials) // First call: server vends credentials. ICEBERG_UNWRAP_OR_FAIL(auto tasks1, scan->PlanFiles()); EXPECT_TRUE(tasks1.empty()); + EXPECT_NE(scan->io().get(), file_io_.get()); // Second call: no credentials. Verifies scan_io_ was cleared so stale // credentials from the first plan do not bleed into the second request. ICEBERG_UNWRAP_OR_FAIL(auto tasks2, scan->PlanFiles()); EXPECT_TRUE(tasks2.empty()); + EXPECT_EQ(scan->io().get(), file_io_.get()); +} + +// -------------------------------------------------------------------------- +// Storage credentials in COMPLETED response: io() returns a credential-scoped IO. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, + StorageCredentialsInPlanResponseUpdatesEffectiveIO) { + constexpr std::string_view kResponseBody = R"({ + "status": "completed", + "storage-credentials": [ + {"prefix": "s3://bucket/prefix", "config": {"key": "value"}} + ] + })"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); + EXPECT_NE(scan->io().get(), file_io_.get()); +} + +// -------------------------------------------------------------------------- +// No storage credentials: io() falls back to the table's FileIO. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, NoStorageCredentialsEffectiveIoFallsBackToTableIO) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); + EXPECT_EQ(scan->io().get(), file_io_.get()); +} + +// -------------------------------------------------------------------------- +// Storage credentials in FetchScanTasksResponse also update io(). +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, + StorageCredentialsInFetchScanTasksResponseUpdatesEffectiveIO) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"incr-plan-1","plan-tasks":["tok-incr-1"]})"; + constexpr std::string_view kTasksResponse = R"({ + "file-scan-tasks": [], + "storage-credentials": [ + {"prefix": "s3://bucket/prefix", "config": {"key": "value"}} + ] + })"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); + EXPECT_NE(scan->io().get(), file_io_.get()); } // ========================================================================== From 4cdd9965790ac86737433eb5dfa4d40390d656bc Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 20:21:28 -0700 Subject: [PATCH 20/36] fix(rest): use table schema for branch scans and keep RestTable after commit ResolveSnapshotSchema() now honors use_snapshot_schema, so branch scans use the current table schema while tags and time travel use the snapshot schema, matching Java's SnapshotUtil.schemaFor(table, ref). MakeTableFromCommitResponse() now goes through the same scan planning mode resolution as LoadTable, so committing through a server-planning table returns a RestTable instead of a plain Table. --- src/iceberg/catalog/rest/rest_catalog.cc | 66 ++++++++++++-------- src/iceberg/catalog/rest/rest_catalog.h | 8 +++ src/iceberg/table_scan.cc | 2 +- src/iceberg/table_scan.h | 2 +- src/iceberg/test/rest_catalog_unit_test.cc | 72 +++++++++++++++++++++- src/iceberg/test/rest_table_scan_test.cc | 69 +++++++++++++++++++++ src/iceberg/test/table_scan_test.cc | 18 +++++- 7 files changed, 205 insertions(+), 32 deletions(-) diff --git a/src/iceberg/catalog/rest/rest_catalog.cc b/src/iceberg/catalog/rest/rest_catalog.cc index b12b2ff795..d919d1892a 100644 --- a/src/iceberg/catalog/rest/rest_catalog.cc +++ b/src/iceberg/catalog/rest/rest_catalog.cc @@ -992,12 +992,10 @@ Result> RestCatalog::MakeTableFromLoadResult( auto table_catalog = std::make_shared( shared_from_this(), context, identifier, table_config, table_session, table_io); - // Determine effective scan planning mode: table config overrides client config. ICEBERG_ASSIGN_OR_RAISE(auto client_mode, RestCatalogProperties::ScanPlanningModeFrom(config_.configs())); ICEBERG_ASSIGN_OR_RAISE(auto server_mode, RestCatalogProperties::ScanPlanningModeFrom(table_config)); - if (client_mode.has_value() && server_mode.has_value() && *client_mode != *server_mode) { Log(LogLevel::kWarn, @@ -1008,6 +1006,38 @@ Result> RestCatalog::MakeTableFromLoadResult( *server_mode == ScanPlanningMode::kClient ? "client" : "server"); } + return MakeTable(identifier, std::move(result.metadata), + std::move(result.metadata_location), std::move(table_io), + std::move(table_catalog), std::move(reporter), table_config, + std::move(table_session)); +} + +Result> RestCatalog::MakeTableFromCommitResponse( + const TableIdentifier& identifier, CommitTableResponse response, + const SessionContext& context, + const std::unordered_map& table_config, + std::shared_ptr table_session, std::shared_ptr table_io) { + ICEBERG_ASSIGN_OR_RAISE(auto reporter, MakeTableReporter(identifier, table_session)); + // Reuse the bound FileIO because commit responses carry no config or credentials. + auto table_catalog = std::make_shared( + shared_from_this(), context, identifier, table_config, table_session, table_io); + return MakeTable(identifier, std::move(response.metadata), + std::move(response.metadata_location), std::move(table_io), + std::move(table_catalog), std::move(reporter), table_config, + std::move(table_session)); +} + +Result> RestCatalog::MakeTable( + const TableIdentifier& identifier, std::shared_ptr metadata, + std::string metadata_location, std::shared_ptr table_io, + std::shared_ptr table_catalog, std::shared_ptr reporter, + const std::unordered_map& table_config, + std::shared_ptr table_session) { + // Table config overrides client config. + ICEBERG_ASSIGN_OR_RAISE(auto client_mode, + RestCatalogProperties::ScanPlanningModeFrom(config_.configs())); + ICEBERG_ASSIGN_OR_RAISE(auto server_mode, + RestCatalogProperties::ScanPlanningModeFrom(table_config)); ScanPlanningMode effective_mode = server_mode.value_or(client_mode.value_or(ScanPlanningMode::kClient)); @@ -1021,37 +1051,21 @@ Result> RestCatalog::MakeTableFromLoadResult( RestScanContext rest_ctx{ .client = client_, .paths = paths_, - .session = table_session, + .session = std::move(table_session), .supported_endpoints = supported_endpoints_, .identifier = identifier, .catalog_config = config_.configs(), .table_config = table_config, }; - return RestTable::Make(identifier, std::move(result.metadata), - std::move(result.metadata_location), std::move(table_io), - std::move(table_catalog), RestTableName(name_, identifier), - reporter, std::move(rest_ctx)); + return RestTable::Make(identifier, std::move(metadata), std::move(metadata_location), + std::move(table_io), std::move(table_catalog), + RestTableName(name_, identifier), std::move(reporter), + std::move(rest_ctx)); } - return Table::Make(identifier, std::move(result.metadata), - std::move(result.metadata_location), std::move(table_io), - std::move(table_catalog), RestTableName(name_, identifier), - std::move(reporter)); -} - -Result> RestCatalog::MakeTableFromCommitResponse( - const TableIdentifier& identifier, CommitTableResponse response, - const SessionContext& context, - const std::unordered_map& table_config, - std::shared_ptr table_session, std::shared_ptr table_io) { - ICEBERG_ASSIGN_OR_RAISE(auto reporter, MakeTableReporter(identifier, table_session)); - // Reuse the bound FileIO because commit responses carry no config or credentials. - auto table_catalog = std::make_shared( - shared_from_this(), context, identifier, table_config, table_session, table_io); - return Table::Make(identifier, std::move(response.metadata), - std::move(response.metadata_location), std::move(table_io), - std::move(table_catalog), RestTableName(name_, identifier), - std::move(reporter)); + return Table::Make(identifier, std::move(metadata), std::move(metadata_location), + std::move(table_io), std::move(table_catalog), + RestTableName(name_, identifier), std::move(reporter)); } } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/rest_catalog.h b/src/iceberg/catalog/rest/rest_catalog.h index 37431ff32d..7b82ab47cd 100644 --- a/src/iceberg/catalog/rest/rest_catalog.h +++ b/src/iceberg/catalog/rest/rest_catalog.h @@ -205,6 +205,14 @@ class ICEBERG_REST_EXPORT RestCatalog final const std::unordered_map& table_config, std::shared_ptr table_session, std::shared_ptr table_io); + /// \brief Returns a RestTable when scan planning mode is server, else a Table. + Result> MakeTable( + const TableIdentifier& identifier, std::shared_ptr metadata, + std::string metadata_location, std::shared_ptr table_io, + std::shared_ptr table_catalog, std::shared_ptr reporter, + const std::unordered_map& table_config, + std::shared_ptr table_session); + RestCatalogProperties config_; std::shared_ptr file_io_; std::shared_ptr client_; diff --git a/src/iceberg/table_scan.cc b/src/iceberg/table_scan.cc index d8bcda903f..636834d622 100644 --- a/src/iceberg/table_scan.cc +++ b/src/iceberg/table_scan.cc @@ -437,7 +437,7 @@ template Result>> TableScanBuilder::ResolveSnapshotSchema() { if (snapshot_schema_ == nullptr) { - if (context_.snapshot_id.has_value()) { + if (context_.snapshot_id.has_value() && context_.use_snapshot_schema) { ICEBERG_ASSIGN_OR_RAISE(auto snapshot, metadata_->SnapshotById(*context_.snapshot_id)); int32_t schema_id = snapshot->schema_id.value_or(Schema::kInitialSchemaId); diff --git a/src/iceberg/table_scan.h b/src/iceberg/table_scan.h index 5abb564b2c..abe93bbffd 100644 --- a/src/iceberg/table_scan.h +++ b/src/iceberg/table_scan.h @@ -413,7 +413,7 @@ class ICEBERG_TEMPLATE_CLASS_EXPORT TableScanBuilder : public ErrorCollector { TableScanBuilder(TableScanBuilder&&) = default; TableScanBuilder& operator=(TableScanBuilder&&) = default; - // Return the schema bound to the specified snapshot. + // Return the snapshot schema for tags and time travel, else the table schema. Result>> ResolveSnapshotSchema(); Status ResolveColumnStatsSelection(); diff --git a/src/iceberg/test/rest_catalog_unit_test.cc b/src/iceberg/test/rest_catalog_unit_test.cc index b23d92a77f..edd840a1ba 100644 --- a/src/iceberg/test/rest_catalog_unit_test.cc +++ b/src/iceberg/test/rest_catalog_unit_test.cc @@ -25,6 +25,7 @@ #include #include +#include "iceberg/catalog.h" #include "iceberg/catalog/rest/catalog_properties.h" #include "iceberg/catalog/rest/endpoint.h" #include "iceberg/catalog/rest/error_handlers.h" @@ -32,8 +33,12 @@ #include "iceberg/catalog/rest/resource_paths.h" #include "iceberg/catalog/rest/rest_catalog.h" #include "iceberg/catalog/rest/rest_table.h" +#include "iceberg/catalog/rest/rest_table_scan.h" #include "iceberg/file_io.h" +#include "iceberg/table.h" #include "iceberg/table_identifier.h" +#include "iceberg/table_requirement.h" +#include "iceberg/table_update.h" #include "iceberg/test/matchers.h" namespace iceberg::rest { @@ -98,6 +103,10 @@ const std::string kLoadTableServerScanResponse = const std::string kLoadTableDefaultResponse = std::string("{") + std::string(kBaseMetadataJson) + "}"; +// UpdateTable (commit) response. +const std::string kCommitTableResponse = + std::string("{") + std::string(kBaseMetadataJson) + "}"; + // -------------------------------------------------------------------------- // Test fixture: builds a RestCatalog via MakeForTesting so no HTTP calls // are made during catalog construction. @@ -114,9 +123,9 @@ class RestCatalogLoadTableTest : public ::testing::Test { identifier_ = TableIdentifier{.ns = Namespace{{"default"}}, .name = "my_table"}; - all_plan_endpoints_ = {Endpoint::LoadTable(), Endpoint::PlanTableScan(), - Endpoint::FetchPlanningResult(), Endpoint::CancelPlanning(), - Endpoint::FetchScanTasks()}; + all_plan_endpoints_ = {Endpoint::LoadTable(), Endpoint::UpdateTable(), + Endpoint::PlanTableScan(), Endpoint::FetchPlanningResult(), + Endpoint::CancelPlanning(), Endpoint::FetchScanTasks()}; no_plan_endpoint_set_ = {Endpoint::LoadTable()}; } @@ -208,4 +217,61 @@ TEST_F(RestCatalogLoadTableTest, DefaultScanModeReturnsPlainTable) { << "Expected plain Table (not RestTable) when no scan-planning-mode is configured"; } +// -------------------------------------------------------------------------- +// Committing through a server-planning table returns a RestTable. +// -------------------------------------------------------------------------- +TEST_F(RestCatalogLoadTableTest, CommitPreservesRestTableFromTableConfig) { + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, kCommitTableResponse))); + + ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(all_plan_endpoints_)); + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); + ICEBERG_UNWRAP_OR_FAIL(auto committed, + table->catalog()->UpdateTable(identifier_, {}, {})); + + EXPECT_NE(dynamic_cast(committed.get()), nullptr); + ICEBERG_UNWRAP_OR_FAIL(auto builder, committed->NewScan()); + EXPECT_NE(dynamic_cast(builder.get()), nullptr); +} + +// -------------------------------------------------------------------------- +// Client config "scan-planning-mode":"server" → commit returns a RestTable. +// -------------------------------------------------------------------------- +TEST_F(RestCatalogLoadTableTest, CommitPreservesRestTableFromClientConfig) { + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableDefaultResponse))); + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, kCommitTableResponse))); + + ICEBERG_UNWRAP_OR_FAIL(auto catalog, + MakeCatalog(all_plan_endpoints_, /*client_scan_mode=*/"server")); + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); + ICEBERG_UNWRAP_OR_FAIL(auto committed, + table->catalog()->UpdateTable(identifier_, {}, {})); + + EXPECT_NE(dynamic_cast(committed.get()), nullptr); +} + +// -------------------------------------------------------------------------- +// Default scan mode → commit returns a plain Table. +// -------------------------------------------------------------------------- +TEST_F(RestCatalogLoadTableTest, CommitWithDefaultScanModeReturnsPlainTable) { + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableDefaultResponse))); + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, kCommitTableResponse))); + + ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(all_plan_endpoints_)); + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); + ICEBERG_UNWRAP_OR_FAIL(auto committed, + table->catalog()->UpdateTable(identifier_, {}, {})); + + EXPECT_EQ(dynamic_cast(committed.get()), nullptr); +} + } // namespace iceberg::rest diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 3f92c56b58..3689f6ad76 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -442,6 +442,75 @@ TEST_F(RestTableScanTest, DefaultScanDoesNotSetUseSnapshotSchema) { EXPECT_TRUE(tasks.empty()); } +// -------------------------------------------------------------------------- +// UseRef(): branches use the table schema, tags use the snapshot schema. +// -------------------------------------------------------------------------- +class RestTableScanRefSchemaTest : public RestTableScanTest { + protected: + static constexpr int64_t kEvolvedSnapshotId = 2000L; + + void SetUp() override { + RestTableScanTest::SetUp(); + evolved_schema_ = std::make_shared( + std::vector{SchemaField::MakeRequired(10, "id", int32()), + SchemaField::MakeRequired(11, "data", string())}, + /*schema_id=*/1); + metadata_->schemas.push_back(evolved_schema_); + metadata_->last_column_id = 11; + metadata_->snapshots.push_back(std::make_shared( + Snapshot{.snapshot_id = kEvolvedSnapshotId, + .parent_snapshot_id = 1000L, + .sequence_number = 2L, + .timestamp_ms = TimePointMsFromUnixMs(1609459201000L), + .manifest_list = "/tmp/manifest-list-2.avro", + .schema_id = evolved_schema_->schema_id()})); + metadata_->refs["evolved-branch"] = std::make_shared(SnapshotRef{ + .snapshot_id = kEvolvedSnapshotId, .retention = SnapshotRef::Branch{}}); + metadata_->refs["evolved-tag"] = std::make_shared( + SnapshotRef{.snapshot_id = kEvolvedSnapshotId, .retention = SnapshotRef::Tag{}}); + } + + std::shared_ptr evolved_schema_; +}; + +TEST_F(RestTableScanRefSchemaTest, UseRefBranchUsesTableSchema) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, + Post(_, + testing::AllOf(JsonBodyHas("snapshot-id", kEvolvedSnapshotId), + JsonBodyHas("use-snapshot-schema", false)), + _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext()); + builder.UseRef("evolved-branch"); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto scan_schema, scan->schema()); + EXPECT_EQ(scan_schema->schema_id(), schema_->schema_id()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +TEST_F(RestTableScanRefSchemaTest, UseRefTagUsesSnapshotSchema) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, + Post(_, + testing::AllOf(JsonBodyHas("snapshot-id", kEvolvedSnapshotId), + JsonBodyHas("use-snapshot-schema", true)), + _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext()); + builder.UseRef("evolved-tag"); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto scan_schema, scan->schema()); + EXPECT_EQ(scan_schema->schema_id(), evolved_schema_->schema_id()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + // -------------------------------------------------------------------------- // Storage credentials in COMPLETED response: io() returns a // credential-scoped IO, not the original table IO. diff --git a/src/iceberg/test/table_scan_test.cc b/src/iceberg/test/table_scan_test.cc index 1214a05360..fa713945b3 100644 --- a/src/iceberg/test/table_scan_test.cc +++ b/src/iceberg/test/table_scan_test.cc @@ -252,6 +252,8 @@ TEST_P(TableScanTest, IncludeColumnStatsUsesFinalSnapshotSchema) { .schema_id = evolved_schema->schema_id()})); table_metadata_->refs["evolved-branch"] = std::make_shared( SnapshotRef{.snapshot_id = kEvolvedSnapshotId, .retention = SnapshotRef::Branch{}}); + table_metadata_->refs["evolved-tag"] = std::make_shared( + SnapshotRef{.snapshot_id = kEvolvedSnapshotId, .retention = SnapshotRef::Tag{}}); { ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); @@ -268,7 +270,7 @@ TEST_P(TableScanTest, IncludeColumnStatsUsesFinalSnapshotSchema) { { ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); - builder->IncludeColumnStats({"id"}).UseRef("evolved-branch"); + builder->IncludeColumnStats({"id"}).UseRef("evolved-tag"); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); ICEBERG_UNWRAP_OR_FAIL(auto scan_schema, scan->schema()); @@ -278,6 +280,20 @@ TEST_P(TableScanTest, IncludeColumnStatsUsesFinalSnapshotSchema) { EXPECT_TRUE(stats_fields.contains(kEvolvedIdFieldId)); EXPECT_FALSE(stats_fields.contains(kBaseIdFieldId)); } + + // Branches use the table schema. + { + ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); + builder->IncludeColumnStats({"id"}).UseRef("evolved-branch"); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); + ICEBERG_UNWRAP_OR_FAIL(auto scan_schema, scan->schema()); + + EXPECT_EQ(scan_schema->schema_id(), schema_->schema_id()); + const auto& stats_fields = scan->context().columns_to_keep_stats; + EXPECT_EQ(stats_fields.size(), 1); + EXPECT_TRUE(stats_fields.contains(kBaseIdFieldId)); + EXPECT_FALSE(stats_fields.contains(kEvolvedIdFieldId)); + } } TEST_P(TableScanTest, IncludeColumnStatsRejectsMissingColumn) { From 7eb96b3004422bb4ca0e399f2992c9dd01eeebc9 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 20:38:00 -0700 Subject: [PATCH 21/36] fix(rest): send stats-fields and projected select in scan plan requests REST scan builders now resolve IncludeColumnStats() columns, so stats-fields is sent and unknown columns are rejected. The plan request's select is derived from the scan's projected schema, matching Java's RESTTableScan, so Project() is honored, filter columns are included, and Select("*") expands to all columns. --- src/iceberg/catalog/rest/rest_table_scan.cc | 27 +++- src/iceberg/test/rest_table_scan_test.cc | 165 ++++++++++++++++++++ 2 files changed, 190 insertions(+), 2 deletions(-) diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 36d871d3cf..2bf2839ff4 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -19,6 +19,7 @@ #include "iceberg/catalog/rest/rest_table_scan.h" +#include #include #include @@ -37,6 +38,7 @@ #include "iceberg/schema.h" #include "iceberg/table_metadata.h" #include "iceberg/util/macros.h" +#include "iceberg/util/type_util.h" namespace iceberg::rest { @@ -64,6 +66,23 @@ using ScanIoSlot = std::shared_ptr>; // Shared HTTP scan planning helpers used by all REST scan implementations. // --------------------------------------------------------------------------- +// Column names of the projected schema, sent as the request's select. +Result> ProjectedColumnNames(const Schema& projected) { + ICEBERG_ASSIGN_OR_RAISE(auto id_set, GetProjectedIdsVisitor::GetProjectedIds( + projected, /*include_struct_ids=*/true)); + std::vector field_ids(id_set.begin(), id_set.end()); + std::ranges::sort(field_ids); + + std::vector names; + names.reserve(field_ids.size()); + for (int32_t field_id : field_ids) { + ICEBERG_ASSIGN_OR_RAISE(auto name, projected.FindColumnNameById(field_id)); + ICEBERG_CHECK(name.has_value(), "Projected field {} not found in schema", field_id); + names.emplace_back(*name); + } + return names; +} + Status ApplyStorageCredentials(const RestScanContext& ctx, const std::vector& credentials, std::shared_ptr& scan_io) { @@ -407,7 +426,8 @@ Result RestTableScan::PlanFilesStream() const { ICEBERG_ASSIGN_OR_RAISE(auto specs, metadata_cache.GetPartitionSpecsById()); PlanTableScanRequest request; - request.select = context_.selected_columns.value_or(std::vector{}); + ICEBERG_ASSIGN_OR_RAISE(auto projected_schema, schema()); + ICEBERG_ASSIGN_OR_RAISE(request.select, ProjectedColumnNames(*projected_schema)); request.filter = context_.filter; request.case_sensitive = context_.case_sensitive; request.min_rows_requested = context_.min_rows_requested; @@ -452,6 +472,7 @@ RestTableScanBuilder::RestTableScanBuilder( Result> RestTableScanBuilder::Build() { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); + ICEBERG_RETURN_UNEXPECTED(ResolveColumnStatsSelection()); ICEBERG_RETURN_UNEXPECTED(context_.Validate()); ICEBERG_ASSIGN_OR_RAISE(auto schema, ResolveSnapshotSchema()); return RestTableScan::Make(metadata_, schema.get(), io_, std::move(context_), @@ -495,7 +516,8 @@ Result>> RestIncrementalAppendScan::Pl ICEBERG_ASSIGN_OR_RAISE(auto specs, metadata_cache.GetPartitionSpecsById()); PlanTableScanRequest request; - request.select = context_.selected_columns.value_or(std::vector{}); + ICEBERG_ASSIGN_OR_RAISE(auto projected_schema, schema()); + ICEBERG_ASSIGN_OR_RAISE(request.select, ProjectedColumnNames(*projected_schema)); request.filter = context_.filter; request.case_sensitive = context_.case_sensitive; request.min_rows_requested = context_.min_rows_requested; @@ -533,6 +555,7 @@ RestIncrementalAppendScanBuilder::RestIncrementalAppendScanBuilder( Result> RestIncrementalAppendScanBuilder::Build() { ICEBERG_RETURN_UNEXPECTED(CheckErrors()); + ICEBERG_RETURN_UNEXPECTED(ResolveColumnStatsSelection()); ICEBERG_RETURN_UNEXPECTED(context_.Validate()); ICEBERG_ASSIGN_OR_RAISE(auto schema, ResolveSnapshotSchema()); return RestIncrementalAppendScan::Make(metadata_, schema.get(), io_, diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 3689f6ad76..58a2cea0e9 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -36,6 +36,8 @@ #include "iceberg/catalog/rest/resource_paths.h" #include "iceberg/catalog/rest/rest_table.h" #include "iceberg/constants.h" +#include "iceberg/expression/expressions.h" +#include "iceberg/expression/literal.h" #include "iceberg/file_io.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/partition_spec.h" @@ -442,6 +444,124 @@ TEST_F(RestTableScanTest, DefaultScanDoesNotSetUseSnapshotSchema) { EXPECT_TRUE(tasks.empty()); } +// -------------------------------------------------------------------------- +// IncludeColumnStats() columns are sent as stats-fields. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, IncludeColumnStatsSendsStatsFields) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL( + *mock_client_, + Post(_, JsonBodyHas("stats-fields", std::vector{"id"}), _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext()); + builder.IncludeColumnStats({"id"}); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// IncludeColumnStats() with an unknown column → ValidationFailed. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, IncludeColumnStatsRejectsMissingColumn) { + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext()); + builder.IncludeColumnStats({"missing"}); + EXPECT_THAT(builder.Build(), + ::testing::AllOf(IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Cannot find stats column: missing"))); +} + +// -------------------------------------------------------------------------- +// Project() columns are sent as select. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, ProjectSendsProjectedColumnsAsSelect) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, + Post(_, JsonBodyHas("select", std::vector{"id"}), _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext()); + builder.Project(std::make_shared( + std::vector{SchemaField::MakeRequired(1, "id", int32())})); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// Select() sends the selected columns plus filter columns. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, SelectIncludesFilterColumns) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL( + *mock_client_, + Post(_, JsonBodyHas("select", std::vector{"id", "data"}), _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext()); + builder.Select({"data"}).Filter(Expressions::Equal("id", Literal::Int(1))); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// Case-insensitive Select() sends the schema's column names. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, SelectResolvesCaseInsensitiveNames) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, + Post(_, JsonBodyHas("select", std::vector{"data"}), _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext()); + builder.CaseSensitive(false).Select({"DATA"}); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// Select("*") sends every column, not a literal "*". +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, SelectAllColumnsSendsAllColumns) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL( + *mock_client_, + Post(_, JsonBodyHas("select", std::vector{"id", "data"}), _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext()); + builder.Select({std::string(Schema::kAllColumns)}); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// No projection sends every column. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, NoProjectionSendsAllColumns) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL( + *mock_client_, + Post(_, JsonBodyHas("select", std::vector{"id", "data"}), _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext()); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + // -------------------------------------------------------------------------- // UseRef(): branches use the table schema, tags use the snapshot schema. // -------------------------------------------------------------------------- @@ -1196,6 +1316,51 @@ TEST_F(RestIncrementalAppendScanTest, RejectsFromSnapshotNotAncestorOfEnd) { EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); } +// -------------------------------------------------------------------------- +// IncludeColumnStats() columns are sent as stats-fields. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, IncludeColumnStatsSendsStatsFields) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL( + *mock_client_, + Post(_, JsonBodyHas("stats-fields", std::vector{"id"}), _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + auto builder = MakeBuilder(MakeContext()); + builder.FromSnapshot(kRootId).IncludeColumnStats({"id"}); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + +// -------------------------------------------------------------------------- +// IncludeColumnStats() with an unknown column → ValidationFailed. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, IncludeColumnStatsRejectsMissingColumn) { + auto builder = MakeBuilder(MakeContext()); + builder.FromSnapshot(kRootId).IncludeColumnStats({"missing"}); + EXPECT_THAT(builder.Build(), + ::testing::AllOf(IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Cannot find stats column: missing"))); +} + +// -------------------------------------------------------------------------- +// Project() columns are sent as select. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, ProjectSendsProjectedColumnsAsSelect) { + constexpr std::string_view kResponseBody = R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, + Post(_, JsonBodyHas("select", std::vector{"id"}), _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + + auto builder = MakeBuilder(MakeContext()); + builder.FromSnapshot(kRootId).Project(std::make_shared( + std::vector{SchemaField::MakeRequired(1, "id", int32())})); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); +} + // -------------------------------------------------------------------------- // PlanFiles: FAILED with plan-id → DELETE /plan called before returning the // error (tests the ExecuteScanPlan kFailed cancel fix). From 0e230fa50d365f37c7bb46b51485802b5ceaf4c1 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 20:47:08 -0700 Subject: [PATCH 22/36] fix(rest): keep plan tasks lazy for submitted plans in PlanFilesStream FetchPlanningResult now returns the completed response instead of resolving its plan tasks, so a SUBMITTED plan builds the same lazy stream as an immediate COMPLETED one. The stream also queues nested plan tasks returned by fetchScanTasks instead of resolving them eagerly, matching Java's ScanTaskIterable. --- src/iceberg/catalog/rest/rest_table_scan.cc | 61 +++++++--- src/iceberg/test/rest_table_scan_test.cc | 127 ++++++++++++++++++++ 2 files changed, 169 insertions(+), 19 deletions(-) diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 2bf2839ff4..5e90cc46fd 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -130,9 +130,12 @@ Result>> ResolveScanTasks( return result; } -Result>> FetchScanTasks( - const RestScanContext& ctx, const Schema& schema, const std::string& plan_task, - const SpecsById& specs, std::shared_ptr& scan_io) { +// Fetches one plan task without resolving any nested plan tasks it returns. +Result FetchScanTasksOnce(const RestScanContext& ctx, + const Schema& schema, + const std::string& plan_task, + const SpecsById& specs, + std::shared_ptr& scan_io) { ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::FetchScanTasks()); ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->FetchScanTasks(ctx.identifier)); @@ -148,12 +151,21 @@ Result>> FetchScanTasks( ICEBERG_RETURN_UNEXPECTED(result.Validate()); ICEBERG_RETURN_UNEXPECTED( ApplyStorageCredentials(ctx, result.storage_credentials, scan_io)); + return result; +} +Result>> FetchScanTasks( + const RestScanContext& ctx, const Schema& schema, const std::string& plan_task, + const SpecsById& specs, std::shared_ptr& scan_io) { + ICEBERG_ASSIGN_OR_RAISE(auto result, + FetchScanTasksOnce(ctx, schema, plan_task, specs, scan_io)); return ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs, scan_io); } -Result>> FetchPlanningResult( +// Polls until the plan completes and returns the completed response. Plan tasks +// are left unresolved. +Result FetchPlanningResult( const RestScanContext& ctx, const Schema& schema, const std::string& plan_id, const SpecsById& specs, std::shared_ptr& scan_io) { ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::FetchPlanningResult()); @@ -197,10 +209,7 @@ Result>> FetchPlanningResult( CancelPlanning(ctx, plan_id); return std::unexpected(s.error()); } - auto tasks = ResolveScanTasks(ctx, schema, result.plan_tasks, - result.file_scan_tasks, specs, scan_io); - if (!tasks) CancelPlanning(ctx, plan_id); - return tasks; + return std::move(result); } case PlanStatus::kSubmitted: { auto elapsed_ms = std::chrono::duration_cast( @@ -233,7 +242,8 @@ Result>> FetchPlanningResult( /// /// The eager POST /plan (or poll until COMPLETED) is done before this stream is /// constructed. The stream then yields the directly-returned file_scan_tasks first, -/// then fetches each plan_task token lazily one at a time as Next() is called. +/// then fetches each plan_task token lazily one at a time as Next() is called, +/// queuing any nested plan tasks returned by the server. /// /// scan_io_slot is shared with the owning RestTableScan so that credentials vended /// by any FetchScanTasks response are visible through RestTableScan::io() even after @@ -267,9 +277,15 @@ class RestFileScanTaskStream final : public FileScanTaskStream { consumed_ = true; return std::nullopt; } - const auto& token = plan_task_tokens_[token_pos_++]; + const std::string token = plan_task_tokens_[token_pos_++]; ICEBERG_ASSIGN_OR_RAISE( - buffer_, FetchScanTasks(ctx_, *schema_, token, specs_, *scan_io_slot_)); + auto result, FetchScanTasksOnce(ctx_, *schema_, token, specs_, *scan_io_slot_)); + if (result.plan_tasks.has_value()) { + plan_task_tokens_.insert(plan_task_tokens_.end(), result.plan_tasks->begin(), + result.plan_tasks->end()); + } + buffer_ = std::move(result.file_scan_tasks) + .value_or(std::vector>{}); buffer_pos_ = 0; } } @@ -310,13 +326,14 @@ Result ExecuteScanPlanStream( const std::string plan_id = result.plan_id; if (result.plan_status == PlanStatus::kSubmitted) { - // Poll until COMPLETED, eagerly collecting all tasks into the buffer. - std::string mutable_plan_id = plan_id; - ICEBERG_ASSIGN_OR_RAISE(auto tasks, FetchPlanningResult(ctx, schema, mutable_plan_id, - specs, *scan_io_slot)); + ICEBERG_ASSIGN_OR_RAISE( + auto completed, FetchPlanningResult(ctx, schema, plan_id, specs, *scan_io_slot)); return std::make_unique( - ctx, schema_ptr, mutable_plan_id, std::move(tasks), std::vector{}, - specs, scan_io_slot); + ctx, std::move(schema_ptr), plan_id, + std::move(completed.file_scan_tasks) + .value_or(std::vector>{}), + std::move(completed.plan_tasks).value_or(std::vector{}), specs, + std::move(scan_io_slot)); } if (result.plan_status == PlanStatus::kFailed) { @@ -380,8 +397,14 @@ Result>> ExecuteScanPlan( if (!tasks) CancelPlanning(ctx, plan_id); return tasks; } - case PlanStatus::kSubmitted: - return FetchPlanningResult(ctx, schema, plan_id, specs, scan_io); + case PlanStatus::kSubmitted: { + ICEBERG_ASSIGN_OR_RAISE(auto completed, + FetchPlanningResult(ctx, schema, plan_id, specs, scan_io)); + auto tasks = ResolveScanTasks(ctx, schema, completed.plan_tasks, + completed.file_scan_tasks, specs, scan_io); + if (!tasks) CancelPlanning(ctx, plan_id); + return tasks; + } case PlanStatus::kFailed: CancelPlanning(ctx, plan_id); return IOError("Scan planning failed: {}", diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 58a2cea0e9..67441dfafc 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -910,6 +910,133 @@ TEST_F(RestTableScanTest, PlanFilesStreamSubmittedThenCompleted) { EXPECT_TRUE(tasks.empty()); } +// -------------------------------------------------------------------------- +// PlanFilesStream: SUBMITTED → COMPLETED with plan tasks; tokens are fetched +// lazily as the stream is consumed. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamSubmittedFetchesTokensLazily) { + constexpr std::string_view kSubmittedBody = + R"({"status":"submitted","plan-id":"plan-poll-lazy"})"; + constexpr std::string_view kCompletedBody = + R"({"status":"completed","plan-tasks":["tok-1","tok-2"]})"; + constexpr std::string_view kTask1Response = R"({ + "file-scan-tasks": [ + {"data-file":{"content":"data","file-path":"s3://b/f1.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} + ] + })"; + constexpr std::string_view kTask2Response = R"({ + "file-scan-tasks": [ + {"data-file":{"content":"data","file-path":"s3://b/f2.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} + ] + })"; + + int fetch_count = 0; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))) + .WillOnce([&](auto&&...) -> Result { + ++fetch_count; + return HttpResponse::MakeForTesting(200, std::string(kTask1Response)); + }) + .WillOnce([&](auto&&...) -> Result { + ++fetch_count; + return HttpResponse::MakeForTesting(200, std::string(kTask2Response)); + }); + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kCompletedBody)))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + EXPECT_EQ(fetch_count, 0); + + ICEBERG_UNWRAP_OR_FAIL(auto t1, stream->Next()); + ASSERT_TRUE(t1.has_value()); + EXPECT_EQ(fetch_count, 1); + EXPECT_EQ((*t1)->data_file()->file_path, "s3://b/f1.parquet"); + + ICEBERG_UNWRAP_OR_FAIL(auto t2, stream->Next()); + ASSERT_TRUE(t2.has_value()); + EXPECT_EQ(fetch_count, 2); + EXPECT_EQ((*t2)->data_file()->file_path, "s3://b/f2.parquet"); + + ICEBERG_UNWRAP_OR_FAIL(auto end, stream->Next()); + EXPECT_FALSE(end.has_value()); +} + +// -------------------------------------------------------------------------- +// PlanFilesStream: SUBMITTED → COMPLETED, stream destroyed before its plan +// tasks are fetched → DELETE /plan called by the destructor. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamSubmittedCancelOnPartialConsumption) { + constexpr std::string_view kSubmittedBody = + R"({"status":"submitted","plan-id":"plan-poll-partial"})"; + constexpr std::string_view kCompletedBody = + R"({"status":"completed","plan-tasks":["tok-1"]})"; + + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kCompletedBody)))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + { + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + } +} + +// -------------------------------------------------------------------------- +// PlanFilesStream: nested plan tasks returned by FetchScanTasks are fetched +// only when the stream reaches them. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamFetchesNestedPlanTasksLazily) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-nested","plan-tasks":["tok-1"]})"; + constexpr std::string_view kTask1Response = R"({ + "plan-tasks": ["tok-nested"], + "file-scan-tasks": [ + {"data-file":{"content":"data","file-path":"s3://b/f1.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} + ] + })"; + constexpr std::string_view kNestedResponse = R"({ + "file-scan-tasks": [ + {"data-file":{"content":"data","file-path":"s3://b/f2.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} + ] + })"; + + int fetch_count = 0; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce([&](auto&&...) -> Result { + ++fetch_count; + return HttpResponse::MakeForTesting(200, std::string(kTask1Response)); + }) + .WillOnce([&](auto&&...) -> Result { + ++fetch_count; + return HttpResponse::MakeForTesting(200, std::string(kNestedResponse)); + }); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + + ICEBERG_UNWRAP_OR_FAIL(auto t1, stream->Next()); + ASSERT_TRUE(t1.has_value()); + EXPECT_EQ(fetch_count, 1); + EXPECT_EQ((*t1)->data_file()->file_path, "s3://b/f1.parquet"); + + ICEBERG_UNWRAP_OR_FAIL(auto t2, stream->Next()); + ASSERT_TRUE(t2.has_value()); + EXPECT_EQ(fetch_count, 2); + EXPECT_EQ((*t2)->data_file()->file_path, "s3://b/f2.parquet"); + + ICEBERG_UNWRAP_OR_FAIL(auto end, stream->Next()); + EXPECT_FALSE(end.has_value()); +} + // -------------------------------------------------------------------------- // PlanFilesStream: FAILED → stream returns an error. // -------------------------------------------------------------------------- From 300cc154350e53d4bb55d3ffcfc0b073e155d48f Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 20:50:22 -0700 Subject: [PATCH 23/36] docs(rest): document serial plan task fetching in REST scans REST scans fetch plan tasks serially and do not use an executor set via PlanWith(). Document this, add a TODO for concurrent fetching, and test that the executor is not used. --- src/iceberg/catalog/rest/rest_table_scan.cc | 2 ++ src/iceberg/catalog/rest/rest_table_scan.h | 4 +++ src/iceberg/test/rest_table_scan_test.cc | 37 +++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 5e90cc46fd..f6cee2c3f5 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -238,6 +238,8 @@ Result FetchPlanningResult( plan_id); } +// TODO(sandeepg): Fetch plan tasks concurrently when an executor is configured via +// PlanWith(), as Java's ScanTaskIterable does. /// A lazy stream that drives FetchScanTasks calls on demand. /// /// The eager POST /plan (or poll until COMPLETED) is done before this stream is diff --git a/src/iceberg/catalog/rest/rest_table_scan.h b/src/iceberg/catalog/rest/rest_table_scan.h index d5e2e5c7b0..1c3c4dc7a6 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.h +++ b/src/iceberg/catalog/rest/rest_table_scan.h @@ -77,6 +77,8 @@ class ICEBERG_REST_EXPORT RestTableScan : public DataTableScan { RestScanContext rest_context); /// \brief Plans files lazily via the REST scan planning endpoints. + /// + /// Plan tasks are fetched serially; an executor set via PlanWith() is not used. Result PlanFilesStream() const override; /// \brief Returns the effective FileIO for reading scan results. @@ -134,6 +136,8 @@ class ICEBERG_REST_EXPORT RestIncrementalAppendScan : public IncrementalAppendSc protected: /// \brief Plans files via the REST scan planning endpoints. + /// + /// Plan tasks are fetched serially; an executor set via PlanWith() is not used. Result>> PlanFiles( std::optional from_snapshot_id_exclusive, int64_t to_snapshot_id_inclusive) const override; diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 67441dfafc..24e8441651 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -46,6 +46,7 @@ #include "iceberg/table_identifier.h" #include "iceberg/table_metadata.h" #include "iceberg/table_scan.h" +#include "iceberg/test/executor.h" #include "iceberg/test/matchers.h" #include "iceberg/type.h" @@ -1037,6 +1038,42 @@ TEST_F(RestTableScanTest, PlanFilesStreamFetchesNestedPlanTasksLazily) { EXPECT_FALSE(end.has_value()); } +// -------------------------------------------------------------------------- +// PlanWith(): plan tasks are fetched serially and the executor is not used. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamIgnoresPlanExecutor) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-exec","plan-tasks":["tok-1","tok-2"]})"; + constexpr std::string_view kTask1Response = R"({ + "file-scan-tasks": [ + {"data-file":{"content":"data","file-path":"s3://b/f1.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} + ] + })"; + constexpr std::string_view kTask2Response = R"({ + "file-scan-tasks": [ + {"data-file":{"content":"data","file-path":"s3://b/f2.parquet", + "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} + ] + })"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTask1Response)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTask2Response)))); + + test::ThreadExecutor executor; + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + MakeContext()); + builder.PlanWith(executor); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + + ASSERT_EQ(tasks.size(), 2); + EXPECT_EQ(tasks[0]->data_file()->file_path, "s3://b/f1.parquet"); + EXPECT_EQ(tasks[1]->data_file()->file_path, "s3://b/f2.parquet"); + EXPECT_EQ(executor.submit_count(), 0); +} + // -------------------------------------------------------------------------- // PlanFilesStream: FAILED → stream returns an error. // -------------------------------------------------------------------------- From e1db3ca8737a24c0c2c5607b4690af23c0a9e436 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 20:53:02 -0700 Subject: [PATCH 24/36] fix(rest): cancel plan when the plan response fails validation PlanTableScanResponseFromJson validates the response, so a failure dropped the plan id before it could be cancelled. Read the plan id from the raw JSON first and cancel the plan if parsing or validation fails. --- src/iceberg/catalog/rest/rest_table_scan.cc | 28 +++++++++++--- src/iceberg/test/rest_table_scan_test.cc | 43 +++++++++++++++++++++ 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index f6cee2c3f5..dbcbbec0c8 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -321,9 +321,17 @@ Result ExecuteScanPlanStream( ctx.client->Post(path, json_request, /*headers=*/{}, *PlanErrorHandler::Instance(), *ctx.session)); ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); - ICEBERG_ASSIGN_OR_RAISE(auto result, - PlanTableScanResponseFromJson(json, specs, schema)); - ICEBERG_RETURN_UNEXPECTED(result.Validate()); + // The parser validates the response, so read the plan id first to cancel on failure. + std::string raw_plan_id; + if (auto it = json.find("plan-id"); it != json.end() && it->is_string()) { + raw_plan_id = it->get(); + } + auto result_or = PlanTableScanResponseFromJson(json, specs, schema); + if (!result_or) { + CancelPlanning(ctx, raw_plan_id); + return std::unexpected(result_or.error()); + } + auto& result = *result_or; const std::string plan_id = result.plan_id; @@ -381,9 +389,17 @@ Result>> ExecuteScanPlan( ctx.client->Post(path, json_request, /*headers=*/{}, *PlanErrorHandler::Instance(), *ctx.session)); ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); - ICEBERG_ASSIGN_OR_RAISE(auto result, - PlanTableScanResponseFromJson(json, specs, schema)); - ICEBERG_RETURN_UNEXPECTED(result.Validate()); + // The parser validates the response, so read the plan id first to cancel on failure. + std::string raw_plan_id; + if (auto it = json.find("plan-id"); it != json.end() && it->is_string()) { + raw_plan_id = it->get(); + } + auto result_or = PlanTableScanResponseFromJson(json, specs, schema); + if (!result_or) { + CancelPlanning(ctx, raw_plan_id); + return std::unexpected(result_or.error()); + } + auto& result = *result_or; const std::string plan_id = result.plan_id; diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 24e8441651..68086a840e 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -1074,6 +1074,34 @@ TEST_F(RestTableScanTest, PlanFilesStreamIgnoresPlanExecutor) { EXPECT_EQ(executor.submit_count(), 0); } +// -------------------------------------------------------------------------- +// PlanFilesStream: invalid plan response with a plan-id → DELETE /plan called. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamInvalidResponseWithPlanIdCancels) { + constexpr std::string_view kInvalidBody = + R"({"status":"submitted","plan-id":"plan-invalid","plan-tasks":["tok-1"]})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kInvalidBody)))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + EXPECT_THAT(scan->PlanFilesStream(), IsError(ErrorKind::kValidationFailed)); +} + +// -------------------------------------------------------------------------- +// PlanFilesStream: invalid plan response without a plan-id → no DELETE. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamInvalidResponseWithoutPlanIdDoesNotCancel) { + constexpr std::string_view kInvalidBody = R"({"status":"failed"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kInvalidBody)))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + EXPECT_THAT(scan->PlanFilesStream(), IsError(ErrorKind::kJsonParseError)); +} + // -------------------------------------------------------------------------- // PlanFilesStream: FAILED → stream returns an error. // -------------------------------------------------------------------------- @@ -1543,6 +1571,21 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesFailedWithPlanIdCancels) { EXPECT_THAT(result, IsError(ErrorKind::kIOError)); } +// -------------------------------------------------------------------------- +// PlanFiles: invalid plan response with a plan-id → DELETE /plan called. +// -------------------------------------------------------------------------- +TEST_F(RestIncrementalAppendScanTest, PlanFilesInvalidResponseWithPlanIdCancels) { + constexpr std::string_view kInvalidBody = + R"({"status":"submitted","plan-id":"plan-invalid-incr","plan-tasks":["tok-1"]})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kInvalidBody)))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); + EXPECT_THAT(scan->PlanFiles(), IsError(ErrorKind::kValidationFailed)); +} + // -------------------------------------------------------------------------- // Stale credentials: second PlanFiles call resets scan_io_ so that credentials // from the first plan response do not persist into the second. From d6c50c4c4d4f911847bf94ffe10561b79e2ddbc2 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 20:56:20 -0700 Subject: [PATCH 25/36] feat(rest): make scan planning poll timeout and retries configurable Add rest-scan-planning.poll-timeout-ms (default 300000, must be positive) and rest-scan-planning.poll-num-retries (default 10, must be non-negative), matching Java. They are validated before the plan is submitted, and polling no longer sleeps after the final attempt. --- .../catalog/rest/catalog_properties.cc | 21 ++++++++++ src/iceberg/catalog/rest/catalog_properties.h | 13 +++++++ src/iceberg/catalog/rest/rest_table_scan.cc | 38 ++++++++++++++----- src/iceberg/test/catalog_properties_test.cc | 36 ++++++++++++++++++ src/iceberg/test/rest_table_scan_test.cc | 33 ++++++++++++++++ 5 files changed, 131 insertions(+), 10 deletions(-) diff --git a/src/iceberg/catalog/rest/catalog_properties.cc b/src/iceberg/catalog/rest/catalog_properties.cc index 91d72e8a49..28a7c10c06 100644 --- a/src/iceberg/catalog/rest/catalog_properties.cc +++ b/src/iceberg/catalog/rest/catalog_properties.cc @@ -20,6 +20,7 @@ #include "iceberg/catalog/rest/catalog_properties.h" #include +#include #include #include #include @@ -72,4 +73,24 @@ Result> RestCatalogProperties::ScanPlanningModeF return InvalidArgument("Invalid scan planning mode: '{}'.", it->second); } +Result RestCatalogProperties::ScanPlanningPollTimeoutMs() const { + const std::string value = Get(kScanPlanningPollTimeoutMs); + auto timeout_ms = StringUtils::ParseNumber(value); + if (!timeout_ms.has_value() || *timeout_ms <= 0) { + return InvalidArgument("Invalid value for {}: {} (must be positive)", + kScanPlanningPollTimeoutMs.key(), value); + } + return *timeout_ms; +} + +Result RestCatalogProperties::ScanPlanningPollNumRetries() const { + const std::string value = Get(kScanPlanningPollNumRetries); + auto num_retries = StringUtils::ParseNumber(value); + if (!num_retries.has_value() || *num_retries < 0) { + return InvalidArgument("Invalid value for {}: {} (must be non-negative)", + kScanPlanningPollNumRetries.key(), value); + } + return *num_retries; +} + } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/catalog_properties.h b/src/iceberg/catalog/rest/catalog_properties.h index 63dcd9a95e..eaef3084d5 100644 --- a/src/iceberg/catalog/rest/catalog_properties.h +++ b/src/iceberg/catalog/rest/catalog_properties.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include @@ -63,6 +64,12 @@ class ICEBERG_REST_EXPORT RestCatalogProperties "rest-metrics-reporting-enabled", "true"}; /// \brief The scan planning mode (client or server). inline static Entry kScanPlanningMode{"scan-planning-mode", "client"}; + /// \brief Maximum time to wait for a submitted scan plan, in milliseconds. + inline static Entry kScanPlanningPollTimeoutMs{ + "rest-scan-planning.poll-timeout-ms", "300000"}; + /// \brief Extra polls after the first fetch of a submitted scan plan. + inline static Entry kScanPlanningPollNumRetries{ + "rest-scan-planning.poll-num-retries", "10"}; /// \brief The prefix for HTTP headers. inline static constexpr std::string_view kHeaderPrefix = "header."; @@ -90,6 +97,12 @@ class ICEBERG_REST_EXPORT RestCatalogProperties /// std::nullopt if the key is absent. static Result> ScanPlanningModeFrom( const std::unordered_map& config); + + /// \brief Get the scan planning poll timeout in milliseconds; must be positive. + Result ScanPlanningPollTimeoutMs() const; + + /// \brief Get the number of scan planning poll retries; must be non-negative. + Result ScanPlanningPollNumRetries() const; }; } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index dbcbbec0c8..3cadbcc4e5 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -21,10 +21,12 @@ #include #include +#include #include #include +#include "iceberg/catalog/rest/catalog_properties.h" #include "iceberg/catalog/rest/endpoint.h" #include "iceberg/catalog/rest/error_handlers.h" #include "iceberg/catalog/rest/http_client.h" @@ -46,8 +48,18 @@ namespace { constexpr int64_t kMinSleepMs = 1'000; constexpr int64_t kMaxSleepMs = 60'000; -constexpr int kMaxRetries = 10; -constexpr int64_t kMaxWaitTimeMs = 5 * 60 * 1'000; + +struct PollConfig { + int64_t max_wait_ms; + int32_t max_retries; +}; + +Result ResolvePollConfig(const RestScanContext& ctx) { + auto properties = RestCatalogProperties::FromMap(ctx.catalog_config); + ICEBERG_ASSIGN_OR_RAISE(auto max_wait_ms, properties.ScanPlanningPollTimeoutMs()); + ICEBERG_ASSIGN_OR_RAISE(auto max_retries, properties.ScanPlanningPollNumRetries()); + return PollConfig{.max_wait_ms = max_wait_ms, .max_retries = max_retries}; +} using SpecsById = std::unordered_map>; // A shared slot that lets the scan and the stream share a single FileIO reference. @@ -167,7 +179,8 @@ Result>> FetchScanTasks( // are left unresolved. Result FetchPlanningResult( const RestScanContext& ctx, const Schema& schema, const std::string& plan_id, - const SpecsById& specs, std::shared_ptr& scan_io) { + const SpecsById& specs, const PollConfig& poll_config, + std::shared_ptr& scan_io) { ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::FetchPlanningResult()); ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->Plan(ctx.identifier, plan_id)); @@ -175,7 +188,7 @@ Result FetchPlanningResult( auto delay_ms = kMinSleepMs; auto start = std::chrono::steady_clock::now(); - for (int retry = 0; retry <= kMaxRetries; ++retry) { + for (int32_t retry = 0; retry <= poll_config.max_retries; ++retry) { auto response_or = ctx.client->Get(path, /*params=*/{}, /*headers=*/{}, *PlanErrorHandler::Instance(), *ctx.session); if (!response_or) { @@ -215,11 +228,12 @@ Result FetchPlanningResult( auto elapsed_ms = std::chrono::duration_cast( std::chrono::steady_clock::now() - start) .count(); - if (elapsed_ms >= kMaxWaitTimeMs) { + if (elapsed_ms >= poll_config.max_wait_ms) { CancelPlanning(ctx, plan_id); return IOError("Scan planning timed out after {}ms waiting for plan_id={}", elapsed_ms, plan_id); } + if (retry == poll_config.max_retries) break; std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); delay_ms = std::min(delay_ms * 2, kMaxSleepMs); continue; @@ -234,8 +248,8 @@ Result FetchPlanningResult( } CancelPlanning(ctx, plan_id); - return IOError("Scan planning exceeded max retries ({}) for plan_id={}", kMaxRetries, - plan_id); + return IOError("Scan planning exceeded max retries ({}) for plan_id={}", + poll_config.max_retries, plan_id); } // TODO(sandeepg): Fetch plan tasks concurrently when an executor is configured via @@ -313,6 +327,7 @@ Result ExecuteScanPlanStream( const RestScanContext& ctx, const Schema& schema, std::shared_ptr schema_ptr, PlanTableScanRequest request, const SpecsById& specs, ScanIoSlot scan_io_slot) { ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::PlanTableScan()); + ICEBERG_ASSIGN_OR_RAISE(auto poll_config, ResolvePollConfig(ctx)); ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->Plan(ctx.identifier)); ICEBERG_ASSIGN_OR_RAISE(auto request_json, ToJson(request)); @@ -337,7 +352,8 @@ Result ExecuteScanPlanStream( if (result.plan_status == PlanStatus::kSubmitted) { ICEBERG_ASSIGN_OR_RAISE( - auto completed, FetchPlanningResult(ctx, schema, plan_id, specs, *scan_io_slot)); + auto completed, + FetchPlanningResult(ctx, schema, plan_id, specs, poll_config, *scan_io_slot)); return std::make_unique( ctx, std::move(schema_ptr), plan_id, std::move(completed.file_scan_tasks) @@ -381,6 +397,7 @@ Result>> ExecuteScanPlan( const RestScanContext& ctx, const Schema& schema, PlanTableScanRequest request, const SpecsById& specs, std::shared_ptr& scan_io) { ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::PlanTableScan()); + ICEBERG_ASSIGN_OR_RAISE(auto poll_config, ResolvePollConfig(ctx)); ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->Plan(ctx.identifier)); ICEBERG_ASSIGN_OR_RAISE(auto request_json, ToJson(request)); @@ -416,8 +433,9 @@ Result>> ExecuteScanPlan( return tasks; } case PlanStatus::kSubmitted: { - ICEBERG_ASSIGN_OR_RAISE(auto completed, - FetchPlanningResult(ctx, schema, plan_id, specs, scan_io)); + ICEBERG_ASSIGN_OR_RAISE( + auto completed, + FetchPlanningResult(ctx, schema, plan_id, specs, poll_config, scan_io)); auto tasks = ResolveScanTasks(ctx, schema, completed.plan_tasks, completed.file_scan_tasks, specs, scan_io); if (!tasks) CancelPlanning(ctx, plan_id); diff --git a/src/iceberg/test/catalog_properties_test.cc b/src/iceberg/test/catalog_properties_test.cc index 5f22e60d45..dc3e282959 100644 --- a/src/iceberg/test/catalog_properties_test.cc +++ b/src/iceberg/test/catalog_properties_test.cc @@ -78,4 +78,40 @@ TEST(ScanPlanningModeTest, OtherKeysAreIgnored) { EXPECT_EQ(**result, ScanPlanningMode::kClient); } +TEST(ScanPlanningPollTest, Defaults) { + auto properties = RestCatalogProperties::default_properties(); + EXPECT_THAT(properties.ScanPlanningPollTimeoutMs(), HasValue(::testing::Eq(300000))); + EXPECT_THAT(properties.ScanPlanningPollNumRetries(), HasValue(::testing::Eq(10))); +} + +TEST(ScanPlanningPollTest, ValidValues) { + auto properties = + RestCatalogProperties::FromMap({{"rest-scan-planning.poll-timeout-ms", "1000"}, + {"rest-scan-planning.poll-num-retries", "0"}}); + EXPECT_THAT(properties.ScanPlanningPollTimeoutMs(), HasValue(::testing::Eq(1000))); + EXPECT_THAT(properties.ScanPlanningPollNumRetries(), HasValue(::testing::Eq(0))); +} + +TEST(ScanPlanningPollTest, InvalidTimeout) { + for (const auto& value : {"0", "-1", "abc"}) { + auto properties = + RestCatalogProperties::FromMap({{"rest-scan-planning.poll-timeout-ms", value}}); + EXPECT_THAT(properties.ScanPlanningPollTimeoutMs(), + ::testing::AllOf(IsError(ErrorKind::kInvalidArgument), + HasErrorMessage("must be positive"))) + << value; + } +} + +TEST(ScanPlanningPollTest, InvalidNumRetries) { + for (const auto& value : {"-1", "abc"}) { + auto properties = + RestCatalogProperties::FromMap({{"rest-scan-planning.poll-num-retries", value}}); + EXPECT_THAT(properties.ScanPlanningPollNumRetries(), + ::testing::AllOf(IsError(ErrorKind::kInvalidArgument), + HasErrorMessage("must be non-negative"))) + << value; + } +} + } // namespace iceberg::rest diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 68086a840e..8e863c6fb8 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -1102,6 +1102,39 @@ TEST_F(RestTableScanTest, PlanFilesStreamInvalidResponseWithoutPlanIdDoesNotCanc EXPECT_THAT(scan->PlanFilesStream(), IsError(ErrorKind::kJsonParseError)); } +// -------------------------------------------------------------------------- +// poll-num-retries=0: a single GET for a SUBMITTED plan, then cancel. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamPollNumRetriesZero) { + constexpr std::string_view kSubmittedBody = + R"({"status":"submitted","plan-id":"plan-no-retry"})"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + + auto ctx = MakeContext(); + ctx.catalog_config = {{"rest-scan-planning.poll-num-retries", "0"}}; + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(std::move(ctx))); + EXPECT_THAT(scan->PlanFilesStream(), + ::testing::AllOf(IsError(ErrorKind::kIOError), + HasErrorMessage("exceeded max retries (0)"))); +} + +// -------------------------------------------------------------------------- +// Invalid poll-timeout-ms fails before the plan is submitted. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, PlanFilesStreamInvalidPollTimeoutFailsBeforePlanning) { + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); + + auto ctx = MakeContext(); + ctx.catalog_config = {{"rest-scan-planning.poll-timeout-ms", "0"}}; + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(std::move(ctx))); + EXPECT_THAT(scan->PlanFilesStream(), IsError(ErrorKind::kInvalidArgument)); +} + // -------------------------------------------------------------------------- // PlanFilesStream: FAILED → stream returns an error. // -------------------------------------------------------------------------- From 1c4237d86bcb05e9b02ddf76f9fbd7a0a36c4db7 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 20:59:41 -0700 Subject: [PATCH 26/36] test: run use_snapshot_schema checks once instead of per format version --- src/iceberg/test/table_scan_test.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/iceberg/test/table_scan_test.cc b/src/iceberg/test/table_scan_test.cc index fa713945b3..cb22997cfc 100644 --- a/src/iceberg/test/table_scan_test.cc +++ b/src/iceberg/test/table_scan_test.cc @@ -858,9 +858,10 @@ TEST_P(TableScanTest, SchemaWithSelectedColumnsAndFilter) { } } -// use_snapshot_schema propagation tests: verify the field is set correctly for -// UseSnapshot, UseRef (tag vs branch), and default/incremental scans. -TEST_P(TableScanTest, UseSnapshotSetsTrueUseSnapshotSchema) { +// use_snapshot_schema checks do not depend on format version, so run them once. +class TableScanContextTest : public TableScanTest {}; + +TEST_F(TableScanContextTest, UseSnapshotSetsTrueUseSnapshotSchema) { constexpr int64_t kSnapshotId = 1000L; ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); builder->UseSnapshot(kSnapshotId); @@ -868,7 +869,7 @@ TEST_P(TableScanTest, UseSnapshotSetsTrueUseSnapshotSchema) { EXPECT_TRUE(scan->context().use_snapshot_schema); } -TEST_P(TableScanTest, UseRefTagSetsTrueUseSnapshotSchema) { +TEST_F(TableScanContextTest, UseRefTagSetsTrueUseSnapshotSchema) { constexpr int64_t kSnapshotId = 1000L; table_metadata_->refs["v1.0"] = std::make_shared( SnapshotRef{.snapshot_id = kSnapshotId, .retention = SnapshotRef::Tag{}}); @@ -879,14 +880,14 @@ TEST_P(TableScanTest, UseRefTagSetsTrueUseSnapshotSchema) { EXPECT_TRUE(scan->context().use_snapshot_schema); } -TEST_P(TableScanTest, UseRefBranchSetsFalseUseSnapshotSchema) { +TEST_F(TableScanContextTest, UseRefBranchSetsFalseUseSnapshotSchema) { ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); builder->UseRef("main"); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); EXPECT_FALSE(scan->context().use_snapshot_schema); } -TEST_P(TableScanTest, DefaultScanHasFalseUseSnapshotSchema) { +TEST_F(TableScanContextTest, DefaultScanHasFalseUseSnapshotSchema) { ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); EXPECT_FALSE(scan->context().use_snapshot_schema); From b09f0add809e343564cfa04e829bd818c0708aa8 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 21:00:13 -0700 Subject: [PATCH 27/36] test(rest): rename stream cancel tests that consume no tasks --- src/iceberg/test/rest_table_scan_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 8e863c6fb8..a49010743c 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -876,7 +876,7 @@ TEST_F(RestTableScanTest, PlanFilesStreamCancelAfterPartialConsumption) { // PlanFilesStream: stream destroyed with no Next() calls at all → // DELETE /plan called by the destructor. // -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamCancelOnPartialConsumption) { +TEST_F(RestTableScanTest, PlanFilesStreamCancelOnNoConsumption) { constexpr std::string_view kPlanResponse = R"({"status":"completed","plan-id":"plan-never-consumed","plan-tasks":["tok-p1"]})"; @@ -966,10 +966,10 @@ TEST_F(RestTableScanTest, PlanFilesStreamSubmittedFetchesTokensLazily) { } // -------------------------------------------------------------------------- -// PlanFilesStream: SUBMITTED → COMPLETED, stream destroyed before its plan -// tasks are fetched → DELETE /plan called by the destructor. +// PlanFilesStream: SUBMITTED → COMPLETED, stream destroyed with no Next() calls +// → DELETE /plan called by the destructor. // -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamSubmittedCancelOnPartialConsumption) { +TEST_F(RestTableScanTest, PlanFilesStreamSubmittedCancelOnNoConsumption) { constexpr std::string_view kSubmittedBody = R"({"status":"submitted","plan-id":"plan-poll-partial"})"; constexpr std::string_view kCompletedBody = From b22effbc9a5b16e8237ae4b309579a1db75cc05f Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 22:16:44 -0700 Subject: [PATCH 28/36] fix(rest): use iceberg::unexpected in REST scan planning Main now aliases Result to ::iceberg::expected so the library builds in C++20 compat mode; std::unexpected no longer converts to it. --- src/iceberg/catalog/rest/rest_table_scan.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 3cadbcc4e5..25054e189c 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -193,24 +193,24 @@ Result FetchPlanningResult( *PlanErrorHandler::Instance(), *ctx.session); if (!response_or) { CancelPlanning(ctx, plan_id); - return std::unexpected(response_or.error()); + return ::iceberg::unexpected(response_or.error()); } auto json_or = FromJsonString(response_or->body()); if (!json_or) { CancelPlanning(ctx, plan_id); - return std::unexpected(json_or.error()); + return ::iceberg::unexpected(json_or.error()); } auto result_or = FetchPlanningResultResponseFromJson(*json_or, specs, schema); if (!result_or) { CancelPlanning(ctx, plan_id); - return std::unexpected(result_or.error()); + return ::iceberg::unexpected(result_or.error()); } if (auto s = result_or->Validate(); !s) { CancelPlanning(ctx, plan_id); - return std::unexpected(s.error()); + return ::iceberg::unexpected(s.error()); } auto& result = *result_or; @@ -220,7 +220,7 @@ Result FetchPlanningResult( if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, scan_io); !s) { CancelPlanning(ctx, plan_id); - return std::unexpected(s.error()); + return ::iceberg::unexpected(s.error()); } return std::move(result); } @@ -344,7 +344,7 @@ Result ExecuteScanPlanStream( auto result_or = PlanTableScanResponseFromJson(json, specs, schema); if (!result_or) { CancelPlanning(ctx, raw_plan_id); - return std::unexpected(result_or.error()); + return ::iceberg::unexpected(result_or.error()); } auto& result = *result_or; @@ -375,7 +375,7 @@ Result ExecuteScanPlanStream( if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, *scan_io_slot); !s) { CancelPlanning(ctx, plan_id); - return std::unexpected(s.error()); + return ::iceberg::unexpected(s.error()); } std::vector> initial_tasks; @@ -414,7 +414,7 @@ Result>> ExecuteScanPlan( auto result_or = PlanTableScanResponseFromJson(json, specs, schema); if (!result_or) { CancelPlanning(ctx, raw_plan_id); - return std::unexpected(result_or.error()); + return ::iceberg::unexpected(result_or.error()); } auto& result = *result_or; @@ -425,7 +425,7 @@ Result>> ExecuteScanPlan( if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, scan_io); !s) { CancelPlanning(ctx, plan_id); - return std::unexpected(s.error()); + return ::iceberg::unexpected(s.error()); } auto tasks = ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs, scan_io); From 91a1e42c3859f75f4398c3eab585bad3d2366aeb Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sat, 10 Oct 2026 22:29:55 -0700 Subject: [PATCH 29/36] feat(rest): install a plan-scoped credential provider on scan FileIO Vended credentials from scan planning responses now get a StorageCredentialProvider that refreshes through the LoadCredentials endpoint with the planId query parameter, matching the table FileIO provider and Java's plan-scoped credential refresh. --- src/iceberg/catalog/rest/rest_catalog.cc | 31 ++++- src/iceberg/catalog/rest/rest_catalog.h | 5 +- src/iceberg/catalog/rest/rest_table_scan.cc | 53 ++++---- src/iceberg/catalog/rest/rest_table_scan.h | 4 + src/iceberg/test/rest_catalog_unit_test.cc | 118 ++++++++++++++++- src/iceberg/test/rest_table_scan_test.cc | 133 ++++++++++++++++++++ 6 files changed, 311 insertions(+), 33 deletions(-) diff --git a/src/iceberg/catalog/rest/rest_catalog.cc b/src/iceberg/catalog/rest/rest_catalog.cc index d919d1892a..a98fefb281 100644 --- a/src/iceberg/catalog/rest/rest_catalog.cc +++ b/src/iceberg/catalog/rest/rest_catalog.cc @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -71,16 +72,18 @@ class RestStorageCredentialProvider final : public StorageCredentialProvider { public: RestStorageCredentialProvider(std::shared_ptr client, std::shared_ptr session, - std::string path) + std::string path, + std::unordered_map params) : client_(std::move(client)), session_(std::move(session)), - path_(std::move(path)) {} + path_(std::move(path)), + params_(std::move(params)) {} Result> Load() override { // The provider can be shared by several FileIOs; keep fetches serialized. std::lock_guard lock(mutex_); ICEBERG_ASSIGN_OR_RAISE(const auto response, - client_->Get(path_, /*params=*/{}, /*headers=*/{}, + client_->Get(path_, params_, /*headers=*/{}, *TableErrorHandler::Instance(), *session_)); // Parse errors can contain credential data; return a fixed message. auto json = FromJsonString(response.body()); @@ -99,6 +102,7 @@ class RestStorageCredentialProvider final : public StorageCredentialProvider { std::shared_ptr client_; std::shared_ptr session_; std::string path_; + std::unordered_map params_; std::mutex mutex_; }; @@ -567,8 +571,8 @@ Result> RestCatalog::TableAuthSession( } std::shared_ptr RestCatalog::MakeStorageCredentialProvider( - const TableIdentifier& identifier, - std::shared_ptr table_session) const { + const TableIdentifier& identifier, std::shared_ptr table_session, + std::optional plan_id) const { if (!supported_endpoints_.contains(Endpoint::TableCredentials())) { // Not an error, but it surfaces much later as credentials expiring. ICEBERG_LOG_DEBUG( @@ -588,8 +592,13 @@ std::shared_ptr RestCatalog::MakeStorageCredentialPro auto client = client_; auto credentials_path = std::move(path.value()); auto session = std::move(table_session); + std::unordered_map params; + if (plan_id.has_value()) { + params.emplace("planId", std::move(*plan_id)); + } return std::make_shared( - std::move(client), std::move(session), std::move(credentials_path)); + std::move(client), std::move(session), std::move(credentials_path), + std::move(params)); } Result> RestCatalog::TableFileIO( @@ -1048,6 +1057,15 @@ Result> RestCatalog::MakeTable( "the PlanTableScan endpoint.", identifier.ToString()); } + auto credential_provider_factory = + [weak_catalog = weak_from_this(), identifier, table_session]( + const std::string& plan_id) -> std::shared_ptr { + auto catalog = weak_catalog.lock(); + if (catalog == nullptr) { + return nullptr; + } + return catalog->MakeStorageCredentialProvider(identifier, table_session, plan_id); + }; RestScanContext rest_ctx{ .client = client_, .paths = paths_, @@ -1056,6 +1074,7 @@ Result> RestCatalog::MakeTable( .identifier = identifier, .catalog_config = config_.configs(), .table_config = table_config, + .credential_provider_factory = std::move(credential_provider_factory), }; return RestTable::Make(identifier, std::move(metadata), std::move(metadata_location), std::move(table_io), std::move(table_catalog), diff --git a/src/iceberg/catalog/rest/rest_catalog.h b/src/iceberg/catalog/rest/rest_catalog.h index 7b82ab47cd..962c5cf28b 100644 --- a/src/iceberg/catalog/rest/rest_catalog.h +++ b/src/iceberg/catalog/rest/rest_catalog.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -101,8 +102,8 @@ class ICEBERG_REST_EXPORT RestCatalog final /// /// Returns nullptr when the catalog does not serve LoadCredentials. std::shared_ptr MakeStorageCredentialProvider( - const TableIdentifier& identifier, - std::shared_ptr table_session) const; + const TableIdentifier& identifier, std::shared_ptr table_session, + std::optional plan_id = std::nullopt) const; Result> ListNamespaces(const Namespace& ns, auth::AuthSession& session) const; diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 25054e189c..4025f76d7b 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -95,12 +95,16 @@ Result> ProjectedColumnNames(const Schema& projected) { return names; } -Status ApplyStorageCredentials(const RestScanContext& ctx, +Status ApplyStorageCredentials(const RestScanContext& ctx, const std::string& plan_id, const std::vector& credentials, std::shared_ptr& scan_io) { if (credentials.empty()) return {}; - ICEBERG_ASSIGN_OR_RAISE( - auto io, MakeTableFileIO(ctx.catalog_config, ctx.table_config, credentials)); + std::shared_ptr provider; + if (ctx.credential_provider_factory) { + provider = ctx.credential_provider_factory(plan_id); + } + ICEBERG_ASSIGN_OR_RAISE(auto io, MakeTableFileIO(ctx.catalog_config, ctx.table_config, + credentials, std::move(provider))); scan_io = std::move(io); return {}; } @@ -117,11 +121,12 @@ void CancelPlanning(const RestScanContext& ctx, const std::string& plan_id) { } Result>> FetchScanTasks( - const RestScanContext& ctx, const Schema& schema, const std::string& plan_task, - const SpecsById& specs, std::shared_ptr& scan_io); + const RestScanContext& ctx, const std::string& plan_id, const Schema& schema, + const std::string& plan_task, const SpecsById& specs, + std::shared_ptr& scan_io); Result>> ResolveScanTasks( - const RestScanContext& ctx, const Schema& schema, + const RestScanContext& ctx, const std::string& plan_id, const Schema& schema, const std::optional>& plan_tasks, const std::optional>>& file_scan_tasks, const SpecsById& specs, std::shared_ptr& scan_io) { @@ -133,8 +138,8 @@ Result>> ResolveScanTasks( if (plan_tasks.has_value()) { for (const auto& token : *plan_tasks) { - ICEBERG_ASSIGN_OR_RAISE(auto tasks, - FetchScanTasks(ctx, schema, token, specs, scan_io)); + ICEBERG_ASSIGN_OR_RAISE( + auto tasks, FetchScanTasks(ctx, plan_id, schema, token, specs, scan_io)); result.insert(result.end(), tasks.begin(), tasks.end()); } } @@ -144,6 +149,7 @@ Result>> ResolveScanTasks( // Fetches one plan task without resolving any nested plan tasks it returns. Result FetchScanTasksOnce(const RestScanContext& ctx, + const std::string& plan_id, const Schema& schema, const std::string& plan_task, const SpecsById& specs, @@ -162,17 +168,18 @@ Result FetchScanTasksOnce(const RestScanContext& ctx, FetchScanTasksResponseFromJson(json, specs, schema)); ICEBERG_RETURN_UNEXPECTED(result.Validate()); ICEBERG_RETURN_UNEXPECTED( - ApplyStorageCredentials(ctx, result.storage_credentials, scan_io)); + ApplyStorageCredentials(ctx, plan_id, result.storage_credentials, scan_io)); return result; } Result>> FetchScanTasks( - const RestScanContext& ctx, const Schema& schema, const std::string& plan_task, - const SpecsById& specs, std::shared_ptr& scan_io) { - ICEBERG_ASSIGN_OR_RAISE(auto result, - FetchScanTasksOnce(ctx, schema, plan_task, specs, scan_io)); - return ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs, - scan_io); + const RestScanContext& ctx, const std::string& plan_id, const Schema& schema, + const std::string& plan_task, const SpecsById& specs, + std::shared_ptr& scan_io) { + ICEBERG_ASSIGN_OR_RAISE( + auto result, FetchScanTasksOnce(ctx, plan_id, schema, plan_task, specs, scan_io)); + return ResolveScanTasks(ctx, plan_id, schema, result.plan_tasks, result.file_scan_tasks, + specs, scan_io); } // Polls until the plan completes and returns the completed response. Plan tasks @@ -217,7 +224,8 @@ Result FetchPlanningResult( switch (result.plan_status) { case PlanStatus::kCompleted: { - if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, scan_io); + if (auto s = ApplyStorageCredentials(ctx, plan_id, result.storage_credentials, + scan_io); !s) { CancelPlanning(ctx, plan_id); return ::iceberg::unexpected(s.error()); @@ -295,7 +303,8 @@ class RestFileScanTaskStream final : public FileScanTaskStream { } const std::string token = plan_task_tokens_[token_pos_++]; ICEBERG_ASSIGN_OR_RAISE( - auto result, FetchScanTasksOnce(ctx_, *schema_, token, specs_, *scan_io_slot_)); + auto result, + FetchScanTasksOnce(ctx_, plan_id_, *schema_, token, specs_, *scan_io_slot_)); if (result.plan_tasks.has_value()) { plan_task_tokens_.insert(plan_task_tokens_.end(), result.plan_tasks->begin(), result.plan_tasks->end()); @@ -372,7 +381,8 @@ Result ExecuteScanPlanStream( } // kCompleted: apply credentials from the initial response, then build the lazy stream. - if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, *scan_io_slot); + if (auto s = ApplyStorageCredentials(ctx, plan_id, result.storage_credentials, + *scan_io_slot); !s) { CancelPlanning(ctx, plan_id); return ::iceberg::unexpected(s.error()); @@ -422,12 +432,13 @@ Result>> ExecuteScanPlan( switch (result.plan_status) { case PlanStatus::kCompleted: { - if (auto s = ApplyStorageCredentials(ctx, result.storage_credentials, scan_io); + if (auto s = + ApplyStorageCredentials(ctx, plan_id, result.storage_credentials, scan_io); !s) { CancelPlanning(ctx, plan_id); return ::iceberg::unexpected(s.error()); } - auto tasks = ResolveScanTasks(ctx, schema, result.plan_tasks, + auto tasks = ResolveScanTasks(ctx, plan_id, schema, result.plan_tasks, result.file_scan_tasks, specs, scan_io); if (!tasks) CancelPlanning(ctx, plan_id); return tasks; @@ -436,7 +447,7 @@ Result>> ExecuteScanPlan( ICEBERG_ASSIGN_OR_RAISE( auto completed, FetchPlanningResult(ctx, schema, plan_id, specs, poll_config, scan_io)); - auto tasks = ResolveScanTasks(ctx, schema, completed.plan_tasks, + auto tasks = ResolveScanTasks(ctx, plan_id, schema, completed.plan_tasks, completed.file_scan_tasks, specs, scan_io); if (!tasks) CancelPlanning(ctx, plan_id); return tasks; diff --git a/src/iceberg/catalog/rest/rest_table_scan.h b/src/iceberg/catalog/rest/rest_table_scan.h index 1c3c4dc7a6..b853a8cb03 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.h +++ b/src/iceberg/catalog/rest/rest_table_scan.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -62,6 +63,9 @@ struct ICEBERG_REST_EXPORT RestScanContext { std::unordered_map catalog_config; /// Table-level config merged with catalog_config for scan-scoped FileIO creation. std::unordered_map table_config; + /// Builds a plan-scoped provider for refreshing vended credentials; may be empty. + std::function(const std::string& plan_id)> + credential_provider_factory; }; /// \brief A DataTableScan that delegates PlanFilesStream() to the REST catalog server diff --git a/src/iceberg/test/rest_catalog_unit_test.cc b/src/iceberg/test/rest_catalog_unit_test.cc index edd840a1ba..1749394c60 100644 --- a/src/iceberg/test/rest_catalog_unit_test.cc +++ b/src/iceberg/test/rest_catalog_unit_test.cc @@ -35,6 +35,8 @@ #include "iceberg/catalog/rest/rest_table.h" #include "iceberg/catalog/rest/rest_table_scan.h" #include "iceberg/file_io.h" +#include "iceberg/file_io_registry.h" +#include "iceberg/storage_credential.h" #include "iceberg/table.h" #include "iceberg/table_identifier.h" #include "iceberg/table_requirement.h" @@ -86,6 +88,55 @@ class NoOpFileIO : public FileIO { Status DeleteFile(const std::string&) override { return {}; } }; +namespace { + +std::shared_ptr installed_provider; + +class CredentialedFileIO : public NoOpFileIO, public SupportsStorageCredentials { + public: + Status InitializeStorageCredentials( + const std::vector& credentials, + std::shared_ptr provider) override { + credentials_ = credentials; + installed_provider = std::move(provider); + return {}; + } + + Status SetStorageCredentials( + const std::vector& credentials) override { + credentials_ = credentials; + return {}; + } + + std::vector credentials() const override { return credentials_; } + + SupportsStorageCredentials* AsSupportsStorageCredentials() override { return this; } + + private: + std::vector credentials_; +}; + +constexpr std::string_view kCredentialedFileIO = "rest-catalog-unit-test-credentialed"; + +void RegisterCredentialedFileIO() { + FileIORegistry::Register( + std::string(kCredentialedFileIO), + {.create = [](const std::unordered_map&) + -> Result> { + return std::make_unique(); + }}); + installed_provider = nullptr; +} + +// Plan response that vends credentials for the scan. +constexpr std::string_view kPlanWithCredentialsResponse = R"({ + "status": "completed", + "plan-id": "plan-1", + "storage-credentials": [{"prefix": "s3://bucket", "config": {"k": "v"}}] +})"; + +} // namespace + // -------------------------------------------------------------------------- // Helper JSON bodies for LoadTable GET responses // -------------------------------------------------------------------------- @@ -134,10 +185,10 @@ class RestCatalogLoadTableTest : public ::testing::Test { // client-side scan-planning-mode setting. Result> MakeCatalog( const std::unordered_set& endpoints, - const std::string& client_scan_mode = "") { - std::unordered_map props{ - {"uri", "http://test-server"}, - }; + const std::string& client_scan_mode = "", + std::unordered_map extra_props = {}) { + std::unordered_map props = std::move(extra_props); + props["uri"] = "http://test-server"; if (!client_scan_mode.empty()) { props["scan-planning-mode"] = client_scan_mode; } @@ -274,4 +325,63 @@ TEST_F(RestCatalogLoadTableTest, CommitWithDefaultScanModeReturnsPlainTable) { EXPECT_EQ(dynamic_cast(committed.get()), nullptr); } +// -------------------------------------------------------------------------- +// Scan-vended credentials get a provider that refreshes via /credentials?planId=. +// -------------------------------------------------------------------------- +TEST_F(RestCatalogLoadTableTest, ScanCredentialProviderRequestsPlanId) { + RegisterCredentialedFileIO(); + EXPECT_CALL(*mock_client_, + Get(::testing::Not(::testing::HasSubstr("/credentials")), _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return( + HttpResponse::MakeForTesting(200, std::string(kPlanWithCredentialsResponse)))); + EXPECT_CALL(*mock_client_, + Get(::testing::HasSubstr("/credentials"), + ::testing::Contains(::testing::Pair("planId", "plan-1")), _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting( + 200, + R"({"storage-credentials":[{"prefix":"s3://bucket","config":{"k":"v2"}}]})"))); + + auto endpoints = all_plan_endpoints_; + endpoints.insert(Endpoint::TableCredentials()); + ICEBERG_UNWRAP_OR_FAIL(auto catalog, + MakeCatalog(endpoints, /*client_scan_mode=*/"", + {{"io-impl", std::string(kCredentialedFileIO)}})); + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); + ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewScan()); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + + ASSERT_NE(installed_provider, nullptr); + ICEBERG_UNWRAP_OR_FAIL(auto refreshed, installed_provider->Load()); + ASSERT_EQ(refreshed.size(), 1); + EXPECT_EQ(refreshed[0].config.at("k"), "v2"); +} + +// -------------------------------------------------------------------------- +// Without the TableCredentials endpoint, scan credentials are not refreshed. +// -------------------------------------------------------------------------- +TEST_F(RestCatalogLoadTableTest, + ScanCredentialsWithoutCredentialsEndpointHaveNoProvider) { + RegisterCredentialedFileIO(); + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return( + HttpResponse::MakeForTesting(200, std::string(kPlanWithCredentialsResponse)))); + + ICEBERG_UNWRAP_OR_FAIL(auto catalog, + MakeCatalog(all_plan_endpoints_, /*client_scan_mode=*/"", + {{"io-impl", std::string(kCredentialedFileIO)}})); + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); + ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewScan()); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + + EXPECT_EQ(installed_provider, nullptr); +} + } // namespace iceberg::rest diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index a49010743c..8fec64cb14 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -39,10 +39,12 @@ #include "iceberg/expression/expressions.h" #include "iceberg/expression/literal.h" #include "iceberg/file_io.h" +#include "iceberg/file_io_registry.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/partition_spec.h" #include "iceberg/schema.h" #include "iceberg/snapshot.h" +#include "iceberg/storage_credential.h" #include "iceberg/table_identifier.h" #include "iceberg/table_metadata.h" #include "iceberg/table_scan.h" @@ -133,6 +135,44 @@ class NoOpFileIO : public FileIO { Status DeleteFile(const std::string&) override { return {}; } }; +namespace { + +// Records the credentials and provider installed on the scan FileIO. +std::vector installed_credentials; +std::shared_ptr installed_provider; + +class CredentialedFileIO : public NoOpFileIO, public SupportsStorageCredentials { + public: + Status InitializeStorageCredentials( + const std::vector& credentials, + std::shared_ptr provider) override { + installed_credentials = credentials; + installed_provider = std::move(provider); + return {}; + } + + Status SetStorageCredentials( + const std::vector& credentials) override { + installed_credentials = credentials; + return {}; + } + + std::vector credentials() const override { + return installed_credentials; + } + + SupportsStorageCredentials* AsSupportsStorageCredentials() override { return this; } +}; + +constexpr std::string_view kCredentialedFileIO = "rest-table-scan-test-credentialed"; + +class StaticCredentialProvider : public StorageCredentialProvider { + public: + Result> Load() override { return {}; } +}; + +} // namespace + // -------------------------------------------------------------------------- // Test fixture shared by RestTableScan tests. // -------------------------------------------------------------------------- @@ -201,6 +241,17 @@ class RestTableScanTest : public ::testing::Test { }; } + static void RegisterCredentialedFileIO() { + FileIORegistry::Register( + std::string(kCredentialedFileIO), + {.create = [](const std::unordered_map&) + -> Result> { + return std::make_unique(); + }}); + installed_credentials.clear(); + installed_provider = nullptr; + } + Result> MakeScan(RestScanContext ctx) { return RestTableScan::Make(metadata_, schema_, file_io_, internal::TableScanContext{}, std::move(ctx)); @@ -1135,6 +1186,88 @@ TEST_F(RestTableScanTest, PlanFilesStreamInvalidPollTimeoutFailsBeforePlanning) EXPECT_THAT(scan->PlanFilesStream(), IsError(ErrorKind::kInvalidArgument)); } +// -------------------------------------------------------------------------- +// Vended credentials in the plan response install a provider scoped to the plan. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, StorageCredentialsInstallPlanScopedProvider) { + constexpr std::string_view kPlanResponse = R"({ + "status": "completed", + "plan-id": "plan-cred", + "storage-credentials": [{"prefix": "s3://bucket", "config": {"k": "v"}}] + })"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))); + RegisterCredentialedFileIO(); + + std::vector requested_plan_ids; + auto provider = std::make_shared(); + auto ctx = MakeContext(); + ctx.catalog_config = {{"io-impl", std::string(kCredentialedFileIO)}}; + ctx.credential_provider_factory = [&](const std::string& plan_id) { + requested_plan_ids.push_back(plan_id); + return provider; + }; + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(std::move(ctx))); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + + EXPECT_THAT(requested_plan_ids, ::testing::ElementsAre("plan-cred")); + EXPECT_EQ(installed_provider, provider); + ASSERT_EQ(installed_credentials.size(), 1); + EXPECT_EQ(installed_credentials[0].prefix, "s3://bucket"); +} + +// -------------------------------------------------------------------------- +// Vended credentials in a FetchScanTasks response also use the plan id. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, FetchScanTasksCredentialsInstallPlanScopedProvider) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-cred-tasks","plan-tasks":["tok-1"]})"; + constexpr std::string_view kTasksResponse = R"({ + "file-scan-tasks": [], + "storage-credentials": [{"prefix": "s3://bucket", "config": {"k": "v"}}] + })"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); + RegisterCredentialedFileIO(); + + std::vector requested_plan_ids; + auto provider = std::make_shared(); + auto ctx = MakeContext(); + ctx.catalog_config = {{"io-impl", std::string(kCredentialedFileIO)}}; + ctx.credential_provider_factory = [&](const std::string& plan_id) { + requested_plan_ids.push_back(plan_id); + return provider; + }; + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(std::move(ctx))); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + + EXPECT_THAT(requested_plan_ids, ::testing::ElementsAre("plan-cred-tasks")); + EXPECT_EQ(installed_provider, provider); +} + +// -------------------------------------------------------------------------- +// Without a provider factory, vended credentials are installed as static. +// -------------------------------------------------------------------------- +TEST_F(RestTableScanTest, StorageCredentialsWithoutFactoryInstallNoProvider) { + constexpr std::string_view kPlanResponse = R"({ + "status": "completed", + "plan-id": "plan-static", + "storage-credentials": [{"prefix": "s3://bucket", "config": {"k": "v"}}] + })"; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))); + RegisterCredentialedFileIO(); + + auto ctx = MakeContext(); + ctx.catalog_config = {{"io-impl", std::string(kCredentialedFileIO)}}; + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(std::move(ctx))); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + + EXPECT_EQ(installed_provider, nullptr); + EXPECT_EQ(installed_credentials.size(), 1); +} + // -------------------------------------------------------------------------- // PlanFilesStream: FAILED → stream returns an error. // -------------------------------------------------------------------------- From 7830f1a63549e1b3e96f1f34c01366af5af7b02e Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 11 Oct 2026 12:33:06 -0700 Subject: [PATCH 30/36] fix(rest): reject plan-id in failed plan responses, matching Java Restore the PlanTableScanResponse validation that allows plan-id only for submitted or completed plans. The plan is still cancelled because the plan-id is read from the raw response before parsing. --- src/iceberg/catalog/rest/types.cc | 6 +++--- src/iceberg/test/rest_json_serde_test.cc | 17 +++++++---------- src/iceberg/test/rest_table_scan_test.cc | 12 ++++++------ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/iceberg/catalog/rest/types.cc b/src/iceberg/catalog/rest/types.cc index 7384e2e393..ef20f44da7 100644 --- a/src/iceberg/catalog/rest/types.cc +++ b/src/iceberg/catalog/rest/types.cc @@ -298,10 +298,10 @@ Status PlanTableScanResponse::Validate() const { "Invalid response: tasks can only be defined when status is 'completed'"); } if (!plan_id.empty() && plan_status != PlanStatus::kSubmitted && - plan_status != PlanStatus::kCompleted && plan_status != PlanStatus::kFailed) { + plan_status != PlanStatus::kCompleted) { return ValidationFailed( - "Invalid response: plan id can only be defined when status is 'submitted', " - "'completed', or 'failed'"); + "Invalid response: plan id can only be defined when status is 'submitted' or " + "'completed'"); } if (!HasNonEmptyFileScanTasks(*this) && !delete_files.empty()) { return ValidationFailed( diff --git a/src/iceberg/test/rest_json_serde_test.cc b/src/iceberg/test/rest_json_serde_test.cc index 9357658f52..34ca4a45b3 100644 --- a/src/iceberg/test/rest_json_serde_test.cc +++ b/src/iceberg/test/rest_json_serde_test.cc @@ -1598,6 +1598,13 @@ INSTANTIATE_TEST_SUITE_P( R"({"status":"submitted","plan-id":"somePlanId","plan-tasks":[]})", .expected_error_kind = ErrorKind::kValidationFailed, .expected_error_msg = "tasks can only be defined when status is 'completed'"}, + PlanTableScanResponseInvalidParam{ + .test_name = "FailedWithPlanId", + .json_str = + R"({"status":"failed","plan-id":"somePlanId","error":{"message":"x","type":"y","code":500}})", + .expected_error_kind = ErrorKind::kValidationFailed, + .expected_error_msg = + "plan id can only be defined when status is 'submitted' or 'completed'"}, PlanTableScanResponseInvalidParam{ .test_name = "FailedWithoutError", .json_str = R"({"status":"failed"})", @@ -2545,16 +2552,6 @@ TEST(PlanTableScanResponseRoundtripTest, FailedWithError) { EXPECT_EQ(*result, *result2); } -TEST(PlanTableScanResponseRoundtripTest, FailedWithPlanId) { - // A server may include a plan-id in a failed response so the client can cancel. - auto json = nlohmann::json::parse( - R"({"status":"failed","plan-id":"plan-123","error":{"message":"Planning failed","type":"PlanningException","code":500}})"); - auto result = PlanTableScanResponseFromJson(json, EmptySpecs(), EmptySchema()); - ASSERT_THAT(result, IsOk()); - EXPECT_EQ(result->plan_id, "plan-123"); - EXPECT_EQ(result->plan_status, PlanStatus::kFailed); -} - TEST(FetchPlanningResultResponseRoundtripTest, CompletedWithPlanTasks) { auto json = nlohmann::json::parse( R"({"status": "completed", "plan-tasks": ["task-1", "task-2"]})"); diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 8fec64cb14..f518ad9ef8 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -1283,8 +1283,8 @@ TEST_F(RestTableScanTest, PlanFilesStreamFailed) { } // -------------------------------------------------------------------------- -// PlanFilesStream: FAILED with plan-id → DELETE /plan called before returning -// the error (tests the ExecuteScanPlanStream kFailed cancel fix). +// PlanFilesStream: FAILED with plan-id is an invalid response → DELETE /plan +// called before returning the error. // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, PlanFilesStreamFailedWithPlanIdCancels) { constexpr std::string_view kFailedBody = @@ -1297,7 +1297,7 @@ TEST_F(RestTableScanTest, PlanFilesStreamFailedWithPlanIdCancels) { ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); auto result = scan->PlanFilesStream(); - EXPECT_THAT(result, IsError(ErrorKind::kIOError)); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); } // -------------------------------------------------------------------------- @@ -1720,8 +1720,8 @@ TEST_F(RestIncrementalAppendScanTest, ProjectSendsProjectedColumnsAsSelect) { } // -------------------------------------------------------------------------- -// PlanFiles: FAILED with plan-id → DELETE /plan called before returning the -// error (tests the ExecuteScanPlan kFailed cancel fix). +// PlanFiles: FAILED with plan-id is an invalid response → DELETE /plan called +// before returning the error. // -------------------------------------------------------------------------- TEST_F(RestIncrementalAppendScanTest, PlanFilesFailedWithPlanIdCancels) { constexpr std::string_view kFailedBody = @@ -1734,7 +1734,7 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesFailedWithPlanIdCancels) { ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); auto result = scan->PlanFiles(); - EXPECT_THAT(result, IsError(ErrorKind::kIOError)); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); } // -------------------------------------------------------------------------- From 80e3de82a58cf574fcdfc64af6e6a29ce3c5ddc8 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 11 Oct 2026 14:56:48 -0700 Subject: [PATCH 31/36] fix(rest): omit planId on credential refresh when the plan has no id A completed plan response may omit plan-id. Refresh credentials table-scoped in that case instead of sending an empty planId query parameter. --- src/iceberg/catalog/rest/rest_catalog.cc | 8 ++++- src/iceberg/test/rest_catalog_unit_test.cc | 34 ++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/iceberg/catalog/rest/rest_catalog.cc b/src/iceberg/catalog/rest/rest_catalog.cc index a98fefb281..e8e0e17bf0 100644 --- a/src/iceberg/catalog/rest/rest_catalog.cc +++ b/src/iceberg/catalog/rest/rest_catalog.cc @@ -1064,7 +1064,13 @@ Result> RestCatalog::MakeTable( if (catalog == nullptr) { return nullptr; } - return catalog->MakeStorageCredentialProvider(identifier, table_session, plan_id); + // Without a plan id, refresh is table-scoped (no planId query parameter). + std::optional scoped_plan_id; + if (!plan_id.empty()) { + scoped_plan_id = plan_id; + } + return catalog->MakeStorageCredentialProvider(identifier, table_session, + std::move(scoped_plan_id)); }; RestScanContext rest_ctx{ .client = client_, diff --git a/src/iceberg/test/rest_catalog_unit_test.cc b/src/iceberg/test/rest_catalog_unit_test.cc index 1749394c60..4b42febb54 100644 --- a/src/iceberg/test/rest_catalog_unit_test.cc +++ b/src/iceberg/test/rest_catalog_unit_test.cc @@ -360,6 +360,40 @@ TEST_F(RestCatalogLoadTableTest, ScanCredentialProviderRequestsPlanId) { EXPECT_EQ(refreshed[0].config.at("k"), "v2"); } +// -------------------------------------------------------------------------- +// A completed plan without a plan-id refreshes credentials without planId. +// -------------------------------------------------------------------------- +TEST_F(RestCatalogLoadTableTest, ScanCredentialProviderWithoutPlanIdOmitsPlanId) { + RegisterCredentialedFileIO(); + EXPECT_CALL(*mock_client_, + Get(::testing::Not(::testing::HasSubstr("/credentials")), _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting( + 200, + R"({"status":"completed","storage-credentials":[{"prefix":"s3://bucket","config":{"k":"v"}}]})"))); + EXPECT_CALL(*mock_client_, + Get(::testing::HasSubstr("/credentials"), ::testing::IsEmpty(), _, _, _)) + .WillOnce(Return(HttpResponse::MakeForTesting( + 200, + R"({"storage-credentials":[{"prefix":"s3://bucket","config":{"k":"v2"}}]})"))); + + auto endpoints = all_plan_endpoints_; + endpoints.insert(Endpoint::TableCredentials()); + ICEBERG_UNWRAP_OR_FAIL(auto catalog, + MakeCatalog(endpoints, /*client_scan_mode=*/"", + {{"io-impl", std::string(kCredentialedFileIO)}})); + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); + ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewScan()); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + + ASSERT_NE(installed_provider, nullptr); + ICEBERG_UNWRAP_OR_FAIL(auto refreshed, installed_provider->Load()); + ASSERT_EQ(refreshed.size(), 1); +} + // -------------------------------------------------------------------------- // Without the TableCredentials endpoint, scan credentials are not refreshed. // -------------------------------------------------------------------------- From 5949bfa2053f6eb046d102b00d65bd6142cc2e19 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 11 Oct 2026 14:56:48 -0700 Subject: [PATCH 32/36] fix(rest): ignore storage credentials in fetchScanTasks responses The spec only defines storage-credentials on completed planning results, and Java ignores them on fetchScanTasks. Drop the field from FetchScanTasksResponse so later responses can no longer replace the plan's scan FileIO, and keep the scan FileIO as a plain member now that the stream no longer writes it. --- src/iceberg/catalog/rest/json_serde.cc | 45 +++++----- src/iceberg/catalog/rest/rest_table_scan.cc | 93 ++++++++------------- src/iceberg/catalog/rest/rest_table_scan.h | 4 +- src/iceberg/catalog/rest/types.cc | 6 +- src/iceberg/catalog/rest/types.h | 1 - src/iceberg/test/rest_table_scan_test.cc | 44 ++-------- 6 files changed, 65 insertions(+), 128 deletions(-) diff --git a/src/iceberg/catalog/rest/json_serde.cc b/src/iceberg/catalog/rest/json_serde.cc index c5bf08a808..83b6e18fa0 100644 --- a/src/iceberg/catalog/rest/json_serde.cc +++ b/src/iceberg/catalog/rest/json_serde.cc @@ -128,8 +128,7 @@ Result StorageCredentialFromJson(const nlohmann::json& json) return credential; } -/// \brief Reads the optional `storage-credentials` array shared by the -/// LoadTable and LoadCredentials responses. +/// \brief Reads the optional `storage-credentials` array. Result> StorageCredentialsFromJson( const nlohmann::json& json) { std::vector credentials; @@ -146,6 +145,21 @@ Result> StorageCredentialsFromJson( } return credentials; } + +/// \brief Writes `storage-credentials` to json if any are present. +Status StorageCredentialsToJson(const std::vector& credentials, + nlohmann::json& json) { + if (credentials.empty()) { + return {}; + } + nlohmann::json creds_json = nlohmann::json::array(); + for (const auto& cred : credentials) { + ICEBERG_ASSIGN_OR_RAISE(auto entry, StorageCredentialToJson(cred)); + creds_json.push_back(std::move(entry)); + } + json[kStorageCredentials] = std::move(creds_json); + return {}; +} } // namespace Result>> FileScanTasksFromJson( @@ -292,15 +306,6 @@ Result ScanTaskFieldsToJson( json[kFileScanTasks] = std::move(tasks_json); } - if (!response.storage_credentials.empty()) { - nlohmann::json creds_json = nlohmann::json::array(); - for (const auto& cred : response.storage_credentials) { - ICEBERG_ASSIGN_OR_RAISE(auto entry, StorageCredentialToJson(cred)); - creds_json.push_back(std::move(entry)); - } - json[kStorageCredentials] = std::move(creds_json); - } - return json; } @@ -343,20 +348,6 @@ Status ScanTaskFieldsFromJson( partition_specs_by_id, schema)); } - // 4. storage_credentials - if (json.contains(kStorageCredentials)) { - ICEBERG_ASSIGN_OR_RAISE(auto creds_json, - GetJsonValue(json, kStorageCredentials)); - if (!creds_json.is_array()) { - return JsonParseError("Cannot parse storage credentials from non-array: {}", - SafeDumpJson(creds_json)); - } - for (const auto& entry : creds_json) { - ICEBERG_ASSIGN_OR_RAISE(auto cred, StorageCredentialFromJson(entry)); - response.storage_credentials.push_back(std::move(cred)); - } - } - return {}; } @@ -908,6 +899,7 @@ Result PlanTableScanResponseFromJson( } ICEBERG_RETURN_UNEXPECTED( ScanTaskFieldsFromJson(json, response, partition_specs_by_id, schema)); + ICEBERG_ASSIGN_OR_RAISE(response.storage_credentials, StorageCredentialsFromJson(json)); ICEBERG_RETURN_UNEXPECTED(response.Validate()); return response; } @@ -919,6 +911,7 @@ Result ToJson( const Schema& schema) { ICEBERG_ASSIGN_OR_RAISE(nlohmann::json json, ScanTaskFieldsToJson(response, partition_specs_by_id, schema)); + ICEBERG_RETURN_UNEXPECTED(StorageCredentialsToJson(response.storage_credentials, json)); json[kPlanStatus] = ToString(response.plan_status); if (!response.plan_id.empty()) { json[kPlanId] = response.plan_id; @@ -945,6 +938,7 @@ Result FetchPlanningResultResponseFromJson( } ICEBERG_RETURN_UNEXPECTED( ScanTaskFieldsFromJson(json, response, partition_specs_by_id, schema)); + ICEBERG_ASSIGN_OR_RAISE(response.storage_credentials, StorageCredentialsFromJson(json)); ICEBERG_RETURN_UNEXPECTED(response.Validate()); return response; } @@ -956,6 +950,7 @@ Result ToJson( const Schema& schema) { ICEBERG_ASSIGN_OR_RAISE(nlohmann::json json, ScanTaskFieldsToJson(response, partition_specs_by_id, schema)); + ICEBERG_RETURN_UNEXPECTED(StorageCredentialsToJson(response.storage_credentials, json)); json[kPlanStatus] = ToString(response.plan_status); if (response.error.has_value()) { json[kError] = ToJson(*response.error)[kError]; diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 4025f76d7b..9dc476be45 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -62,10 +62,6 @@ Result ResolvePollConfig(const RestScanContext& ctx) { } using SpecsById = std::unordered_map>; -// A shared slot that lets the scan and the stream share a single FileIO reference. -// Credentials vended by any lazy FetchScanTasks response are written into *slot, -// making them visible to RestTableScan::io() even after the stream is consumed. -using ScanIoSlot = std::shared_ptr>; #define ICEBERG_ENDPOINT_CHECK(endpoints, endpoint) \ do { \ @@ -121,15 +117,14 @@ void CancelPlanning(const RestScanContext& ctx, const std::string& plan_id) { } Result>> FetchScanTasks( - const RestScanContext& ctx, const std::string& plan_id, const Schema& schema, - const std::string& plan_task, const SpecsById& specs, - std::shared_ptr& scan_io); + const RestScanContext& ctx, const Schema& schema, const std::string& plan_task, + const SpecsById& specs); Result>> ResolveScanTasks( - const RestScanContext& ctx, const std::string& plan_id, const Schema& schema, + const RestScanContext& ctx, const Schema& schema, const std::optional>& plan_tasks, const std::optional>>& file_scan_tasks, - const SpecsById& specs, std::shared_ptr& scan_io) { + const SpecsById& specs) { std::vector> result; if (file_scan_tasks.has_value()) { @@ -138,8 +133,7 @@ Result>> ResolveScanTasks( if (plan_tasks.has_value()) { for (const auto& token : *plan_tasks) { - ICEBERG_ASSIGN_OR_RAISE( - auto tasks, FetchScanTasks(ctx, plan_id, schema, token, specs, scan_io)); + ICEBERG_ASSIGN_OR_RAISE(auto tasks, FetchScanTasks(ctx, schema, token, specs)); result.insert(result.end(), tasks.begin(), tasks.end()); } } @@ -149,11 +143,9 @@ Result>> ResolveScanTasks( // Fetches one plan task without resolving any nested plan tasks it returns. Result FetchScanTasksOnce(const RestScanContext& ctx, - const std::string& plan_id, const Schema& schema, const std::string& plan_task, - const SpecsById& specs, - std::shared_ptr& scan_io) { + const SpecsById& specs) { ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::FetchScanTasks()); ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->FetchScanTasks(ctx.identifier)); @@ -167,19 +159,14 @@ Result FetchScanTasksOnce(const RestScanContext& ctx, ICEBERG_ASSIGN_OR_RAISE(auto result, FetchScanTasksResponseFromJson(json, specs, schema)); ICEBERG_RETURN_UNEXPECTED(result.Validate()); - ICEBERG_RETURN_UNEXPECTED( - ApplyStorageCredentials(ctx, plan_id, result.storage_credentials, scan_io)); return result; } Result>> FetchScanTasks( - const RestScanContext& ctx, const std::string& plan_id, const Schema& schema, - const std::string& plan_task, const SpecsById& specs, - std::shared_ptr& scan_io) { - ICEBERG_ASSIGN_OR_RAISE( - auto result, FetchScanTasksOnce(ctx, plan_id, schema, plan_task, specs, scan_io)); - return ResolveScanTasks(ctx, plan_id, schema, result.plan_tasks, result.file_scan_tasks, - specs, scan_io); + const RestScanContext& ctx, const Schema& schema, const std::string& plan_task, + const SpecsById& specs) { + ICEBERG_ASSIGN_OR_RAISE(auto result, FetchScanTasksOnce(ctx, schema, plan_task, specs)); + return ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs); } // Polls until the plan completes and returns the completed response. Plan tasks @@ -268,24 +255,18 @@ Result FetchPlanningResult( /// constructed. The stream then yields the directly-returned file_scan_tasks first, /// then fetches each plan_task token lazily one at a time as Next() is called, /// queuing any nested plan tasks returned by the server. -/// -/// scan_io_slot is shared with the owning RestTableScan so that credentials vended -/// by any FetchScanTasks response are visible through RestTableScan::io() even after -/// the stream has been consumed. class RestFileScanTaskStream final : public FileScanTaskStream { public: RestFileScanTaskStream(RestScanContext ctx, std::shared_ptr schema, std::string plan_id, std::vector> initial_tasks, - std::vector plan_task_tokens, SpecsById specs, - ScanIoSlot scan_io_slot) + std::vector plan_task_tokens, SpecsById specs) : ctx_(std::move(ctx)), schema_(std::move(schema)), plan_id_(std::move(plan_id)), buffer_(std::move(initial_tasks)), plan_task_tokens_(std::move(plan_task_tokens)), - specs_(std::move(specs)), - scan_io_slot_(std::move(scan_io_slot)) {} + specs_(std::move(specs)) {} ~RestFileScanTaskStream() override { if (!consumed_) CancelPlanning(ctx_, plan_id_); @@ -302,9 +283,8 @@ class RestFileScanTaskStream final : public FileScanTaskStream { return std::nullopt; } const std::string token = plan_task_tokens_[token_pos_++]; - ICEBERG_ASSIGN_OR_RAISE( - auto result, - FetchScanTasksOnce(ctx_, plan_id_, *schema_, token, specs_, *scan_io_slot_)); + ICEBERG_ASSIGN_OR_RAISE(auto result, + FetchScanTasksOnce(ctx_, *schema_, token, specs_)); if (result.plan_tasks.has_value()) { plan_task_tokens_.insert(plan_task_tokens_.end(), result.plan_tasks->begin(), result.plan_tasks->end()); @@ -324,17 +304,16 @@ class RestFileScanTaskStream final : public FileScanTaskStream { std::vector plan_task_tokens_; size_t token_pos_ = 0; SpecsById specs_; - ScanIoSlot scan_io_slot_; bool consumed_ = false; }; /// POST /plan (polling if SUBMITTED), apply credentials, return a lazy stream. -/// -/// scan_io_slot is shared with the caller (RestTableScan) so that credentials -/// vended by FetchScanTasks responses update RestTableScan::io() in place. -Result ExecuteScanPlanStream( - const RestScanContext& ctx, const Schema& schema, std::shared_ptr schema_ptr, - PlanTableScanRequest request, const SpecsById& specs, ScanIoSlot scan_io_slot) { +Result ExecuteScanPlanStream(const RestScanContext& ctx, + const Schema& schema, + std::shared_ptr schema_ptr, + PlanTableScanRequest request, + const SpecsById& specs, + std::shared_ptr& scan_io) { ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::PlanTableScan()); ICEBERG_ASSIGN_OR_RAISE(auto poll_config, ResolvePollConfig(ctx)); @@ -362,13 +341,12 @@ Result ExecuteScanPlanStream( if (result.plan_status == PlanStatus::kSubmitted) { ICEBERG_ASSIGN_OR_RAISE( auto completed, - FetchPlanningResult(ctx, schema, plan_id, specs, poll_config, *scan_io_slot)); + FetchPlanningResult(ctx, schema, plan_id, specs, poll_config, scan_io)); return std::make_unique( ctx, std::move(schema_ptr), plan_id, std::move(completed.file_scan_tasks) .value_or(std::vector>{}), - std::move(completed.plan_tasks).value_or(std::vector{}), specs, - std::move(scan_io_slot)); + std::move(completed.plan_tasks).value_or(std::vector{}), specs); } if (result.plan_status == PlanStatus::kFailed) { @@ -381,8 +359,7 @@ Result ExecuteScanPlanStream( } // kCompleted: apply credentials from the initial response, then build the lazy stream. - if (auto s = ApplyStorageCredentials(ctx, plan_id, result.storage_credentials, - *scan_io_slot); + if (auto s = ApplyStorageCredentials(ctx, plan_id, result.storage_credentials, scan_io); !s) { CancelPlanning(ctx, plan_id); return ::iceberg::unexpected(s.error()); @@ -397,9 +374,9 @@ Result ExecuteScanPlanStream( plan_task_tokens = std::move(*result.plan_tasks); } - return std::make_unique( - ctx, std::move(schema_ptr), plan_id, std::move(initial_tasks), - std::move(plan_task_tokens), specs, std::move(scan_io_slot)); + return std::make_unique(ctx, std::move(schema_ptr), plan_id, + std::move(initial_tasks), + std::move(plan_task_tokens), specs); } /// Eager batch planning: used by RestIncrementalAppendScan which has no stream path. @@ -438,8 +415,8 @@ Result>> ExecuteScanPlan( CancelPlanning(ctx, plan_id); return ::iceberg::unexpected(s.error()); } - auto tasks = ResolveScanTasks(ctx, plan_id, schema, result.plan_tasks, - result.file_scan_tasks, specs, scan_io); + auto tasks = + ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs); if (!tasks) CancelPlanning(ctx, plan_id); return tasks; } @@ -447,8 +424,8 @@ Result>> ExecuteScanPlan( ICEBERG_ASSIGN_OR_RAISE( auto completed, FetchPlanningResult(ctx, schema, plan_id, specs, poll_config, scan_io)); - auto tasks = ResolveScanTasks(ctx, plan_id, schema, completed.plan_tasks, - completed.file_scan_tasks, specs, scan_io); + auto tasks = ResolveScanTasks(ctx, schema, completed.plan_tasks, + completed.file_scan_tasks, specs); if (!tasks) CancelPlanning(ctx, plan_id); return tasks; } @@ -474,8 +451,7 @@ RestTableScan::RestTableScan(std::shared_ptr metadata, RestScanContext rest_context) : DataTableScan(std::move(metadata), std::move(schema), std::move(io), std::move(context)), - rest_context_(std::move(rest_context)), - scan_io_slot_(std::make_shared>()) {} + rest_context_(std::move(rest_context)) {} Result> RestTableScan::Make( std::shared_ptr metadata, std::shared_ptr schema, @@ -490,8 +466,7 @@ Result> RestTableScan::Make( } Result RestTableScan::PlanFilesStream() const { - *scan_io_slot_ = - nullptr; // reset so stale credentials from a prior plan are not reused + scan_io_ = nullptr; // reset so stale credentials from a prior plan are not reused TableMetadataCache metadata_cache(metadata_.get()); ICEBERG_ASSIGN_OR_RAISE(auto specs, metadata_cache.GetPartitionSpecsById()); @@ -521,11 +496,11 @@ Result RestTableScan::PlanFilesStream() const { } return ExecuteScanPlanStream(rest_context_, *schema_, schema_, std::move(request), - specs, scan_io_slot_); + specs, scan_io_); } const std::shared_ptr& RestTableScan::io() const { - return *scan_io_slot_ ? *scan_io_slot_ : io_; + return scan_io_ ? scan_io_ : io_; } // --------------------------------------------------------------------------- diff --git a/src/iceberg/catalog/rest/rest_table_scan.h b/src/iceberg/catalog/rest/rest_table_scan.h index b853a8cb03..2280263dca 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.h +++ b/src/iceberg/catalog/rest/rest_table_scan.h @@ -97,9 +97,7 @@ class ICEBERG_REST_EXPORT RestTableScan : public DataTableScan { RestScanContext rest_context); RestScanContext rest_context_; - /// Shared slot so credentials vended by any lazy FetchScanTasks response are - /// visible through io() even after the stream has been consumed. - mutable std::shared_ptr> scan_io_slot_; + mutable std::shared_ptr scan_io_; }; /// \brief Builder that produces a RestTableScan with the REST HTTP context injected. diff --git a/src/iceberg/catalog/rest/types.cc b/src/iceberg/catalog/rest/types.cc index ef20f44da7..27cd7b1705 100644 --- a/src/iceberg/catalog/rest/types.cc +++ b/src/iceberg/catalog/rest/types.cc @@ -210,7 +210,6 @@ bool OptionalSharedPtrVectorEqual( template bool ScanTaskFieldsEqual(const Response& lhs, const Response& rhs) { return lhs.plan_tasks == rhs.plan_tasks && - lhs.storage_credentials == rhs.storage_credentials && SharedPtrVectorEqual(lhs.delete_files, rhs.delete_files) && OptionalSharedPtrVectorEqual(lhs.file_scan_tasks, rhs.file_scan_tasks, FileScanTaskEqual); @@ -230,13 +229,14 @@ bool HasNonEmptyFileScanTasks(const Response& response) { bool PlanTableScanResponse::operator==(const PlanTableScanResponse& other) const { return ScanTaskFieldsEqual(*this, other) && plan_status == other.plan_status && - plan_id == other.plan_id && error == other.error; + plan_id == other.plan_id && error == other.error && + storage_credentials == other.storage_credentials; } bool FetchPlanningResultResponse::operator==( const FetchPlanningResultResponse& other) const { return ScanTaskFieldsEqual(*this, other) && plan_status == other.plan_status && - error == other.error; + error == other.error && storage_credentials == other.storage_credentials; } bool FetchScanTasksRequest::operator==(const FetchScanTasksRequest& other) const { diff --git a/src/iceberg/catalog/rest/types.h b/src/iceberg/catalog/rest/types.h index aa2b57d17b..f048d6e7d6 100644 --- a/src/iceberg/catalog/rest/types.h +++ b/src/iceberg/catalog/rest/types.h @@ -388,7 +388,6 @@ struct ICEBERG_REST_EXPORT FetchScanTasksResponse { std::optional> plan_tasks; std::optional>> file_scan_tasks; std::vector> delete_files; - std::vector storage_credentials; Status Validate() const; diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index f518ad9ef8..ec31bfccd0 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -721,9 +721,9 @@ TEST_F(RestTableScanTest, NoStorageCredentialsEffectiveIoFallsBackToTableIO) { } // -------------------------------------------------------------------------- -// Storage credentials returned in FetchScanTasksResponse also update io(). +// Storage credentials in a FetchScanTasks response are ignored. // -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, StorageCredentialsInFetchScanTasksResponseUpdatesEffectiveIO) { +TEST_F(RestTableScanTest, StorageCredentialsInFetchScanTasksResponseAreIgnored) { constexpr std::string_view kPlanResponse = R"({"status":"completed","plan-id":"plan-cred","plan-tasks":["tok-cred"]})"; constexpr std::string_view kTasksResponse = R"({ @@ -741,7 +741,7 @@ TEST_F(RestTableScanTest, StorageCredentialsInFetchScanTasksResponseUpdatesEffec ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); - EXPECT_NE(scan->io().get(), file_io_.get()); + EXPECT_EQ(scan->io().get(), file_io_.get()); } // -------------------------------------------------------------------------- @@ -1216,36 +1216,6 @@ TEST_F(RestTableScanTest, StorageCredentialsInstallPlanScopedProvider) { EXPECT_EQ(installed_credentials[0].prefix, "s3://bucket"); } -// -------------------------------------------------------------------------- -// Vended credentials in a FetchScanTasks response also use the plan id. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, FetchScanTasksCredentialsInstallPlanScopedProvider) { - constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"plan-cred-tasks","plan-tasks":["tok-1"]})"; - constexpr std::string_view kTasksResponse = R"({ - "file-scan-tasks": [], - "storage-credentials": [{"prefix": "s3://bucket", "config": {"k": "v"}}] - })"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); - RegisterCredentialedFileIO(); - - std::vector requested_plan_ids; - auto provider = std::make_shared(); - auto ctx = MakeContext(); - ctx.catalog_config = {{"io-impl", std::string(kCredentialedFileIO)}}; - ctx.credential_provider_factory = [&](const std::string& plan_id) { - requested_plan_ids.push_back(plan_id); - return provider; - }; - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(std::move(ctx))); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - - EXPECT_THAT(requested_plan_ids, ::testing::ElementsAre("plan-cred-tasks")); - EXPECT_EQ(installed_provider, provider); -} - // -------------------------------------------------------------------------- // Without a provider factory, vended credentials are installed as static. // -------------------------------------------------------------------------- @@ -1321,7 +1291,7 @@ TEST_F(RestTableScanTest, CancelCalledOnFetchPlanningResultGetError) { } // -------------------------------------------------------------------------- -// Stale credentials: second PlanFilesStream call resets scan_io_slot_ so that +// Stale credentials: second PlanFilesStream call resets the scan FileIO so that // credentials from the first plan response do not persist into the second. // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, SecondPlanFilesStreamCallClearsStaleCredentials) { @@ -1818,10 +1788,10 @@ TEST_F(RestIncrementalAppendScanTest, NoStorageCredentialsEffectiveIoFallsBackTo } // -------------------------------------------------------------------------- -// Storage credentials in FetchScanTasksResponse also update io(). +// Storage credentials in a FetchScanTasks response are ignored. // -------------------------------------------------------------------------- TEST_F(RestIncrementalAppendScanTest, - StorageCredentialsInFetchScanTasksResponseUpdatesEffectiveIO) { + StorageCredentialsInFetchScanTasksResponseAreIgnored) { constexpr std::string_view kPlanResponse = R"({"status":"completed","plan-id":"incr-plan-1","plan-tasks":["tok-incr-1"]})"; constexpr std::string_view kTasksResponse = R"({ @@ -1837,7 +1807,7 @@ TEST_F(RestIncrementalAppendScanTest, ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); - EXPECT_NE(scan->io().get(), file_io_.get()); + EXPECT_EQ(scan->io().get(), file_io_.get()); } // ========================================================================== From bfc92667a0f288c84a692e8aaed62d3936ab30a4 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 11 Oct 2026 14:56:48 -0700 Subject: [PATCH 33/36] test(rest): use braced return in incremental scan test builder --- src/iceberg/test/rest_table_scan_test.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index ec31bfccd0..7c827690c1 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -1362,8 +1362,7 @@ class RestIncrementalAppendScanTest : public RestTableScanTest { } RestIncrementalAppendScanBuilder MakeBuilder(RestScanContext ctx) { - return RestIncrementalAppendScanBuilder(metadata_, file_io_, "test.my_table", nullptr, - std::move(ctx)); + return {metadata_, file_io_, "test.my_table", nullptr, std::move(ctx)}; } Result> MakeIncrementalScan( From ee0b14697c3c2898c52a4274b26743573927d87e Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 11 Oct 2026 16:13:33 -0700 Subject: [PATCH 34/36] build: add REST table scan headers to the C++20 compatibility header --- src/iceberg/cpp20_compatibility_internal.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/iceberg/cpp20_compatibility_internal.h b/src/iceberg/cpp20_compatibility_internal.h index b5f46e488d..3f9b5b5773 100644 --- a/src/iceberg/cpp20_compatibility_internal.h +++ b/src/iceberg/cpp20_compatibility_internal.h @@ -240,6 +240,8 @@ # include "iceberg/catalog/rest/resource_paths.h" # include "iceberg/catalog/rest/rest_catalog.h" # include "iceberg/catalog/rest/rest_file_io.h" +# include "iceberg/catalog/rest/rest_table.h" +# include "iceberg/catalog/rest/rest_table_scan.h" # include "iceberg/catalog/rest/rest_util.h" # include "iceberg/catalog/rest/type_fwd.h" # include "iceberg/catalog/rest/types.h" From 9d92d39bb9eb769427d2471343f6bd68381f76a5 Mon Sep 17 00:00:00 2001 From: Sandeep Gottimukkala Date: Sun, 11 Oct 2026 16:33:59 -0700 Subject: [PATCH 35/36] fix: export GetProjectedIdsVisitor for use from iceberg_rest --- src/iceberg/util/type_util.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iceberg/util/type_util.h b/src/iceberg/util/type_util.h index 8623ad254f..72e6bacee6 100644 --- a/src/iceberg/util/type_util.h +++ b/src/iceberg/util/type_util.h @@ -127,7 +127,7 @@ class PruneColumnVisitor { }; /// \brief Visitor for getting projected field IDs. -class GetProjectedIdsVisitor { +class ICEBERG_EXPORT GetProjectedIdsVisitor { public: explicit GetProjectedIdsVisitor(bool include_struct_ids = false); From b40f915dac8c64d0aa85b51624df937509253192 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Wed, 14 Oct 2026 14:07:54 +0800 Subject: [PATCH 36/36] fix(rest): fix scan planning and simplify tests Preserve residual filters, keep scan sessions alive, and centralize plan cleanup. Share planning context and consolidate REST tests. --- .../catalog/rest/catalog_properties.cc | 20 - src/iceberg/catalog/rest/catalog_properties.h | 14 +- src/iceberg/catalog/rest/http_client.cc | 17 +- src/iceberg/catalog/rest/http_client.h | 4 +- src/iceberg/catalog/rest/json_serde.cc | 3 +- src/iceberg/catalog/rest/rest_catalog.cc | 107 +- src/iceberg/catalog/rest/rest_catalog.h | 27 +- src/iceberg/catalog/rest/rest_table.cc | 2 +- src/iceberg/catalog/rest/rest_table.h | 5 +- src/iceberg/catalog/rest/rest_table_scan.cc | 379 ++-- src/iceberg/catalog/rest/rest_table_scan.h | 41 +- src/iceberg/catalog/rest/type_fwd.h | 1 + src/iceberg/expression/expression.h | 2 +- src/iceberg/test/auth_manager_test.cc | 2 +- src/iceberg/test/catalog_properties_test.cc | 91 +- src/iceberg/test/metrics_reporter_test.cc | 9 +- src/iceberg/test/rest_catalog_unit_test.cc | 382 +++-- src/iceberg/test/rest_table_scan_test.cc | 1526 ++++++----------- src/iceberg/test/table_scan_test.cc | 35 - src/iceberg/type_fwd.h | 1 + 20 files changed, 1066 insertions(+), 1602 deletions(-) diff --git a/src/iceberg/catalog/rest/catalog_properties.cc b/src/iceberg/catalog/rest/catalog_properties.cc index 28a7c10c06..759c3f1712 100644 --- a/src/iceberg/catalog/rest/catalog_properties.cc +++ b/src/iceberg/catalog/rest/catalog_properties.cc @@ -73,24 +73,4 @@ Result> RestCatalogProperties::ScanPlanningModeF return InvalidArgument("Invalid scan planning mode: '{}'.", it->second); } -Result RestCatalogProperties::ScanPlanningPollTimeoutMs() const { - const std::string value = Get(kScanPlanningPollTimeoutMs); - auto timeout_ms = StringUtils::ParseNumber(value); - if (!timeout_ms.has_value() || *timeout_ms <= 0) { - return InvalidArgument("Invalid value for {}: {} (must be positive)", - kScanPlanningPollTimeoutMs.key(), value); - } - return *timeout_ms; -} - -Result RestCatalogProperties::ScanPlanningPollNumRetries() const { - const std::string value = Get(kScanPlanningPollNumRetries); - auto num_retries = StringUtils::ParseNumber(value); - if (!num_retries.has_value() || *num_retries < 0) { - return InvalidArgument("Invalid value for {}: {} (must be non-negative)", - kScanPlanningPollNumRetries.key(), value); - } - return *num_retries; -} - } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/catalog_properties.h b/src/iceberg/catalog/rest/catalog_properties.h index eaef3084d5..da5bd550aa 100644 --- a/src/iceberg/catalog/rest/catalog_properties.h +++ b/src/iceberg/catalog/rest/catalog_properties.h @@ -65,11 +65,11 @@ class ICEBERG_REST_EXPORT RestCatalogProperties /// \brief The scan planning mode (client or server). inline static Entry kScanPlanningMode{"scan-planning-mode", "client"}; /// \brief Maximum time to wait for a submitted scan plan, in milliseconds. - inline static Entry kScanPlanningPollTimeoutMs{ - "rest-scan-planning.poll-timeout-ms", "300000"}; + inline static Entry kScanPlanningPollTimeoutMs{ + "rest-scan-planning.poll-timeout-ms", 300000}; /// \brief Extra polls after the first fetch of a submitted scan plan. - inline static Entry kScanPlanningPollNumRetries{ - "rest-scan-planning.poll-num-retries", "10"}; + inline static Entry kScanPlanningPollNumRetries{ + "rest-scan-planning.poll-num-retries", 10}; /// \brief The prefix for HTTP headers. inline static constexpr std::string_view kHeaderPrefix = "header."; @@ -97,12 +97,6 @@ class ICEBERG_REST_EXPORT RestCatalogProperties /// std::nullopt if the key is absent. static Result> ScanPlanningModeFrom( const std::unordered_map& config); - - /// \brief Get the scan planning poll timeout in milliseconds; must be positive. - Result ScanPlanningPollTimeoutMs() const; - - /// \brief Get the number of scan planning poll retries; must be non-negative. - Result ScanPlanningPollNumRetries() const; }; } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/http_client.cc b/src/iceberg/catalog/rest/http_client.cc index ec870b87d3..d48471cde3 100644 --- a/src/iceberg/catalog/rest/http_client.cc +++ b/src/iceberg/catalog/rest/http_client.cc @@ -50,6 +50,14 @@ class HttpResponse::Impl { }; HttpResponse::HttpResponse() = default; + +HttpResponse::HttpResponse(int32_t status_code, std::string body) { + cpr::Response cpr_response; + cpr_response.status_code = status_code; + cpr_response.text = std::move(body); + impl_ = std::make_unique(std::move(cpr_response)); +} + HttpResponse::~HttpResponse() = default; HttpResponse::HttpResponse(HttpResponse&&) noexcept = default; HttpResponse& HttpResponse::operator=(HttpResponse&&) noexcept = default; @@ -62,15 +70,6 @@ std::unordered_map HttpResponse::headers() const { return impl_->headers(); } -HttpResponse HttpResponse::MakeForTesting(int32_t status_code, std::string body) { - cpr::Response cpr_response; - cpr_response.status_code = status_code; - cpr_response.text = std::move(body); - HttpResponse response; - response.impl_ = std::make_unique(std::move(cpr_response)); - return response; -} - namespace { /// \brief Default error type for unparseable REST responses. diff --git a/src/iceberg/catalog/rest/http_client.h b/src/iceberg/catalog/rest/http_client.h index d42798faea..9400202dc8 100644 --- a/src/iceberg/catalog/rest/http_client.h +++ b/src/iceberg/catalog/rest/http_client.h @@ -45,6 +45,7 @@ namespace iceberg::rest { class ICEBERG_REST_EXPORT HttpResponse { public: HttpResponse(); + HttpResponse(int32_t status_code, std::string body); ~HttpResponse(); HttpResponse(const HttpResponse&) = delete; @@ -61,9 +62,6 @@ class ICEBERG_REST_EXPORT HttpResponse { /// \brief Get the headers of the response as a map. std::unordered_map headers() const; - /// \brief Create a response for use in unit tests. - static HttpResponse MakeForTesting(int32_t status_code, std::string body); - private: friend class HttpClient; class Impl; diff --git a/src/iceberg/catalog/rest/json_serde.cc b/src/iceberg/catalog/rest/json_serde.cc index 83b6e18fa0..c7d4662dd9 100644 --- a/src/iceberg/catalog/rest/json_serde.cc +++ b/src/iceberg/catalog/rest/json_serde.cc @@ -347,7 +347,6 @@ Status ScanTaskFieldsFromJson( FileScanTasksFromJson(file_scan_tasks_json, response.delete_files, partition_specs_by_id, schema)); } - return {}; } @@ -844,6 +843,8 @@ Result ToJson(const PlanTableScanRequest& request) { if (request.snapshot_id.has_value()) { json[kSnapshotId] = request.snapshot_id.value(); } + // FIXME: Preserve explicit empty projections as select: []; omitting select + // means the default projection. Java currently has the same issue. if (!request.select.empty()) { json[kSelect] = request.select; } diff --git a/src/iceberg/catalog/rest/rest_catalog.cc b/src/iceberg/catalog/rest/rest_catalog.cc index e8e0e17bf0..767b6eef69 100644 --- a/src/iceberg/catalog/rest/rest_catalog.cc +++ b/src/iceberg/catalog/rest/rest_catalog.cc @@ -31,7 +31,6 @@ #include -#include "iceberg/catalog/rest/auth/auth_manager_internal.h" #include "iceberg/catalog/rest/auth/auth_managers.h" #include "iceberg/catalog/rest/auth/auth_session.h" #include "iceberg/catalog/rest/catalog_properties.h" @@ -44,6 +43,7 @@ #include "iceberg/catalog/rest/rest_file_io.h" #include "iceberg/catalog/rest/rest_metrics_reporter_internal.h" #include "iceberg/catalog/rest/rest_table.h" +#include "iceberg/catalog/rest/rest_table_scan.h" #include "iceberg/catalog/rest/rest_util.h" #include "iceberg/catalog/rest/types.h" #include "iceberg/json_serde_internal.h" @@ -129,11 +129,10 @@ std::string RestTableName(std::string_view catalog_name, } /// \brief Fetch server configuration from the REST catalog server. -Result FetchServerConfig(const ResourcePaths& paths, +Result FetchServerConfig(HttpClient& client, const ResourcePaths& paths, const RestCatalogProperties& current_config, auth::AuthSession& session) { ICEBERG_ASSIGN_OR_RAISE(auto config_path, paths.Config()); - HttpClient client(current_config.ExtractHeaders()); // Send the client's warehouse location to the service to keep in sync. // This is needed for cases where the warehouse is configured client side, but may @@ -432,7 +431,14 @@ RestCatalog::~RestCatalog() { } Result> RestCatalog::Make( - const RestCatalogProperties& config, Executor* metrics_executor) { + const RestCatalogProperties& config, Executor* metrics_executor, + ClientFactory client_factory) { + if (!client_factory) { + client_factory = [](const RestCatalogProperties& properties) + -> Result> { + return std::make_shared(properties.ExtractHeaders()); + }; + } ICEBERG_ASSIGN_OR_RAISE(auto uri, config.Uri()); std::string catalog_name = config.Get(RestCatalogProperties::kName); @@ -445,11 +451,12 @@ Result> RestCatalog::Make( config.Get(RestCatalogProperties::kNamespaceSeparator))); // Create init session for fetching server configuration - auto init_client = std::make_shared(config.ExtractHeaders()); + ICEBERG_ASSIGN_OR_RAISE(auto init_client, client_factory(config)); + ICEBERG_PRECHECK(init_client != nullptr, "HTTP client factory returned null"); ICEBERG_ASSIGN_OR_RAISE(auto init_session, auth_manager->InitSession(init_client, config.configs())); ICEBERG_ASSIGN_OR_RAISE(auto server_config, - FetchServerConfig(*paths, config, *init_session)); + FetchServerConfig(*init_client, *paths, config, *init_session)); RestCatalogProperties final_config = RestCatalogProperties::FromMap( MergeConfigs(server_config.defaults, config.configs(), server_config.overrides)); @@ -475,8 +482,12 @@ Result> RestCatalog::Make( // Get snapshot loading mode ICEBERG_ASSIGN_OR_RAISE(auto snapshot_mode, final_config.SnapshotLoadingMode()); + ICEBERG_ASSIGN_OR_RAISE( + auto client_scan_planning_mode, + RestCatalogProperties::ScanPlanningModeFrom(final_config.configs())); - auto client = std::make_shared(final_config.ExtractHeaders()); + ICEBERG_ASSIGN_OR_RAISE(auto client, client_factory(final_config)); + ICEBERG_PRECHECK(client != nullptr, "HTTP client factory returned null"); ICEBERG_ASSIGN_OR_RAISE(auto catalog_session, auth_manager->CatalogSession(client, final_config.configs())); @@ -495,23 +506,8 @@ Result> RestCatalog::Make( return std::shared_ptr(new RestCatalog( std::move(final_config), std::move(file_io), std::move(client), std::move(paths), std::move(endpoints), std::move(auth_manager), std::move(catalog_session), - snapshot_mode, std::move(default_context), std::move(reporter), metrics_executor)); -} - -Result> RestCatalog::MakeForTesting( - RestCatalogProperties config, std::shared_ptr file_io, - std::shared_ptr client, std::shared_ptr paths, - std::unordered_set endpoints) { - std::string catalog_name = config.Get(RestCatalogProperties::kName); - ICEBERG_ASSIGN_OR_RAISE(auto auth_manager, - auth::MakeNoopAuthManager(catalog_name, config.configs())); - auto session = auth::AuthSession::MakeDefault({}); - ICEBERG_ASSIGN_OR_RAISE(auto snapshot_mode, config.SnapshotLoadingMode()); - return std::shared_ptr( - new RestCatalog(std::move(config), std::move(file_io), std::move(client), - std::move(paths), std::move(endpoints), std::move(auth_manager), - std::move(session), snapshot_mode, SessionContext::Empty(), - /*reporter=*/nullptr, /*metrics_executor=*/nullptr)); + snapshot_mode, client_scan_planning_mode, std::move(default_context), + std::move(reporter), metrics_executor)); } RestCatalog::RestCatalog(RestCatalogProperties config, std::shared_ptr file_io, @@ -520,7 +516,9 @@ RestCatalog::RestCatalog(RestCatalogProperties config, std::shared_ptr f std::unordered_set endpoints, std::unique_ptr auth_manager, std::shared_ptr catalog_session, - SnapshotMode snapshot_mode, SessionContext default_context, + SnapshotMode snapshot_mode, + std::optional client_scan_planning_mode, + SessionContext default_context, std::shared_ptr reporter, Executor* metrics_executor) : config_(std::move(config)), @@ -532,6 +530,7 @@ RestCatalog::RestCatalog(RestCatalogProperties config, std::shared_ptr f auth_manager_(std::move(auth_manager)), catalog_session_(std::move(catalog_session)), snapshot_mode_(snapshot_mode), + client_scan_planning_mode_(client_scan_planning_mode), default_context_(std::move(default_context)), reporter_(std::move(reporter)), metrics_executor_(metrics_executor) { @@ -570,7 +569,8 @@ Result> RestCatalog::TableAuthSession( std::move(contextual_session)); } -std::shared_ptr RestCatalog::MakeStorageCredentialProvider( +Result> +RestCatalog::MakeStorageCredentialProvider( const TableIdentifier& identifier, std::shared_ptr table_session, std::optional plan_id) const { if (!supported_endpoints_.contains(Endpoint::TableCredentials())) { @@ -579,18 +579,10 @@ std::shared_ptr RestCatalog::MakeStorageCredentialPro "Catalog does not advertise {}; vended credentials for '{}' will not be " "refreshed", Endpoint::TableCredentials().ToString(), ToString(identifier)); - return nullptr; - } - auto path = paths_->Credentials(identifier); - if (!path.has_value()) { - ICEBERG_LOG_WARN( - "Cannot build the credentials path for '{}' ({}); its vended credentials " - "will not be refreshed", - ToString(identifier), path.error().message); - return nullptr; + return std::shared_ptr{}; } + ICEBERG_ASSIGN_OR_RAISE(auto credentials_path, paths_->Credentials(identifier)); auto client = client_; - auto credentials_path = std::move(path.value()); auto session = std::move(table_session); std::unordered_map params; if (plan_id.has_value()) { @@ -610,7 +602,8 @@ Result> RestCatalog::TableFileIO( // Only vended credentials expire, so only they need a provider. std::shared_ptr provider; if (!storage_credentials.empty()) { - provider = MakeStorageCredentialProvider(identifier, std::move(table_session)); + ICEBERG_ASSIGN_OR_RAISE( + provider, MakeStorageCredentialProvider(identifier, std::move(table_session))); } return MakeTableFileIO(config_.configs(), table_config, storage_credentials, std::move(provider)); @@ -1001,20 +994,6 @@ Result> RestCatalog::MakeTableFromLoadResult( auto table_catalog = std::make_shared( shared_from_this(), context, identifier, table_config, table_session, table_io); - ICEBERG_ASSIGN_OR_RAISE(auto client_mode, - RestCatalogProperties::ScanPlanningModeFrom(config_.configs())); - ICEBERG_ASSIGN_OR_RAISE(auto server_mode, - RestCatalogProperties::ScanPlanningModeFrom(table_config)); - if (client_mode.has_value() && server_mode.has_value() && - *client_mode != *server_mode) { - Log(LogLevel::kWarn, - "Scan planning mode mismatch for table {}: client config={}, server config={}. " - "Server config will take precedence.", - identifier.ToString(), - *client_mode == ScanPlanningMode::kClient ? "client" : "server", - *server_mode == ScanPlanningMode::kClient ? "client" : "server"); - } - return MakeTable(identifier, std::move(result.metadata), std::move(result.metadata_location), std::move(table_io), std::move(table_catalog), std::move(reporter), table_config, @@ -1043,12 +1022,19 @@ Result> RestCatalog::MakeTable( const std::unordered_map& table_config, std::shared_ptr table_session) { // Table config overrides client config. - ICEBERG_ASSIGN_OR_RAISE(auto client_mode, - RestCatalogProperties::ScanPlanningModeFrom(config_.configs())); ICEBERG_ASSIGN_OR_RAISE(auto server_mode, RestCatalogProperties::ScanPlanningModeFrom(table_config)); - ScanPlanningMode effective_mode = - server_mode.value_or(client_mode.value_or(ScanPlanningMode::kClient)); + if (client_scan_planning_mode_.has_value() && server_mode.has_value() && + *client_scan_planning_mode_ != *server_mode) { + Log(LogLevel::kWarn, + "Scan planning mode mismatch for table {}: client config={}, server config={}. " + "Server config will take precedence.", + identifier.ToString(), + *client_scan_planning_mode_ == ScanPlanningMode::kClient ? "client" : "server", + *server_mode == ScanPlanningMode::kClient ? "client" : "server"); + } + ScanPlanningMode effective_mode = server_mode.value_or( + client_scan_planning_mode_.value_or(ScanPlanningMode::kClient)); if (effective_mode == ScanPlanningMode::kServer) { if (!supported_endpoints_.contains(Endpoint::PlanTableScan())) { @@ -1057,13 +1043,10 @@ Result> RestCatalog::MakeTable( "the PlanTableScan endpoint.", identifier.ToString()); } - auto credential_provider_factory = - [weak_catalog = weak_from_this(), identifier, table_session]( - const std::string& plan_id) -> std::shared_ptr { - auto catalog = weak_catalog.lock(); - if (catalog == nullptr) { - return nullptr; - } + // Scans can outlive the table; keep its authentication sessions alive too. + auto credential_provider_factory = [catalog = shared_from_this(), identifier, + table_session](const std::string& plan_id) + -> Result> { // Without a plan id, refresh is table-scoped (no planId query parameter). std::optional scoped_plan_id; if (!plan_id.empty()) { diff --git a/src/iceberg/catalog/rest/rest_catalog.h b/src/iceberg/catalog/rest/rest_catalog.h index 962c5cf28b..967e7cc9f6 100644 --- a/src/iceberg/catalog/rest/rest_catalog.h +++ b/src/iceberg/catalog/rest/rest_catalog.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include #include @@ -52,19 +53,18 @@ class ICEBERG_REST_EXPORT RestCatalog final RestCatalog(RestCatalog&&) = delete; RestCatalog& operator=(RestCatalog&&) = delete; + using ClientFactory = + std::function>(const RestCatalogProperties&)>; + /// \brief Create a RestCatalog instance. /// \param metrics_executor Optional non-owning executor for asynchronous REST metrics. /// It must outlive the catalog and all tables created by it. When null, REST metrics /// are reported synchronously. + /// \param client_factory Optional HTTP client factory, called with the initial config + /// and then the merged server config. When empty, uses the default HTTP client. static Result> Make(const RestCatalogProperties& config, - Executor* metrics_executor = nullptr); - - /// \brief Test-only factory that constructs a RestCatalog with pre-built dependencies, - /// bypassing the FetchServerConfig HTTP exchange performed by Make(). - static Result> MakeForTesting( - RestCatalogProperties config, std::shared_ptr file_io, - std::shared_ptr client, std::shared_ptr paths, - std::unordered_set endpoints); + Executor* metrics_executor = nullptr, + ClientFactory client_factory = {}); std::string_view name() const override; @@ -81,8 +81,10 @@ class ICEBERG_REST_EXPORT RestCatalog final std::unordered_set endpoints, std::unique_ptr auth_manager, std::shared_ptr catalog_session, - SnapshotMode snapshot_mode, SessionContext default_context, - std::shared_ptr reporter, Executor* metrics_executor); + SnapshotMode snapshot_mode, + std::optional client_scan_planning_mode, + SessionContext default_context, std::shared_ptr reporter, + Executor* metrics_executor); Result> ContextualAuthSession( const SessionContext& context); @@ -100,8 +102,8 @@ class ICEBERG_REST_EXPORT RestCatalog final /// \brief Build a provider for this table's vended credentials. /// - /// Returns nullptr when the catalog does not serve LoadCredentials. - std::shared_ptr MakeStorageCredentialProvider( + /// Returns a null provider when the catalog does not serve LoadCredentials. + Result> MakeStorageCredentialProvider( const TableIdentifier& identifier, std::shared_ptr table_session, std::optional plan_id = std::nullopt) const; @@ -223,6 +225,7 @@ class ICEBERG_REST_EXPORT RestCatalog final std::unique_ptr auth_manager_; std::shared_ptr catalog_session_; SnapshotMode snapshot_mode_; + std::optional client_scan_planning_mode_; SessionContext default_context_; std::weak_ptr default_catalog_; std::shared_ptr reporter_; diff --git a/src/iceberg/catalog/rest/rest_table.cc b/src/iceberg/catalog/rest/rest_table.cc index dd2b43b960..dda33316fc 100644 --- a/src/iceberg/catalog/rest/rest_table.cc +++ b/src/iceberg/catalog/rest/rest_table.cc @@ -36,7 +36,7 @@ RestTable::RestTable(TableIdentifier identifier, std::shared_ptr RestScanContext rest_context) : Table(std::move(identifier), std::move(metadata), std::move(metadata_location), std::move(io), std::move(catalog), std::move(full_name), std::move(reporter)), - rest_context_(std::move(rest_context)) {} + rest_context_(std::make_shared(std::move(rest_context))) {} RestTable::~RestTable() = default; diff --git a/src/iceberg/catalog/rest/rest_table.h b/src/iceberg/catalog/rest/rest_table.h index f47dbe5b35..fdfa7d9526 100644 --- a/src/iceberg/catalog/rest/rest_table.h +++ b/src/iceberg/catalog/rest/rest_table.h @@ -23,8 +23,7 @@ #include #include "iceberg/catalog/rest/iceberg_rest_export.h" -#include "iceberg/catalog/rest/rest_table_scan.h" -#include "iceberg/metrics/metrics_reporter.h" +#include "iceberg/catalog/rest/type_fwd.h" #include "iceberg/result.h" #include "iceberg/table.h" #include "iceberg/type_fwd.h" @@ -60,7 +59,7 @@ class ICEBERG_REST_EXPORT RestTable final : public Table { std::shared_ptr catalog, std::string full_name, std::shared_ptr reporter, RestScanContext rest_context); - RestScanContext rest_context_; + std::shared_ptr rest_context_; }; } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/rest_table_scan.cc b/src/iceberg/catalog/rest/rest_table_scan.cc index 9dc476be45..e7dae0f492 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.cc +++ b/src/iceberg/catalog/rest/rest_table_scan.cc @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include @@ -34,12 +36,15 @@ #include "iceberg/catalog/rest/resource_paths.h" #include "iceberg/catalog/rest/rest_file_io.h" #include "iceberg/catalog/rest/types.h" +#include "iceberg/expression/expression.h" #include "iceberg/json_serde_internal.h" +#include "iceberg/logging/log_macros.h" #include "iceberg/partition_spec.h" #include "iceberg/result.h" #include "iceberg/schema.h" #include "iceberg/table_metadata.h" #include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" #include "iceberg/util/type_util.h" namespace iceberg::rest { @@ -55,10 +60,29 @@ struct PollConfig { }; Result ResolvePollConfig(const RestScanContext& ctx) { - auto properties = RestCatalogProperties::FromMap(ctx.catalog_config); - ICEBERG_ASSIGN_OR_RAISE(auto max_wait_ms, properties.ScanPlanningPollTimeoutMs()); - ICEBERG_ASSIGN_OR_RAISE(auto max_retries, properties.ScanPlanningPollNumRetries()); - return PollConfig{.max_wait_ms = max_wait_ms, .max_retries = max_retries}; + const auto& timeout = RestCatalogProperties::kScanPlanningPollTimeoutMs; + const auto& retries = RestCatalogProperties::kScanPlanningPollNumRetries; + PollConfig config{ + .max_wait_ms = timeout.value(), + .max_retries = retries.value(), + }; + + if (auto it = ctx.catalog_config.find(timeout.key()); it != ctx.catalog_config.end()) { + ICEBERG_ASSIGN_OR_RAISE(config.max_wait_ms, + StringUtils::ParseNumber(it->second)); + } + ICEBERG_PRECHECK(config.max_wait_ms > 0, "Invalid value for {}: {} (must be positive)", + timeout.key(), config.max_wait_ms); + + if (auto it = ctx.catalog_config.find(retries.key()); it != ctx.catalog_config.end()) { + ICEBERG_ASSIGN_OR_RAISE(config.max_retries, + StringUtils::ParseNumber(it->second)); + } + ICEBERG_PRECHECK(config.max_retries >= 0, + "Invalid value for {}: {} (must be non-negative)", retries.key(), + config.max_retries); + + return config; } using SpecsById = std::unordered_map>; @@ -74,10 +98,13 @@ using SpecsById = std::unordered_map>; // Shared HTTP scan planning helpers used by all REST scan implementations. // --------------------------------------------------------------------------- -// Column names of the projected schema, sent as the request's select. -Result> ProjectedColumnNames(const Schema& projected) { +// Column names in field ID order. +// FIXME: Sending both "s" and "s.x" asks the server for all of "s". +// Preserve partial struct projections; Java currently has the same issue. +Result> ProjectedColumnNames(const Schema& projected, + bool include_struct_ids = true) { ICEBERG_ASSIGN_OR_RAISE(auto id_set, GetProjectedIdsVisitor::GetProjectedIds( - projected, /*include_struct_ids=*/true)); + projected, include_struct_ids)); std::vector field_ids(id_set.begin(), id_set.end()); std::ranges::sort(field_ids); @@ -91,55 +118,56 @@ Result> ProjectedColumnNames(const Schema& projected) { return names; } -Status ApplyStorageCredentials(const RestScanContext& ctx, const std::string& plan_id, - const std::vector& credentials, - std::shared_ptr& scan_io) { - if (credentials.empty()) return {}; +Result> MakeScanFileIO( + const RestScanContext& ctx, const std::string& plan_id, + const std::vector& credentials) { + if (credentials.empty()) return std::shared_ptr{}; std::shared_ptr provider; if (ctx.credential_provider_factory) { - provider = ctx.credential_provider_factory(plan_id); + ICEBERG_ASSIGN_OR_RAISE(provider, ctx.credential_provider_factory(plan_id)); } - ICEBERG_ASSIGN_OR_RAISE(auto io, MakeTableFileIO(ctx.catalog_config, ctx.table_config, - credentials, std::move(provider))); - scan_io = std::move(io); - return {}; + ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr io, + MakeTableFileIO(ctx.catalog_config, ctx.table_config, + credentials, std::move(provider))); + return io; } -void CancelPlanning(const RestScanContext& ctx, const std::string& plan_id) { - if (plan_id.empty()) return; - if (!ctx.supported_endpoints.contains(Endpoint::CancelPlanning())) return; - - auto path = ctx.paths->Plan(ctx.identifier, plan_id); - if (!path.has_value()) return; +Status CancelPlanning(const RestScanContext& ctx, const std::string& plan_id) { + if (plan_id.empty()) return {}; + if (!ctx.supported_endpoints.contains(Endpoint::CancelPlanning())) return {}; - std::ignore = ctx.client->Delete(*path, /*params=*/{}, /*headers=*/{}, - *PlanErrorHandler::Instance(), *ctx.session); + ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->Plan(ctx.identifier, plan_id)); + ICEBERG_RETURN_UNEXPECTED(ctx.client->Delete( + path, /*params=*/{}, /*headers=*/{}, *PlanErrorHandler::Instance(), *ctx.session)); + return {}; } -Result>> FetchScanTasks( - const RestScanContext& ctx, const Schema& schema, const std::string& plan_task, - const SpecsById& specs); - -Result>> ResolveScanTasks( - const RestScanContext& ctx, const Schema& schema, - const std::optional>& plan_tasks, - const std::optional>>& file_scan_tasks, - const SpecsById& specs) { - std::vector> result; - - if (file_scan_tasks.has_value()) { - result.insert(result.end(), file_scan_tasks->begin(), file_scan_tasks->end()); - } - - if (plan_tasks.has_value()) { - for (const auto& token : *plan_tasks) { - ICEBERG_ASSIGN_OR_RAISE(auto tasks, FetchScanTasks(ctx, schema, token, specs)); - result.insert(result.end(), tasks.begin(), tasks.end()); +// Cancels the plan on scope exit unless released. +class PlanCancellationGuard { + public: + PlanCancellationGuard(const RestScanContext& ctx, const std::string& plan_id) + : ctx_(ctx), plan_id_(plan_id) {} + + PlanCancellationGuard(const PlanCancellationGuard&) = delete; + PlanCancellationGuard& operator=(const PlanCancellationGuard&) = delete; + + ~PlanCancellationGuard() { + if (!active_) return; + // Cleanup is best-effort; a missing plan has already been released. + if (auto status = CancelPlanning(ctx_, plan_id_); + !status && status.error().kind != ErrorKind::kNoSuchPlanId) { + ICEBERG_LOG_WARN("Failed to cancel scan plan {}: {}", plan_id_, + status.error().message); } } - return result; -} + void Release() { active_ = false; } + + private: + const RestScanContext& ctx_; + const std::string& plan_id_; + bool active_ = true; +}; // Fetches one plan task without resolving any nested plan tasks it returns. Result FetchScanTasksOnce(const RestScanContext& ctx, @@ -162,69 +190,38 @@ Result FetchScanTasksOnce(const RestScanContext& ctx, return result; } -Result>> FetchScanTasks( - const RestScanContext& ctx, const Schema& schema, const std::string& plan_task, - const SpecsById& specs) { - ICEBERG_ASSIGN_OR_RAISE(auto result, FetchScanTasksOnce(ctx, schema, plan_task, specs)); - return ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs); -} - // Polls until the plan completes and returns the completed response. Plan tasks // are left unresolved. Result FetchPlanningResult( const RestScanContext& ctx, const Schema& schema, const std::string& plan_id, const SpecsById& specs, const PollConfig& poll_config, - std::shared_ptr& scan_io) { + std::shared_ptr& scan_io, PlanCancellationGuard& cleanup) { ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::FetchPlanningResult()); - ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->Plan(ctx.identifier, plan_id)); auto delay_ms = kMinSleepMs; auto start = std::chrono::steady_clock::now(); for (int32_t retry = 0; retry <= poll_config.max_retries; ++retry) { - auto response_or = ctx.client->Get(path, /*params=*/{}, /*headers=*/{}, - *PlanErrorHandler::Instance(), *ctx.session); - if (!response_or) { - CancelPlanning(ctx, plan_id); - return ::iceberg::unexpected(response_or.error()); - } - - auto json_or = FromJsonString(response_or->body()); - if (!json_or) { - CancelPlanning(ctx, plan_id); - return ::iceberg::unexpected(json_or.error()); - } - - auto result_or = FetchPlanningResultResponseFromJson(*json_or, specs, schema); - if (!result_or) { - CancelPlanning(ctx, plan_id); - return ::iceberg::unexpected(result_or.error()); - } - - if (auto s = result_or->Validate(); !s) { - CancelPlanning(ctx, plan_id); - return ::iceberg::unexpected(s.error()); - } - - auto& result = *result_or; + ICEBERG_ASSIGN_OR_RAISE(auto response, + ctx.client->Get(path, /*params=*/{}, /*headers=*/{}, + *PlanErrorHandler::Instance(), *ctx.session)); + ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); + ICEBERG_ASSIGN_OR_RAISE(auto result, + FetchPlanningResultResponseFromJson(json, specs, schema)); + ICEBERG_RETURN_UNEXPECTED(result.Validate()); switch (result.plan_status) { case PlanStatus::kCompleted: { - if (auto s = ApplyStorageCredentials(ctx, plan_id, result.storage_credentials, - scan_io); - !s) { - CancelPlanning(ctx, plan_id); - return ::iceberg::unexpected(s.error()); - } - return std::move(result); + ICEBERG_ASSIGN_OR_RAISE(scan_io, + MakeScanFileIO(ctx, plan_id, result.storage_credentials)); + return result; } case PlanStatus::kSubmitted: { auto elapsed_ms = std::chrono::duration_cast( std::chrono::steady_clock::now() - start) .count(); if (elapsed_ms >= poll_config.max_wait_ms) { - CancelPlanning(ctx, plan_id); return IOError("Scan planning timed out after {}ms waiting for plan_id={}", elapsed_ms, plan_id); } @@ -234,15 +231,14 @@ Result FetchPlanningResult( continue; } case PlanStatus::kFailed: - CancelPlanning(ctx, plan_id); return IOError("Scan planning failed: {}", result.error ? result.error->message : "unknown error"); case PlanStatus::kCancelled: + cleanup.Release(); return IOError("Scan planning was cancelled for plan_id={}", plan_id); } } - CancelPlanning(ctx, plan_id); return IOError("Scan planning exceeded max retries ({}) for plan_id={}", poll_config.max_retries, plan_id); } @@ -257,37 +253,49 @@ Result FetchPlanningResult( /// queuing any nested plan tasks returned by the server. class RestFileScanTaskStream final : public FileScanTaskStream { public: - RestFileScanTaskStream(RestScanContext ctx, std::shared_ptr schema, - std::string plan_id, + RestFileScanTaskStream(std::shared_ptr ctx, + std::shared_ptr schema, std::string plan_id, std::vector> initial_tasks, - std::vector plan_task_tokens, SpecsById specs) + std::vector plan_task_tokens, SpecsById specs, + std::shared_ptr filter, bool ignore_residuals) : ctx_(std::move(ctx)), schema_(std::move(schema)), plan_id_(std::move(plan_id)), buffer_(std::move(initial_tasks)), - plan_task_tokens_(std::move(plan_task_tokens)), - specs_(std::move(specs)) {} + plan_task_tokens_(std::make_move_iterator(plan_task_tokens.begin()), + std::make_move_iterator(plan_task_tokens.end())), + specs_(std::move(specs)), + filter_(filter ? std::move(filter) : True::Instance()), + ignore_residuals_(ignore_residuals), + cleanup_(*ctx_, plan_id_) {} - ~RestFileScanTaskStream() override { - if (!consumed_) CancelPlanning(ctx_, plan_id_); - } + ~RestFileScanTaskStream() override = default; protected: Result>> NextImpl() override { while (true) { if (buffer_pos_ < buffer_.size()) { - return buffer_[buffer_pos_++]; + auto task = std::move(buffer_[buffer_pos_++]); + if (ignore_residuals_ || !task->residual_filter()) { + // The server may leave row filtering to the client. + return std::make_shared( + task->data_file(), task->delete_files(), + ignore_residuals_ ? True::Instance() : filter_); + } + return task; } - if (token_pos_ >= plan_task_tokens_.size()) { - consumed_ = true; + if (plan_task_tokens_.empty()) { + cleanup_.Release(); return std::nullopt; } - const std::string token = plan_task_tokens_[token_pos_++]; + const std::string token = std::move(plan_task_tokens_.front()); + plan_task_tokens_.pop_front(); ICEBERG_ASSIGN_OR_RAISE(auto result, - FetchScanTasksOnce(ctx_, *schema_, token, specs_)); + FetchScanTasksOnce(*ctx_, *schema_, token, specs_)); if (result.plan_tasks.has_value()) { - plan_task_tokens_.insert(plan_task_tokens_.end(), result.plan_tasks->begin(), - result.plan_tasks->end()); + plan_task_tokens_.insert(plan_task_tokens_.end(), + std::make_move_iterator(result.plan_tasks->begin()), + std::make_move_iterator(result.plan_tasks->end())); } buffer_ = std::move(result.file_scan_tasks) .value_or(std::vector>{}); @@ -296,24 +304,24 @@ class RestFileScanTaskStream final : public FileScanTaskStream { } private: - RestScanContext ctx_; + std::shared_ptr ctx_; std::shared_ptr schema_; std::string plan_id_; std::vector> buffer_; size_t buffer_pos_ = 0; - std::vector plan_task_tokens_; - size_t token_pos_ = 0; + std::deque plan_task_tokens_; SpecsById specs_; - bool consumed_ = false; + std::shared_ptr filter_; + bool ignore_residuals_; + PlanCancellationGuard cleanup_; }; /// POST /plan (polling if SUBMITTED), apply credentials, return a lazy stream. -Result ExecuteScanPlanStream(const RestScanContext& ctx, - const Schema& schema, - std::shared_ptr schema_ptr, - PlanTableScanRequest request, - const SpecsById& specs, - std::shared_ptr& scan_io) { +Result ExecuteScanPlanStream( + const std::shared_ptr& rest_context, const Schema& schema, + std::shared_ptr schema_ptr, PlanTableScanRequest request, + const SpecsById& specs, std::shared_ptr& scan_io, bool ignore_residuals) { + const auto& ctx = *rest_context; ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::PlanTableScan()); ICEBERG_ASSIGN_OR_RAISE(auto poll_config, ResolvePollConfig(ctx)); @@ -329,41 +337,39 @@ Result ExecuteScanPlanStream(const RestScanContext& ctx, if (auto it = json.find("plan-id"); it != json.end() && it->is_string()) { raw_plan_id = it->get(); } - auto result_or = PlanTableScanResponseFromJson(json, specs, schema); - if (!result_or) { - CancelPlanning(ctx, raw_plan_id); - return ::iceberg::unexpected(result_or.error()); - } - auto& result = *result_or; + PlanCancellationGuard cleanup(ctx, raw_plan_id); + ICEBERG_ASSIGN_OR_RAISE(auto result, + PlanTableScanResponseFromJson(json, specs, schema)); const std::string plan_id = result.plan_id; if (result.plan_status == PlanStatus::kSubmitted) { ICEBERG_ASSIGN_OR_RAISE( auto completed, - FetchPlanningResult(ctx, schema, plan_id, specs, poll_config, scan_io)); - return std::make_unique( - ctx, std::move(schema_ptr), plan_id, + FetchPlanningResult(ctx, schema, plan_id, specs, poll_config, scan_io, cleanup)); + FileScanTaskStreamPtr stream = std::make_unique( + rest_context, std::move(schema_ptr), plan_id, std::move(completed.file_scan_tasks) .value_or(std::vector>{}), - std::move(completed.plan_tasks).value_or(std::vector{}), specs); + std::move(completed.plan_tasks).value_or(std::vector{}), specs, + std::move(request.filter), ignore_residuals); + cleanup.Release(); + return stream; } if (result.plan_status == PlanStatus::kFailed) { - CancelPlanning(ctx, plan_id); return IOError("Scan planning failed: {}", result.error ? result.error->message : "unknown error"); } if (result.plan_status == PlanStatus::kCancelled) { + cleanup.Release(); return IOError("Scan planning was cancelled for plan_id={}", plan_id); } - // kCompleted: apply credentials from the initial response, then build the lazy stream. - if (auto s = ApplyStorageCredentials(ctx, plan_id, result.storage_credentials, scan_io); - !s) { - CancelPlanning(ctx, plan_id); - return ::iceberg::unexpected(s.error()); - } + ICEBERG_CHECK(result.plan_status == PlanStatus::kCompleted, + "Unexpected scan planning status: {}", ToString(result.plan_status)); + ICEBERG_ASSIGN_OR_RAISE(scan_io, + MakeScanFileIO(ctx, plan_id, result.storage_credentials)); std::vector> initial_tasks; if (result.file_scan_tasks.has_value()) { @@ -374,69 +380,11 @@ Result ExecuteScanPlanStream(const RestScanContext& ctx, plan_task_tokens = std::move(*result.plan_tasks); } - return std::make_unique(ctx, std::move(schema_ptr), plan_id, - std::move(initial_tasks), - std::move(plan_task_tokens), specs); -} - -/// Eager batch planning: used by RestIncrementalAppendScan which has no stream path. -Result>> ExecuteScanPlan( - const RestScanContext& ctx, const Schema& schema, PlanTableScanRequest request, - const SpecsById& specs, std::shared_ptr& scan_io) { - ICEBERG_ENDPOINT_CHECK(ctx.supported_endpoints, Endpoint::PlanTableScan()); - ICEBERG_ASSIGN_OR_RAISE(auto poll_config, ResolvePollConfig(ctx)); - - ICEBERG_ASSIGN_OR_RAISE(auto path, ctx.paths->Plan(ctx.identifier)); - ICEBERG_ASSIGN_OR_RAISE(auto request_json, ToJson(request)); - ICEBERG_ASSIGN_OR_RAISE(auto json_request, ToJsonString(request_json)); - ICEBERG_ASSIGN_OR_RAISE(const auto response, - ctx.client->Post(path, json_request, /*headers=*/{}, - *PlanErrorHandler::Instance(), *ctx.session)); - ICEBERG_ASSIGN_OR_RAISE(auto json, FromJsonString(response.body())); - // The parser validates the response, so read the plan id first to cancel on failure. - std::string raw_plan_id; - if (auto it = json.find("plan-id"); it != json.end() && it->is_string()) { - raw_plan_id = it->get(); - } - auto result_or = PlanTableScanResponseFromJson(json, specs, schema); - if (!result_or) { - CancelPlanning(ctx, raw_plan_id); - return ::iceberg::unexpected(result_or.error()); - } - auto& result = *result_or; - - const std::string plan_id = result.plan_id; - - switch (result.plan_status) { - case PlanStatus::kCompleted: { - if (auto s = - ApplyStorageCredentials(ctx, plan_id, result.storage_credentials, scan_io); - !s) { - CancelPlanning(ctx, plan_id); - return ::iceberg::unexpected(s.error()); - } - auto tasks = - ResolveScanTasks(ctx, schema, result.plan_tasks, result.file_scan_tasks, specs); - if (!tasks) CancelPlanning(ctx, plan_id); - return tasks; - } - case PlanStatus::kSubmitted: { - ICEBERG_ASSIGN_OR_RAISE( - auto completed, - FetchPlanningResult(ctx, schema, plan_id, specs, poll_config, scan_io)); - auto tasks = ResolveScanTasks(ctx, schema, completed.plan_tasks, - completed.file_scan_tasks, specs); - if (!tasks) CancelPlanning(ctx, plan_id); - return tasks; - } - case PlanStatus::kFailed: - CancelPlanning(ctx, plan_id); - return IOError("Scan planning failed: {}", - result.error ? result.error->message : "unknown error"); - case PlanStatus::kCancelled: - return IOError("Scan planning was cancelled for plan_id={}", plan_id); - } - return IOError("Unexpected plan status"); + FileScanTaskStreamPtr stream = std::make_unique( + rest_context, std::move(schema_ptr), plan_id, std::move(initial_tasks), + std::move(plan_task_tokens), specs, std::move(request.filter), ignore_residuals); + cleanup.Release(); + return stream; } } // namespace @@ -448,7 +396,7 @@ Result>> ExecuteScanPlan( RestTableScan::RestTableScan(std::shared_ptr metadata, std::shared_ptr schema, std::shared_ptr io, internal::TableScanContext context, - RestScanContext rest_context) + std::shared_ptr rest_context) : DataTableScan(std::move(metadata), std::move(schema), std::move(io), std::move(context)), rest_context_(std::move(rest_context)) {} @@ -456,10 +404,14 @@ RestTableScan::RestTableScan(std::shared_ptr metadata, Result> RestTableScan::Make( std::shared_ptr metadata, std::shared_ptr schema, std::shared_ptr io, internal::TableScanContext context, - RestScanContext rest_context) { + std::shared_ptr rest_context) { ICEBERG_PRECHECK(metadata != nullptr, "Table metadata cannot be null"); ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null"); ICEBERG_PRECHECK(io != nullptr, "FileIO cannot be null"); + ICEBERG_PRECHECK(rest_context != nullptr, "REST scan context cannot be null"); + ICEBERG_CHECK( + !context.from_snapshot_id.has_value() && !context.to_snapshot_id.has_value(), + "Use RestIncrementalAppendScan for incremental scans"); return std::unique_ptr( new RestTableScan(std::move(metadata), std::move(schema), std::move(io), std::move(context), std::move(rest_context))); @@ -477,16 +429,15 @@ Result RestTableScan::PlanFilesStream() const { request.case_sensitive = context_.case_sensitive; request.min_rows_requested = context_.min_rows_requested; - if (context_.from_snapshot_id.has_value() && context_.to_snapshot_id.has_value()) { - request.start_snapshot_id = context_.from_snapshot_id; - request.end_snapshot_id = context_.to_snapshot_id; - request.use_snapshot_schema = true; - } else if (context_.snapshot_id.has_value()) { + if (context_.snapshot_id.has_value()) { request.snapshot_id = context_.snapshot_id; request.use_snapshot_schema = context_.use_snapshot_schema; } - if (!context_.columns_to_keep_stats.empty()) { + if (context_.return_column_stats && context_.columns_to_keep_stats.empty()) { + ICEBERG_ASSIGN_OR_RAISE(request.stats_fields, + ProjectedColumnNames(*schema_, /*include_struct_ids=*/false)); + } else { for (int32_t field_id : context_.columns_to_keep_stats) { ICEBERG_ASSIGN_OR_RAISE(auto name, schema_->FindColumnNameById(field_id)); if (name.has_value()) { @@ -496,7 +447,7 @@ Result RestTableScan::PlanFilesStream() const { } return ExecuteScanPlanStream(rest_context_, *schema_, schema_, std::move(request), - specs, scan_io_); + specs, scan_io_, context_.ignore_residuals); } const std::shared_ptr& RestTableScan::io() const { @@ -510,7 +461,7 @@ const std::shared_ptr& RestTableScan::io() const { RestTableScanBuilder::RestTableScanBuilder( std::shared_ptr metadata, std::shared_ptr io, std::string table_name, std::shared_ptr metrics_reporter, - RestScanContext rest_context) + std::shared_ptr rest_context) : DataTableScanBuilder(std::move(metadata), std::move(io), std::move(table_name), std::move(metrics_reporter)), rest_context_(std::move(rest_context)) {} @@ -531,7 +482,7 @@ Result> RestTableScanBuilder::Build() { RestIncrementalAppendScan::RestIncrementalAppendScan( std::shared_ptr metadata, std::shared_ptr schema, std::shared_ptr io, internal::TableScanContext context, - RestScanContext rest_context) + std::shared_ptr rest_context) : IncrementalAppendScan(std::move(metadata), std::move(schema), std::move(io), std::move(context)), rest_context_(std::move(rest_context)) {} @@ -539,10 +490,11 @@ RestIncrementalAppendScan::RestIncrementalAppendScan( Result> RestIncrementalAppendScan::Make( std::shared_ptr metadata, std::shared_ptr schema, std::shared_ptr io, internal::TableScanContext context, - RestScanContext rest_context) { + std::shared_ptr rest_context) { ICEBERG_PRECHECK(metadata != nullptr, "Table metadata cannot be null"); ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null"); ICEBERG_PRECHECK(io != nullptr, "FileIO cannot be null"); + ICEBERG_PRECHECK(rest_context != nullptr, "REST scan context cannot be null"); return std::unique_ptr( new RestIncrementalAppendScan(std::move(metadata), std::move(schema), std::move(io), std::move(context), std::move(rest_context))); @@ -568,9 +520,14 @@ Result>> RestIncrementalAppendScan::Pl request.min_rows_requested = context_.min_rows_requested; request.start_snapshot_id = from_snapshot_id_exclusive; request.end_snapshot_id = to_snapshot_id_inclusive; + // Match Java's RESTTableScan: schema selection for incremental ranges is + // unspecified in the REST spec. request.use_snapshot_schema = true; - if (!context_.columns_to_keep_stats.empty()) { + if (context_.return_column_stats && context_.columns_to_keep_stats.empty()) { + ICEBERG_ASSIGN_OR_RAISE(request.stats_fields, + ProjectedColumnNames(*schema_, /*include_struct_ids=*/false)); + } else { for (int32_t field_id : context_.columns_to_keep_stats) { ICEBERG_ASSIGN_OR_RAISE(auto name, schema_->FindColumnNameById(field_id)); if (name.has_value()) { @@ -579,7 +536,11 @@ Result>> RestIncrementalAppendScan::Pl } } - return ExecuteScanPlan(rest_context_, *schema_, std::move(request), specs, scan_io_); + ICEBERG_ASSIGN_OR_RAISE( + auto stream, + ExecuteScanPlanStream(rest_context_, *schema_, schema_, std::move(request), specs, + scan_io_, context_.ignore_residuals)); + return stream->ToVector(); } const std::shared_ptr& RestIncrementalAppendScan::io() const { @@ -593,7 +554,7 @@ const std::shared_ptr& RestIncrementalAppendScan::io() const { RestIncrementalAppendScanBuilder::RestIncrementalAppendScanBuilder( std::shared_ptr metadata, std::shared_ptr io, std::string table_name, std::shared_ptr metrics_reporter, - RestScanContext rest_context) + std::shared_ptr rest_context) : IncrementalAppendScanBuilder(std::move(metadata), std::move(io), std::move(table_name), std::move(metrics_reporter)), rest_context_(std::move(rest_context)) {} diff --git a/src/iceberg/catalog/rest/rest_table_scan.h b/src/iceberg/catalog/rest/rest_table_scan.h index 2280263dca..c366e10c67 100644 --- a/src/iceberg/catalog/rest/rest_table_scan.h +++ b/src/iceberg/catalog/rest/rest_table_scan.h @@ -30,11 +30,10 @@ #include "iceberg/catalog/rest/endpoint.h" #include "iceberg/catalog/rest/iceberg_rest_export.h" +#include "iceberg/catalog/rest/type_fwd.h" #include "iceberg/data_table_scan.h" #include "iceberg/incremental_append_scan.h" -#include "iceberg/metrics/metrics_reporter.h" #include "iceberg/result.h" -#include "iceberg/storage_credential.h" #include "iceberg/table_identifier.h" #include "iceberg/table_scan.h" #include "iceberg/type_fwd.h" @@ -44,14 +43,7 @@ namespace iceberg::rest { -class HttpClient; -class ResourcePaths; - -namespace auth { -class AuthSession; -} // namespace auth - -/// \brief HTTP context shared between RestTable and REST scan classes. +/// \brief REST table context shared by its builders, scans and task streams. struct ICEBERG_REST_EXPORT RestScanContext { std::shared_ptr client; std::shared_ptr paths; @@ -64,7 +56,9 @@ struct ICEBERG_REST_EXPORT RestScanContext { /// Table-level config merged with catalog_config for scan-scoped FileIO creation. std::unordered_map table_config; /// Builds a plan-scoped provider for refreshing vended credentials; may be empty. - std::function(const std::string& plan_id)> + /// A null provider means credential refresh is unavailable. + std::function>( + const std::string& plan_id)> credential_provider_factory; }; @@ -75,10 +69,12 @@ class ICEBERG_REST_EXPORT RestTableScan : public DataTableScan { public: ~RestTableScan() override = default; + /// \brief Create a point-in-time scan. Incremental ranges must use + /// RestIncrementalAppendScan. static Result> Make( std::shared_ptr metadata, std::shared_ptr schema, std::shared_ptr io, internal::TableScanContext context, - RestScanContext rest_context); + std::shared_ptr rest_context); /// \brief Plans files lazily via the REST scan planning endpoints. /// @@ -94,9 +90,9 @@ class ICEBERG_REST_EXPORT RestTableScan : public DataTableScan { private: RestTableScan(std::shared_ptr metadata, std::shared_ptr schema, std::shared_ptr io, internal::TableScanContext context, - RestScanContext rest_context); + std::shared_ptr rest_context); - RestScanContext rest_context_; + std::shared_ptr rest_context_; mutable std::shared_ptr scan_io_; }; @@ -106,17 +102,20 @@ class ICEBERG_REST_EXPORT RestTableScanBuilder : public DataTableScanBuilder { RestTableScanBuilder(std::shared_ptr metadata, std::shared_ptr io, std::string table_name, std::shared_ptr metrics_reporter, - RestScanContext rest_context); + std::shared_ptr rest_context); /// \brief Resolves schema/context via parent logic then creates a RestTableScan. Result> Build() override; private: - RestScanContext rest_context_; + std::shared_ptr rest_context_; }; /// \brief An IncrementalAppendScan that delegates PlanFiles() to the REST catalog server. /// +/// Scans without an exclusive start snapshot, including an inclusive root snapshot, +/// return NotSupported. +/// /// IncrementalChangelogScan is not delegated because the REST planTableScan response /// only carries FileScanTask objects; reconstructing ChangelogScanTask entries /// (AddedRowsScanTask vs DeletedDataFileScanTask) requires per-snapshot operation @@ -128,7 +127,7 @@ class ICEBERG_REST_EXPORT RestIncrementalAppendScan : public IncrementalAppendSc static Result> Make( std::shared_ptr metadata, std::shared_ptr schema, std::shared_ptr io, internal::TableScanContext context, - RestScanContext rest_context); + std::shared_ptr rest_context); using IncrementalAppendScan::PlanFiles; @@ -148,9 +147,9 @@ class ICEBERG_REST_EXPORT RestIncrementalAppendScan : public IncrementalAppendSc RestIncrementalAppendScan(std::shared_ptr metadata, std::shared_ptr schema, std::shared_ptr io, internal::TableScanContext context, - RestScanContext rest_context); + std::shared_ptr rest_context); - RestScanContext rest_context_; + std::shared_ptr rest_context_; mutable std::shared_ptr scan_io_; }; @@ -161,12 +160,12 @@ class ICEBERG_REST_EXPORT RestIncrementalAppendScanBuilder RestIncrementalAppendScanBuilder(std::shared_ptr metadata, std::shared_ptr io, std::string table_name, std::shared_ptr metrics_reporter, - RestScanContext rest_context); + std::shared_ptr rest_context); Result> Build() override; private: - RestScanContext rest_context_; + std::shared_ptr rest_context_; }; } // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/type_fwd.h b/src/iceberg/catalog/rest/type_fwd.h index ee684b245d..ba8aa51962 100644 --- a/src/iceberg/catalog/rest/type_fwd.h +++ b/src/iceberg/catalog/rest/type_fwd.h @@ -28,6 +28,7 @@ struct ErrorResponse; struct CommitTableResponse; struct LoadTableResult; struct OAuthTokenResponse; +struct RestScanContext; class Endpoint; class ErrorHandler; diff --git a/src/iceberg/expression/expression.h b/src/iceberg/expression/expression.h index 1663c69f8e..43e4a2c1f1 100644 --- a/src/iceberg/expression/expression.h +++ b/src/iceberg/expression/expression.h @@ -64,7 +64,7 @@ class ICEBERG_EXPORT Expression : public util::Formattable { kMin }; - virtual ~Expression() = default; + ~Expression() override = default; /// \brief Returns the operation for an expression node. virtual Operation op() const = 0; diff --git a/src/iceberg/test/auth_manager_test.cc b/src/iceberg/test/auth_manager_test.cc index 61962d1c3c..fd9605fc31 100644 --- a/src/iceberg/test/auth_manager_test.cc +++ b/src/iceberg/test/auth_manager_test.cc @@ -788,7 +788,7 @@ TEST(OAuth2SessionTest, InitialTokenIsUsed) { ASSERT_THAT(auth_result, IsOk()); EXPECT_EQ(auth_result.value().headers.at("Authorization"), "Bearer initial-token-123"); - session->Close(); + ASSERT_THAT(session->Close(), IsOk()); } TEST(OAuth2SessionTest, InitTokenExpirationUsesRequestStartTime) { diff --git a/src/iceberg/test/catalog_properties_test.cc b/src/iceberg/test/catalog_properties_test.cc index dc3e282959..0100c14882 100644 --- a/src/iceberg/test/catalog_properties_test.cc +++ b/src/iceberg/test/catalog_properties_test.cc @@ -19,6 +19,9 @@ #include "iceberg/catalog/rest/catalog_properties.h" +#include +#include + #include #include "iceberg/test/matchers.h" @@ -32,36 +35,20 @@ TEST(ScanPlanningModeTest, MissingKeyReturnsNullopt) { EXPECT_FALSE(result->has_value()); } -TEST(ScanPlanningModeTest, ClientLowercaseReturnsKClient) { - auto result = - RestCatalogProperties::ScanPlanningModeFrom({{"scan-planning-mode", "client"}}); - ASSERT_THAT(result, IsOk()); - ASSERT_TRUE(result->has_value()); - EXPECT_EQ(**result, ScanPlanningMode::kClient); -} - -TEST(ScanPlanningModeTest, ServerLowercaseReturnsKServer) { - auto result = - RestCatalogProperties::ScanPlanningModeFrom({{"scan-planning-mode", "server"}}); - ASSERT_THAT(result, IsOk()); - ASSERT_TRUE(result->has_value()); - EXPECT_EQ(**result, ScanPlanningMode::kServer); -} - -TEST(ScanPlanningModeTest, ClientUppercaseReturnsKClient) { - auto result = - RestCatalogProperties::ScanPlanningModeFrom({{"scan-planning-mode", "CLIENT"}}); - ASSERT_THAT(result, IsOk()); - ASSERT_TRUE(result->has_value()); - EXPECT_EQ(**result, ScanPlanningMode::kClient); -} - -TEST(ScanPlanningModeTest, ServerUppercaseReturnsKServer) { - auto result = - RestCatalogProperties::ScanPlanningModeFrom({{"scan-planning-mode", "SERVER"}}); - ASSERT_THAT(result, IsOk()); - ASSERT_TRUE(result->has_value()); - EXPECT_EQ(**result, ScanPlanningMode::kServer); +TEST(ScanPlanningModeTest, ParsesExplicitModes) { + const std::array, 4> cases{ + {{"client", ScanPlanningMode::kClient}, + {"CLIENT", ScanPlanningMode::kClient}, + {"server", ScanPlanningMode::kServer}, + {"SERVER", ScanPlanningMode::kServer}}}; + for (const auto& [value, expected] : cases) { + SCOPED_TRACE(value); + auto result = + RestCatalogProperties::ScanPlanningModeFrom({{"scan-planning-mode", value}}); + ASSERT_THAT(result, IsOk()); + ASSERT_TRUE(result->has_value()); + EXPECT_EQ(**result, expected); + } } TEST(ScanPlanningModeTest, InvalidValueReturnsError) { @@ -70,48 +57,4 @@ TEST(ScanPlanningModeTest, InvalidValueReturnsError) { EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); } -TEST(ScanPlanningModeTest, OtherKeysAreIgnored) { - auto result = RestCatalogProperties::ScanPlanningModeFrom( - {{"other-key", "server"}, {"scan-planning-mode", "client"}}); - ASSERT_THAT(result, IsOk()); - ASSERT_TRUE(result->has_value()); - EXPECT_EQ(**result, ScanPlanningMode::kClient); -} - -TEST(ScanPlanningPollTest, Defaults) { - auto properties = RestCatalogProperties::default_properties(); - EXPECT_THAT(properties.ScanPlanningPollTimeoutMs(), HasValue(::testing::Eq(300000))); - EXPECT_THAT(properties.ScanPlanningPollNumRetries(), HasValue(::testing::Eq(10))); -} - -TEST(ScanPlanningPollTest, ValidValues) { - auto properties = - RestCatalogProperties::FromMap({{"rest-scan-planning.poll-timeout-ms", "1000"}, - {"rest-scan-planning.poll-num-retries", "0"}}); - EXPECT_THAT(properties.ScanPlanningPollTimeoutMs(), HasValue(::testing::Eq(1000))); - EXPECT_THAT(properties.ScanPlanningPollNumRetries(), HasValue(::testing::Eq(0))); -} - -TEST(ScanPlanningPollTest, InvalidTimeout) { - for (const auto& value : {"0", "-1", "abc"}) { - auto properties = - RestCatalogProperties::FromMap({{"rest-scan-planning.poll-timeout-ms", value}}); - EXPECT_THAT(properties.ScanPlanningPollTimeoutMs(), - ::testing::AllOf(IsError(ErrorKind::kInvalidArgument), - HasErrorMessage("must be positive"))) - << value; - } -} - -TEST(ScanPlanningPollTest, InvalidNumRetries) { - for (const auto& value : {"-1", "abc"}) { - auto properties = - RestCatalogProperties::FromMap({{"rest-scan-planning.poll-num-retries", value}}); - EXPECT_THAT(properties.ScanPlanningPollNumRetries(), - ::testing::AllOf(IsError(ErrorKind::kInvalidArgument), - HasErrorMessage("must be non-negative"))) - << value; - } -} - } // namespace iceberg::rest diff --git a/src/iceberg/test/metrics_reporter_test.cc b/src/iceberg/test/metrics_reporter_test.cc index 385e20c231..ecddde07e7 100644 --- a/src/iceberg/test/metrics_reporter_test.cc +++ b/src/iceberg/test/metrics_reporter_test.cc @@ -31,6 +31,7 @@ #include "iceberg/metrics/counter.h" #include "iceberg/metrics/metrics_reporters.h" +#include "iceberg/test/matchers.h" namespace iceberg { @@ -75,7 +76,7 @@ TEST(CustomMetricsReporterTest, RegisterAndLoad) { ASSERT_NE(reporter, nullptr); ScanReport scan_report{.table_name = "test.table"}; - reporter->Report(scan_report); + ASSERT_THAT(reporter->Report(scan_report), IsOk()); EXPECT_EQ(reporter->reports().size(), 1); EXPECT_EQ(GetReportType(reporter->reports()[0]), MetricsReportType::kScanReport); @@ -163,7 +164,7 @@ class CollectorTest : public ::testing::TestWithParam {}; TEST_P(CollectorTest, CollectsAndPreservesReport) { const auto& param = GetParam(); CollectingMetricsReporter reporter; - reporter.Report(param.report); + ASSERT_THAT(reporter.Report(param.report), IsOk()); ASSERT_EQ(reporter.reports().size(), 1); EXPECT_EQ(GetReportType(reporter.reports()[0]), param.expected_type); @@ -228,7 +229,7 @@ TEST(CombineTest, FlattenNestedComposite) { EXPECT_EQ(dynamic_cast(r.get()), nullptr); } - abc->Report(CommitReport{.table_name = "db.t2"}); + ASSERT_THAT(abc->Report(CommitReport{.table_name = "db.t2"}), IsOk()); EXPECT_EQ(a->reports().size(), 1u); EXPECT_EQ(b->reports().size(), 1u); EXPECT_EQ(c->reports().size(), 1u); @@ -246,7 +247,7 @@ TEST(CombineTest, DeduplicateByIdentity) { ASSERT_NE(composite, nullptr); EXPECT_EQ(composite->Reporters().size(), 2u); - result->Report(ScanReport{}); + ASSERT_THAT(result->Report(ScanReport{}), IsOk()); EXPECT_EQ(a->reports().size(), 1u); EXPECT_EQ(b->reports().size(), 1u); // delivered once, not twice } diff --git a/src/iceberg/test/rest_catalog_unit_test.cc b/src/iceberg/test/rest_catalog_unit_test.cc index 4b42febb54..bb76a59a4f 100644 --- a/src/iceberg/test/rest_catalog_unit_test.cc +++ b/src/iceberg/test/rest_catalog_unit_test.cc @@ -21,19 +21,22 @@ #include #include #include +#include #include #include +#include #include "iceberg/catalog.h" #include "iceberg/catalog/rest/catalog_properties.h" #include "iceberg/catalog/rest/endpoint.h" #include "iceberg/catalog/rest/error_handlers.h" #include "iceberg/catalog/rest/http_client.h" -#include "iceberg/catalog/rest/resource_paths.h" +#include "iceberg/catalog/rest/json_serde_internal.h" #include "iceberg/catalog/rest/rest_catalog.h" #include "iceberg/catalog/rest/rest_table.h" #include "iceberg/catalog/rest/rest_table_scan.h" +#include "iceberg/catalog/rest/types.h" #include "iceberg/file_io.h" #include "iceberg/file_io_registry.h" #include "iceberg/storage_credential.h" @@ -45,6 +48,8 @@ namespace iceberg::rest { +namespace { + using ::testing::_; using ::testing::Return; @@ -159,18 +164,20 @@ const std::string kCommitTableResponse = std::string("{") + std::string(kBaseMetadataJson) + "}"; // -------------------------------------------------------------------------- -// Test fixture: builds a RestCatalog via MakeForTesting so no HTTP calls -// are made during catalog construction. +// Use the regular catalog initialization with a mock HTTP client. // -------------------------------------------------------------------------- class RestCatalogLoadTableTest : public ::testing::Test { protected: + void TearDown() override { installed_provider.reset(); } + void SetUp() override { mock_client_ = std::make_shared(); - file_io_ = std::make_shared(); - - ICEBERG_UNWRAP_OR_FAIL(paths_, - ResourcePaths::Make("http://test-server", /*prefix=*/"", - /*namespace_separator=*/"%1F")); + FileIORegistry::Register( + "rest-catalog-unit-test-noop", + {.create = [](const std::unordered_map&) + -> Result> { + return std::make_unique(); + }}); identifier_ = TableIdentifier{.ns = Namespace{{"default"}}, .name = "my_table"}; @@ -189,52 +196,99 @@ class RestCatalogLoadTableTest : public ::testing::Test { std::unordered_map extra_props = {}) { std::unordered_map props = std::move(extra_props); props["uri"] = "http://test-server"; + props.try_emplace("io-impl", "rest-catalog-unit-test-noop"); if (!client_scan_mode.empty()) { props["scan-planning-mode"] = client_scan_mode; } - auto config = RestCatalogProperties::FromMap(props); - return RestCatalog::MakeForTesting(std::move(config), file_io_, mock_client_, paths_, - endpoints); + CatalogConfig server_config{.endpoints = {endpoints.begin(), endpoints.end()}}; + EXPECT_CALL(*mock_client_, Get(::testing::EndsWith("/v1/config"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, ToJson(server_config).dump()))); + auto config = RestCatalogProperties::FromMap(std::move(props)); + return RestCatalog::Make(config, /*metrics_executor=*/nullptr, + [client = mock_client_](const auto&) { return client; }); } std::shared_ptr mock_client_; - std::shared_ptr file_io_; - std::shared_ptr paths_; TableIdentifier identifier_; std::unordered_set all_plan_endpoints_; std::unordered_set no_plan_endpoint_set_; }; -// -------------------------------------------------------------------------- -// Table config "scan-planning-mode":"server" → LoadTable returns a RestTable. -// -------------------------------------------------------------------------- -TEST_F(RestCatalogLoadTableTest, TableConfigServerScanReturnsRestTable) { - EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); - - ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(all_plan_endpoints_)); - ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); - ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); - - EXPECT_NE(dynamic_cast(table.get()), nullptr) - << "Expected RestTable when server config requests server-side scan planning"; +TEST_F(RestCatalogLoadTableTest, InitializationPropagatesClientFactoryErrors) { + auto config = RestCatalogProperties::FromMap( + {{"uri", "http://test-server"}, {"io-impl", "rest-catalog-unit-test-noop"}}); + for (int fail_on_call : {1, 2}) { + SCOPED_TRACE(fail_on_call); + if (fail_on_call == 1) { + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)).Times(0); + } else { + EXPECT_CALL(*mock_client_, Get(::testing::EndsWith("/v1/config"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, ToJson(CatalogConfig{}).dump()))); + } + int calls = 0; + auto result = RestCatalog::Make( + config, /*metrics_executor=*/nullptr, + [&](const RestCatalogProperties&) -> Result> { + if (++calls == fail_on_call) { + return IOError("HTTP client setup failed"); + } + return mock_client_; + }); + ASSERT_THAT(result, IsError(ErrorKind::kIOError)); + EXPECT_EQ(result.error().message, "HTTP client setup failed"); + EXPECT_EQ(calls, fail_on_call); + ASSERT_TRUE(::testing::Mock::VerifyAndClearExpectations(mock_client_.get())); + } } -// -------------------------------------------------------------------------- -// Client config "scan-planning-mode":"server", table config absent → -// LoadTable returns a RestTable. -// -------------------------------------------------------------------------- -TEST_F(RestCatalogLoadTableTest, ClientConfigServerScanReturnsRestTable) { - EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableDefaultResponse))); - - ICEBERG_UNWRAP_OR_FAIL(auto catalog, - MakeCatalog(all_plan_endpoints_, /*client_scan_mode=*/"server")); - ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); - ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); - - EXPECT_NE(dynamic_cast(table.get()), nullptr) - << "Expected RestTable when client config requests server-side scan planning"; +TEST_F(RestCatalogLoadTableTest, InitializationUsesMergedServerConfig) { + CatalogConfig server_config{ + .defaults = {{"header.X-Default", "default"}, {"header.X-Shared", "default"}}, + .overrides = {{"prefix", "catalog"}, + {"snapshot-loading-mode", "REFS"}, + {"scan-planning-mode", "server"}, + {"header.X-Shared", "server"}}, + .endpoints = {all_plan_endpoints_.begin(), all_plan_endpoints_.end()}}; + EXPECT_CALL( + *mock_client_, + Get(::testing::EndsWith("/v1/config"), + ::testing::Contains(::testing::Pair("warehouse", "client-warehouse")), _, _, _)) + .WillOnce(Return(HttpResponse(200, ToJson(server_config).dump()))); + EXPECT_CALL(*mock_client_, + Get(::testing::EndsWith("/v1/catalog/namespaces/default/tables/my_table"), + ::testing::Contains(::testing::Pair("snapshots", "refs")), _, _, _)) + .WillOnce(Return(HttpResponse(200, kLoadTableDefaultResponse))); + EXPECT_CALL( + *mock_client_, + Post(::testing::EndsWith("/v1/catalog/namespaces/default/tables/my_table/plan"), _, + _, _, _)) + .WillOnce(Return(HttpResponse(200, R"({"status":"completed"})"))); + + auto config = + RestCatalogProperties::FromMap({{"uri", "http://test-server/"}, + {"warehouse", "client-warehouse"}, + {"io-impl", "rest-catalog-unit-test-noop"}, + {"scan-planning-mode", "client"}, + {"header.X-Shared", "client"}}); + std::vector client_configs; + ICEBERG_UNWRAP_OR_FAIL(auto root, + RestCatalog::Make(config, /*metrics_executor=*/nullptr, + [&](const RestCatalogProperties& properties) { + client_configs.push_back(properties); + return mock_client_; + })); + ASSERT_EQ(client_configs.size(), 2); + EXPECT_EQ(client_configs[0].ExtractHeaders().at("X-Shared"), "client"); + EXPECT_FALSE(client_configs[0].ExtractHeaders().contains("X-Default")); + EXPECT_EQ(client_configs[1].ExtractHeaders().at("X-Shared"), "server"); + EXPECT_EQ(client_configs[1].ExtractHeaders().at("X-Default"), "default"); + EXPECT_EQ(client_configs[1].Get(RestCatalogProperties::kPrefix), "catalog"); + ICEBERG_UNWRAP_OR_FAIL(auto catalog, root->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, catalog->LoadTable(identifier_)); + ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewScan()); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); } // -------------------------------------------------------------------------- @@ -243,7 +297,7 @@ TEST_F(RestCatalogLoadTableTest, ClientConfigServerScanReturnsRestTable) { // -------------------------------------------------------------------------- TEST_F(RestCatalogLoadTableTest, ServerScanWithMissingEndpointReturnsNotSupported) { EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); + .WillOnce(Return(HttpResponse(200, kLoadTableServerScanResponse))); ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(no_plan_endpoint_set_)); ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); @@ -252,114 +306,6 @@ TEST_F(RestCatalogLoadTableTest, ServerScanWithMissingEndpointReturnsNotSupporte EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); } -// -------------------------------------------------------------------------- -// No scan-planning-mode set anywhere → LoadTable returns a plain Table -// (not RestTable). -// -------------------------------------------------------------------------- -TEST_F(RestCatalogLoadTableTest, DefaultScanModeReturnsPlainTable) { - EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableDefaultResponse))); - - ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(all_plan_endpoints_)); - ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); - ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); - - EXPECT_EQ(dynamic_cast(table.get()), nullptr) - << "Expected plain Table (not RestTable) when no scan-planning-mode is configured"; -} - -// -------------------------------------------------------------------------- -// Committing through a server-planning table returns a RestTable. -// -------------------------------------------------------------------------- -TEST_F(RestCatalogLoadTableTest, CommitPreservesRestTableFromTableConfig) { - EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kCommitTableResponse))); - - ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(all_plan_endpoints_)); - ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); - ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); - ICEBERG_UNWRAP_OR_FAIL(auto committed, - table->catalog()->UpdateTable(identifier_, {}, {})); - - EXPECT_NE(dynamic_cast(committed.get()), nullptr); - ICEBERG_UNWRAP_OR_FAIL(auto builder, committed->NewScan()); - EXPECT_NE(dynamic_cast(builder.get()), nullptr); -} - -// -------------------------------------------------------------------------- -// Client config "scan-planning-mode":"server" → commit returns a RestTable. -// -------------------------------------------------------------------------- -TEST_F(RestCatalogLoadTableTest, CommitPreservesRestTableFromClientConfig) { - EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableDefaultResponse))); - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kCommitTableResponse))); - - ICEBERG_UNWRAP_OR_FAIL(auto catalog, - MakeCatalog(all_plan_endpoints_, /*client_scan_mode=*/"server")); - ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); - ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); - ICEBERG_UNWRAP_OR_FAIL(auto committed, - table->catalog()->UpdateTable(identifier_, {}, {})); - - EXPECT_NE(dynamic_cast(committed.get()), nullptr); -} - -// -------------------------------------------------------------------------- -// Default scan mode → commit returns a plain Table. -// -------------------------------------------------------------------------- -TEST_F(RestCatalogLoadTableTest, CommitWithDefaultScanModeReturnsPlainTable) { - EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableDefaultResponse))); - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kCommitTableResponse))); - - ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(all_plan_endpoints_)); - ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); - ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); - ICEBERG_UNWRAP_OR_FAIL(auto committed, - table->catalog()->UpdateTable(identifier_, {}, {})); - - EXPECT_EQ(dynamic_cast(committed.get()), nullptr); -} - -// -------------------------------------------------------------------------- -// Scan-vended credentials get a provider that refreshes via /credentials?planId=. -// -------------------------------------------------------------------------- -TEST_F(RestCatalogLoadTableTest, ScanCredentialProviderRequestsPlanId) { - RegisterCredentialedFileIO(); - EXPECT_CALL(*mock_client_, - Get(::testing::Not(::testing::HasSubstr("/credentials")), _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return( - HttpResponse::MakeForTesting(200, std::string(kPlanWithCredentialsResponse)))); - EXPECT_CALL(*mock_client_, - Get(::testing::HasSubstr("/credentials"), - ::testing::Contains(::testing::Pair("planId", "plan-1")), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting( - 200, - R"({"storage-credentials":[{"prefix":"s3://bucket","config":{"k":"v2"}}]})"))); - - auto endpoints = all_plan_endpoints_; - endpoints.insert(Endpoint::TableCredentials()); - ICEBERG_UNWRAP_OR_FAIL(auto catalog, - MakeCatalog(endpoints, /*client_scan_mode=*/"", - {{"io-impl", std::string(kCredentialedFileIO)}})); - ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); - ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); - ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewScan()); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - - ASSERT_NE(installed_provider, nullptr); - ICEBERG_UNWRAP_OR_FAIL(auto refreshed, installed_provider->Load()); - ASSERT_EQ(refreshed.size(), 1); - EXPECT_EQ(refreshed[0].config.at("k"), "v2"); -} - // -------------------------------------------------------------------------- // A completed plan without a plan-id refreshes credentials without planId. // -------------------------------------------------------------------------- @@ -367,14 +313,14 @@ TEST_F(RestCatalogLoadTableTest, ScanCredentialProviderWithoutPlanIdOmitsPlanId) RegisterCredentialedFileIO(); EXPECT_CALL(*mock_client_, Get(::testing::Not(::testing::HasSubstr("/credentials")), _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); + .WillOnce(Return(HttpResponse(200, kLoadTableServerScanResponse))); EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting( + .WillOnce(Return(HttpResponse( 200, R"({"status":"completed","storage-credentials":[{"prefix":"s3://bucket","config":{"k":"v"}}]})"))); EXPECT_CALL(*mock_client_, Get(::testing::HasSubstr("/credentials"), ::testing::IsEmpty(), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting( + .WillOnce(Return(HttpResponse( 200, R"({"storage-credentials":[{"prefix":"s3://bucket","config":{"k":"v2"}}]})"))); @@ -401,10 +347,9 @@ TEST_F(RestCatalogLoadTableTest, ScanCredentialsWithoutCredentialsEndpointHaveNoProvider) { RegisterCredentialedFileIO(); EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, kLoadTableServerScanResponse))); + .WillOnce(Return(HttpResponse(200, kLoadTableServerScanResponse))); EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return( - HttpResponse::MakeForTesting(200, std::string(kPlanWithCredentialsResponse)))); + .WillOnce(Return(HttpResponse(200, std::string(kPlanWithCredentialsResponse)))); ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(all_plan_endpoints_, /*client_scan_mode=*/"", @@ -416,6 +361,131 @@ TEST_F(RestCatalogLoadTableTest, ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_EQ(installed_provider, nullptr); + auto* credentialed = scan->io()->AsSupportsStorageCredentials(); + ASSERT_NE(credentialed, nullptr); + ASSERT_EQ(credentialed->credentials().size(), 1); + EXPECT_EQ(credentialed->credentials()[0].config.at("k"), "v"); +} + +struct PlanningModeCase { + std::string client; + std::string table; + bool server; +}; + +class RestCatalogPlanningModeTest + : public RestCatalogLoadTableTest, + public ::testing::WithParamInterface {}; + +TEST_P(RestCatalogPlanningModeTest, LoadAndCommitKeepTheEffectivePlanningMode) { + const auto& config = GetParam(); + auto response = nlohmann::json::parse(kLoadTableDefaultResponse); + if (!config.table.empty()) response["config"]["scan-planning-mode"] = config.table; + EXPECT_CALL(*mock_client_, Get(::testing::EndsWith("/tables/my_table"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, response.dump()))); + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/tables/my_table"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, kCommitTableResponse))); + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .Times(config.server ? 2 : 0) + .WillRepeatedly([](auto&&...) -> Result { + return HttpResponse(200, R"({"status":"completed"})"); + }); + + ICEBERG_UNWRAP_OR_FAIL(auto catalog, MakeCatalog(all_plan_endpoints_, config.client)); + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); + ICEBERG_UNWRAP_OR_FAIL(auto committed, + table->catalog()->UpdateTable(identifier_, {}, {})); + for (const auto& current : {table, committed}) { + ICEBERG_UNWRAP_OR_FAIL(auto builder, current->NewScan()); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + EXPECT_TRUE(tasks.empty()); + } } +INSTANTIATE_TEST_SUITE_P( + PlanningModes, RestCatalogPlanningModeTest, + ::testing::Values( + PlanningModeCase{.client = "", .table = "", .server = false}, + PlanningModeCase{.client = "server", .table = "", .server = true}, + PlanningModeCase{.client = "", .table = "server", .server = true}, + PlanningModeCase{.client = "server", .table = "client", .server = false}, + PlanningModeCase{.client = "client", .table = "server", .server = true})); + +class RestCatalogScanLifetimeTest : public RestCatalogLoadTableTest, + public ::testing::WithParamInterface {}; + +TEST_P(RestCatalogScanLifetimeTest, ScanKeepsCredentialRefreshAlive) { + RegisterCredentialedFileIO(); + auto response = nlohmann::json::parse(kLoadTableServerScanResponse); + auto& metadata = response["metadata"]; + metadata["current-snapshot-id"] = 200; + metadata["last-sequence-number"] = 2; + metadata["snapshots"] = + nlohmann::json::array({{{"snapshot-id", 100}, + {"sequence-number", 1}, + {"timestamp-ms", 1}, + {"manifest-list", "s3://bucket/manifests-1.avro"}, + {"schema-id", 1}}, + {{"snapshot-id", 200}, + {"parent-snapshot-id", 100}, + {"sequence-number", 2}, + {"timestamp-ms", 2}, + {"manifest-list", "s3://bucket/manifests-2.avro"}, + {"schema-id", 1}}}); + EXPECT_CALL(*mock_client_, Get(::testing::EndsWith("/tables/my_table"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, response.dump()))); + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, std::string(kPlanWithCredentialsResponse)))); + EXPECT_CALL(*mock_client_, + Get(::testing::EndsWith("/credentials"), + ::testing::Contains(::testing::Pair("planId", "plan-1")), _, _, _)) + .WillOnce(Return(HttpResponse( + 200, + R"({"storage-credentials":[{"prefix":"s3://bucket","config":{"k":"v2"}}]})"))); + + auto endpoints = all_plan_endpoints_; + endpoints.insert(Endpoint::TableCredentials()); + ICEBERG_UNWRAP_OR_FAIL( + auto catalog, + MakeCatalog(endpoints, "", {{"io-impl", std::string(kCredentialedFileIO)}})); + std::weak_ptr weak_catalog = catalog; + ICEBERG_UNWRAP_OR_FAIL(auto as_catalog, catalog->AsCatalog()); + ICEBERG_UNWRAP_OR_FAIL(auto table, as_catalog->LoadTable(identifier_)); + auto exercise = [&](auto& builder) { + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); + builder.reset(); + table.reset(); + as_catalog.reset(); + catalog.reset(); + EXPECT_FALSE(weak_catalog.expired()); + + ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); + ASSERT_NE(installed_provider, nullptr); + auto* credentialed = scan->io()->AsSupportsStorageCredentials(); + ASSERT_NE(credentialed, nullptr); + ASSERT_EQ(credentialed->credentials().size(), 1); + EXPECT_EQ(credentialed->credentials()[0].config.at("k"), "v"); + ICEBERG_UNWRAP_OR_FAIL(auto refreshed, installed_provider->Load()); + ASSERT_EQ(refreshed.size(), 1); + EXPECT_EQ(refreshed[0].config.at("k"), "v2"); + + scan.reset(); + EXPECT_TRUE(weak_catalog.expired()); + }; + if (GetParam()) { + ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewIncrementalAppendScan()); + builder->FromSnapshot(100); + exercise(builder); + } else { + ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewScan()); + exercise(builder); + } +} + +INSTANTIATE_TEST_SUITE_P(ScanModes, RestCatalogScanLifetimeTest, ::testing::Bool()); + +} // namespace + } // namespace iceberg::rest diff --git a/src/iceberg/test/rest_table_scan_test.cc b/src/iceberg/test/rest_table_scan_test.cc index 7c827690c1..a1b8e6e867 100644 --- a/src/iceberg/test/rest_table_scan_test.cc +++ b/src/iceberg/test/rest_table_scan_test.cc @@ -19,11 +19,15 @@ #include "iceberg/catalog/rest/rest_table_scan.h" +#include #include #include #include +#include #include #include +#include +#include #include #include @@ -35,8 +39,10 @@ #include "iceberg/catalog/rest/http_client.h" #include "iceberg/catalog/rest/resource_paths.h" #include "iceberg/catalog/rest/rest_table.h" +#include "iceberg/catalog/rest/rest_table_scan.h" #include "iceberg/constants.h" #include "iceberg/expression/expressions.h" +#include "iceberg/expression/json_serde_internal.h" #include "iceberg/expression/literal.h" #include "iceberg/file_io.h" #include "iceberg/file_io_registry.h" @@ -54,6 +60,8 @@ namespace iceberg::rest { +namespace { + using ::testing::_; using ::testing::Return; @@ -166,10 +174,16 @@ class CredentialedFileIO : public NoOpFileIO, public SupportsStorageCredentials constexpr std::string_view kCredentialedFileIO = "rest-table-scan-test-credentialed"; -class StaticCredentialProvider : public StorageCredentialProvider { - public: - Result> Load() override { return {}; } -}; +nlohmann::json FileTaskJson(const std::string& path) { + return {{"data-file", + {{"content", "data"}, + {"file-path", path}, + {"file-format", "PARQUET"}, + {"spec-id", 0}, + {"partition", nlohmann::json::array()}, + {"file-size-in-bytes", 100}, + {"record-count", 10}}}}; +} } // namespace @@ -178,6 +192,8 @@ class StaticCredentialProvider : public StorageCredentialProvider { // -------------------------------------------------------------------------- class RestTableScanTest : public ::testing::Test { protected: + void TearDown() override { installed_provider.reset(); } + void SetUp() override { schema_ = std::make_shared( std::vector{SchemaField::MakeRequired(1, "id", int32()), @@ -254,7 +270,7 @@ class RestTableScanTest : public ::testing::Test { Result> MakeScan(RestScanContext ctx) { return RestTableScan::Make(metadata_, schema_, file_io_, internal::TableScanContext{}, - std::move(ctx)); + std::make_shared(std::move(ctx))); } std::shared_ptr schema_; @@ -267,114 +283,6 @@ class RestTableScanTest : public ::testing::Test { std::unordered_set all_plan_endpoints_; }; -// -------------------------------------------------------------------------- -// PlanFiles: server returns COMPLETED immediately, no file scan tasks. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesCompleted) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// PlanFiles: server returns COMPLETED with a non-empty plan-id (still valid). -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesCompletedWithPlanId) { - constexpr std::string_view kResponseBody = - R"({"status":"completed","plan-id":"plan-abc"})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// PlanFiles: server returns SUBMITTED → poll returns COMPLETED. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesSubmittedThenCompleted) { - constexpr std::string_view kSubmittedBody = - R"({"status":"submitted","plan-id":"plan-poll-1"})"; - constexpr std::string_view kCompletedBody = R"({"status":"completed"})"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); - EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kCompletedBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// PlanFiles: server returns FAILED → scan returns IOError. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesFailed) { - constexpr std::string_view kFailedBody = - R"({"status":"failed","error":{"message":"server error","type":"ServerError","code":500}})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFailedBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - auto result = scan->PlanFiles(); - EXPECT_THAT(result, IsError(ErrorKind::kIOError)); -} - -// -------------------------------------------------------------------------- -// PlanFiles: PlanTableScan endpoint missing → NotSupported error. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesEndpointNotSupported) { - ICEBERG_UNWRAP_OR_FAIL(auto scan, - MakeScan(MakeContext(std::unordered_set{}))); - auto result = scan->PlanFiles(); - EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); -} - -// -------------------------------------------------------------------------- -// PlanFiles with plan-tasks: server returns COMPLETED with opaque task token, -// then FetchScanTasks is called and returns no file scan tasks. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesWithPlanTasks) { - constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"plan-1","plan-tasks":["tok-1"]})"; - // FetchScanTasksResponse requires at least one of plan-tasks or file-scan-tasks - // present. - constexpr std::string_view kTasksResponse = R"({"file-scan-tasks":[]})"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// Cancel is called when FetchScanTasks fails after a COMPLETED response that -// included plan-tasks. This mirrors the Java cancelPlan-on-close behavior. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, CancelCalledWhenFetchScanTasksFails) { - constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"plan-cancel-1","plan-tasks":["tok-a"]})"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce(Return(IOError("FetchScanTasks failed"))); - EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - auto result = scan->PlanFiles(); - EXPECT_THAT(result, IsError(ErrorKind::kIOError)); -} - // -------------------------------------------------------------------------- // Cancel is a no-op when plan_id is empty (server returned no plan-id). // -------------------------------------------------------------------------- @@ -383,7 +291,7 @@ TEST_F(RestTableScanTest, CancelIsNoOpWithEmptyPlanId) { R"({"status":"completed","plan-tasks":["tok-b"]})"; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse(200, std::string(kPlanResponse)))) .WillOnce(Return(IOError("FetchScanTasks failed"))); EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); @@ -404,7 +312,7 @@ TEST_F(RestTableScanTest, CancelIsNoOpWhenEndpointNotAdvertised) { Endpoint::FetchScanTasks()}; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse(200, std::string(kPlanResponse)))) .WillOnce(Return(IOError("FetchScanTasks failed"))); EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); @@ -413,45 +321,6 @@ TEST_F(RestTableScanTest, CancelIsNoOpWhenEndpointNotAdvertised) { EXPECT_THAT(result, IsError(ErrorKind::kIOError)); } -// -------------------------------------------------------------------------- -// FetchPlanningResult: FetchPlanningResult endpoint missing → NotSupported. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, FetchPlanningResultEndpointNotSupported) { - constexpr std::string_view kSubmittedBody = - R"({"status":"submitted","plan-id":"plan-3"})"; - - std::unordered_set endpoints_without_fetch = { - Endpoint::PlanTableScan(), Endpoint::CancelPlanning(), Endpoint::FetchScanTasks()}; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext(endpoints_without_fetch))); - auto result = scan->PlanFiles(); - EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); -} - -// -------------------------------------------------------------------------- -// FetchScanTasks: endpoint missing → NotSupported. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, FetchScanTasksEndpointNotSupported) { - constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"plan-4","plan-tasks":["tok-d"]})"; - - std::unordered_set endpoints_without_tasks = {Endpoint::PlanTableScan(), - Endpoint::FetchPlanningResult(), - Endpoint::CancelPlanning()}; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))); - EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext(endpoints_without_tasks))); - auto result = scan->PlanFiles(); - EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); -} - // -------------------------------------------------------------------------- // UseSnapshot(): the POST body sent to the server must contain both // "snapshot-id" and "use-snapshot-schema": true. @@ -465,120 +334,17 @@ TEST_F(RestTableScanTest, UseSnapshotPropagatesUseSnapshotSchemaInContext) { testing::AllOf(JsonBodyHas("snapshot-id", kSnapshotId), JsonBodyHas("use-snapshot-schema", true)), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kResponseBody)))); - RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext(std::nullopt)); + RestTableScanBuilder builder( + metadata_, file_io_, "test.my_table", nullptr, + std::make_shared(MakeContext(std::nullopt))); builder.UseSnapshot(kSnapshotId); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); } -// -------------------------------------------------------------------------- -// Default scan: POST body must have "use-snapshot-schema": false and no -// "snapshot-id" field. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, DefaultScanDoesNotSetUseSnapshotSchema) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - - EXPECT_CALL(*mock_client_, - Post(_, - testing::AllOf(JsonBodyHas("use-snapshot-schema", false), - JsonBodyLacks("snapshot-id")), - _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext(std::nullopt)); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// IncludeColumnStats() columns are sent as stats-fields. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, IncludeColumnStatsSendsStatsFields) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL( - *mock_client_, - Post(_, JsonBodyHas("stats-fields", std::vector{"id"}), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext()); - builder.IncludeColumnStats({"id"}); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// IncludeColumnStats() with an unknown column → ValidationFailed. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, IncludeColumnStatsRejectsMissingColumn) { - RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext()); - builder.IncludeColumnStats({"missing"}); - EXPECT_THAT(builder.Build(), - ::testing::AllOf(IsError(ErrorKind::kValidationFailed), - HasErrorMessage("Cannot find stats column: missing"))); -} - -// -------------------------------------------------------------------------- -// Project() columns are sent as select. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, ProjectSendsProjectedColumnsAsSelect) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, - Post(_, JsonBodyHas("select", std::vector{"id"}), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext()); - builder.Project(std::make_shared( - std::vector{SchemaField::MakeRequired(1, "id", int32())})); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// Select() sends the selected columns plus filter columns. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, SelectIncludesFilterColumns) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL( - *mock_client_, - Post(_, JsonBodyHas("select", std::vector{"id", "data"}), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext()); - builder.Select({"data"}).Filter(Expressions::Equal("id", Literal::Int(1))); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// Case-insensitive Select() sends the schema's column names. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, SelectResolvesCaseInsensitiveNames) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, - Post(_, JsonBodyHas("select", std::vector{"data"}), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext()); - builder.CaseSensitive(false).Select({"DATA"}); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - // -------------------------------------------------------------------------- // Select("*") sends every column, not a literal "*". // -------------------------------------------------------------------------- @@ -587,33 +353,16 @@ TEST_F(RestTableScanTest, SelectAllColumnsSendsAllColumns) { EXPECT_CALL( *mock_client_, Post(_, JsonBodyHas("select", std::vector{"id", "data"}), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kResponseBody)))); RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext()); + std::make_shared(MakeContext())); builder.Select({std::string(Schema::kAllColumns)}); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); } -// -------------------------------------------------------------------------- -// No projection sends every column. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, NoProjectionSendsAllColumns) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL( - *mock_client_, - Post(_, JsonBodyHas("select", std::vector{"id", "data"}), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext()); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - // -------------------------------------------------------------------------- // UseRef(): branches use the table schema, tags use the snapshot schema. // -------------------------------------------------------------------------- @@ -652,10 +401,10 @@ TEST_F(RestTableScanRefSchemaTest, UseRefBranchUsesTableSchema) { testing::AllOf(JsonBodyHas("snapshot-id", kEvolvedSnapshotId), JsonBodyHas("use-snapshot-schema", false)), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kResponseBody)))); RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext()); + std::make_shared(MakeContext())); builder.UseRef("evolved-branch"); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); ICEBERG_UNWRAP_OR_FAIL(auto scan_schema, scan->schema()); @@ -671,10 +420,10 @@ TEST_F(RestTableScanRefSchemaTest, UseRefTagUsesSnapshotSchema) { testing::AllOf(JsonBodyHas("snapshot-id", kEvolvedSnapshotId), JsonBodyHas("use-snapshot-schema", true)), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kResponseBody)))); RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext()); + std::make_shared(MakeContext())); builder.UseRef("evolved-tag"); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); ICEBERG_UNWRAP_OR_FAIL(auto scan_schema, scan->schema()); @@ -683,43 +432,6 @@ TEST_F(RestTableScanRefSchemaTest, UseRefTagUsesSnapshotSchema) { EXPECT_TRUE(tasks.empty()); } -// -------------------------------------------------------------------------- -// Storage credentials in COMPLETED response: io() returns a -// credential-scoped IO, not the original table IO. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, StorageCredentialsInPlanResponseUpdatesEffectiveIO) { - constexpr std::string_view kResponseBody = R"({ - "status": "completed", - "storage-credentials": [ - {"prefix": "s3://bucket/prefix", "config": {"key": "value"}} - ] - })"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); - - // io() must return a credential-scoped IO without requiring a downcast. - EXPECT_NE(scan->io().get(), file_io_.get()); -} - -// -------------------------------------------------------------------------- -// No storage credentials: io() falls back to the table's FileIO. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, NoStorageCredentialsEffectiveIoFallsBackToTableIO) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); - - EXPECT_EQ(scan->io().get(), file_io_.get()); -} - // -------------------------------------------------------------------------- // Storage credentials in a FetchScanTasks response are ignored. // -------------------------------------------------------------------------- @@ -734,8 +446,8 @@ TEST_F(RestTableScanTest, StorageCredentialsInFetchScanTasksResponseAreIgnored) })"; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); + .WillOnce(Return(HttpResponse(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse(200, std::string(kTasksResponse)))); ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); @@ -745,303 +457,58 @@ TEST_F(RestTableScanTest, StorageCredentialsInFetchScanTasksResponseAreIgnored) } // -------------------------------------------------------------------------- -// RestTable::NewScan returns a RestTableScanBuilder (not a plain builder). +// PlanFilesStream: Next() propagates a FetchScanTasks error and DELETE /plan +// is called via the stream destructor since consumed_ is never set. // -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, RestTableNewScanReturnsRestTableScanBuilder) { - ICEBERG_UNWRAP_OR_FAIL( - auto table, RestTable::Make(identifier_, metadata_, "/tmp/metadata.json", file_io_, - /*catalog=*/nullptr, "test.my_table", nullptr, - MakeContext(std::nullopt))); - ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewScan()); - auto* typed = dynamic_cast(builder.get()); - EXPECT_NE(typed, nullptr); -} - -// ========================================================================== -// PlanFilesStream tests -// ========================================================================== +TEST_F(RestTableScanTest, PlanFilesStreamNextReturnsErrorOnFetchFailure) { + constexpr std::string_view kPlanResponse = + R"({"status":"completed","plan-id":"plan-next-err","plan-tasks":["tok-err"]})"; -// -------------------------------------------------------------------------- -// PlanFilesStream: COMPLETED immediately, stream yields no tasks. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamCompleted) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kPlanResponse)))) + .WillOnce(Return(IOError("FetchScanTasks network error"))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse(200, "{}"))); ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, stream->ToVector()); - EXPECT_TRUE(tasks.empty()); + auto result = stream->Next(); + EXPECT_THAT(result, IsError(ErrorKind::kIOError)); } // -------------------------------------------------------------------------- -// PlanFilesStream: two plan-task tokens each trigger a separate FetchScanTasks -// POST, one per token, and the combined task set is returned. +// PlanFilesStream: stream destroyed after consuming the first task but before +// the second token is fetched → DELETE /plan called by the destructor. // -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamFetchesEachTokenSeparately) { +TEST_F(RestTableScanTest, PlanFilesStreamCancelAfterPartialConsumption) { constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"plan-stream","plan-tasks":["tok-s1","tok-s2"]})"; - constexpr std::string_view kTask1Response = R"({ - "file-scan-tasks": [ - {"data-file":{"content":"data","file-path":"s3://b/f1.parquet", - "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} - ] - })"; - constexpr std::string_view kTask2Response = R"({ + R"({"status":"completed","plan-id":"plan-partial","plan-tasks":["tok-p1","tok-p2"]})"; + constexpr std::string_view kTaskResponse = R"({ "file-scan-tasks": [ - {"data-file":{"content":"data","file-path":"s3://b/f2.parquet", + {"data-file":{"content":"data","file-path":"s3://b/fp1.parquet", "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} ] })"; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTask1Response)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTask2Response)))); + .WillOnce(Return(HttpResponse(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse(200, std::string(kTaskResponse)))); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) + .WillOnce(Return(HttpResponse(200, "{}"))); ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, stream->ToVector()); - ASSERT_EQ(tasks.size(), 2u); - EXPECT_EQ(tasks[0]->data_file()->file_path, "s3://b/f1.parquet"); - EXPECT_EQ(tasks[1]->data_file()->file_path, "s3://b/f2.parquet"); + { + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + // Consume the first task from tok-p1; tok-p2 has never been fetched. + ICEBERG_UNWRAP_OR_FAIL(auto task, stream->Next()); + ASSERT_TRUE(task.has_value()); + // Destroy stream here — tok-p2 is still pending, so destructor calls DELETE. + } } // -------------------------------------------------------------------------- -// PlanFilesStream: the second FetchScanTasks POST is not made until the first -// token's buffer is exhausted. Verified by counting POST calls between Next() -// invocations. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamFetchesTokenOnlyWhenBufferExhausted) { - constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"plan-lazy","plan-tasks":["tok-1","tok-2"]})"; - // tok-1 returns 2 tasks; tok-2 must not be fetched until both are consumed. - constexpr std::string_view kTwoTasksResponse = R"({ - "file-scan-tasks": [ - {"data-file":{"content":"data","file-path":"s3://b/f1.parquet", - "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}}, - {"data-file":{"content":"data","file-path":"s3://b/f2.parquet", - "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} - ] - })"; - constexpr std::string_view kOneTaskResponse = R"({ - "file-scan-tasks": [ - {"data-file":{"content":"data","file-path":"s3://b/f3.parquet", - "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} - ] - })"; - - int fetch_count = 0; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce([&](auto&&...) -> Result { - ++fetch_count; - return HttpResponse::MakeForTesting(200, std::string(kTwoTasksResponse)); - }) - .WillOnce([&](auto&&...) -> Result { - ++fetch_count; - return HttpResponse::MakeForTesting(200, std::string(kOneTaskResponse)); - }); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); - - // No FetchScanTasks call yet — stream has not been driven. - EXPECT_EQ(fetch_count, 0); - - // First Next(): fetches tok-1 (2 tasks buffered), returns f1. - ICEBERG_UNWRAP_OR_FAIL(auto t1, stream->Next()); - ASSERT_TRUE(t1.has_value()); - EXPECT_EQ(fetch_count, 1); - EXPECT_EQ((*t1)->data_file()->file_path, "s3://b/f1.parquet"); - - // Second Next(): served from buffer; tok-2 not fetched yet. - ICEBERG_UNWRAP_OR_FAIL(auto t2, stream->Next()); - ASSERT_TRUE(t2.has_value()); - EXPECT_EQ(fetch_count, 1); - EXPECT_EQ((*t2)->data_file()->file_path, "s3://b/f2.parquet"); - - // Third Next(): buffer exhausted, fetches tok-2, returns f3. - ICEBERG_UNWRAP_OR_FAIL(auto t3, stream->Next()); - ASSERT_TRUE(t3.has_value()); - EXPECT_EQ(fetch_count, 2); - EXPECT_EQ((*t3)->data_file()->file_path, "s3://b/f3.parquet"); - - // Fourth Next(): all tokens consumed, stream terminates. - ICEBERG_UNWRAP_OR_FAIL(auto end, stream->Next()); - EXPECT_FALSE(end.has_value()); -} - -// -------------------------------------------------------------------------- -// PlanFilesStream: Next() propagates a FetchScanTasks error and DELETE /plan -// is called via the stream destructor since consumed_ is never set. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamNextReturnsErrorOnFetchFailure) { - constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"plan-next-err","plan-tasks":["tok-err"]})"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce(Return(IOError("FetchScanTasks network error"))); - EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); - auto result = stream->Next(); - EXPECT_THAT(result, IsError(ErrorKind::kIOError)); -} - -// -------------------------------------------------------------------------- -// PlanFilesStream: stream destroyed after consuming the first task but before -// the second token is fetched → DELETE /plan called by the destructor. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamCancelAfterPartialConsumption) { - constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"plan-partial","plan-tasks":["tok-p1","tok-p2"]})"; - constexpr std::string_view kTaskResponse = R"({ - "file-scan-tasks": [ - {"data-file":{"content":"data","file-path":"s3://b/fp1.parquet", - "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} - ] - })"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTaskResponse)))); - EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - { - ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); - // Consume the first task from tok-p1; tok-p2 has never been fetched. - ICEBERG_UNWRAP_OR_FAIL(auto task, stream->Next()); - ASSERT_TRUE(task.has_value()); - // Destroy stream here — tok-p2 is still pending, so destructor calls DELETE. - } -} - -// -------------------------------------------------------------------------- -// PlanFilesStream: stream destroyed with no Next() calls at all → -// DELETE /plan called by the destructor. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamCancelOnNoConsumption) { - constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"plan-never-consumed","plan-tasks":["tok-p1"]})"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))); - EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - { - ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); - // Destroy without any Next() call — destructor must call DELETE /plan. - } -} - -// -------------------------------------------------------------------------- -// PlanFilesStream: SUBMITTED → poll until COMPLETED, stream yields all tasks. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamSubmittedThenCompleted) { - constexpr std::string_view kSubmittedBody = - R"({"status":"submitted","plan-id":"plan-poll-stream"})"; - constexpr std::string_view kCompletedBody = R"({"status":"completed"})"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); - EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kCompletedBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, stream->ToVector()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// PlanFilesStream: SUBMITTED → COMPLETED with plan tasks; tokens are fetched -// lazily as the stream is consumed. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamSubmittedFetchesTokensLazily) { - constexpr std::string_view kSubmittedBody = - R"({"status":"submitted","plan-id":"plan-poll-lazy"})"; - constexpr std::string_view kCompletedBody = - R"({"status":"completed","plan-tasks":["tok-1","tok-2"]})"; - constexpr std::string_view kTask1Response = R"({ - "file-scan-tasks": [ - {"data-file":{"content":"data","file-path":"s3://b/f1.parquet", - "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} - ] - })"; - constexpr std::string_view kTask2Response = R"({ - "file-scan-tasks": [ - {"data-file":{"content":"data","file-path":"s3://b/f2.parquet", - "file-format":"PARQUET","spec-id":0,"partition":[],"file-size-in-bytes":1,"record-count":1}} - ] - })"; - - int fetch_count = 0; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))) - .WillOnce([&](auto&&...) -> Result { - ++fetch_count; - return HttpResponse::MakeForTesting(200, std::string(kTask1Response)); - }) - .WillOnce([&](auto&&...) -> Result { - ++fetch_count; - return HttpResponse::MakeForTesting(200, std::string(kTask2Response)); - }); - EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kCompletedBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); - EXPECT_EQ(fetch_count, 0); - - ICEBERG_UNWRAP_OR_FAIL(auto t1, stream->Next()); - ASSERT_TRUE(t1.has_value()); - EXPECT_EQ(fetch_count, 1); - EXPECT_EQ((*t1)->data_file()->file_path, "s3://b/f1.parquet"); - - ICEBERG_UNWRAP_OR_FAIL(auto t2, stream->Next()); - ASSERT_TRUE(t2.has_value()); - EXPECT_EQ(fetch_count, 2); - EXPECT_EQ((*t2)->data_file()->file_path, "s3://b/f2.parquet"); - - ICEBERG_UNWRAP_OR_FAIL(auto end, stream->Next()); - EXPECT_FALSE(end.has_value()); -} - -// -------------------------------------------------------------------------- -// PlanFilesStream: SUBMITTED → COMPLETED, stream destroyed with no Next() calls -// → DELETE /plan called by the destructor. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamSubmittedCancelOnNoConsumption) { - constexpr std::string_view kSubmittedBody = - R"({"status":"submitted","plan-id":"plan-poll-partial"})"; - constexpr std::string_view kCompletedBody = - R"({"status":"completed","plan-tasks":["tok-1"]})"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); - EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kCompletedBody)))); - EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - { - ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); - } -} - -// -------------------------------------------------------------------------- -// PlanFilesStream: nested plan tasks returned by FetchScanTasks are fetched -// only when the stream reaches them. +// PlanFilesStream: nested plan tasks returned by FetchScanTasks are fetched +// only when the stream reaches them. // -------------------------------------------------------------------------- TEST_F(RestTableScanTest, PlanFilesStreamFetchesNestedPlanTasksLazily) { constexpr std::string_view kPlanResponse = @@ -1061,15 +528,19 @@ TEST_F(RestTableScanTest, PlanFilesStreamFetchesNestedPlanTasksLazily) { })"; int fetch_count = 0; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, std::string(kPlanResponse)))); + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/tasks"), + JsonBodyHas("plan-task", "tok-1"), _, _, _)) .WillOnce([&](auto&&...) -> Result { ++fetch_count; - return HttpResponse::MakeForTesting(200, std::string(kTask1Response)); - }) + return HttpResponse(200, std::string(kTask1Response)); + }); + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/tasks"), + JsonBodyHas("plan-task", "tok-nested"), _, _, _)) .WillOnce([&](auto&&...) -> Result { ++fetch_count; - return HttpResponse::MakeForTesting(200, std::string(kNestedResponse)); + return HttpResponse(200, std::string(kNestedResponse)); }); ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); @@ -1108,13 +579,13 @@ TEST_F(RestTableScanTest, PlanFilesStreamIgnoresPlanExecutor) { ] })"; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTask1Response)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTask2Response)))); + .WillOnce(Return(HttpResponse(200, std::string(kPlanResponse)))) + .WillOnce(Return(HttpResponse(200, std::string(kTask1Response)))) + .WillOnce(Return(HttpResponse(200, std::string(kTask2Response)))); test::ThreadExecutor executor; RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, - MakeContext()); + std::make_shared(MakeContext())); builder.PlanWith(executor); ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); @@ -1125,34 +596,6 @@ TEST_F(RestTableScanTest, PlanFilesStreamIgnoresPlanExecutor) { EXPECT_EQ(executor.submit_count(), 0); } -// -------------------------------------------------------------------------- -// PlanFilesStream: invalid plan response with a plan-id → DELETE /plan called. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamInvalidResponseWithPlanIdCancels) { - constexpr std::string_view kInvalidBody = - R"({"status":"submitted","plan-id":"plan-invalid","plan-tasks":["tok-1"]})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kInvalidBody)))); - EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - EXPECT_THAT(scan->PlanFilesStream(), IsError(ErrorKind::kValidationFailed)); -} - -// -------------------------------------------------------------------------- -// PlanFilesStream: invalid plan response without a plan-id → no DELETE. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamInvalidResponseWithoutPlanIdDoesNotCancel) { - constexpr std::string_view kInvalidBody = R"({"status":"failed"})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kInvalidBody)))); - EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - EXPECT_THAT(scan->PlanFilesStream(), IsError(ErrorKind::kJsonParseError)); -} - // -------------------------------------------------------------------------- // poll-num-retries=0: a single GET for a SUBMITTED plan, then cancel. // -------------------------------------------------------------------------- @@ -1160,11 +603,11 @@ TEST_F(RestTableScanTest, PlanFilesStreamPollNumRetriesZero) { constexpr std::string_view kSubmittedBody = R"({"status":"submitted","plan-id":"plan-no-retry"})"; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kSubmittedBody)))); EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kSubmittedBody)))); EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + .WillOnce(Return(HttpResponse(200, "{}"))); auto ctx = MakeContext(); ctx.catalog_config = {{"rest-scan-planning.poll-num-retries", "0"}}; @@ -1174,48 +617,6 @@ TEST_F(RestTableScanTest, PlanFilesStreamPollNumRetriesZero) { HasErrorMessage("exceeded max retries (0)"))); } -// -------------------------------------------------------------------------- -// Invalid poll-timeout-ms fails before the plan is submitted. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamInvalidPollTimeoutFailsBeforePlanning) { - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); - - auto ctx = MakeContext(); - ctx.catalog_config = {{"rest-scan-planning.poll-timeout-ms", "0"}}; - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(std::move(ctx))); - EXPECT_THAT(scan->PlanFilesStream(), IsError(ErrorKind::kInvalidArgument)); -} - -// -------------------------------------------------------------------------- -// Vended credentials in the plan response install a provider scoped to the plan. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, StorageCredentialsInstallPlanScopedProvider) { - constexpr std::string_view kPlanResponse = R"({ - "status": "completed", - "plan-id": "plan-cred", - "storage-credentials": [{"prefix": "s3://bucket", "config": {"k": "v"}}] - })"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))); - RegisterCredentialedFileIO(); - - std::vector requested_plan_ids; - auto provider = std::make_shared(); - auto ctx = MakeContext(); - ctx.catalog_config = {{"io-impl", std::string(kCredentialedFileIO)}}; - ctx.credential_provider_factory = [&](const std::string& plan_id) { - requested_plan_ids.push_back(plan_id); - return provider; - }; - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(std::move(ctx))); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - - EXPECT_THAT(requested_plan_ids, ::testing::ElementsAre("plan-cred")); - EXPECT_EQ(installed_provider, provider); - ASSERT_EQ(installed_credentials.size(), 1); - EXPECT_EQ(installed_credentials[0].prefix, "s3://bucket"); -} - // -------------------------------------------------------------------------- // Without a provider factory, vended credentials are installed as static. // -------------------------------------------------------------------------- @@ -1226,7 +627,7 @@ TEST_F(RestTableScanTest, StorageCredentialsWithoutFactoryInstallNoProvider) { "storage-credentials": [{"prefix": "s3://bucket", "config": {"k": "v"}}] })"; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))); + .WillOnce(Return(HttpResponse(200, std::string(kPlanResponse)))); RegisterCredentialedFileIO(); auto ctx = MakeContext(); @@ -1238,38 +639,6 @@ TEST_F(RestTableScanTest, StorageCredentialsWithoutFactoryInstallNoProvider) { EXPECT_EQ(installed_credentials.size(), 1); } -// -------------------------------------------------------------------------- -// PlanFilesStream: FAILED → stream returns an error. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamFailed) { - constexpr std::string_view kFailedBody = - R"({"status":"failed","error":{"message":"server error","type":"ServerError","code":500}})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFailedBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - auto result = scan->PlanFilesStream(); - EXPECT_THAT(result, IsError(ErrorKind::kIOError)); -} - -// -------------------------------------------------------------------------- -// PlanFilesStream: FAILED with plan-id is an invalid response → DELETE /plan -// called before returning the error. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, PlanFilesStreamFailedWithPlanIdCancels) { - constexpr std::string_view kFailedBody = - R"({"status":"failed","plan-id":"plan-fail-stream","error":{"message":"server error","type":"ServerError","code":500}})"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFailedBody)))); - EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - auto result = scan->PlanFilesStream(); - EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); -} - // -------------------------------------------------------------------------- // FetchPlanningResult: GET fails after SUBMITTED → DELETE /plan called before // returning the error (tests the FetchPlanningResult error-path cancel fix). @@ -1279,46 +648,16 @@ TEST_F(RestTableScanTest, CancelCalledOnFetchPlanningResultGetError) { R"({"status":"submitted","plan-id":"plan-fetch-err"})"; EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kSubmittedBody)))); EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) .WillOnce(Return(IOError("network failure"))); EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); + .WillOnce(Return(IOError("cancel failure"))); ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); auto result = scan->PlanFiles(); - EXPECT_THAT(result, IsError(ErrorKind::kIOError)); -} - -// -------------------------------------------------------------------------- -// Stale credentials: second PlanFilesStream call resets the scan FileIO so that -// credentials from the first plan response do not persist into the second. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, SecondPlanFilesStreamCallClearsStaleCredentials) { - constexpr std::string_view kFirstResponse = R"({ - "status": "completed", - "storage-credentials": [ - {"prefix": "s3://bucket/prefix", "config": {"key": "value"}} - ] - })"; - constexpr std::string_view kSecondResponse = R"({"status":"completed"})"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFirstResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSecondResponse)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); - - // First call: server vends credentials → io() returns a credential-scoped IO. - ICEBERG_UNWRAP_OR_FAIL(auto tasks1, scan->PlanFiles()); - EXPECT_TRUE(tasks1.empty()); - EXPECT_NE(scan->io().get(), file_io_.get()); - - // Second call: no credentials returned → io() must revert to the table IO, - // not retain the credentials from the first plan. - ICEBERG_UNWRAP_OR_FAIL(auto tasks2, scan->PlanFiles()); - EXPECT_TRUE(tasks2.empty()); - EXPECT_EQ(scan->io().get(), file_io_.get()); + EXPECT_THAT(result, ::testing::AllOf(IsError(ErrorKind::kIOError), + HasErrorMessage("network failure"))); } // ========================================================================== @@ -1362,7 +701,8 @@ class RestIncrementalAppendScanTest : public RestTableScanTest { } RestIncrementalAppendScanBuilder MakeBuilder(RestScanContext ctx) { - return {metadata_, file_io_, "test.my_table", nullptr, std::move(ctx)}; + return {metadata_, file_io_, "test.my_table", nullptr, + std::make_shared(std::move(ctx))}; } Result> MakeIncrementalScan( @@ -1377,79 +717,26 @@ class RestIncrementalAppendScanTest : public RestTableScanTest { } return builder.Build(); } -}; - -// -------------------------------------------------------------------------- -// PlanFiles: server returns COMPLETED immediately, no tasks. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, PlanFilesCompleted) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// PlanFiles: COMPLETED with a plan-task token; FetchScanTasks is called. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, PlanFilesWithPlanTasks) { - constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"incr-plan-1","plan-tasks":["tok-incr-1"]})"; - constexpr std::string_view kTasksResponse = R"({"file-scan-tasks":[]})"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// PlanFiles: SUBMITTED → poll → COMPLETED. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, PlanFilesSubmittedThenCompleted) { - constexpr std::string_view kSubmittedBody = - R"({"status":"submitted","plan-id":"incr-poll-1"})"; - constexpr std::string_view kCompletedBody = R"({"status":"completed"})"; - - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSubmittedBody)))); - EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kCompletedBody)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); -} - -// -------------------------------------------------------------------------- -// PlanFiles: FAILED → IOError. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, PlanFilesFailed) { - constexpr std::string_view kFailedBody = - R"({"status":"failed","error":{"message":"server error","type":"ServerError","code":500}})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFailedBody)))); - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); - auto result = scan->PlanFiles(); - EXPECT_THAT(result, IsError(ErrorKind::kIOError)); -} - -// -------------------------------------------------------------------------- -// PlanFiles: PlanTableScan endpoint missing → NotSupported. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, PlanFilesEndpointNotSupported) { - ICEBERG_UNWRAP_OR_FAIL( - auto scan, MakeIncrementalScan(MakeContext(std::unordered_set{}))); - auto result = scan->PlanFiles(); - EXPECT_THAT(result, IsError(ErrorKind::kNotSupported)); -} + template + Result>> PlanScan(bool incremental, + RestScanContext ctx, + Configure configure) { + auto plan = [&](auto& builder) -> Result>> { + configure(builder); + ICEBERG_ASSIGN_OR_RAISE(auto scan, builder.Build()); + return scan->PlanFiles(); + }; + if (incremental) { + auto builder = MakeBuilder(std::move(ctx)); + builder.FromSnapshot(kRootId); + return plan(builder); + } + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + std::make_shared(std::move(ctx))); + return plan(builder); + } +}; // -------------------------------------------------------------------------- // No current snapshot → PlanFiles returns empty without calling the server. @@ -1474,10 +761,10 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesEmptyWhenNoCurrentSnapshot) { EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); - // Use Make() directly to bypass builder validation that requires a snapshot. - ICEBERG_UNWRAP_OR_FAIL(auto scan, RestIncrementalAppendScan::Make( - empty_metadata, schema_, file_io_, - internal::TableScanContext{}, MakeContext())); + RestIncrementalAppendScanBuilder builder( + empty_metadata, file_io_, "test.my_table", nullptr, + std::make_shared(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); EXPECT_TRUE(tasks.empty()); } @@ -1493,7 +780,7 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesSendsResolvedRange) { JsonBodyHas("end-snapshot-id", kMainHeadId), JsonBodyHas("use-snapshot-schema", true)), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kResponseBody)))); ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); @@ -1510,7 +797,7 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesWithExplicitToSnapshotId) { testing::AllOf(JsonBodyHas("start-snapshot-id", kRootId), JsonBodyHas("end-snapshot-id", kMiddleId)), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kResponseBody)))); ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext(), kRootId, @@ -1529,7 +816,7 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdExclusive) { testing::AllOf(JsonBodyHas("start-snapshot-id", kMiddleId), JsonBodyHas("end-snapshot-id", kMainHeadId)), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kResponseBody)))); ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext(), kMiddleId, /*from_inclusive=*/false)); @@ -1547,7 +834,7 @@ TEST_F(RestIncrementalAppendScanTest, PlanFilesWithFromSnapshotIdInclusiveWithPa testing::AllOf(JsonBodyHas("start-snapshot-id", kRootId), JsonBodyHas("end-snapshot-id", kMainHeadId)), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kResponseBody)))); ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext(), kMiddleId, /*from_inclusive=*/true)); @@ -1589,7 +876,7 @@ TEST_F(RestIncrementalAppendScanTest, UseBranchWithoutToSnapshotUsesBranchHead) testing::AllOf(JsonBodyHas("start-snapshot-id", kRootId), JsonBodyHas("end-snapshot-id", kBranchHeadId)), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kResponseBody)))); auto builder = MakeBuilder(MakeContext()); builder.FromSnapshot(kRootId).UseBranch(std::string(kBranch)); @@ -1608,7 +895,7 @@ TEST_F(RestIncrementalAppendScanTest, UseBranchWithToSnapshotOnBranch) { testing::AllOf(JsonBodyHas("start-snapshot-id", kRootId), JsonBodyHas("end-snapshot-id", kMiddleId)), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); + .WillOnce(Return(HttpResponse(200, std::string(kResponseBody)))); auto builder = MakeBuilder(MakeContext()); builder.FromSnapshot(kRootId).ToSnapshot(kMiddleId).UseBranch(std::string(kBranch)); @@ -1643,187 +930,466 @@ TEST_F(RestIncrementalAppendScanTest, RejectsFromSnapshotNotAncestorOfEnd) { EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); } -// -------------------------------------------------------------------------- -// IncludeColumnStats() columns are sent as stats-fields. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, IncludeColumnStatsSendsStatsFields) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL( - *mock_client_, - Post(_, JsonBodyHas("stats-fields", std::vector{"id"}), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); +// Exercise residual handling for both scan types and each response path. +class RestScanResidualTest + : public RestIncrementalAppendScanTest, + public ::testing::WithParamInterface> {}; + +TEST_P(RestScanResidualTest, PreservesFilteringAndDeleteFiles) { + const auto [incremental, submitted, fetch_token, server_residual, ignore_residuals] = + GetParam(); + auto filter = Expressions::GreaterThan("id", Literal::Int(10)); + auto residual = Expressions::GreaterThan("id", Literal::Int(20)); + ICEBERG_UNWRAP_OR_FAIL(auto filter_json, iceberg::ToJson(*filter)); + ICEBERG_UNWRAP_OR_FAIL(auto residual_json, iceberg::ToJson(*residual)); + + auto task = FileTaskJson("s3://bucket/data.parquet"); + task["delete-file-references"] = {0}; + if (server_residual) task["residual-filter"] = residual_json; + auto delete_file = FileTaskJson("s3://bucket/delete.parquet")["data-file"]; + delete_file["content"] = "position-deletes"; + nlohmann::json tasks_response = { + {"file-scan-tasks", nlohmann::json::array({task})}, + {"delete-files", nlohmann::json::array({delete_file})}}; + auto completed = + fetch_token ? nlohmann::json{{"plan-tasks", {"residual-token"}}} : tasks_response; + completed["status"] = "completed"; + completed["plan-id"] = "residual-plan"; + auto initial = + submitted ? nlohmann::json{{"status", "submitted"}, {"plan-id", "residual-plan"}} + : completed; + EXPECT_CALL(*mock_client_, + Post(::testing::EndsWith("/plan"), + ::testing::AllOf( + JsonBodyHas("filter", filter_json), + JsonBodyHas("select", std::vector{"id", "data"})), + _, _, _)) + .WillOnce(Return(HttpResponse(200, initial.dump()))); + if (submitted) { + completed.erase("plan-id"); + EXPECT_CALL(*mock_client_, + Get(::testing::EndsWith("/plan/residual-plan"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, completed.dump()))); + } + if (fetch_token) { + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/tasks"), + JsonBodyHas("plan-task", "residual-token"), _, _, _)) + .WillOnce(Return(HttpResponse(200, tasks_response.dump()))); + } + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); - auto builder = MakeBuilder(MakeContext()); - builder.FromSnapshot(kRootId).IncludeColumnStats({"id"}); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); - EXPECT_TRUE(tasks.empty()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, + PlanScan(incremental, MakeContext(), [&](auto& builder) { + builder.Select({"data"}).Filter(filter); + if (ignore_residuals) builder.IgnoreResiduals(); + })); + ASSERT_EQ(tasks.size(), 1); + EXPECT_EQ(tasks[0]->data_file()->file_path, "s3://bucket/data.parquet"); + ASSERT_EQ(tasks[0]->delete_files().size(), 1); + EXPECT_EQ(tasks[0]->delete_files()[0]->file_path, "s3://bucket/delete.parquet"); + ASSERT_NE(tasks[0]->residual_filter(), nullptr); + if (ignore_residuals) { + EXPECT_EQ(tasks[0]->residual_filter()->op(), Expression::Operation::kTrue); + } else { + ICEBERG_UNWRAP_OR_FAIL(auto actual, iceberg::ToJson(*tasks[0]->residual_filter())); + EXPECT_EQ(actual, server_residual ? residual_json : filter_json); + } } -// -------------------------------------------------------------------------- -// IncludeColumnStats() with an unknown column → ValidationFailed. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, IncludeColumnStatsRejectsMissingColumn) { - auto builder = MakeBuilder(MakeContext()); - builder.FromSnapshot(kRootId).IncludeColumnStats({"missing"}); - EXPECT_THAT(builder.Build(), - ::testing::AllOf(IsError(ErrorKind::kValidationFailed), - HasErrorMessage("Cannot find stats column: missing"))); -} +INSTANTIATE_TEST_SUITE_P(ScanModes, RestScanResidualTest, + ::testing::Combine(::testing::Bool(), ::testing::Bool(), + ::testing::Bool(), ::testing::Bool(), + ::testing::Bool())); -// -------------------------------------------------------------------------- -// Project() columns are sent as select. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, ProjectSendsProjectedColumnsAsSelect) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, - Post(_, JsonBodyHas("select", std::vector{"id"}), _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); +class RestScanRegressionTest : public RestIncrementalAppendScanTest, + public ::testing::WithParamInterface {}; - auto builder = MakeBuilder(MakeContext()); - builder.FromSnapshot(kRootId).Project(std::make_shared( - std::vector{SchemaField::MakeRequired(1, "id", int32())})); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); +TEST_P(RestScanRegressionTest, NoFilterProducesTrueResidual) { + nlohmann::json response = { + {"status", "completed"}, + {"file-scan-tasks", + nlohmann::json::array({FileTaskJson("s3://bucket/data.parquet")})}}; + EXPECT_CALL( + *mock_client_, + Post(::testing::EndsWith("/plan"), + ::testing::AllOf(JsonBodyHas("select", std::vector{"id", "data"}), + JsonBodyHas("use-snapshot-schema", GetParam()), + JsonBodyLacks("snapshot-id")), + _, _, _)) + .WillOnce(Return(HttpResponse(200, response.dump()))); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, PlanScan(GetParam(), MakeContext(), [](auto&) {})); + ASSERT_EQ(tasks.size(), 1); + ASSERT_NE(tasks[0]->residual_filter(), nullptr); + EXPECT_EQ(tasks[0]->residual_filter()->op(), Expression::Operation::kTrue); +} + +TEST_P(RestScanRegressionTest, AllColumnStatsIncludeUnprojectedColumns) { + EXPECT_CALL( + *mock_client_, + Post(::testing::EndsWith("/plan"), + ::testing::AllOf( + JsonBodyHas("select", std::vector{"data"}), + JsonBodyHas("stats-fields", std::vector{"id", "data"})), + _, _, _)) + .WillOnce(Return(HttpResponse(200, R"({"status":"completed"})"))); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, + PlanScan(GetParam(), MakeContext(), [](auto& builder) { + builder.Select({"data"}).IncludeColumnStats(); + })); EXPECT_TRUE(tasks.empty()); } -// -------------------------------------------------------------------------- -// PlanFiles: FAILED with plan-id is an invalid response → DELETE /plan called -// before returning the error. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, PlanFilesFailedWithPlanIdCancels) { - constexpr std::string_view kFailedBody = - R"({"status":"failed","plan-id":"plan-fail-incr","error":{"message":"server error","type":"ServerError","code":500}})"; +TEST_P(RestScanRegressionTest, InvalidPollingConfigFailsBeforePlanning) { + struct PropertyCase { + std::string_view key; + std::string_view value; + std::string_view error; + }; + const std::array cases{{{.key = "rest-scan-planning.poll-timeout-ms", + .value = "0", + .error = "must be positive"}, + {.key = "rest-scan-planning.poll-timeout-ms", + .value = "-1", + .error = "must be positive"}, + {.key = "rest-scan-planning.poll-timeout-ms", + .value = "abc", + .error = "invalid argument"}, + {.key = "rest-scan-planning.poll-timeout-ms", + .value = "9223372036854775808", + .error = "out of range"}, + {.key = "rest-scan-planning.poll-num-retries", + .value = "-1", + .error = "must be non-negative"}, + {.key = "rest-scan-planning.poll-num-retries", + .value = "abc", + .error = "invalid argument"}, + {.key = "rest-scan-planning.poll-num-retries", + .value = "2147483648", + .error = "out of range"}}}; + for (const auto& [key, value, error] : cases) { + SCOPED_TRACE(std::string(key) + "=" + std::string(value)); + auto ctx = MakeContext(); + ctx.catalog_config = {{std::string(key), std::string(value)}}; + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); + EXPECT_THAT( + PlanScan(GetParam(), std::move(ctx), [](auto&) {}), + ::testing::AllOf(IsError(ErrorKind::kInvalidArgument), HasErrorMessage(error))); + ASSERT_TRUE(::testing::Mock::VerifyAndClearExpectations(mock_client_.get())); + } +} - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFailedBody)))); - EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); +TEST_P(RestScanRegressionTest, MissingPollingEndpointCancelsSubmittedPlan) { + auto ctx = MakeContext(); + ctx.supported_endpoints.erase(Endpoint::FetchPlanningResult()); + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .WillOnce(Return( + HttpResponse(200, R"({"status":"submitted","plan-id":"missing-poll"})"))); + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)).Times(0); + EXPECT_CALL(*mock_client_, + Delete(::testing::EndsWith("/plan/missing-poll"), _, _, _, _)) + .WillOnce(Return(HttpResponse(204, ""))); + EXPECT_THAT(PlanScan(GetParam(), std::move(ctx), [](auto&) {}), + IsError(ErrorKind::kNotSupported)); +} + +TEST_P(RestScanRegressionTest, LongContinuationChain) { + constexpr int kPages = 2048; + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .WillOnce(Return(HttpResponse( + 200, R"({"status":"completed","plan-id":"pages","plan-tasks":["0"]})"))); + int next_page = 0; + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/tasks"), _, _, _, _)) + .Times(kPages) + .WillRepeatedly([&](const std::string&, const std::string& body, + auto&&...) -> Result { + EXPECT_EQ(nlohmann::json::parse(body).at("plan-task"), std::to_string(next_page)); + auto task = FileTaskJson("s3://bucket/" + std::to_string(next_page) + ".parquet"); + nlohmann::json response = { + {"file-scan-tasks", nlohmann::json::array({std::move(task)})}}; + if (++next_page < kPages) { + response["plan-tasks"] = {std::to_string(next_page)}; + } + return HttpResponse(200, response.dump()); + }); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); - auto result = scan->PlanFiles(); - EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + ICEBERG_UNWRAP_OR_FAIL( + auto table, RestTable::Make(identifier_, metadata_, "/tmp/metadata.json", file_io_, + nullptr, "test.my_table", nullptr, MakeContext())); + auto plan = [&]() -> Result>> { + if (GetParam()) { + ICEBERG_ASSIGN_OR_RAISE(auto builder, table->NewIncrementalAppendScan()); + builder->FromSnapshot(kRootId); + ICEBERG_ASSIGN_OR_RAISE(auto scan, builder->Build()); + return scan->PlanFiles(); + } + ICEBERG_ASSIGN_OR_RAISE(auto builder, table->NewScan()); + ICEBERG_ASSIGN_OR_RAISE(auto scan, builder->Build()); + ICEBERG_ASSIGN_OR_RAISE(auto stream, scan->PlanFilesStream()); + scan.reset(); + table.reset(); + return stream->ToVector(); + }; + ICEBERG_UNWRAP_OR_FAIL(auto tasks, plan()); + ASSERT_EQ(tasks.size(), kPages); + for (int i = 0; i < kPages; ++i) { + EXPECT_EQ(tasks[i]->data_file()->file_path, + "s3://bucket/" + std::to_string(i) + ".parquet"); + } } -// -------------------------------------------------------------------------- -// PlanFiles: invalid plan response with a plan-id → DELETE /plan called. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, PlanFilesInvalidResponseWithPlanIdCancels) { - constexpr std::string_view kInvalidBody = - R"({"status":"submitted","plan-id":"plan-invalid-incr","plan-tasks":["tok-1"]})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kInvalidBody)))); - EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, "{}"))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); - EXPECT_THAT(scan->PlanFiles(), IsError(ErrorKind::kValidationFailed)); +TEST_P(RestScanRegressionTest, TaskFetchFailureCancelsPlan) { + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .WillOnce(Return(HttpResponse( + 200, R"({"status":"completed","plan-id":"failed-page","plan-tasks":["0"]})"))); + EXPECT_CALL(*mock_client_, + Post(::testing::EndsWith("/tasks"), JsonBodyHas("plan-task", "0"), _, _, _)) + .WillOnce(Return(IOError("task fetch failed"))); + EXPECT_CALL(*mock_client_, Delete(::testing::EndsWith("/plan/failed-page"), _, _, _, _)) + .WillOnce(Return(HttpResponse(204, ""))); + EXPECT_THAT(PlanScan(GetParam(), MakeContext(), [](auto&) {}), + IsError(ErrorKind::kIOError)); } -// -------------------------------------------------------------------------- -// Stale credentials: second PlanFiles call resets scan_io_ so that credentials -// from the first plan response do not persist into the second. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, SecondPlanFilesCallClearsStaleCredentials) { - constexpr std::string_view kFirstResponse = R"({ - "status": "completed", - "storage-credentials": [ - {"prefix": "s3://bucket/prefix", "config": {"key": "value"}} - ] - })"; - constexpr std::string_view kSecondResponse = R"({"status":"completed"})"; +class RestScanStreamTest : public RestTableScanTest, + public ::testing::WithParamInterface { + protected: + void ExpectPlan(nlohmann::json completed) { + completed["status"] = "completed"; + completed["plan-id"] = "stream"; + auto initial = GetParam() + ? nlohmann::json{{"status", "submitted"}, {"plan-id", "stream"}} + : completed; + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, initial.dump()))); + if (GetParam()) { + completed.erase("plan-id"); + EXPECT_CALL(*mock_client_, Get(::testing::EndsWith("/plan/stream"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, completed.dump()))); + } else { + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)).Times(0); + } + } +}; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kFirstResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kSecondResponse)))); +TEST_P(RestScanStreamTest, FetchesOnlyWhenBufferIsExhausted) { + ExpectPlan( + {{"file-scan-tasks", nlohmann::json::array({FileTaskJson("s3://b/0.parquet")})}, + {"plan-tasks", {"first", "second"}}}); + int fetch_count = 0; + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/tasks"), + JsonBodyHas("plan-task", "first"), _, _, _)) + .WillOnce([&](auto&&...) -> Result { + EXPECT_EQ(fetch_count++, 0); + return HttpResponse(200, R"({"plan-tasks":["third"]})"); + }); + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/tasks"), + JsonBodyHas("plan-task", "second"), _, _, _)) + .WillOnce([&](auto&&...) -> Result { + EXPECT_EQ(fetch_count++, 1); + nlohmann::json response = { + {"file-scan-tasks", + nlohmann::json::array( + {FileTaskJson("s3://b/1.parquet"), FileTaskJson("s3://b/2.parquet")})}}; + return HttpResponse(200, response.dump()); + }); + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/tasks"), + JsonBodyHas("plan-task", "third"), _, _, _)) + .WillOnce([&](auto&&...) -> Result { + EXPECT_EQ(fetch_count++, 2); + nlohmann::json response = { + {"file-scan-tasks", + nlohmann::json::array({FileTaskJson("s3://b/3.parquet")})}}; + return HttpResponse(200, response.dump()); + }); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + EXPECT_EQ(fetch_count, 0); + for (int i = 0; i <= 3; ++i) { + ICEBERG_UNWRAP_OR_FAIL(auto task, stream->Next()); + ASSERT_TRUE(task.has_value()); + EXPECT_EQ((*task)->data_file()->file_path, + "s3://b/" + std::to_string(i) + ".parquet"); + EXPECT_EQ(fetch_count, i == 0 ? 0 : (i <= 2 ? 2 : 3)); + } + ICEBERG_UNWRAP_OR_FAIL(auto end, stream->Next()); + EXPECT_FALSE(end.has_value()); +} - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); +TEST_P(RestScanStreamTest, UnconsumedPlanIsCancelled) { + ExpectPlan({{"plan-tasks", {"pending"}}}); + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/tasks"), _, _, _, _)).Times(0); + EXPECT_CALL(*mock_client_, Delete(::testing::EndsWith("/plan/stream"), _, _, _, _)) + .WillOnce(Return(HttpResponse(204, ""))); + ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeScan(MakeContext())); + ICEBERG_UNWRAP_OR_FAIL(auto stream, scan->PlanFilesStream()); + stream.reset(); +} + +INSTANTIATE_TEST_SUITE_P(PlanningResponses, RestScanStreamTest, ::testing::Bool()); + +TEST_P(RestScanRegressionTest, EmptyCompletedPlan) { + for (bool submitted : {false, true}) { + SCOPED_TRACE(submitted); + std::string initial = submitted ? R"({"status":"submitted","plan-id":"empty"})" + : R"({"status":"completed"})"; + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, initial))); + if (submitted) { + EXPECT_CALL(*mock_client_, Get(::testing::EndsWith("/plan/empty"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, R"({"status":"completed"})"))); + } else { + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)).Times(0); + } + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, PlanScan(GetParam(), MakeContext(), [](auto&) {})); + EXPECT_TRUE(tasks.empty()); + ASSERT_TRUE(::testing::Mock::VerifyAndClearExpectations(mock_client_.get())); + } +} - // First call: server vends credentials. - ICEBERG_UNWRAP_OR_FAIL(auto tasks1, scan->PlanFiles()); - EXPECT_TRUE(tasks1.empty()); - EXPECT_NE(scan->io().get(), file_io_.get()); +TEST_P(RestScanRegressionTest, FailedPlanReturnsError) { + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .WillOnce(Return(HttpResponse( + 200, + R"({"status":"failed","error":{"message":"server error","type":"ServerError","code":500}})"))); + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)).Times(0); + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); + EXPECT_THAT( + PlanScan(GetParam(), MakeContext(), [](auto&) {}), + ::testing::AllOf(IsError(ErrorKind::kIOError), HasErrorMessage("server error"))); +} - // Second call: no credentials. Verifies scan_io_ was cleared so stale - // credentials from the first plan do not bleed into the second request. - ICEBERG_UNWRAP_OR_FAIL(auto tasks2, scan->PlanFiles()); - EXPECT_TRUE(tasks2.empty()); - EXPECT_EQ(scan->io().get(), file_io_.get()); +TEST_P(RestScanRegressionTest, MissingPlanningEndpointDoesNotSendRequest) { + auto ctx = MakeContext(); + ctx.supported_endpoints.erase(Endpoint::PlanTableScan()); + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); + EXPECT_THAT(PlanScan(GetParam(), std::move(ctx), [](auto&) {}), + IsError(ErrorKind::kNotSupported)); } -// -------------------------------------------------------------------------- -// Storage credentials in COMPLETED response: io() returns a credential-scoped IO. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, - StorageCredentialsInPlanResponseUpdatesEffectiveIO) { - constexpr std::string_view kResponseBody = R"({ - "status": "completed", - "storage-credentials": [ - {"prefix": "s3://bucket/prefix", "config": {"key": "value"}} - ] - })"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); +TEST_P(RestScanRegressionTest, MissingTaskEndpointCancelsPlan) { + auto ctx = MakeContext(); + ctx.supported_endpoints.erase(Endpoint::FetchScanTasks()); + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .WillOnce(Return(HttpResponse( + 200, + R"({"status":"completed","plan-id":"no-tasks","plan-tasks":["pending"]})"))); + EXPECT_CALL(*mock_client_, Delete(::testing::EndsWith("/plan/no-tasks"), _, _, _, _)) + .WillOnce(Return(HttpResponse(204, ""))); + EXPECT_THAT(PlanScan(GetParam(), std::move(ctx), [](auto&) {}), + IsError(ErrorKind::kNotSupported)); +} + +TEST_P(RestScanRegressionTest, InvalidPlanResponseCancelsKnownId) { + struct Case { + std::string body; + ErrorKind error; + bool cancel; + }; + const std::array cases{ + {{.body = R"({"status":"submitted","plan-id":"invalid","plan-tasks":["token"]})", + .error = ErrorKind::kValidationFailed, + .cancel = true}, + {.body = + R"({"status":"failed","plan-id":"invalid","error":{"message":"error","type":"ServerError","code":500}})", + .error = ErrorKind::kValidationFailed, + .cancel = true}, + {.body = R"({"status":"failed"})", + .error = ErrorKind::kJsonParseError, + .cancel = false}}}; + for (const auto& test_case : cases) { + SCOPED_TRACE(test_case.body); + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .WillOnce(Return(HttpResponse(200, test_case.body))); + EXPECT_CALL(*mock_client_, Get(_, _, _, _, _)).Times(0); + if (test_case.cancel) { + EXPECT_CALL(*mock_client_, Delete(::testing::EndsWith("/plan/invalid"), _, _, _, _)) + .WillOnce(Return(HttpResponse(204, ""))); + } else { + EXPECT_CALL(*mock_client_, Delete(_, _, _, _, _)).Times(0); + } + EXPECT_THAT(PlanScan(GetParam(), MakeContext(), [](auto&) {}), + IsError(test_case.error)); + ASSERT_TRUE(::testing::Mock::VerifyAndClearExpectations(mock_client_.get())); + } +} - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); +TEST_P(RestScanRegressionTest, RequestedColumnStatsAreSent) { + EXPECT_CALL(*mock_client_, + Post(::testing::EndsWith("/plan"), + JsonBodyHas("stats-fields", std::vector{"id"}), _, _, _)) + .WillOnce(Return(HttpResponse(200, R"({"status":"completed"})"))); + ICEBERG_UNWRAP_OR_FAIL( + auto tasks, PlanScan(GetParam(), MakeContext(), + [](auto& builder) { builder.IncludeColumnStats({"id"}); })); EXPECT_TRUE(tasks.empty()); - EXPECT_NE(scan->io().get(), file_io_.get()); } -// -------------------------------------------------------------------------- -// No storage credentials: io() falls back to the table's FileIO. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, NoStorageCredentialsEffectiveIoFallsBackToTableIO) { - constexpr std::string_view kResponseBody = R"({"status":"completed"})"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kResponseBody)))); +TEST_P(RestScanRegressionTest, UnknownStatsColumnFailsBeforePlanning) { + EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)).Times(0); + EXPECT_THAT(PlanScan(GetParam(), MakeContext(), + [](auto& builder) { builder.IncludeColumnStats({"missing"}); }), + ::testing::AllOf(IsError(ErrorKind::kValidationFailed), + HasErrorMessage("Cannot find stats column: missing"))); +} - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); +TEST_P(RestScanRegressionTest, ProjectUsesTheProjectedSchema) { + EXPECT_CALL(*mock_client_, + Post(::testing::EndsWith("/plan"), + JsonBodyHas("select", std::vector{"id"}), _, _, _)) + .WillOnce(Return(HttpResponse(200, R"({"status":"completed"})"))); + ICEBERG_UNWRAP_OR_FAIL( + auto tasks, PlanScan(GetParam(), MakeContext(), [](auto& builder) { + builder.Project(std::make_shared( + std::vector{SchemaField::MakeRequired(1, "id", int32())})); + })); EXPECT_TRUE(tasks.empty()); - EXPECT_EQ(scan->io().get(), file_io_.get()); } -// -------------------------------------------------------------------------- -// Storage credentials in a FetchScanTasks response are ignored. -// -------------------------------------------------------------------------- -TEST_F(RestIncrementalAppendScanTest, - StorageCredentialsInFetchScanTasksResponseAreIgnored) { - constexpr std::string_view kPlanResponse = - R"({"status":"completed","plan-id":"incr-plan-1","plan-tasks":["tok-incr-1"]})"; - constexpr std::string_view kTasksResponse = R"({ - "file-scan-tasks": [], - "storage-credentials": [ - {"prefix": "s3://bucket/prefix", "config": {"key": "value"}} - ] - })"; - EXPECT_CALL(*mock_client_, Post(_, _, _, _, _)) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kPlanResponse)))) - .WillOnce(Return(HttpResponse::MakeForTesting(200, std::string(kTasksResponse)))); - - ICEBERG_UNWRAP_OR_FAIL(auto scan, MakeIncrementalScan(MakeContext())); - ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles()); +TEST_P(RestScanRegressionTest, SelectResolvesCaseInsensitiveNames) { + EXPECT_CALL(*mock_client_, + Post(::testing::EndsWith("/plan"), + JsonBodyHas("select", std::vector{"id"}), _, _, _)) + .WillOnce(Return(HttpResponse(200, R"({"status":"completed"})"))); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, + PlanScan(GetParam(), MakeContext(), [](auto& builder) { + builder.CaseSensitive(false).Select({"ID"}); + })); EXPECT_TRUE(tasks.empty()); - EXPECT_EQ(scan->io().get(), file_io_.get()); } -// ========================================================================== -// RestTable::NewIncrementalAppendScan -// ========================================================================== - -// -------------------------------------------------------------------------- -// RestTable::NewIncrementalAppendScan returns a RestIncrementalAppendScanBuilder. -// -------------------------------------------------------------------------- -TEST_F(RestTableScanTest, RestTableNewIncrementalAppendScanReturnsRestBuilder) { - ICEBERG_UNWRAP_OR_FAIL( - auto table, RestTable::Make(identifier_, metadata_, "/tmp/metadata.json", file_io_, - /*catalog=*/nullptr, "test.my_table", nullptr, - MakeContext(std::nullopt))); - ICEBERG_UNWRAP_OR_FAIL(auto builder, table->NewIncrementalAppendScan()); - auto* typed = dynamic_cast(builder.get()); - EXPECT_NE(typed, nullptr); +TEST_P(RestScanRegressionTest, ReplanningDropsOldCredentials) { + EXPECT_CALL(*mock_client_, Post(::testing::EndsWith("/plan"), _, _, _, _)) + .WillOnce(Return(HttpResponse( + 200, + R"({"status":"completed","storage-credentials":[{"prefix":"s3://bucket","config":{"key":"value"}}]})"))) + .WillOnce(Return(HttpResponse(200, R"({"status":"completed"})"))); + auto exercise = [&](auto& builder) { + ICEBERG_UNWRAP_OR_FAIL(auto scan, builder.Build()); + EXPECT_EQ(scan->io(), file_io_); + ICEBERG_UNWRAP_OR_FAIL(auto first, scan->PlanFiles()); + auto* credentialed = scan->io()->AsSupportsStorageCredentials(); + ASSERT_NE(credentialed, nullptr); + ASSERT_EQ(credentialed->credentials().size(), 1); + EXPECT_EQ(credentialed->credentials()[0].prefix, "s3://bucket"); + ICEBERG_UNWRAP_OR_FAIL(auto second, scan->PlanFiles()); + EXPECT_EQ(scan->io(), file_io_); + }; + if (GetParam()) { + auto builder = MakeBuilder(MakeContext()); + builder.FromSnapshot(kRootId); + exercise(builder); + } else { + RestTableScanBuilder builder(metadata_, file_io_, "test.my_table", nullptr, + std::make_shared(MakeContext())); + exercise(builder); + } } +INSTANTIATE_TEST_SUITE_P(ScanModes, RestScanRegressionTest, ::testing::Bool()); + +} // namespace + } // namespace iceberg::rest diff --git a/src/iceberg/test/table_scan_test.cc b/src/iceberg/test/table_scan_test.cc index cb22997cfc..40b51bd5ef 100644 --- a/src/iceberg/test/table_scan_test.cc +++ b/src/iceberg/test/table_scan_test.cc @@ -858,41 +858,6 @@ TEST_P(TableScanTest, SchemaWithSelectedColumnsAndFilter) { } } -// use_snapshot_schema checks do not depend on format version, so run them once. -class TableScanContextTest : public TableScanTest {}; - -TEST_F(TableScanContextTest, UseSnapshotSetsTrueUseSnapshotSchema) { - constexpr int64_t kSnapshotId = 1000L; - ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); - builder->UseSnapshot(kSnapshotId); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); - EXPECT_TRUE(scan->context().use_snapshot_schema); -} - -TEST_F(TableScanContextTest, UseRefTagSetsTrueUseSnapshotSchema) { - constexpr int64_t kSnapshotId = 1000L; - table_metadata_->refs["v1.0"] = std::make_shared( - SnapshotRef{.snapshot_id = kSnapshotId, .retention = SnapshotRef::Tag{}}); - - ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); - builder->UseRef("v1.0"); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); - EXPECT_TRUE(scan->context().use_snapshot_schema); -} - -TEST_F(TableScanContextTest, UseRefBranchSetsFalseUseSnapshotSchema) { - ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); - builder->UseRef("main"); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); - EXPECT_FALSE(scan->context().use_snapshot_schema); -} - -TEST_F(TableScanContextTest, DefaultScanHasFalseUseSnapshotSchema) { - ICEBERG_UNWRAP_OR_FAIL(auto builder, MakeScanBuilder(table_metadata_)); - ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build()); - EXPECT_FALSE(scan->context().use_snapshot_schema); -} - INSTANTIATE_TEST_SUITE_P(TableScanVersions, TableScanTest, testing::Values(1, 2, 3)); } // namespace iceberg diff --git a/src/iceberg/type_fwd.h b/src/iceberg/type_fwd.h index 84a9919097..511b14eb05 100644 --- a/src/iceberg/type_fwd.h +++ b/src/iceberg/type_fwd.h @@ -204,6 +204,7 @@ struct ReaderOptions; struct WriterOptions; struct StorageCredential; class FileIO; +class StorageCredentialProvider; class Reader; class Writer;