From 11990315bf5fac692c4237899635035eed13c26d Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Mon, 17 Aug 2026 06:35:28 +0000 Subject: [PATCH 01/22] feat: Resumable Media Upload functionality implementation Public API `Gapic::Rest::ResumableUpload::Session` is the entry point. `Session#run` for initial upload, and `Session#resume` for resuming after error. Also public: `Progress`, `ResumeHandle`, `HasResumeHandle` module, and a typed error hierarchy. Changes to existing files: * `StubLogger#warn` * request-payload abridging in `ClientStub` so that binary upload bodies don't land in logs * `REST_ERROR_PREFIX` constant extracted in Rest::Error Integration testing against Showcase, for manual confirmation (disabled in CI). CI integration tests are planned to land in the generator. --- gapic-common/.toys/test-integration.rb | 139 +++ .../resumable_upload/implementation-guide.md | 687 +++++++++++ .../resumable_upload/integration-test-plan.md | 286 +++++ gapic-common/integration/README.md | 51 + .../integration/integration_helper.rb | 204 ++++ .../chunk_granularity_test.rb | 48 + .../resumable_upload/error_on_start_test.rb | 159 +++ .../resumable_upload/error_recovery_test.rb | 144 +++ .../resumable_upload/golden_path_test.rb | 111 ++ .../resumable_upload/resume_test.rb | 229 ++++ gapic-common/lib/gapic/logging_concerns.rb | 4 + gapic-common/lib/gapic/rest.rb | 1 + gapic-common/lib/gapic/rest/client_stub.rb | 16 +- gapic-common/lib/gapic/rest/error.rb | 5 +- .../lib/gapic/rest/resumable_upload.rb | 73 ++ .../lib/gapic/rest/resumable_upload/core.rb | 77 ++ .../gapic/rest/resumable_upload/data_types.rb | 415 +++++++ .../lib/gapic/rest/resumable_upload/driver.rb | 780 +++++++++++++ .../rest/resumable_upload/driver/abridge.rb | 168 +++ .../resumable_upload/driver/upload_log.rb | 326 ++++++ .../lib/gapic/rest/resumable_upload/errors.rb | 565 +++++++++ .../lib/gapic/rest/resumable_upload/events.rb | 128 +++ .../rest/resumable_upload/instructions.rb | 265 +++++ .../rest/resumable_upload/retry_policies.rb | 165 +++ .../lib/gapic/rest/resumable_upload/rules.rb | 1016 +++++++++++++++++ .../gapic/rest/resumable_upload/session.rb | 546 +++++++++ .../test/gapic/rest/client_stub_test.rb | 41 + gapic-common/test/gapic/rest/error_test.rb | 13 + .../gapic/rest/resumable_upload/core_test.rb | 85 ++ .../rest/resumable_upload/data_types_test.rb | 231 ++++ .../resumable_upload/driver/abridge_test.rb | 101 ++ .../driver/upload_log_test.rb | 216 ++++ .../resumable_upload/driver_buffer_test.rb | 378 ++++++ .../resumable_upload/driver_config_test.rb | 295 +++++ .../driver_error_mapping_test.rb | 209 ++++ .../resumable_upload/driver_logging_test.rb | 574 ++++++++++ .../resumable_upload/driver_progress_test.rb | 178 +++ .../driver_retry_policy_test.rb | 143 +++ .../resumable_upload/driver_retry_test.rb | 187 +++ .../rest/resumable_upload/driver_test.rb | 430 +++++++ .../resumable_upload/retry_policies_test.rb | 222 ++++ .../rules_classification_test.rb | 351 ++++++ .../resumable_upload/rules_decide_test.rb | 334 ++++++ .../rest/resumable_upload/rules_error_test.rb | 474 ++++++++ .../resumable_upload/rules_recovery_test.rb | 134 +++ .../gapic/rest/resumable_upload/rules_test.rb | 279 +++++ .../rest/resumable_upload/session_test.rb | 704 ++++++++++++ gapic-common/test/test_helper.rb | 25 + 48 files changed, 12208 insertions(+), 4 deletions(-) create mode 100644 gapic-common/.toys/test-integration.rb create mode 100644 gapic-common/design/resumable_upload/implementation-guide.md create mode 100644 gapic-common/design/resumable_upload/integration-test-plan.md create mode 100644 gapic-common/integration/README.md create mode 100644 gapic-common/integration/integration_helper.rb create mode 100644 gapic-common/integration/resumable_upload/chunk_granularity_test.rb create mode 100644 gapic-common/integration/resumable_upload/error_on_start_test.rb create mode 100644 gapic-common/integration/resumable_upload/error_recovery_test.rb create mode 100644 gapic-common/integration/resumable_upload/golden_path_test.rb create mode 100644 gapic-common/integration/resumable_upload/resume_test.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/core.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/data_types.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/driver.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/errors.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/events.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/instructions.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/rules.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/session.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/core_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver/abridge_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver/upload_log_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/rules_recovery_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/rules_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/session_test.rb diff --git a/gapic-common/.toys/test-integration.rb b/gapic-common/.toys/test-integration.rb new file mode 100644 index 0000000..7929523 --- /dev/null +++ b/gapic-common/.toys/test-integration.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "socket" +require "tmpdir" + +expand :minitest, name: "" do |t| + t.libs = ["lib", "integration"] + t.files = ["integration/**/*_test.rb"] + t.bundler = true +end + +alias_method :run_minitest, :run + +def run + if !ENV["SHOWCASE_ENDPOINT"].to_s.empty? + run_minitest + return + end + + bin = resolve_showcase_bin + if bin.nil? + if !ENV["CI"].to_s.empty? + logger.error "No SHOWCASE_ENDPOINT or gapic-showcase binary found in CI environment." + exit 1 + else + logger.warn "Skipping integration tests: no SHOWCASE_ENDPOINT or gapic-showcase binary found." + return + end + end + + verify_showcase_version! bin + + port = allocate_port + fallback_port = allocate_port + log_path = File.join Dir.tmpdir, "gapic-showcase-#{Process.pid}-#{Time.now.to_i}.log" + + pid = Process.spawn( + bin, "run", + "--port", ":#{port}", + "--fallback-port", ":#{fallback_port}", + out: log_path, + err: log_path, + pgroup: true + ) + + begin + wait_for_showcase! pid, port, log_path + ENV["SHOWCASE_ENDPOINT"] = "http://localhost:#{port}" + run_minitest + ensure + if pid + begin + Process.kill "-TERM", pid + Process.waitpid pid + rescue Errno::ESRCH, Errno::ECHILD + # Process already terminated + end + end + end +end + +def resolve_showcase_bin + env_bin = ENV["SHOWCASE_BIN"].to_s + return env_bin unless env_bin.empty? + + ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).each do |dir| + candidate = File.join dir, "gapic-showcase" + return candidate if File.executable?(candidate) && !File.directory?(candidate) + end + nil +end + +def verify_showcase_version!(bin) + output = begin + IO.popen([bin, "--version"], err: [:child, :out], &:read).strip + rescue StandardError => e + logger.error "Failed to execute '#{bin} --version': #{e.message}" + exit 1 + end + + version_match = output[/\d+\.\d+(?:\.\d+)*/] + if version_match.nil? + logger.error "Could not parse version from '#{bin} --version' output: #{output.inspect}" + exit 1 + end + + if Gem::Version.new(version_match) < Gem::Version.new("0.43") + logger.error "gapic-showcase version #{version_match} is too old (minimum required is 0.43)." + exit 1 + end +end + +def allocate_port + server = TCPServer.open "127.0.0.1", 0 + port = server.addr[1] + server.close + port +end + +def wait_for_showcase!(pid, port, log_path) + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 10.0 + + loop do + exited_pid, status = Process.waitpid2 pid, Process::WNOHANG + if exited_pid + logger.error "gapic-showcase exited prematurely (status: #{status.exitstatus}). Log file: #{log_path}" + exit 1 + end + + begin + sock = TCPSocket.new "127.0.0.1", port + sock.close + return + rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH + # Server not ready yet + end + + if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + logger.error "Timed out waiting 10s for gapic-showcase to listen on port #{port}. Log file: #{log_path}" + exit 1 + end + + sleep 0.1 + end +end diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md new file mode 100644 index 0000000..3a561a9 --- /dev/null +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -0,0 +1,687 @@ +# Resumable Upload Protocol (RUP) Implementation Guide + +## 1. System Architecture + +The Resumable Upload Protocol (RUP) implementation in `gapic-common` is structured across three distinct tiers to separate network execution, protocol state progression, and state transition decision logic: + +```mermaid +graph TD + Client[Client Code] -->|StartUploadConfig| Driver + subgraph Gapic::Rest::ResumableUpload + Driver[Driver
Synchronous I/O Adapter] -->|Events| Core[Core
State Container] + Core -->|Instructions| Driver + Core -->|state, event| Rules[Rules
Pure Decision Function] + Rules -->|next_state, instructions| Core + end + Driver -->|RetryPolicy / Faraday| Server[Upload Backend / GCS] + Driver -->|IO#read| Stream[Local Stream] +``` + +### 1.0 Domain Vocabulary +* **Upload**: Server-side entity created by a successful session initiation (`start`), identified by `upload_url`. +* **Resume Handle (`ResumeHandle`)**: An immutable snapshot (`upload_url`, `chunk_size`) identifying an upload for resumption. +* **Session (`Session`)**: Client-side transfer coordinator; performs exactly one run (`start` or `resume`), never both, never twice. It owns the input stream and configuration options. +* **Run**: One invocation of `Driver#run` (either a start or resume execution). +* **Bound**: The property that a session has executed a run or is bound to an upload (`session.bound?`). A session becomes bound when `start` or `resume` begins execution. A bound session never runs again. + +### 1.1 Driver (Synchronous I/O Adapter) +The `Driver` executes all operations with side-effects. It interacts with HTTP transport via `Gapic::Rest::ClientStub`, reads binary data from local input streams, tracks monotonic execution deadlines, and dispatches progress callbacks. + +Crucially, the Driver delegates all **Category 1 (Transient)** transport retries directly to `Gapic::Common::RetryPolicy`. Transient retries occur entirely within the Driver's network execution wrapper. The `Core` state machine is never exposed to transient noise, receiving only verified successful HTTP responses or terminal transport exceptions. + +The Driver exposes: +* `Driver#resume_handle`: Returns a `ResumeHandle` (or `nil` if initiation has not established an upload URL, or if the session is `:rejected`, `:cancelled`, or `:success`; completed uploads are not resumable). Reading this property mid-run provides a best-effort snapshot of current session parameters. +* `Driver#upload_url`: Returns the raw protocol state upload URL under any status (`:active`, `:success`, `:rejected`, `:cancelled`). + +### 1.2 Core (State Container) +The `Core` maintains the immutable `State` snapshot. When `Core#dispatch(event)` is invoked by the Driver, Core forwards `@state`, the event, and static configuration to `Rules.decide`. Core mutates `@state` to `decision.next_state`, records the decision in `@last_decision`, and returns `decision.instructions` back to the Driver. Core contains zero protocol branching logic and zero side effects. + +### 1.3 Rules (Pure Decision Function) +The `Rules` module encapsulates the Resumable Upload Protocol state transitions as a pure functional module. Given a state snapshot, an input event, and configuration, `Rules.decide` evaluates the transition router and returns a `Decision` snapshot containing `from_status`, `shape`, `recipe`, `next_state`, and `instructions`. + +### 1.4 Stream Buffering +Because arbitrary Ruby `IO` objects (network sockets, pipes, `STDIN`) do not support seeking (`#seek`), the Driver buffers the current in-flight chunk in memory (bounded by chunk size, default: 8MB). When `RetryPolicy` executes transport retries, or when `Core` triggers Category 2 recovery realignments within the buffered range, the Driver retransmits directly from memory. The buffer is discarded only after receiving a `200 OK` durably confirming receipt of the chunk. + +### 1.5 Session (Transfer Coordinator) +The `Session` (`Gapic::Rest::ResumableUpload::Session`) provides a client-facing coordinator that encapsulates shared transfer configuration, owns the input stream, and manages upload execution across a strict single-run lifecycle. + +#### Single-Run Contract & Two-State Model +A session adheres to a two-state model with a strict single-run contract: a session performs exactly one run (`start` or `resume`), never both, never twice. + +1. **Unbound (`!session.bound?`)**: + * Initial state upon construction (`Session.new`). The session has not yet executed a run. + * Permitted operations: `start(...)` or `resume(...)`. +2. **Bound (`session.bound?`)**: + * Transitions to bound as soon as `start` or `resume` begins execution. + * The session has executed its run and cannot be reused. + * Both `start` and `resume` raise `SessionStateError` ("Session has already executed a run"). + +#### Constructor & Initiation Signatures +Configuration is split between transfer-wide options passed to `Session.new` (`COMMON_MEMBERS` plus `client_stub` and `logger`) and initiation-only arguments passed to `Session#start`: +* **Constructor (`Session#initialize`)**: + `Session.new(client_stub:, stream:, upload_size: nil, content_type: nil, timeout: nil, control_plane_retry_policy: nil, data_plane_retry_policy: nil, on_progress: nil, logger: nil)` +* **Initiation (`Session#start`)**: + `session.start(initial_url:, initial_body: nil, initial_headers: {}, chunk_size: nil, start_retry_policy: nil)` + * `initial_url` is required (`ArgumentError` if missing or blank). + * `initial_headers` accepts caller-supplied HTTP headers for the initiation request, merged over the driver's headers. Any key in `RESERVED_INITIAL_HEADERS` (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, `X-Goog-Upload-Header-Content-Length`, in any casing) raises an `ArgumentError`; callers influence `X-Goog-Upload-Header-Content-Type` and `X-Goog-Upload-Header-Content-Length` through `content_type:` and `upload_size:` on the constructor. Pass-through headers such as `X-Goog-Upload-Header-Content-Disposition` remain permitted. + +#### Resumability (`session.resumable?`) +* Reports whether a *new* session can resume the transfer (`!session.resume_handle.nil?`). +* Completed uploads are finalized: once a transfer succeeds, `resume_handle` returns `nil` and `resumable?` returns `false`, so there is no handle to resume from. Calling `#resume` on the session that completed the run raises `SessionStateError`. Resuming a *fresh* session against a finalized `upload_url` is undefined behavior: it queries the server and might return the response body or raise an error, depending on the server response. +* When a run fails with a recoverable error, `resume_handle` captures the upload parameters (`upload_url`, `chunk_size`) and `resumable?` returns `true`. + +#### Precondition on Stream Position for Resume +* Before executing `resume`, the caller must ensure the input stream is positioned at byte 0. +* If `stream.respond_to?(:pos) && !stream.pos.zero?`, `Session#resume` raises `ArgumentError` ("Input stream must be at byte 0 to resume; rewind the stream before resuming"). +* For unseekable streams without `:pos` (or streams at `pos == 0`), `Session` trusts the stream is at byte 0 and delegates to `Driver`, which fast-forwards to the server-confirmed offset by seeking or reading and discarding bytes. + +#### Resume Invocations & Forms +The `Session#resume` method accepts strictly keyword-only arguments: `upload_url: nil, chunk_size: nil, resume_handle: nil`. +Resumption always requires an unbound session with one of two mutually exclusive parameter forms: +1. **Explicit URL & Chunk Size**: `session.resume(upload_url: url, chunk_size: size)` +2. **Resume Handle**: `session.resume(resume_handle: handle)` + +Calling `resume` without arguments (bare resume), calling `resume` with `upload_url` but omitting `chunk_size`, or mixing `resume_handle` with other parameters raises `ArgumentError`. + +#### Cross-Session Resumption Flow +Because a session performs only a single run, resuming an interrupted upload requires instantiating a fresh session: +1. Session 1 encounters a recoverable error. +2. Caller extracts `resume_handle = session1.resume_handle` (or from the error's `#resume_handle`). +3. Caller rewinds the stream to byte 0 (if seekable, or provides an equivalent stream starting at byte 0). +4. Caller instantiates Session 2 and invokes `session2.resume(resume_handle: resume_handle)`. + +#### Concurrency & Execution Model +* At most one run (`Driver#run`) may execute at any time. +* `@running` is checked and toggled exclusively inside a `Mutex`. +* Network execution (`driver.run`) occurs outside the mutex to prevent blocking reader threads. +* Invoking `start` or `resume` while `@running` is `true` raises `SessionStateError`. +* Errors propagate unchanged. The failed `Driver` remains referenced so `upload_url`, `resume_handle`, and `bound?` remain inspectable after an exception. + +--- + +## 2. Component Interfaces & Data Models + +### 2.1 Initiation Configuration (`StartUploadConfig`) +```ruby +module Gapic + module Rest + module ResumableUpload + COMMON_MEMBERS = [ + :stream, # [IO] Binary input stream to upload + :upload_size, # [Integer, nil] Total upload bytes if known upfront + :content_type, # [String, nil] MIME type of uploaded media + :timeout, # [Numeric, nil] Total upload timeout in seconds (zero/negative treated as nil) + :control_plane_retry_policy, # [Gapic::Common::RetryPolicy, Hash, nil] Policy or hash override for query/cancel commands + :data_plane_retry_policy, # [Gapic::Common::RetryPolicy, Hash, nil] Policy or hash override for upload/finalize + :on_progress # [Proc, nil] Callback: ->(progress) with a Progress instance + ].freeze + + RESERVED_INITIAL_HEADERS = [ + "x-goog-upload-protocol", + "x-goog-upload-command", + "x-goog-upload-offset", + "x-goog-upload-header-content-type", + "x-goog-upload-header-content-length" + ].freeze + + StartUploadConfig = Data.define( + *COMMON_MEMBERS, + :initial_url, # [String] Initial endpoint URI for session initiation + :initial_body, # [String, nil] Request payload for session initiation + :initial_headers, # [Hash] Additional headers for initiation (RESERVED_INITIAL_HEADERS rejected) + :chunk_size, # [Integer, nil] Explicit chunk size in bytes + :start_retry_policy # [Gapic::Common::RetryPolicy, Hash, nil] Policy or hash override for start command + ) + + Progress = Data.define( + :phase, # [Symbol] Upload lifecycle phase, one of Progress::PHASES + :bytes_uploaded, # [Integer] Cumulative bytes acknowledged by the server (may decrease on recovery rewind) + :total_bytes # [Integer, nil] Total upload size in bytes if known + ) do + # Important to define it via `self.`, since this block is not a class body + self::PHASES = %i[initiating uploading recovering finalizing cancelling completed].freeze + end + end + end +end +``` + +**Reserved Initial Headers Rule (`RESERVED_INITIAL_HEADERS`):** +* The five headers in `RESERVED_INITIAL_HEADERS` (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, `X-Goog-Upload-Header-Content-Length`) are protocol machinery owned by the driver. +* Any key in `initial_headers` matching those five names (case-insensitively) is rejected at configuration construction with an `ArgumentError`. Callers shape media descriptors exclusively through `content_type` and `upload_size`. Pass-through headers under the prefix such as `X-Goog-Upload-Header-Content-Disposition` remain permitted. + +**Progress Notification Contract (`on_progress`):** +* `on_progress` fires whenever upload status or server-confirmed byte offset changes. Sequential callbacks may report the same `bytes_uploaded`. +* `bytes_uploaded` represents the server-confirmed offset and is **not guaranteed to be monotonic** — a server rewind during recovery can decrease this value. +* Terminal failures and completed cancellations do not emit `Progress` notifications; however, entering the `:cancelling` phase does. +* Public phases (`Progress::PHASES`): `:initiating`, `:uploading`, `:recovering`, `:finalizing`, `:cancelling`, `:completed`. + +### 2.2 Resume Configuration (`ResumeUploadConfig`) +```ruby +module Gapic + module Rest + module ResumableUpload + ResumeUploadConfig = Data.define( + *COMMON_MEMBERS, + :upload_url, # [String] Upload session URL returned by the upload backend + :chunk_size # [Integer] Chunk size in bytes (> 0) + ) + end + end +end +``` +`ResumeUploadConfig` allows resuming an existing session directly using the session URL (typically obtained from `ResumeHandle#upload_url` or an error's `#resume_handle`). Because a resumed run skips session initiation, `start_retry_policy` and initiation headers/URL are absent. + +### 2.3 Protocol State (`State`) & Decisions (`Decision`) +```ruby +module Gapic + module Rest + module ResumableUpload + State = Data.define( + :status, # [Symbol] :initializing, :starting, :transmission_reading, :transmission_sending, + # :finalizing_sending_upload, :finalizing_sending_finalize, + # :recovery, :cancelling, :cancelled, :success, :error, :rejected + :upload_url, # [String, nil] Session upload URL returned by the upload backend + :offset, # [Integer] Contiguous bytes confirmed by server (protocol_state_offset) + :chunk_size, # [Integer] Resolved effective chunk size + :chunk_granularity, # [Integer, nil] Alignment modulus returned by server + :in_flight_length, # [Integer] Byte length of in-flight chunk currently being transmitted + :last_error # [StandardError, nil] Terminal exception + ) do + end + + Decision = Data.define( + :from_status, # [Symbol] Status before transition + :shape, # [Symbol] Classified canonical event shape + :recipe, # [Symbol] Selected transition recipe method name + :next_state, # [State] Resulting protocol state snapshot + :instructions # [Array] Emitted instructions for the Driver + ) + end + end +end +``` + +### 2.4 Resume Handle (`ResumeHandle`) +```ruby +module Gapic + module Rest + module ResumableUpload + ResumeHandle = Data.define( + :upload_url, # [String] Upload session URL provided by the server + :chunk_size # [Integer] Effective chunk size in bytes + ) + end + end +end +``` +`ResumeHandle` captures server-provided parameters that can be persisted to resume the upload session at a later time. + +### 2.5 Events Vocabulary (Driver -> Core) +* `Event::StartUpload`: Start a new upload session. +* `Event::ResumeUpload.new(upload_url:, chunk_size:, upload_size:)`: Resume an existing upload session with a known upload URL. +* `Event::ChunkRead.new(bytes_buffered:, eof:)`: Binary data buffered in Driver memory; reports total bytes ready in buffer and whether the stream hit EOF. +* `Event::HttpResponse.new(status:, headers:, body:, error: nil)`: Dispatched for any completed HTTP exchange over the wire (including 2xx, 4xx, 5xx, or responses with missing/unexpected headers). Carries optional parsed `error` (`Gapic::Rest::Error`) when rescued from transport errors. `Core` inspects status and headers to determine protocol progression or recovery. +* `Event::RequestFailed.new(kind:, message:, source_error:)`: Dispatched when an HTTP request fails to produce a usable HTTP response (e.g., request timeout, transport connection errors, or `RetryPolicy` exhaustion). + * `kind`: Normalized Symbol enum (`:timeout`, `:connection_failed`, `:retries_exhausted`). `Core` branches on `kind` and treats other fields as opaque. + * `message`: Human-readable summary string. + * `source_error`: Original underlying exception, preserved for terminal error propagation and logging. +* `Event::Cancel`: Caller requested session cancellation. +* `Event::GlobalDeadlineExceeded`: Absolute monotonic clock exceeded the session deadline (`@deadline`) computed at the start of `Driver#run`. + +### 2.6 Instructions Vocabulary (Core -> Driver) +* `Instruction::SendStart.new(url:, headers:, body:)`: Execute initiation request to establish upload session. +* `Instruction::SendChunk.new(url:, offset:, length:, finalize:)`: Transmit buffered chunk of specified `length` starting at `offset`. If `finalize` is true, sends command `upload, finalize`. +* `Instruction::SendFinalize.new(url:)`: Send standalone `finalize` command when all data bytes were already acknowledged. +* `Instruction::SendQuery.new(url:)`: Query backend for current acknowledged offset (`query` command). +* `Instruction::SendCancel.new(url:)`: Cancel upload session on server (`cancel` command). +* `Instruction::RealignBuffer.new(server_offset:)`: Realign Driver in-memory buffer and stream position to match `server_offset`. +* `Instruction::FillBuffer.new(target_bytesize:)`: Read from stream until in-memory buffer reaches `target_bytesize` bytes or stream encounters EOF. +* `Instruction::NotifyProgress.new(progress:)`: Invoke `on_progress` callback with a `Progress` instance. +* `Instruction::TerminateSuccess.new(response:)`: Upload finalized cleanly; Driver returns `response.body`. +* `Instruction::TerminateFailure.new(error:)`: Raise terminal exception. + +### 2.5 Driver Buffer Invariants & Stream Position Model + +The Driver coordinates stream reading and in-memory buffering using four explicit offset markers: +* `server_offset`: Contiguous byte count acknowledged by the server (extracted from `X-Goog-Upload-Size-Received`). +* `protocol_state_offset`: Byte offset maintained in `State.offset`. +* `buffer_start_offset`: Absolute stream offset corresponding to the first byte in the Driver's `@buffer`. +* `buffer_end_offset`: `buffer_start_offset + @buffer.bytesize`. + +```text +Stream Offset: 0 -----------------> buffer_start_offset -------------------> buffer_end_offset ----> (Stream EOF) + |----------------- @buffer -------------| + ^ + server_offset +``` + +#### Buffer Alignment Strategy (`Instruction::RealignBuffer`) +When `Core` resolves a recovery query or offset realignment, the Driver executes one of three alignment paths based on `server_offset`: + +1. **Case 1: Within Buffer Range (`buffer_start_offset <= server_offset <= buffer_end_offset`)** + * The required offset is already buffered in memory. + * Driver trims already-persisted bytes: `@buffer = @buffer.byteslice((server_offset - buffer_start_offset)..-1)`. + * Driver updates `buffer_start_offset = server_offset`. + * When subsequently executing `Instruction::FillBuffer(target_bytesize)`, Driver calculates `needed = target_bytesize - @buffer.bytesize` and reads only the missing difference from `stream` to complete the chunk to full `chunk_size` (unless stream reaches EOF). +2. **Case 2: Server Offset Behind Buffer (`server_offset < buffer_start_offset`)** + * Occurs if the server rolls back beyond the retained buffer window. + * If `stream.respond_to?(:seek)`: Driver seeks the stream back to `server_offset`, resets `@buffer = "".b`, and sets `buffer_start_offset = server_offset`. + * If `stream` is unseekable (e.g. Socket, Pipe, STDIN): Driver raises a terminal `UnseekableStreamError` (Category 3 failure), attaching `resume_handle`. +3. **Case 3: Server Offset Ahead of Buffer (`server_offset > buffer_end_offset`)** + * Occurs when resuming an existing session or when the server processed a previously timed-out request ahead of local state. + * If total `upload_size` is known and `server_offset > upload_size`, Driver raises a terminal `StreamMismatchError` with `resume_handle`. + * If `upload_size` is `nil` and `stream.respond_to?(:size)` and `server_offset > stream.size`, Driver raises a terminal `StreamMismatchError` with `resume_handle` (preventing seek past EOF from silently succeeding on seekable streams). + * Driver resets `@buffer = "".b`. + * Driver advances the stream to `server_offset`: + * If seekable: `stream.seek(server_offset)`. + * If unseekable: Driver reads and discards bytes from `stream` until reaching `server_offset` (reading `server_offset - buffer_end` bytes). If the stream encounters an unexpected EOF before reaching `server_offset`, Driver raises a terminal `StreamMismatchError` with `resume_handle`. + * Driver sets `buffer_start_offset = server_offset`. + +--- + +## 3. Component Architecture + +The authoritative implementation is the source itself, under `lib/gapic/rest/resumable_upload/`. This section describes the contract each component honours; the code is normative where the two disagree. + +### 3.1 Rules Module (`Gapic::Rest::ResumableUpload::Rules`) +The `Rules` module is a pure functional transition engine with zero state awareness and zero side effects. It provides two primary entry points: +* `Rules.shape_of(event)`: Classifies raw input events (`Event::StartUpload`, `Event::ChunkRead`, `Event::HttpResponse`, `Event::RequestFailed`, `Event::Cancel`, `Event::GlobalDeadlineExceeded`) into canonical symbols. +* `Rules.decide(state, event, config)`: Evaluates `case [state.status, shape]` pattern matching to select a transition recipe symbol, dispatches via `public_send(recipe, state, event, config)`, and returns a `Decision` snapshot (`from_status`, `shape`, `recipe`, `next_state`, `instructions`). +* `Rules.step(state, event, config)`: Convenience tuple wrapper around `Rules.decide` returning `[decision.next_state, decision.instructions]`. + +Source: `lib/gapic/rest/resumable_upload/rules.rb` + +### 3.2 Core Class (`Gapic::Rest::ResumableUpload::Core`) +The `Core` class is the state container holding the immutable `State` snapshot. It exposes: +* `#state`: Reader for the current `State` snapshot. +* `#last_decision`: Reader for the `Decision` recorded during the most recent `#dispatch` (or `nil`). +* `#dispatch(event)`: Invokes `Rules.decide(@state, event, @config)`, updates `@state = decision.next_state` and `@last_decision = decision`, and returns `decision.instructions` to the Driver. + +Source: `lib/gapic/rest/resumable_upload/core.rb` + +### 3.3 Driver Class (`Gapic::Rest::ResumableUpload::Driver`) +The `Driver` is the synchronous execution engine for the pure protocol state machine. When `Core#dispatch(event)` is invoked, it returns an ordered list (`Array`) of commands that the Driver executes in sequence. + +#### Instruction Processing Semantics +The Driver categorizes instructions into three execution types: +1. **Synchronous Side-Effects** (`NotifyProgress`, `RealignBuffer`): + * Executed immediately in-process. + * Do not yield a new `Event` and do not break the batch loop. Exceptions raised within user callbacks (e.g. `on_progress`) are not swallowed and immediately propagate to the caller. +2. **I/O & Network Operations** (`FillBuffer`, `SendStart`, `SendChunk`, `SendFinalize`, `SendQuery`, `SendCancel`): + * Execute physical stream reads or HTTP requests (wrapped in `Gapic::Common::RetryPolicy` for Category 1 transient errors). + * Yield a single resulting `Event` (`ChunkRead`, `HttpResponse`, or `RequestFailed`) that becomes the input for the next cycle. +3. **Terminal Handlers** (`TerminateSuccess`, `TerminateFailure`): + * Break the event loop and return the final response body string (`response.body`) or raise the terminal exception. + +Source: `lib/gapic/rest/resumable_upload/driver.rb` + +--- + +## 4. State Machine Protocol Rules + +### 4.1 Upstream Protocol Contract +1. **Logical Header Prefixing**: In the `start` request, logical headers describing the uploaded object must be prefixed with `X-Goog-Upload-Header-`. Specifically: + * `X-Goog-Upload-Header-Content-Type: config.content_type` + * `X-Goog-Upload-Header-Content-Length: config.upload_size` (if known upfront). + * Callers cannot supply either of these two headers via `initial_headers`; see the reserved-headers rule in Section 2 (doing so raises an `ArgumentError`). Other `X-Goog-Upload-Header-*` pass-through headers are permitted. +2. **Offset Extraction**: On `query` responses, the acknowledged byte count is extracted from `X-Goog-Upload-Size-Received` as an integer (`server_offset`). +3. **Request Modification on 4xx**: Retrying Category 2 errors requires querying the backend for `server_offset` first. +4. **Standard Retry Configuration & Distinct Policies**: The Driver manages distinct retry policy configurations for Category 1 transient errors: + * **Start Policy (`start_retry_policy`)**: Applies specifically to session initiation (`start`). Configured with standard retry codes (`["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"]`) and network errors (`[Faraday::ConnectionFailed, Faraday::TimeoutError, SocketError]`). A missing or empty `X-Goog-Upload-Status` header is treated as **retriable** (predicate returns `true`) across **any response code, including 200 OK**. + * **Control Plane Policy (`control_plane_retry_policy`)**: Applies to session control requests (`query`, `cancel`). Configured with standard retry codes and network errors. It does **not** retry on a missing `X-Goog-Upload-Status` header, allowing `Core` to evaluate responses immediately. + * **Data Plane Policy (`data_plane_retry_policy`)**: Applies to data transmission requests (`upload`, `upload,finalize`, and standalone `finalize`). Shares the standard retry codes and network errors, but treats a missing `X-Goog-Upload-Status` header as **unretriable** (predicate returns `false`). This prevents blind chunk re-transmission and returns `Event::HttpResponse` immediately to `Core` so it can initiate Category 2 `Recovery`. + * **Retry Policy Override Contract**: Each retry policy configuration field accepts a `Gapic::Common::RetryPolicy` instance, a `Hash`, or `nil`. Passing a `RetryPolicy` instance replaces the default policy entirely. Passing a `Hash` constructs a new `RetryPolicy` and applies the category's defaults (`RetryPolicy.new(**hash).apply_defaults(defaults)`), overriding the specified fields while preserving unspecified defaults such as `retry_codes` and `retry_predicate`. Passing `nil` constructs the default policy directly from the category defaults. + +### 4.2 State Transition & Data Mutation Specification + +**State Classification:** +* **Non-Terminal States**: `Initializing`, `Starting`, `Transmission | Reading from stream`, `Transmission | Sending`, `Finalizing | Sending with upload`, `Finalizing | Sending finalize`, `Recovery`, `Cancelling`. +* **Terminal States**: `Success`, `Cancelled`, `Error`, `Rejected`. + +| From State | Event Shape | Event & Input Payload | State Mutations | To State | Emitted Instructions & Parameters | +| :--- | :--- | :--- | :--- | :--- | :--- | +| **`Initializing`** | `:start_upload` | `Event::StartUpload` | `status = :starting` | `Starting` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::SendStart.new(url: config.initial_url, headers: config.initial_headers, body: config.initial_body)` | +| **`Initializing`** | `:resume_upload` | `Event::ResumeUpload` | `upload_url = event.upload_url`
`chunk_size = event.chunk_size`
`offset = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: event.upload_size))`
`Instruction::SendQuery.new(url: event.upload_url)` | +| **`Starting`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `upload_url = headers['X-Goog-Upload-URL']`
`chunk_granularity = headers['...-Granularity']&.to_i`
`chunk_size = resolve(config, chunk_granularity)`
`offset = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Starting`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Starting`** | `:response_cat2` / `:response_fatal_bad_response` | `Event::HttpResponse` (Non-200; see Section 6.1) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Starting`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind:, message:, source_error:)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Transmission \| Reading from stream`** | `:chunk_read_full` | `Event::ChunkRead(bytes_buffered, eof: false)` | `in_flight_length = event.bytes_buffered`
`status = :transmission_sending` | `Transmission \| Sending` | `Instruction::SendChunk.new(url: state.upload_url, offset: state.offset, length: event.bytes_buffered, finalize: false)` | +| **`Transmission \| Reading from stream`** | `:chunk_read_eof_with_data` | `Event::ChunkRead(bytes_buffered, eof: true)` where `bytes_buffered > 0` | `in_flight_length = event.bytes_buffered`
`status = :finalizing_sending_upload` | `Finalizing \| Sending with upload` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :finalizing, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendChunk.new(url: state.upload_url, offset: state.offset, length: event.bytes_buffered, finalize: true)` | +| **`Transmission \| Reading from stream`** | `:chunk_read_eof_empty` | `Event::ChunkRead(bytes_buffered: 0, eof: true)` | `in_flight_length = 0`
`status = :finalizing_sending_finalize` | `Finalizing \| Sending finalize` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :finalizing, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendFinalize.new(url: state.upload_url)` | +| **`Transmission \| Sending`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = state.offset + state.in_flight_length`
`in_flight_length = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Transmission \| Sending`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Transmission \| Sending`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Transmission \| Sending`** | `:request_retries_exhausted` | `Event::RequestFailed(kind: :retries_exhausted)` | `in_flight_length = 0`
`last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Transmission \| Sending`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `in_flight_length = 0`
`status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Transmission \| Sending`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Finalizing \| Sending with upload`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `offset = state.offset + state.in_flight_length`
`in_flight_length = 0`
`status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | +| **`Finalizing \| Sending with upload`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Finalizing \| Sending with upload`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Finalizing \| Sending with upload`** | `:request_retries_exhausted` | `Event::RequestFailed(kind: :retries_exhausted)` | `in_flight_length = 0`
`last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Finalizing \| Sending with upload`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `in_flight_length = 0`
`status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Finalizing \| Sending with upload`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Finalizing \| Sending finalize`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | +| **`Finalizing \| Sending finalize`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Finalizing \| Sending finalize`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Finalizing \| Sending finalize`** | `:request_retries_exhausted` | `Event::RequestFailed(kind: :retries_exhausted)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Finalizing \| Sending finalize`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Finalizing \| Sending finalize`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Recovery`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = headers['X-Goog-Upload-Size-Received'].to_i`
`in_flight_length = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Recovery`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `in_flight_length = 0`
`status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | +| **`Recovery`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `status = :recovery` | `Recovery` | `Instruction::SendQuery.new(url: state.upload_url)` | +| **`Recovery`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind:, ...)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Recovery`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Recovery`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Transmission \| Reading from stream` / `Transmission \| Sending chunk` / `Finalizing \| Sending with upload` / `Finalizing \| Sending finalize` / `Recovery`** | `:user_cancel` | `Event::Cancel` | `status = :cancelling` | `Cancelling` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :cancelling, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendCancel.new(url: state.upload_url)` | +| **`Cancelling`** | `:response_cancelled` | `Event::HttpResponse(200, headers, _)` with `Status: cancelled` | `status = :cancelled` | `Cancelled` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadCancelledError.from(event))` | +| **`Cancelling`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Cancelling`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:response_fatal_bad_response` | `Event::RequestFailed` or HTTP failure | `last_error = error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **Any Non-Terminal** | `:global_deadline_exceeded` | `Event::GlobalDeadlineExceeded` | `last_error = Gapic::Rest::ResumableUpload::DeadlineExceededError.new`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **Any State** | *Unmatched* | Any event not matched above | — | — | `fail_with_unmatched_transition(state, event)`: raises `InvalidTransitionError` stating in human terms what the protocol was doing (e.g. sending a chunk of data), what happened including HTTP status and `X-Goog-Upload-Status` header, and attaches the response. | + +### 4.3 State Transition Graph + +```mermaid +stateDiagram-v2 + [*] --> Initializing + Initializing --> Starting : Event::StartUpload + Initializing --> Recovery : Event::ResumeUpload + Starting --> Transmission_Reading : Event::HttpResponse(200, active) + + state Transmission { + Transmission_Reading --> Transmission_Sending : Event::ChunkRead(eof: false) + Transmission_Sending --> Transmission_Reading : Event::HttpResponse(200, active) + } + + Transmission_Reading --> Finalizing_Sending_Upload : Event::ChunkRead(eof: true, buffered > 0) + Transmission_Reading --> Finalizing_Sending_Finalize : Event::ChunkRead(eof: true, buffered == 0) + + state Finalizing { + Finalizing_Sending_Upload --> Success : Event::HttpResponse(200, final) + Finalizing_Sending_Finalize --> Success : Event::HttpResponse(200, final) + } + + Transmission_Sending --> Recovery : Event::HttpResponse(recoverable) / Event::RequestFailed + Finalizing_Sending_Upload --> Recovery : Event::HttpResponse(recoverable) / Event::RequestFailed + Finalizing_Sending_Finalize --> Recovery : Event::HttpResponse(recoverable) / Event::RequestFailed + + Recovery --> Transmission_Reading : Event::HttpResponse(200, active, server_offset) + Recovery --> Success : Event::HttpResponse(200, final) + + Starting --> Rejected : Event::HttpResponse(non-200, final) + Transmission_Sending --> Rejected : Event::HttpResponse(non-200, final) + Finalizing_Sending_Upload --> Rejected : Event::HttpResponse(non-200, final) + Finalizing_Sending_Finalize --> Rejected : Event::HttpResponse(non-200, final) + Recovery --> Rejected : Event::HttpResponse(non-200, final) + + Starting --> Error : Event::RequestFailed / 4xx / 5xx + Recovery --> Error : Event::RequestFailed + + Success --> [*] + Rejected --> [*] + Error --> [*] +``` + +--- + +## 5. Chunk Size Adjustment Rules + +Upon receiving `200 OK` from the `start` request, `Core` inspects the response headers for `X-Goog-Upload-Chunk-Granularity`. The effective chunk size (`effective_chunk_size`) stored in `State` is resolved using the following variable definitions and rules: + +### 5.1 Variable Definitions +* `DEFAULT_CHUNK_SIZE`: Default chunk size of `8_388_608` bytes (8 MB). +* `user_chunk_size`: Explicit chunk size specified in `StartUploadConfig.chunk_size` (or `nil` if unspecified). +* `chunk_granularity`: Required byte alignment modulus parsed from header `X-Goog-Upload-Chunk-Granularity` as an Integer (or `nil` if header is absent). +* `effective_chunk_size`: Final calculated byte size used by Driver for in-memory buffering and chunk transmission. + +### 5.2 Resolution Rules + +#### Rule 1: No Server Granularity Specified (`chunk_granularity` is nil or 0) +When the server does not specify a granularity requirement: +* If `user_chunk_size` is provided: `effective_chunk_size = user_chunk_size`. +* If `user_chunk_size` is omitted: `effective_chunk_size = DEFAULT_CHUNK_SIZE`. + +#### Rule 2: Default Chunk Size with Server Granularity (`user_chunk_size` is nil, `chunk_granularity > 0`) +When the user does not specify a chunk size, the default 8 MB chunk size is aligned down to the nearest multiple of `chunk_granularity`: +* `effective_chunk_size = DEFAULT_CHUNK_SIZE - (DEFAULT_CHUNK_SIZE % chunk_granularity)`. +* If `DEFAULT_CHUNK_SIZE < chunk_granularity`, `effective_chunk_size` is promoted to `chunk_granularity`. + +#### Rule 3: User Specified Chunk Size with Server Granularity (`user_chunk_size > 0`, `chunk_granularity > 0`) +When an explicit `user_chunk_size` is supplied alongside a server `chunk_granularity`: +* **Case 3A (Standard Alignment: `user_chunk_size >= chunk_granularity`)**: + * The user chunk size is aligned down to the nearest integer multiple of `chunk_granularity`: + * `effective_chunk_size = user_chunk_size - (user_chunk_size % chunk_granularity)`. + * If `user_chunk_size` is already a multiple of `chunk_granularity` (`user_chunk_size % chunk_granularity == 0`), `effective_chunk_size = user_chunk_size`. +* **Case 3B (User Size Below Granularity: `user_chunk_size < chunk_granularity`)**: + * If `user_chunk_size` is strictly less than `chunk_granularity`, downward alignment would produce `0` bytes (an invalid chunk size). + * To satisfy the server's mandatory granularity constraint, `effective_chunk_size` is promoted to `chunk_granularity`. + +### 5.3 Reference Implementation +```ruby +def self.resolve_chunk_size(user_chunk_size, chunk_granularity) + base_size = user_chunk_size || DEFAULT_CHUNK_SIZE + return base_size if chunk_granularity.nil? || chunk_granularity <= 0 + return chunk_granularity if base_size <= chunk_granularity + + base_size - (base_size % chunk_granularity) +end +``` + +--- + +## 6. Error Classification & Recovery Flows + +### 6.1 Error Categories +The implementation distinguishes three categories of network and protocol-level failures: + +#### 6.1.1 Category 1: Transient Transport Failures +* **Definition**: Standard TCP, network connection timeout, DNS, or server load-shedding errors that do not compromise the protocol session. +* **Examples**: `503 Service Unavailable`, `408 Request Timeout`, `429 Too Many Requests`, `Faraday::ConnectionFailed`, `Faraday::TimeoutError`. +* **Resolution**: The `Driver` intercepts these errors inside the physical execution wrapper and delegates directly to `Gapic::Common::RetryPolicy`. If retries succeed, `Core` receives `Event::HttpResponse`. If retries exhaust attempt/timeout limits, Driver emits `Event::RequestFailed(kind: :retries_exhausted, ...)`. + +#### 6.1.2 Category 2: Recoverable Protocol Failures +* **Definition**: Responses indicating that the client's current offset is misaligned with the server, protocol headers are missing/stripped on completed requests, or unretried transport connection failures during data transmission. +* **Conditions Producing `:response_cat2`**: + 1. **Non-200 Active Responses**: Any response with `X-Goog-Upload-Status: active` where HTTP status is non-200. + 2. **Missing or Empty `X-Goog-Upload-Status` Header**: Any response lacking `X-Goog-Upload-Status` (or empty) whose HTTP status is **not** in `FATAL_STATUS_CODES` (Section 6.1.3). This includes HTTP 200, 5xx server/gateway errors (`500`, `502`, `503`, `504`), and recoverable client errors (`400`, `408`, `409`, `412`, `416`, `429`, `499`). + 3. **Unretried Data Plane Connection Drops or Request Timeouts**: `Event::RequestFailed(kind: :connection_failed)` or `Event::RequestFailed(kind: :timeout)` (`:request_connection_failed`, `:request_timeout`) occurring during `Transmission` or `Finalizing`. +* **Missing Header Handling & Retry Policy Contract**: + * *Why Headers Go Missing*: Intermediate proxies, reverse-proxies, or Google Front End (GFE) edge proxies can strip the protocol response headers or return raw HTML/text error pages on failure. + * *Session Initiation (`start`)*: Missing `X-Goog-Upload-Status` is treated as **retriable** by `start_retry_policy` (retry predicate returns `true`) across **any response code, including 200 OK**. Driver retries transparently to smooth over transient gateway noise. If retries exhaust, `Starting` transitions to `:error` via `fail_with_request_error` or `fail_with_bad_response` (cannot recover a session before an upload URL is obtained). + * *Session Control (`query`, `cancel`)*: `control_plane_retry_policy` does **not** treat missing status headers as retriable, returning the completed `Event::HttpResponse` immediately to `Core` so it can manage protocol recovery or fail fast. + * *Data Plane (`upload`, `upload, finalize`, standalone `finalize`)*: Missing `X-Goog-Upload-Status` is treated as **unretriable** by `data_plane_retry_policy` (retry predicate returns `false`). The Driver immediately returns `Event::HttpResponse` to `Core` so it classifies as `:response_cat2` and initiates Category 2 `Recovery` via `Instruction::SendQuery` rather than blindly re-transmitting data. +* **Resolution**: Core transitions to `Recovery` and emits `Instruction::SendQuery.new(url: state.upload_url)` to obtain `server_offset`. + +#### 6.1.3 Category 3: Terminal Failures & Fatal Status Codes +* **Definition**: Irrecoverable errors where either the request is structurally invalid, unauthorized, transport retry limits are exhausted, unseekable rewind is needed, or the server has explicitly aborted/rejected the session. +* **Canonical Fatal Status Codes (`FATAL_STATUS_CODES`)**: + The following status codes indicate structural or authentication failures that cannot be resolved by querying the upload backend: + * `401 Unauthorized`: Authentication token is expired, invalid, or missing. + * `403 Forbidden`: Caller lacks required IAM permissions for the upload destination. + * `404 Not Found`: Session upload URL does not exist or has expired. + * `405 Method Not Allowed`: HTTP method is rejected by the server. + * `410 Gone`: Upload session has been permanently removed. + * `413 Payload Too Large`: Upload chunk or overall size exceeds server limit. + * `415 Unsupported Media Type`: Object content type is rejected. + Responses with these status codes are classified as `:response_fatal_bad_response` even if the `X-Goog-Upload-Status` header is absent. +* **Other Terminal Conditions**: + * **Retry Exhaustion**: Any `Event::RequestFailed(kind: :retries_exhausted)` occurring at any stage. When Category 1 transport retries are exhausted by the `RetryPolicy`, failure is immediate and terminal; it does not enter Category 2 `Recovery`. + * **Session Rejection**: Any response with `X-Goog-Upload-Status: final` and non-2xx status code (`:response_rejected` -> raises `Gapic::Rest::ResumableUpload::UploadRejectedError`). + * **Initiation Failure**: Any 4xx/5xx or `Event::RequestFailed` during `Starting` (`:error` -> raises `Gapic::Rest::ResumableUpload::BadResponseError` or source error). + * **Session Cancellation**: Cancelled upload sessions raise `Gapic::Rest::ResumableUpload::UploadCancelledError`. + * **Global Deadline Expiration**: Monotonic clock exceeding session deadline raises `Gapic::Rest::ResumableUpload::DeadlineExceededError`. + * **Unseekable Stream Rewind**: Server offset rolled back behind retained buffer (`server_offset < buffer_start_offset`) on an unseekable stream (raises `Gapic::Rest::ResumableUpload::UnseekableStreamError`). +* **Resolution**: Core transitions to `:rejected` or `:error` and emits `Instruction::TerminateFailure`. + +#### 6.1.4 Actionable Terminal Errors & Metadata Propagation +Terminal errors provide actionable context so downstream SDK callers can inspect error metadata: +* **Error Classes**: + * `BadResponseError < Gapic::Rest::Error`: Unrecoverable non-2xx HTTP responses or invalid payloads. Retains `attr_reader :response_body` returning `event.body`, and includes `HasResumeHandle`. + * `UploadRejectedError < Gapic::Rest::Error`: Backend explicitly rejected the session with `X-Goog-Upload-Status: final`. Retains `attr_reader :response_body` returning `event.body`. Does NOT include `HasResumeHandle` (session is terminated permanently). + * `UploadCancelledError < Gapic::Common::Error`: Upload session cancelled by caller. Does NOT include `HasResumeHandle` (session is terminated permanently). + * `DeadlineExceededError < Gapic::Common::Error`: Upload deadline exceeded with optional root cause (`attr_reader :root_cause`), and includes `HasResumeHandle`. + * `UnseekableStreamError < Gapic::Common::Error`: Stream rewind required on an unseekable stream; includes `HasResumeHandle`. + * `InvalidTransitionError < Gapic::Common::Error`: Unexpected event dispatched for state; includes `HasResumeHandle`. + * `StreamMismatchError < Gapic::Common::Error`: Stream content or length does not match resumed upload specifications; includes `HasResumeHandle`. + * `RequestFailedError < Gapic::Common::Error`: Terminal HTTP request failure (e.g. transport connection failure, request timeout, or retries exhausted). Retains `attr_reader :cause` returning the underlying error, preserves REST error attributes (`status_code`, `status`, `details`, `headers`) when available, and includes `HasResumeHandle`. + * `SessionStateError < Gapic::Common::Error`: Raised when an operation violates Session lifecycle rules (e.g. attempting to start an already-bound session, resuming an unbound session without a target upload, re-binding to a different upload, resuming a finalized/dead session, or concurrent run invocations). Distinguished from `ArgumentError`, which is raised strictly for invalid argument shapes. +* **Resume Handle Propagation (`HasResumeHandle`)**: + * The `HasResumeHandle` mixin exposes `attr_reader :resume_handle` returning a `ResumeHandle` (or `nil` if session initiation was incomplete or if the session was `:rejected` or `:cancelled`). + * Whenever `resume_handle` is non-nil, the uniform suffix `" (upload session is resumable: see #resume_handle)"` is automatically appended to the error message. +* **Metadata Sourcing & De-prefixing**: + * When `event.error` is present (from `Gapic::Rest::Error.wrap_faraday_error`), factories source `status_code`, `status`, `details`/`status_details`, and `headers`/`header`. + * The prefix literal `Gapic::Rest::Error::REST_ERROR_PREFIX` (`"An error has occurred when making a REST request"`) is stripped from `event.error.message` to avoid redundant prefixes. + * The resulting actionable message follows the format: + * For `UploadRejectedError`: `"Upload rejected by server with HTTP #{status_code} #{status_name}: #{inner_message}"` (e.g., `"Upload rejected by server with HTTP 403 PERMISSION_DENIED: The caller does not have permission"`). + * For `BadResponseError`: `"Resumable upload failed with HTTP #{status_code} #{status_name}: #{inner_message}"` (e.g., `"Resumable upload failed with HTTP 429 RESOURCE_EXHAUSTED: Quota limit reached"`). +* **Fallback Formatting**: + * When `event.error` is absent, factories fall back to `event.status` and `event.headers`, naming the status and including the detailed `X-Goog-Upload-Status` header: + * For `UploadRejectedError`: `"Upload rejected by server with HTTP #{event.status} #{status_name} (X-Goog-Upload-Status: 'final')"`. + * For `BadResponseError`: `"Resumable upload failed with HTTP #{event.status} #{status_name} (X-Goog-Upload-Status: #{upload_status_desc})"`. + * `response_body` on `BadResponseError` and `UploadRejectedError` returns `event.body`. + +### 6.2 Recovery and Buffer Alignment +When `Core` resolves a `query` response in the `Recovery` state, it updates `State.offset` (`protocol_state_offset`) to `server_offset` (extracted from `X-Goog-Upload-Size-Received`) and transitions to `Transmission | Reading from stream`. + +To realign the upload state, the `Driver` processes `Instruction::RealignBuffer(server_offset)` using its in-memory buffer and stream position tracking: +1. **Within-Buffer Alignment (`buffer_start_offset <= server_offset <= buffer_end_offset`)**: + * The Driver trims already-persisted bytes: `@buffer = @buffer.byteslice((server_offset - buffer_start_offset)..-1)`. + * The Driver updates `buffer_start_offset = server_offset`. + * Upon executing the accompanying `Instruction::FillBuffer(target_bytesize)`, the Driver reads `target_bytesize - @buffer.bytesize` bytes from `stream` to restore `@buffer` to full `chunk_size` before transmitting. +2. **Rewind Required (`server_offset < buffer_start_offset`)**: + * If `stream.respond_to?(:seek)`: the Driver seeks to `server_offset`, clears `@buffer = "".b`, and sets `buffer_start_offset = server_offset`. + * If `stream` is unseekable (e.g. Socket, Pipe, STDIN): the Driver raises terminal `UnseekableStreamError` (Category 3), attaching `resume_handle`. +3. **Fast-Forward Required (`server_offset > buffer_end_offset`)**: + * If total `upload_size` is known and `server_offset > upload_size`: Driver raises terminal `StreamMismatchError` with `resume_handle`. + * If `upload_size` is `nil` and `stream.respond_to?(:size)` and `server_offset > stream.size`: Driver raises terminal `StreamMismatchError` with `resume_handle`. + * The Driver clears `@buffer = "".b`. + * If `stream.respond_to?(:seek)`: seeks to `server_offset`. + * If unseekable: reads and discards `server_offset - current_stream_pos` bytes from `stream`. If the stream encounters unexpected EOF before reaching `server_offset`, Driver raises terminal `StreamMismatchError` with `resume_handle`. + * The Driver sets `buffer_start_offset = server_offset`. + +### 6.3 Sensible Defaults for Global Deadline +Every upload session executed via `Driver#run` must have a finite, guaranteed upper bound on total wall-clock execution time. Without a mandatory global deadline, a session encountering repeated Category 2 protocol recoveries or intermittent network stalls could hang indefinitely. + +To guarantee termination, `Driver#run` establishes an absolute monotonic deadline at the very start of execution: +```ruby +@deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + resolve_timeout +``` + +#### Timeout Resolution Algorithm (`resolve_timeout`) +The total session timeout is resolved in priority order: +1. **Explicit User Timeout (`config.timeout`)**: If `config.timeout` is present and strictly positive (`config.timeout&.positive?`), that value is used directly. Zero or negative values are treated as unset (`nil`). +2. **Size-Proportional Timeout (`config.upload_size`)**: If total `upload_size` is known upfront, the timeout is computed assuming a minimum sustained upload throughput of `MIN_ASSUMED_THROUGHPUT = 1_048_576` bytes/sec (1 MB/s), floored by `BASE_TIMEOUT = 3_600` seconds (1 hour): + ```ruby + [config.upload_size.fdiv(MIN_ASSUMED_THROUGHPUT), BASE_TIMEOUT].max + ``` + *Rationale*: Using `BASE_TIMEOUT` as a floor prevents sub-millisecond timeouts for small payloads while scaling linearly for multi-gigabyte uploads. +3. **Default Base Timeout (`BASE_TIMEOUT`)**: If neither a positive timeout nor `upload_size` is provided (e.g., streaming uploads of unknown length), the timeout defaults to `BASE_TIMEOUT` (`3_600` seconds). + +#### Bounding Transport Retries by Global Deadline +Transport retries and individual HTTP exchanges must never exceed the remaining global deadline. When `Driver#make_post_request` invokes `ClientStub#make_post_request`, it computes the per-request timeout from the remaining session budget (`max(deadline - monotonic_now, 0)`), additionally capped by `retry_policy.timeout`: +```ruby +remaining = [@deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max +timeout = retry_policy&.timeout ? [remaining, retry_policy.timeout].min : remaining +``` +This timeout is passed in `options[:timeout]`, ensuring that underlying Faraday requests and `Gapic::Common::RetryPolicy` evaluations always respect the remaining upload budget. + +--- + +## 7. Observability Standards + +### 7.1 Architecture & Separation of Concerns +Because `Rules` is a pure decision engine and `Core` is a side-effect-free state container, protocol decisions are encoded as immutable `Decision` data structures and logged exclusively by the `Driver` via `Driver::UploadLog`. + +Each invocation of `Driver#run` generates a fresh UUIDv4 session identifier (`uploadId`) that is attached to every log entry emitted during that run. Structured log entries are constructed using `Gapic::LoggingConcerns` (`StubLogger` yielding a `LogEntryBuilder` producing `Google::Logging::Message` instances). Machine-readable state and telemetry are stored in `Google::Logging::Message#fields`, allowing log message text to evolve independently without breaking structured queries. + +### 7.2 Log Level & Recipe Mapping +The `Driver` emits structured logs across three severity levels (`INFO`, `DEBUG`, `WARN`). High-frequency per-chunk acknowledgements (`:ack_chunk`) are suppressed from `INFO` lifecycle logs to avoid log volume bloat on multi-gigabyte uploads. + +| Severity | Category | Trigger / Recipe | Message Summary | +| :--- | :--- | :--- | :--- | +| `INFO` | Lifecycle | `:start_session` | Initiating resumable upload | +| `INFO` | Lifecycle | `:begin_transmission` | Upload session established | +| `INFO` | Lifecycle | `:send_upload_finalize` | Sending final upload chunk | +| `INFO` | Lifecycle | `:send_finalize` | Sending finalize command | +| `INFO` | Lifecycle | `:enter_recovery` | Entering upload recovery | +| `INFO` | Lifecycle | `:retry_recovery` | Retrying upload recovery query | +| `INFO` | Lifecycle | `:realign_from_recovery` | Resuming upload from server offset | +| `INFO` | Lifecycle | `:complete_upload_with_data`, `:complete_upload_finalized` | Resumable upload completed | +| `INFO` | Lifecycle | `:cancel_session` | Canceling resumable upload | +| `INFO` | Lifecycle | `:complete_cancellation` | Resumable upload canceled | +| `DEBUG` | Lifecycle | `:send_chunk` | Sending upload chunk | +| `DEBUG` | Decision | Every `Core#dispatch` transition | `Rules: + -> -> ` | +| `DEBUG` | Wire | Outbound HTTP request (`wire_send`) | `Sending request` | +| `DEBUG` | Wire | Inbound HTTP response (`wire_receive`) | `Received HTTP ` | +| `DEBUG` | Wire | Transport exception (`wire_failure`) | `Request failed: ` | +| `DEBUG` | Buffer | Stream/buffer realignment (`buffer_realign`) | `Buffer realignment: ` | +| `WARN` | Lifecycle | `:fail_with_deadline_exceeded`, `:fail_with_rejected`, `:fail_with_bad_response`, `:fail_with_request_error` | Resumable upload failed | +| `WARN` | Transition | `InvalidTransitionError` (`unmatched_transition`) | Unmatched transition | +| `WARN` | Buffer | Backward server offset rewind on unseekable stream | Server offset rewind on unseekable stream | + +### 7.3 Structured Field Glossary +All log entries emitted by `UploadLog` populate structured fields in `Google::Logging::Message#fields`: + +* **Common Context Fields** (present on all entries): + * `system`: `"gapic-common"` + * `serviceName`: `"ResumableUpload"` + * `clientId`: Object ID of the underlying `Gapic::Rest::ClientStub`. + * `uploadId`: Unique UUIDv4 identifying the specific `Driver#run` execution. +* **Decision & Lifecycle Fields**: + * `fromStatus`: Protocol status symbol prior to event dispatch. + * `toStatus`: Resulting protocol status symbol (`decision.next_state.status`). + * `shape`: Canonical event shape symbol classified by `Rules.shape_of`. + * `recipe`: Transition recipe method symbol executed by `Rules`. + * `offset`: Current server-confirmed byte offset (`Integer`). + * `inFlightLength`: Byte length of the chunk currently in flight (`Integer`). + * `instructions`: Array of abridged instruction hashes emitted by the transition. + * `uploadSize`: Total expected upload size in bytes from `config.upload_size` (on `:start_session`). + * `requestedChunkSize`: Configured chunk size in bytes from `config.chunk_size` (on `:start_session`). + * `effectiveChunkSize`: Negotiated chunk size aligned to server granularity (on `:begin_transmission`). + * `granularity`: Server chunk alignment modulus from `X-Goog-Upload-Chunk-Granularity` (on `:begin_transmission`). + * `uploadUrl`: Abridged session upload URL (on `:begin_transmission` and `:cancel_session`). + * `status`: Current protocol status symbol (on `unmatched_transition`). + * `error`: Exception message string (on `fail_with_*` and `unmatched_transition`). + * `responseBody`: Abridged error response body from `last_error.response_body` when present (on `fail_with_*`). +* **Wire & Transport Fields**: + * `method`: Always the string `"POST"`. + * `url`: Abridged request target URI. + * `headers`: Redacted HTTP header hash. + * `startAttempt`: Retry attempt counter (`Integer`). + * `command`: Value of `X-Goog-Upload-Command` request header. + * `offset`: Parsed integer value of `X-Goog-Upload-Offset` request header (`wire_send`). + * `bodySize`: Total byte length of request payload (`Integer`). + * `body`: Abridged payload or error body snippet. + * `status`: HTTP response status code (`Integer`, on `wire_receive`). + * `uploadStatus`: Value of `X-Goog-Upload-Status` response header. + * `sizeReceived`: Parsed integer value of `X-Goog-Upload-Size-Received` response header. + * `granularity`: Parsed integer value of `X-Goog-Upload-Chunk-Granularity` response header (`wire_receive`). + * `kind`: Transport failure classification symbol (`:timeout`, `:connection_failed`, `:retries_exhausted`). + * `error`: Exception message string (`wire_failure`). +* **Buffer Realignment Fields**: + * `action`: Realignment strategy string (`"within_buffer"`, `"rewind"`, or `"fast_forward"`). + * `serverOffset`: Target byte offset reported by the server (`Integer`). + * `currentOffset`: Local buffer start offset before realignment (`Integer`). + +### 7.4 Redaction & Payload Abridgement +To prevent credential leakage and ensure log volume is proportional to the number of requests and independent of payload size, `Driver::Abridge` and `ClientStub` enforce strict sanitization rules before any entry is passed to the logger: + +1. **URL Query Elision (`Abridge.url`)**: Upload session URLs contain capability tokens in their query parameters (e.g., `upload_id`, `sid`). `Abridge.url` parses the URI and replaces every query parameter value with `<...>` (e.g., `https://storage.googleapis.com/upload?upload_id=<...>`). +2. **Header Allowlisting (`Abridge.headers`)**: Only protocol control headers prefixed with `x-goog-upload-` retain their values in log entries (with `x-goog-upload-url` passed through `Abridge.url`). All other request and response headers—including `Authorization` or custom metadata—are replaced with `"<...>"`. Note that Faraday injects `Authorization` headers below the `ClientStub` logging layer; tests verify that bearer tokens never appear in logs. +3. **Binary Payload Abridgement (`Abridge.bytes` & `ClientStub#abridge_request_body`)**: + * In `Driver::Abridge.bytes`, binary payloads of 64 bytes or more are abridged to their first 32 bytes encoded in hexadecimal followed by the total byte size: `"... "`. + * In `Gapic::Rest::ClientStub#log_request`, any request body exceeding 1 KiB (1024 bytes) or containing non-UTF-8 binary data is abridged to `">"`, preventing 8 MiB upload chunks from being dumped into `DEBUG` logs. +4. **Error Body Truncation (`Abridge.error_body`)**: HTTP error response bodies (status $\ge 400$) are forced to UTF-8 encoding with invalid byte sequences scrubbed and truncated to at most 512 characters. + +### 7.5 Enabling & Configuring Logging +Logging is disabled by default (`logger: nil`) and incurs negligible allocation overhead when inactive. Users and test harnesses can enable logging via two mechanisms: + +1. **Environment Variable Opt-In (`GOOGLE_SDK_RUBY_LOGGING_GEMS`)**: + Setting the `GOOGLE_SDK_RUBY_LOGGING_GEMS` environment variable activates default `Logger` instances writing to `$stderr` at `DEBUG` level (using `Google::Logging::StructuredFormatter` when running in a Google Cloud environment): + * `GOOGLE_SDK_RUBY_LOGGING_GEMS=all` or `GOOGLE_SDK_RUBY_LOGGING_GEMS=true`: Enables logging across all Google Cloud Ruby SDK components. + * `GOOGLE_SDK_RUBY_LOGGING_GEMS=gapic-common`: Enables logging specifically for `gapic-common` (including `ResumableUpload::Driver` and `ClientStub`). + * `GOOGLE_SDK_RUBY_LOGGING_GEMS=false` or `none`: Explicitly disables SDK logging even if a default logger is configured. +2. **Explicit Logger Injection**: + Pass any Ruby `::Logger`-compatible instance directly to `Driver.new(client_stub: stub, config: config, logger: my_logger)` or configure it on the parent service client config. \ No newline at end of file diff --git a/gapic-common/design/resumable_upload/integration-test-plan.md b/gapic-common/design/resumable_upload/integration-test-plan.md new file mode 100644 index 0000000..46a5750 --- /dev/null +++ b/gapic-common/design/resumable_upload/integration-test-plan.md @@ -0,0 +1,286 @@ +# Resumable Upload Integration Test Plan + +This document outlines the integration test architecture and test suites for the Resumable Upload protocol implementation in `gapic-common`. Unlike the unit test suite under `test/gapic/rest/resumable_upload/`, which isolates protocol state transitions and driver components against test doubles, the integration test suite exercises the full stack end-to-end over real HTTP/REST connections against a live `gapic-showcase` server. + +--- + +## 1. Integration Test Architecture Overview + +```mermaid +flowchart TD + subgraph Runner["Test Runner (.toys/test-integration.rb)"] + Toys["toys test-integration"] + Lifecycle["Showcase Lifecycle Manager
(Verify version >= 0.43, allocate ports, spawn & health-check)"] + end + + subgraph Harness["Test Harness (integration/integration_helper.rb)"] + BaseClass["ShowcaseIntegrationTest
(Minitest::Test)"] + PayloadGen["payload(size)
Deterministic binary stream"] + TraceLog["StringIO Logger
(Emits debug wire traces on test failure)"] + end + + subgraph SUT["System Under Test"] + Driver["Gapic::Rest::ResumableUpload::Driver"] + Stub["Gapic::Rest::ClientStub
(raise_faraday_errors: false)"] + end + + subgraph Server["External Server"] + Showcase["gapic-showcase
(/resumable/upload/v1beta1/files:upload)"] + end + + Toys --> Lifecycle + Lifecycle --> Showcase + Toys --> BaseClass + BaseClass --> PayloadGen + BaseClass --> TraceLog + BaseClass --> Driver + Driver --> Stub + Stub <-->|"HTTP POST / PUT (REST)"| Showcase +``` + +### 1.1 Execution & Server Lifecycle (`.toys/test-integration.rb`) +* **External Endpoint Override**: If `SHOWCASE_ENDPOINT` is set in the environment, the runner connects directly to that address without spawning a local server. +* **Automatic Server Provisioning**: When `SHOWCASE_ENDPOINT` is unset, the runner locates the `gapic-showcase` binary on `PATH` (or via `SHOWCASE_BIN`), verifies that its version is at least `0.43`, allocates ephemeral ports, and spawns `gapic-showcase run`. +* **Health Check & Teardown**: Polls the TCP socket with a 10-second deadline before invoking Minitest, and guarantees process cleanup (`SIGTERM` + `waitpid`) in an `ensure` block. + +### 1.2 Test Harness (`integration/integration_helper.rb`) +* **`ShowcaseIntegrationTest`**: Base class providing helper methods for test configuration: + * `showcase_client_stub`: Instantiates a real `Gapic::Rest::ClientStub` targeting `SHOWCASE_ENDPOINT` with `raise_faraday_errors: false` and an attached `DEBUG` logger. + * `build_config(scenario: nil, scenario_config: {}, **overrides)`: Creates a `StartUploadConfig` targeting `/resumable/upload/v1beta1/files:upload`. When `scenario` is provided, injects `X-Goog-Test-Scenario` and `X-Goog-Test-Scenario-Config` (with a generated `client_uuid` merged with `scenario_config`) into `initial_headers` (these test headers are unaffected by `RESERVED_INITIAL_HEADERS` since they are not in the five reserved protocol headers). Configures fast retry policies (`FAST_RETRY = { initial_delay: 0.01, max_delay: 0.05, multiplier: 1, timeout: 2 }`), a default 10-second session timeout, a default payload of `786_432` bytes (`3 * 262_144`), default chunk size of `262_144` bytes, and an `on_progress` callback appending each `Progress` struct to `@progress_records`. + * `build_session(scenario: nil, scenario_config: {}, **overrides)` & `start_session(session, **overrides)`: Partitions overrides using `START_ONLY_KEYS` (`[:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy]`). `build_session` instantiates a `Gapic::Rest::ResumableUpload::Session` with the common members (`client_stub`, `stream`, `upload_size`, `content_type`, `timeout`, `control_plane_retry_policy`, `data_plane_retry_policy`, `on_progress`, `logger`) and stores initiation arguments in `@start_args`, while `start_session` invokes `session.start(**@start_args, **overrides)`. + * `phases` & `offsets`: Convenience accessors returning `@progress_records.map(&:phase)` and `@progress_records.map(&:bytes_uploaded)`. + * `payload(size)`: Generates deterministic binary strings of arbitrary byte length for stream uploads. + * `UnseekableStream`: Stream wrapper around `StringIO` that exposes `#read` and `#pos` while omitting `#seek` (`respond_to?(:seek)` is `false`). + * **Diagnostic Trace Capture**: Buffers `DEBUG`-level driver logs in memory during each test run and dumps the full trace to `stderr` only if a test fails (or when `SHOWCASE_LOG` is set). + +--- + +## 2. Detailed Test Suites & Cases + +### 2.1 Golden Path Suite (`integration/resumable_upload/golden_path_test.rb`) + +Tests standard, uninterrupted resumable upload workflows against `gapic-showcase`. + +#### Case 1. Multi-chunk upload with known size (`test_multi_chunk_known_size`) +* **Scenario**: Uploads a 1.5 MB (`1_500_000` bytes) stream with an explicit `upload_size: 1_500_000` and `chunk_size: 524_288` (512 KiB). +* **Protocol Flow**: + 1. `start` command initiates the session with `X-Goog-Upload-Header-Content-Length: 1500000`. + 2. Chunk 1 transmits bytes `0..524287` (`upload`). + 3. Chunk 2 transmits bytes `524288..1048575` (`upload`). + 4. Chunk 3 transmits remaining bytes `1048576..1499999` with `upload, finalize`. +* **Assertions**: + * Returned JSON body parses cleanly and reports `"size" == 1_500_000`. + * `progress_records` contains 6 `Progress` notifications across lifecycle phases: + * `Progress(phase: :initiating, bytes_uploaded: 0, total_bytes: 1_500_000)` + * `Progress(phase: :uploading, bytes_uploaded: 0, total_bytes: 1_500_000)` + * `Progress(phase: :uploading, bytes_uploaded: 524_288, total_bytes: 1_500_000)` + * `Progress(phase: :uploading, bytes_uploaded: 1_048_576, total_bytes: 1_500_000)` + * `Progress(phase: :finalizing, bytes_uploaded: 1_048_576, total_bytes: 1_500_000)` + * `Progress(phase: :completed, bytes_uploaded: 1_500_000, total_bytes: 1_500_000)` + +#### Case 2. Default chunk size on small upload (`test_small_upload_default_chunk_size`) +* **Scenario**: Uploads a ~100 KB (`100_000` bytes) payload with `upload_size: 100_000` and no `chunk_size` specified. +* **Protocol Flow**: + 1. `start` command initiates the session; chunk size defaults to 8 MiB (`8_388_608` bytes). + 2. The entire 100,000-byte payload fits within a single buffer read and is transmitted in one `upload, finalize` request. +* **Assertions**: + * Returned JSON body reports `"size" == 100_000`. + * `progress_records` contains 4 `Progress` notifications: + * `Progress(phase: :initiating, bytes_uploaded: 0, total_bytes: 100_000)` + * `Progress(phase: :uploading, bytes_uploaded: 0, total_bytes: 100_000)` + * `Progress(phase: :finalizing, bytes_uploaded: 0, total_bytes: 100_000)` + * `Progress(phase: :completed, bytes_uploaded: 100_000, total_bytes: 100_000)` + +#### Case 3. Standalone finalize on unseekable stream (`test_standalone_finalize_unseekable_stream`) +* **Scenario**: Uploads a `786_432`-byte payload (`3 * 262_144` bytes) wrapped in an `UnseekableStream`, with `chunk_size: 262_144` (256 KiB) and `upload_size` omitted (`nil`). +* **Coverage**: + * Unknown total upload size on `start` (omitted `X-Goog-Upload-Header-Content-Length`). + * End-of-exact-boundary stream reading path (payload is an exact multiple of `chunk_size`, so EOF is not detected until the subsequent buffer fill). + * Standalone `SendFinalize` instruction (`upload_command: "finalize"` with empty body). +* **Protocol Flow**: + 1. `start` command initiates the session without a total content length header. + 2. Chunk 1 transmits bytes `0..262143` (`upload`). + 3. Chunk 2 transmits bytes `262144..524287` (`upload`). + 4. Chunk 3 transmits bytes `524288..786431` (`upload`). + 5. Next buffer read returns 0 bytes at EOF (`:chunk_read_eof_empty`), emitting `SendFinalize` to send a standalone `finalize` request at offset `786432`. +* **Assertions**: + * Returned JSON body reports `"size" == 786_432`. + * `progress_records` contains 7 `Progress` notifications: + * `Progress(phase: :initiating, bytes_uploaded: 0, total_bytes: nil)` + * `Progress(phase: :uploading, bytes_uploaded: 0, total_bytes: nil)` + * `Progress(phase: :uploading, bytes_uploaded: 262_144, total_bytes: nil)` + * `Progress(phase: :uploading, bytes_uploaded: 524_288, total_bytes: nil)` + * `Progress(phase: :uploading, bytes_uploaded: 786_432, total_bytes: nil)` + * `Progress(phase: :finalizing, bytes_uploaded: 786_432, total_bytes: nil)` + * `Progress(phase: :completed, bytes_uploaded: 786_432, total_bytes: 786_432)` + +### 2.2 Chunk Granularity Suite (`integration/resumable_upload/chunk_granularity_test.rb`) + +Tests dynamic chunk size resolution when the server mandates a byte alignment modulus via `X-Goog-Upload-Chunk-Granularity`. + +#### Case 1. Downward alignment to server granularity (`test_chunk_granularity_alignment`) +* **Scenario**: Uploads a `1_000_000`-byte payload with `scenario: "chunk_granularity"`, an explicit unaligned user `chunk_size: 300_000`, and `timeout: 5`. +* **Protocol Flow**: + 1. `start` command initiates the session; Showcase returns `X-Goog-Upload-Chunk-Granularity: 256`. + 2. Client resolves the effective chunk size down to the nearest multiple of 256: `300_000 - (300_000 % 256) = 299_776` bytes. + 3. Chunks 1, 2, and 3 transmit `299_776` bytes each (`upload`), advancing confirmed offsets to `299_776`, `599_552`, and `899_328`. + 4. Final chunk transmits the remaining `100_672` bytes (`899_328..999_999`) with `upload, finalize`. +* **Assertions**: + * Returned JSON body reports `"size" == 1_000_000`. + * `offsets` equals `[0, 0, 299_776, 599_552, 899_328, 899_328, 1_000_000]`. + * `phases` equals `[:initiating, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed]`. + +### 2.3 Error Recovery Suite (`integration/resumable_upload/error_recovery_test.rb`) + +Tests Category 1 transient transport retries and Category 2 protocol recovery workflows against `scenario: "non_fatal_error_on_chunk_upload"`. + +#### Case 1. Category 1 transient error retried transparently (`test_cat1_error_retried_transparently`) +* **Scenario**: Injects a single `503 Service Unavailable` response at offset `0` (`error_code: 503, failure_count: 1, after_offset: 0`). +* **Protocol Flow**: + 1. `start` establishes the session (`200 active`). + 2. First attempt to upload chunk 1 (`0..262143`) receives `503`. + 3. `Driver` intercepts the transient error via `data_plane_retry_policy` (`FAST_RETRY`) and retries the chunk transparently without entering protocol `Recovery`. + 4. Subsequent chunks and standalone `finalize` succeed normally. +* **Assertions**: + * Returned JSON body reports `"size" == 786_432`. + * `phases` does not include `:recovering` (`[:initiating, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed]`). + * `offsets` equals `[0, 0, 262_144, 524_288, 786_432, 786_432, 786_432]`. + +#### Case 2. Simple Category 2 error recovery at offset 0 (`test_simple_cat2_error_recovery`) +* **Scenario**: Injects a single `409 Conflict` with `X-Goog-Upload-Status: active` at offset `0` (`error_code: 409, failure_count: 1, after_offset: 0`). +* **Protocol Flow**: + 1. First upload chunk receives `409` (`:response_cat2`). + 2. `Core` transitions to `Recovery` (`:recovering`) and issues `SendQuery`. + 3. Server responds with `X-Goog-Upload-Size-Received: 0`; client realigns buffer to offset `0`, retransmits chunk 1, and completes the upload. +* **Assertions**: + * Returned JSON body reports `"size" == 786_432`. + * `phases` equals `[:initiating, :uploading, :recovering, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed]`. + * `offsets` equals `[0, 0, 0, 0, 262_144, 524_288, 786_432, 786_432, 786_432]`. + +#### Case 3. Two consecutive Category 2 recoveries on chunk 2 (`test_two_consecutive_cat2_recoveries_on_chunk_2`) +* **Scenario**: Injects two consecutive `409` errors at offset `262_144` (`error_code: 409, failure_count: 2, after_offset: 262_144`). +* **Protocol Flow**: + 1. Chunk 1 (`0..262143`) succeeds. + 2. First attempt at chunk 2 (`262144..524287`) fails with `409` -> `:recovering` -> `query` (offset `262_144`) -> `:uploading`. + 3. Second attempt at chunk 2 fails with `409` -> `:recovering` -> `query` (offset `262_144`) -> `:uploading`. + 4. Third attempt at chunk 2 succeeds; chunk 3 and `finalize` complete normally. +* **Assertions**: + * Returned JSON body reports `"size" == 786_432`. + * `phases` equals `[:initiating, :uploading, :uploading, :recovering, :uploading, :recovering, :uploading, :uploading, :uploading, :finalizing, :completed]`. + * `offsets` equals `[0, 0, 262_144, 262_144, 262_144, 262_144, 262_144, 524_288, 786_432, 786_432, 786_432]`. + +#### Case 4. Category 2 failure on finalizing chunk (`test_cat2_failure_on_finalizing_chunk`) +* **Scenario**: Uploads a `786_332`-byte payload (`3 * 262_144 - 100`) where chunk 3 (`524288..786331`) carries `upload, finalize`, injecting a `409` error at offset `524_288` (`error_code: 409, failure_count: 1, after_offset: 524_288`). +* **Protocol Flow**: + 1. Chunks 1 and 2 succeed, advancing offset to `524_288`. + 2. Client enters `:finalizing` and transmits chunk 3 with `upload, finalize`. + 3. Server returns `409` (`:response_cat2`); client transitions from `:finalizing` to `:recovering`, queries server (`X-Goog-Upload-Size-Received: 524288`), realigns buffer, re-enters `:finalizing`, and retransmits `upload, finalize` to completion. +* **Assertions**: + * Returned JSON body reports `"size" == 786_332`. + * `phases` equals `[:initiating, :uploading, :uploading, :uploading, :finalizing, :recovering, :uploading, :finalizing, :completed]`. + * `offsets` equals `[0, 0, 262_144, 524_288, 524_288, 524_288, 524_288, 524_288, 786_332]`. + +#### Case 5. Repeated no-header failures until global deadline exceeded (`test_no_headers_failure_recovers_until_deadline_exceeded`) +* **Scenario**: Configures `failure_count: 0, action_after_failures: "terminate"` with a 1-second session `timeout`. +* **Protocol Flow**: + 1. Server responds to every upload chunk with HTTP `500` and no `X-Goog-Upload-Status` header. + 2. `data_plane_retry_policy` treats missing `X-Goog-Upload-Status` as unretriable (`predicate` returns `false`), yielding `Event::HttpResponse(500)` to `Core`. + 3. `Core` classifies the response as Category 2 (`:response_cat2`), enters `:recovering`, queries the server (which returns `200 active` at offset `0`), and retries the upload. + 4. This recovery loop repeats until the 1-second global session deadline expires and `Driver#run` raises `Gapic::Rest::ResumableUpload::DeadlineExceededError`. +* **Assertions**: + * Raises `Gapic::Rest::ResumableUpload::DeadlineExceededError`. + * `phases.count(:recovering) >= 2`. + +### 2.4 Error on Start Suite (`integration/resumable_upload/error_on_start_test.rb`) + +Tests non-fatal transient retries, missing status headers, retry exhaustion, fatal errors, and session isolation during the session initiation (`start`) phase. Uses a 100-byte payload, configuring `FAST_RETRY` for non-exhaustion retry cases to minimize test execution latency while retaining default policies for exhaustion and fatal checks. + +#### Case 1. Non-fatal transient error on start (`test_non_fatal_error_on_start_503`) +* **Scenario**: Injects a single `503 Service Unavailable` on the initial `start` request (`scenario: "non_fatal_error_on_start"`, `error_code: 503, failure_count: 1`, `start_retry_policy: FAST_RETRY`). +* **Protocol Flow**: + 1. First `start` POST request receives `503`. + 2. `start_retry_policy` transparently retries the initiation request. + 3. Second attempt succeeds (`200 active`), returning session URL. + 4. 100-byte upload completes normally. +* **Assertions**: + * Returned JSON body reports `"size" == 100`. + * Exactly 1 `:initiating` notification in `phases` (`phases.count(:initiating) == 1`). + * `phases.last == :completed`. + +#### Case 2. Missing status header / 400 on start (`test_missing_header_retriable_on_start_400`) +* **Scenario**: Injects a single `400 Bad Request` without an `X-Goog-Upload-Status` header on `start` (`scenario: "non_fatal_error_on_start"`, `error_code: 400, failure_count: 1`, `start_retry_policy: FAST_RETRY`). +* **Protocol Flow**: + 1. First `start` POST request receives `400` with no upload status header. + 2. `START_PREDICATE` identifies the missing status header on start as retriable (for non-fatal status codes) and retries the initiation request. + 3. Second attempt succeeds (`200 active`). + 4. 100-byte payload is transmitted and finalized. +* **Assertions**: + * Returned JSON body reports `"size" == 100`. + * `phases.count(:initiating) == 1`. + * `phases.last == :completed`. + +#### Case 3. Retry exhaustion and session deadline on start (`test_retry_exhaustion_on_start_times_out`) +* **Scenario**: Injects repeated `503 Service Unavailable` responses (`failure_count: 10_000`) with default start retry policy and a 3-second session `timeout` (`scenario: "non_fatal_error_on_start"`). +* **Protocol Flow**: + 1. `start` command encounters continuous 503 errors. + 2. Client retries with default exponential backoff until the 3-second global session deadline expires. + 3. Client terminates failure before entering transmission. +* **Assertions**: + * Raises a `Gapic::Common::Error` (`BadResponseError` or `DeadlineExceededError`). + * Total elapsed time is close to 3 seconds (`2.5s <= elapsed <= 6.0s`, accounting for default backoff delays and network latency). + * `phases` contains no `:uploading` entries (`refute_includes phases, :uploading`). + +#### Case 4. Fatal errors on start (`test_fatal_error_on_start_raises_bad_response_immediately`) +* **Scenario**: Injects fatal HTTP status codes (`403 Forbidden` and `404 Not Found`) on `start` (`scenario: "fatal_error_on_start"`). +* **Protocol Flow**: + 1. Initial `start` request receives a fatal status code (`403` or `404`). + 2. `START_PREDICATE` refutes retry on fatal status codes (`Rules::FATAL_STATUS_CODES`). + 3. `Driver#execute_send_start` returns the fatal response directly to `Core`. + 4. `Rules` classifies the response as `:response_fatal_bad_response` and emits `:fail_with_bad_response`. + 5. Session terminates immediately with `BadResponseError`. +* **Assertions**: + * Raises `Gapic::Rest::ResumableUpload::BadResponseError` with error message containing the HTTP status code. + * Elapsed time is < 0.5s, confirming zero retries were attempted. + * Refutes any `:uploading` phases. + +#### Case 5. Sequential session isolation (`test_sequential_runs_session_isolation`) +* **Scenario**: Executes two sequential upload runs under Case 1 (`non_fatal_error_on_start`, 503, failure_count: 1, `start_retry_policy: FAST_RETRY`) with distinct client UUIDs. +* **Protocol Flow**: + 1. First run executes and succeeds. + 2. Second run executes with a newly generated `client_uuid` and independent progress tracking. +* **Assertions**: + * Both runs successfully complete uploading 100 bytes. + * Demonstrates Showcase session state isolation across sequential client sessions. + +--- + +### 2.5 Resumption Suite (`integration/resumable_upload/resume_test.rb`) + +Tests `Gapic::Rest::ResumableUpload::Session` resumption capabilities against Showcase. + +#### Case 1. Resume in-progress upload on seekable stream (`test_resume_in_progress_upload`) +* Uploads chunk 1 via `raw_upload`, then resumes with a fresh session and full stream. +* Asserts `phases == [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed]` and offsets align correctly. + +#### Case 2. Resume already finalized upload (`test_resume_finalized_upload`) +* Finalizes upload upfront via `raw_upload(finalize: true)`, then attempts to resume. +* Asserts direct completion with `phases == [:initiating, :completed]` and final payload parsed cleanly. + +#### Case 3. Non-fatal errors on query during resume +* **Case 3a (`test_resume_query_503_absorbed_by_retry`)**: 503 on resume query is absorbed transparently by `control_plane_retry_policy`. +* **Case 3b (`test_resume_query_409_triggers_retry_recovery`)**: 409 Category 2 error on query triggers `retry_recovery` re-querying without progress notification (`refute_includes phases, :recovering`). + +#### Case 4. Unseekable stream fast-forward on resume (`test_resume_unseekable_stream_fast_forwards`) +* Resumes using an `UnseekableStream` starting at byte 0. +* Verifies Driver fast-forwards by discarding bytes up to server offset, then uploads remaining chunks. + +#### Case 5. Stream mismatch errors on resume +* **Case 5a (`test_resume_wrong_stream_unseekable_mismatch`)**: Unseekable stream hitting unexpected EOF during discard raises `StreamMismatchError`. +* **Case 5b (`test_resume_wrong_stream_seekable_size_guard`)**: Seekable `StringIO` with `server_offset > stream.size` (and `upload_size: nil`) raises `StreamMismatchError` via stream size guard. + +#### Case 6. Golden user-style resume (`test_golden_user_style_resume_seekable`, `test_golden_user_style_resume_unseekable`) +* User raises exception in `on_progress` carrying `session.resume_handle` on first upload ack. +* Fresh session resumes via `resume_handle: handle` and completes the transfer. Tested on both seekable streams and fresh unseekable streams starting at byte 0. + +#### Case 7. Lifecycle and contract violations (`test_lifecycle_violations`) +* Verifies second `start` and `resume` on bound session raise `SessionStateError`, and bare `resume` raises `ArgumentError`. diff --git a/gapic-common/integration/README.md b/gapic-common/integration/README.md new file mode 100644 index 0000000..d699fd3 --- /dev/null +++ b/gapic-common/integration/README.md @@ -0,0 +1,51 @@ +# Integration Tests + +This directory contains integration tests for `gapic-common`, executed against a running `gapic-showcase` server via the `toys test-integration` command. + +## Running Integration Tests + +```bash +toys test-integration +``` + +You can pass standard Minitest flags to filter or seed test runs: + +```bash +toys test-integration --name /resumable_upload/ --seed 1234 +``` + +## Showcase Server Management & Lifecycle + +The `toys test-integration` command (`.toys/test-integration.rb`) manages the `gapic-showcase` server lifecycle automatically: + +1. **Existing Endpoint (`SHOWCASE_ENDPOINT`)**: + - If `ENV["SHOWCASE_ENDPOINT"]` is present and non-empty, `toys test-integration` skips binary resolution and runs the Minitest suite directly against that endpoint. + +2. **Binary Resolution (`SHOWCASE_BIN` / `PATH`)**: + - When `SHOWCASE_ENDPOINT` is not set, the runner checks `ENV["SHOWCASE_BIN"]` first, then searches `ENV["PATH"]` for an executable `gapic-showcase` binary. + - **Version Check**: The runner executes ` --version` and verifies that the version is at least `0.43`. If the version is older than `0.43`, it logs an error and exits with status `1`. + +3. **Missing Endpoint and Binary**: + - **CI Environment (`ENV["CI"]` set)**: Fails immediately with exit status `1`. + - **Local Environment (`ENV["CI"]` unset)**: Logs an informational message and skips integration tests cleanly (exit status `0`). + +4. **Ephemeral Port Allocation & Polling**: + - Allocates two ephemeral TCP ports on `127.0.0.1` for `--port :` and `--fallback-port :` to avoid port collisions across concurrent runs. + - Spawns `gapic-showcase run --port : --fallback-port :` in a dedicated process group (`pgroup: true`) with `stdout` and `stderr` redirected to a temporary log file in `Dir.tmpdir`. + - Polls `127.0.0.1:` with a 10-second monotonic clock budget while checking `Process.waitpid2` (`WNOHANG`) on each iteration. If the process exits prematurely or fails to accept TCP connections within 10 seconds, the runner prints the path to the log file and exits with status `1`. + +5. **Execution & Teardown**: + - Sets `ENV["SHOWCASE_ENDPOINT"] = "http://localhost:#{port}"` and runs the Minitest suite (`integration/**/*_test.rb`). + - An `ensure` block sends `SIGTERM` to the entire process group (`-TERM`) and reaps the child process so no background showcase processes are leaked. + +## Logging and Diagnostics + +Each integration test captures `DEBUG`-level client and driver logs into an in-memory buffer during execution: + +- **Automatic Failure Dump**: If a test fails or raises an unhandled exception, the captured trace is automatically dumped to `stderr` during `teardown`. +- **Force Log Dump (`SHOWCASE_LOG`)**: Set `SHOWCASE_LOG=1` (or any non-empty value) to dump the captured trace for all executed tests, including passing ones: + +```bash +SHOWCASE_LOG=1 toys test-integration +``` + diff --git a/gapic-common/integration/integration_helper.rb b/gapic-common/integration/integration_helper.rb new file mode 100644 index 0000000..0af5c0b --- /dev/null +++ b/gapic-common/integration/integration_helper.rb @@ -0,0 +1,204 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "json" +require "logger" +require "securerandom" +require "stringio" +require "minitest/autorun" +require "minitest/focus" +require "minitest/mock" + +require "gapic/common" +require "gapic/rest" +require "gapic/rest/resumable_upload" + +## +# Base class for Showcase integration tests. +# +class ShowcaseIntegrationTest < Minitest::Test + UPLOAD_PATH = "/resumable/upload/v1beta1/files:upload" + FAST_RETRY = { initial_delay: 0.01, max_delay: 0.05, multiplier: 1, timeout: 2 }.freeze + DEFAULT_CHUNK_SIZE = 262_144 + DEFAULT_PAYLOAD_SIZE = DEFAULT_CHUNK_SIZE * 3 + + ## + # Stream double that intentionally does not implement #seek. + # + class UnseekableStream + def initialize data + @io = StringIO.new data + end + + def read length = nil + @io.read length + end + + def pos + @io.pos + end + end + + attr_reader :logger + attr_reader :progress_records + + def phases + @progress_records.map(&:phase) + end + + def offsets + @progress_records.map(&:bytes_uploaded) + end + + def showcase_endpoint + ENV["SHOWCASE_ENDPOINT"] + end + + def setup + skip "SHOWCASE_ENDPOINT is not set" if showcase_endpoint.to_s.empty? + @log_output = StringIO.new + @logger = Logger.new @log_output, level: Logger::DEBUG + super + end + + def teardown + if (!passed? || !ENV["SHOWCASE_LOG"].to_s.empty?) && @log_output && !@log_output.string.empty? + warn "\n--- Captured trace for #{name} ---\n#{@log_output.string}--- End trace ---\n" + end + super + end + + def payload size + pattern = "0123456789".b + (pattern * ((size / pattern.bytesize) + 1)).byteslice 0, size + end + + def showcase_client_stub + Gapic::Rest::ClientStub.new( + endpoint: showcase_endpoint, + credentials: :dummy_credentials, + raise_faraday_errors: false, + logger: @logger + ) + end + + def build_config scenario: nil, scenario_config: {}, **overrides + @progress_records = [] + headers = (overrides[:initial_headers] || {}).dup + if scenario + headers["X-Goog-Test-Scenario"] = scenario + headers["X-Goog-Test-Scenario-Config"] = JSON.generate( + { "client_uuid" => SecureRandom.uuid }.merge(scenario_config) + ) + end + + defaults = { + initial_url: UPLOAD_PATH, + initial_headers: headers, + start_retry_policy: FAST_RETRY, + control_plane_retry_policy: FAST_RETRY, + data_plane_retry_policy: FAST_RETRY, + timeout: 10, + chunk_size: DEFAULT_CHUNK_SIZE, + on_progress: ->(progress) { @progress_records << progress } + } + unless overrides.key? :stream + defaults[:stream] = StringIO.new payload(DEFAULT_PAYLOAD_SIZE) + defaults[:upload_size] = DEFAULT_PAYLOAD_SIZE + end + + Gapic::Rest::ResumableUpload::StartUploadConfig.new(**defaults, **overrides, initial_headers: headers) + end + + START_ONLY_KEYS = [:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy].freeze + + # Builds a session from the shared arguments and remembers the per-run arguments that #start needs, + # so callers can run it with `start_session session`. + def build_session scenario: nil, scenario_config: {}, **overrides + @progress_records = [] + headers = (overrides.delete(:initial_headers) || {}).dup + if scenario + headers["X-Goog-Test-Scenario"] = scenario + headers["X-Goog-Test-Scenario-Config"] = JSON.generate( + { "client_uuid" => SecureRandom.uuid }.merge(scenario_config) + ) + end + + @start_args = { + initial_url: UPLOAD_PATH, + initial_headers: headers, + start_retry_policy: FAST_RETRY, + chunk_size: DEFAULT_CHUNK_SIZE + }.merge(overrides.slice(*START_ONLY_KEYS)) + + defaults = { + client_stub: showcase_client_stub, + control_plane_retry_policy: FAST_RETRY, + data_plane_retry_policy: FAST_RETRY, + timeout: 10, + on_progress: ->(progress) { @progress_records << progress }, + logger: @logger + } + unless overrides.key? :stream + defaults[:stream] = StringIO.new payload(DEFAULT_PAYLOAD_SIZE) + defaults[:upload_size] = DEFAULT_PAYLOAD_SIZE + end + + Gapic::Rest::ResumableUpload::Session.new(**defaults, **overrides.except(*START_ONLY_KEYS)) + end + + def start_session session, **overrides + session.start(**@start_args, **overrides) + end + + def raw_start scenario: nil, scenario_config: {}, upload_size: nil, headers: {} + req_headers = { + "X-Goog-Upload-Protocol" => "resumable", + "X-Goog-Upload-Command" => "start" + } + req_headers["X-Goog-Upload-Header-Content-Length"] = upload_size.to_s if upload_size + if scenario + req_headers["X-Goog-Test-Scenario"] = scenario + req_headers["X-Goog-Test-Scenario-Config"] = JSON.generate( + { "client_uuid" => SecureRandom.uuid }.merge(scenario_config) + ) + end + req_headers.merge! headers + + response = showcase_client_stub.make_post_request( + uri: UPLOAD_PATH, + body: nil, + options: { metadata: req_headers } + ) + Gapic::Rest::ResumableUpload::Rules.header_value response.headers, "x-goog-upload-url" + end + + def raw_upload upload_url:, offset:, bytes:, finalize: false, headers: {} + req_headers = { + "X-Goog-Upload-Command" => finalize ? "upload, finalize" : "upload", + "X-Goog-Upload-Offset" => offset.to_s, + "Content-Type" => "application/octet-stream", + "Content-Length" => bytes.bytesize.to_s + } + req_headers.merge! headers + + showcase_client_stub.make_post_request( + uri: upload_url, + body: bytes, + options: { metadata: req_headers } + ) + end +end diff --git a/gapic-common/integration/resumable_upload/chunk_granularity_test.rb b/gapic-common/integration/resumable_upload/chunk_granularity_test.rb new file mode 100644 index 0000000..ea05965 --- /dev/null +++ b/gapic-common/integration/resumable_upload/chunk_granularity_test.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "integration_helper" +require "json" +require "stringio" + +## +# Integration tests for chunk granularity alignment against Showcase. +# +class ChunkGranularityTest < ShowcaseIntegrationTest + # Verifies chunk size alignment to server-specified granularity (300_000 -> 299_776) and progress notifications. + def test_chunk_granularity_alignment + size = 1_000_000 + config = build_config( + scenario: "chunk_granularity", + stream: StringIO.new(payload(size)), + upload_size: size, + chunk_size: 300_000, + timeout: 5 + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal size, parsed["size"] + assert_equal [0, 0, 299_776, 599_552, 899_328, 899_328, 1_000_000], offsets + assert_equal [:initiating, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed], phases + end +end diff --git a/gapic-common/integration/resumable_upload/error_on_start_test.rb b/gapic-common/integration/resumable_upload/error_on_start_test.rb new file mode 100644 index 0000000..f43d23d --- /dev/null +++ b/gapic-common/integration/resumable_upload/error_on_start_test.rb @@ -0,0 +1,159 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "integration_helper" +require "json" +require "securerandom" +require "stringio" + +## +# Suite A: Integration tests for non-fatal and fatal errors on session initiation (`start`). +# +class ErrorOnStartTest < ShowcaseIntegrationTest + PAYLOAD_SIZE = 100 + + def build_start_error_config scenario:, scenario_config: {}, start_retry_policy: nil, **overrides + build_config( + scenario: scenario, + scenario_config: scenario_config, + start_retry_policy: start_retry_policy, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + stream: StringIO.new(payload(PAYLOAD_SIZE)), + upload_size: PAYLOAD_SIZE, + **overrides + ) + end + + # A1. Verifies non-fatal transient error (503) on start is retried and upload completes. + def test_non_fatal_error_on_start_503 + config = build_start_error_config( + scenario: "non_fatal_error_on_start", + scenario_config: { error_code: 503, failure_count: 1 }, + start_retry_policy: FAST_RETRY + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal PAYLOAD_SIZE, parsed["size"] + assert_equal 1, phases.count(:initiating) + assert_equal :completed, phases.last + end + + # A2. Verifies missing status header / 400 on start is retried and upload completes. + def test_missing_header_retriable_on_start_400 + config = build_start_error_config( + scenario: "non_fatal_error_on_start", + scenario_config: { error_code: 400, failure_count: 1 }, + start_retry_policy: FAST_RETRY + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal PAYLOAD_SIZE, parsed["size"] + assert_equal 1, phases.count(:initiating) + assert_equal :completed, phases.last + end + + # A3. Verifies retry exhaustion on start with high failure count times out within ~3s without uploading. + def test_retry_exhaustion_on_start_times_out + config = build_start_error_config( + scenario: "non_fatal_error_on_start", + scenario_config: { error_code: 503, failure_count: 10_000 }, + timeout: 3 + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + t0 = Process.clock_gettime Process::CLOCK_MONOTONIC + err = assert_raises Gapic::Common::Error do + driver.run + end + t1 = Process.clock_gettime Process::CLOCK_MONOTONIC + + elapsed = t1 - t0 + assert_operator elapsed, :>=, 2.5 + assert_operator elapsed, :<=, 6.0 + is_expected_error = err.is_a?(Gapic::Rest::ResumableUpload::BadResponseError) || + err.is_a?(Gapic::Rest::ResumableUpload::DeadlineExceededError) + assert is_expected_error, "Expected BadResponseError or DeadlineExceededError, got #{err.class}" + refute_includes phases, :uploading + end + + # A4. Verifies fatal errors on start (403 and 404) immediately raise BadResponseError in < 0.5s without retrying. + def test_fatal_error_on_start_raises_bad_response_immediately + [403, 404].each do |code| + config = build_start_error_config( + scenario: "fatal_error_on_start", + scenario_config: { error_code: code } + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + t0 = Process.clock_gettime Process::CLOCK_MONOTONIC + err = assert_raises Gapic::Rest::ResumableUpload::BadResponseError do + driver.run + end + t1 = Process.clock_gettime Process::CLOCK_MONOTONIC + + elapsed = t1 - t0 + assert_operator elapsed, :<, 0.5, "Expected failure in < 0.5s for HTTP #{code}, took #{elapsed}s" + assert_match(/#{code}/, err.message) + refute_includes phases, :uploading + end + end + + # A5. Verifies sequential executions with fresh client UUIDs remain isolated. + def test_sequential_runs_session_isolation + 2.times do + config = build_start_error_config( + scenario: "non_fatal_error_on_start", + scenario_config: { error_code: 503, failure_count: 1 }, + start_retry_policy: FAST_RETRY + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal PAYLOAD_SIZE, parsed["size"] + assert_equal 1, phases.count(:initiating) + assert_equal :completed, phases.last + end + end +end diff --git a/gapic-common/integration/resumable_upload/error_recovery_test.rb b/gapic-common/integration/resumable_upload/error_recovery_test.rb new file mode 100644 index 0000000..fd0f948 --- /dev/null +++ b/gapic-common/integration/resumable_upload/error_recovery_test.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "integration_helper" +require "json" +require "stringio" + +## +# Suite B: Integration tests for Category 1 transient retries and Category 2 error recovery against Showcase. +# +class ErrorRecoveryTest < ShowcaseIntegrationTest + SCENARIO = "non_fatal_error_on_chunk_upload" + + # B1. Verifies Category 1 transient error (503) is retried transparently by FAST_RETRY without entering recovery. + def test_cat1_error_retried_transparently + config = build_config( + scenario: SCENARIO, + scenario_config: { error_code: 503, failure_count: 1, after_offset: 0 } + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + refute_includes phases, :recovering + assert_equal [:initiating, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed], phases + assert_equal [0, 0, 262_144, 524_288, 786_432, 786_432, 786_432], offsets + end + + # B2. Verifies simple Category 2 error (409) at offset 0 triggers recovery query and resumes upload to completion. + def test_simple_cat2_error_recovery + config = build_config( + scenario: SCENARIO, + scenario_config: { error_code: 409, failure_count: 1, after_offset: 0 } + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal( + [:initiating, :uploading, :recovering, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed], + phases + ) + assert_equal [0, 0, 0, 0, 262_144, 524_288, 786_432, 786_432, 786_432], offsets + end + + # B3. Verifies two consecutive Category 2 recoveries (409) on chunk 2 at offset 262_144. + def test_two_consecutive_cat2_recoveries_on_chunk_2 + config = build_config( + scenario: SCENARIO, + scenario_config: { error_code: 409, failure_count: 2, after_offset: 262_144 } + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal( + [ + :initiating, :uploading, :uploading, + :recovering, :uploading, + :recovering, :uploading, + :uploading, :uploading, :finalizing, :completed + ], + phases + ) + assert_equal [0, 0, 262_144, 262_144, 262_144, 262_144, 262_144, 524_288, 786_432, 786_432, 786_432], offsets + end + + # B4. Verifies Category 2 error (409) on the finalizing chunk (upload, finalize) recovers and completes. + def test_cat2_failure_on_finalizing_chunk + size = (DEFAULT_CHUNK_SIZE * 3) - 100 + config = build_config( + scenario: SCENARIO, + scenario_config: { error_code: 409, failure_count: 1, after_offset: DEFAULT_CHUNK_SIZE * 2 }, + stream: StringIO.new(payload(size)), + upload_size: size + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal size, parsed["size"] + assert_equal( + [:initiating, :uploading, :uploading, :uploading, :finalizing, :recovering, :uploading, :finalizing, :completed], + phases + ) + assert_equal [0, 0, 262_144, 524_288, 524_288, 524_288, 524_288, 524_288, size], offsets + end + + # B5. Verifies unrecoverable 500 without status header triggers repeated recovery until DeadlineExceededError. + def test_no_headers_failure_recovers_until_deadline_exceeded + config = build_config( + scenario: SCENARIO, + scenario_config: { failure_count: 0, action_after_failures: "terminate" }, + timeout: 1 + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + assert_raises Gapic::Rest::ResumableUpload::DeadlineExceededError do + driver.run + end + + assert_operator phases.count(:recovering), :>=, 2 + end +end diff --git a/gapic-common/integration/resumable_upload/golden_path_test.rb b/gapic-common/integration/resumable_upload/golden_path_test.rb new file mode 100644 index 0000000..d40c31d --- /dev/null +++ b/gapic-common/integration/resumable_upload/golden_path_test.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "integration_helper" +require "json" +require "stringio" + +## +# Golden path integration tests for ResumableUpload Driver against Showcase. +# +class GoldenPathTest < ShowcaseIntegrationTest + def test_multi_chunk_known_size + size = 1_500_000 + chunk_size = 524_288 + stream = StringIO.new payload(size) + + config = build_config( + stream: stream, + upload_size: size, + chunk_size: chunk_size + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal size, parsed["size"] + assert_equal [ + Gapic::Rest::ResumableUpload::Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 1_500_000), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: 1_500_000), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 524_288, total_bytes: 1_500_000), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 1_048_576, total_bytes: 1_500_000), + Gapic::Rest::ResumableUpload::Progress.new(phase: :finalizing, bytes_uploaded: 1_048_576, total_bytes: 1_500_000), + Gapic::Rest::ResumableUpload::Progress.new(phase: :completed, bytes_uploaded: 1_500_000, total_bytes: 1_500_000) + ], progress_records + end + + def test_small_upload_default_chunk_size + size = 100_000 + stream = StringIO.new payload(size) + + config = build_config( + stream: stream, + upload_size: size, + chunk_size: nil # use default chunk size + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal size, parsed["size"] + assert_equal [ + Gapic::Rest::ResumableUpload::Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: size), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: size), + Gapic::Rest::ResumableUpload::Progress.new(phase: :finalizing, bytes_uploaded: 0, total_bytes: size), + Gapic::Rest::ResumableUpload::Progress.new(phase: :completed, bytes_uploaded: size, total_bytes: size) + ], progress_records + end + + def test_standalone_finalize_unseekable_stream + chunk_size = 262_144 + size = 3 * chunk_size + stream = UnseekableStream.new payload(size) + + config = build_config( + stream: stream, + chunk_size: chunk_size + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal size, parsed["size"] + assert_equal [ + Gapic::Rest::ResumableUpload::Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 262_144, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 524_288, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 786_432, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :finalizing, bytes_uploaded: 786_432, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :completed, bytes_uploaded: 786_432, total_bytes: 786_432) + ], progress_records + end +end diff --git a/gapic-common/integration/resumable_upload/resume_test.rb b/gapic-common/integration/resumable_upload/resume_test.rb new file mode 100644 index 0000000..5c4dc5f --- /dev/null +++ b/gapic-common/integration/resumable_upload/resume_test.rb @@ -0,0 +1,229 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "integration_helper" +require "json" +require "stringio" + +## +# Suite D: Integration tests for Resumable Upload Session resumption against Showcase. +# +class ResumeTest < ShowcaseIntegrationTest + ## + # Custom error to simulate a user aborting an in-progress transfer from inside on_progress. + # + class UserPauseError < StandardError + attr_reader :resume_handle + + def initialize message, resume_handle + super message + @resume_handle = resume_handle + end + end + + # D1. Resume an in-progress upload on a seekable stream. + def test_resume_in_progress_upload + upload_url = raw_start upload_size: DEFAULT_PAYLOAD_SIZE + chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false + + session = build_session + result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed], phases + assert_equal [0, 262_144, 524_288, 786_432, 786_432, 786_432], offsets + end + + # D2. Resuming an already-finalized upload terminates cleanly and returns final body. + def test_resume_finalized_upload + upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: true + + session = build_session stream: StringIO.new(payload(DEFAULT_CHUNK_SIZE)), upload_size: DEFAULT_CHUNK_SIZE + result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + parsed = JSON.parse result + + assert_equal DEFAULT_CHUNK_SIZE, parsed["size"] + assert_equal [:initiating, :completed], phases + end + + # D3a. Non-fatal 503 error on query during resume is absorbed by control plane retry policy. + def test_resume_query_503_absorbed_by_retry + upload_url = raw_start( + scenario: "non_fatal_error_on_query", + scenario_config: { error_code: 503, failure_count: 1 }, + upload_size: DEFAULT_PAYLOAD_SIZE + ) + chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false + + session = build_session + result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed], phases + refute_includes @log_output.string, "retry_recovery" + end + + # D3b. Non-fatal 409 error on query during resume triggers protocol retry_recovery without progress notification. + def test_resume_query_409_triggers_retry_recovery + upload_url = raw_start( + scenario: "non_fatal_error_on_query", + scenario_config: { error_code: 409, failure_count: 1 }, + upload_size: DEFAULT_PAYLOAD_SIZE + ) + chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false + + session = build_session + result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed], phases + refute_includes phases, :recovering + assert_includes @log_output.string, "retry_recovery" + end + + # D4. Resume fast-forwards by discarding bytes on an unseekable stream starting at byte 0. + def test_resume_unseekable_stream_fast_forwards + upload_url = raw_start upload_size: DEFAULT_PAYLOAD_SIZE + chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false + + stream = UnseekableStream.new payload(DEFAULT_PAYLOAD_SIZE) + session = build_session stream: stream + result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed], phases + assert_equal [0, 262_144, 524_288, 786_432, 786_432, 786_432], offsets + end + + # D5a. Resume with unseekable stream shorter than acknowledged server offset raises StreamMismatchError. + def test_resume_wrong_stream_unseekable_mismatch + upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: false + + stream = UnseekableStream.new payload(100) + session = build_session stream: stream, upload_size: nil + + assert_raises Gapic::Rest::ResumableUpload::StreamMismatchError do + session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + end + refute_includes phases, :finalizing + end + + # D5b. Resume with seekable stream shorter than server offset raises StreamMismatchError via stream.size guard. + def test_resume_wrong_stream_seekable_size_guard + upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: false + + stream = StringIO.new payload(100) + session = build_session stream: stream, upload_size: nil + + assert_raises Gapic::Rest::ResumableUpload::StreamMismatchError do + session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + end + refute_includes phases, :finalizing + end + + # D6a. Golden user-style resume on a seekable stream after user abort in on_progress. + def test_golden_user_style_resume_seekable + stream = StringIO.new payload(DEFAULT_PAYLOAD_SIZE) + session1 = nil + on_progress = lambda do |progress| + if progress.phase == :uploading && progress.bytes_uploaded == DEFAULT_CHUNK_SIZE + raise UserPauseError.new("user paused", session1.resume_handle) + end + end + + session1 = build_session stream: stream, on_progress: on_progress + err = assert_raises UserPauseError do + start_session session1 + end + + assert session1.bound? + assert session1.resumable? + handle = err.resume_handle + refute_nil handle + + stream.rewind + session2 = build_session stream: stream + result = session2.resume resume_handle: handle + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert session2.bound? + refute session2.resumable? + end + + # D6b. Golden user-style resume with a fresh unseekable stream starting at byte 0. + def test_golden_user_style_resume_unseekable + session1 = nil + on_progress = lambda do |progress| + if progress.phase == :uploading && progress.bytes_uploaded == DEFAULT_CHUNK_SIZE + raise UserPauseError.new("user paused", session1.resume_handle) + end + end + + session1 = build_session stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)), on_progress: on_progress + err = assert_raises UserPauseError do + start_session session1 + end + + assert session1.bound? + assert session1.resumable? + handle = err.resume_handle + refute_nil handle + + session2 = build_session stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)) + result = session2.resume resume_handle: handle + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert session2.bound? + refute session2.resumable? + end + + # D7. Lifecycle and contract violations on session runs. + def test_lifecycle_violations + session = build_session stream: StringIO.new(payload(100)), upload_size: 100 + start_session session + + assert session.bound? + + # Second start on executed session raises SessionStateError + assert_raises Gapic::Rest::ResumableUpload::SessionStateError do + start_session session + end + + # Resume on already bound/executed session raises SessionStateError + assert_raises Gapic::Rest::ResumableUpload::SessionStateError do + session.resume upload_url: "https://example.com/test", chunk_size: DEFAULT_CHUNK_SIZE + end + + # Resume without parameters on fresh session raises ArgumentError + fresh_session = build_session + assert_raises ArgumentError do + fresh_session.resume + end + end +end diff --git a/gapic-common/lib/gapic/logging_concerns.rb b/gapic-common/lib/gapic/logging_concerns.rb index 78ae13d..8be096f 100644 --- a/gapic-common/lib/gapic/logging_concerns.rb +++ b/gapic-common/lib/gapic/logging_concerns.rb @@ -67,6 +67,10 @@ def debug(&) log(Logger::DEBUG, &) end + def warn(&) + log(Logger::WARN, &) + end + ## # @private # Builder for a log entry, passed to {StubLogger#log}. diff --git a/gapic-common/lib/gapic/rest.rb b/gapic-common/lib/gapic/rest.rb index ae4ccbb..693aba2 100644 --- a/gapic-common/lib/gapic/rest.rb +++ b/gapic-common/lib/gapic/rest.rb @@ -28,6 +28,7 @@ require "gapic/rest/http_binding_override_configuration" require "gapic/rest/operation" require "gapic/rest/paged_enumerable" +require "gapic/rest/resumable_upload" require "gapic/rest/server_stream" require "gapic/rest/threaded_enumerator" require "gapic/rest/transport_operation" diff --git a/gapic-common/lib/gapic/rest/client_stub.rb b/gapic-common/lib/gapic/rest/client_stub.rb index 0ec818c..1fd05b4 100644 --- a/gapic-common/lib/gapic/rest/client_stub.rb +++ b/gapic-common/lib/gapic/rest/client_stub.rb @@ -287,17 +287,27 @@ def log_request method_name, request_id, try_number, body, metadata entry.set "requestId", request_id entry.message = "Sending request to #{entry.service}.#{method_name} (try #{try_number})" end - body = body.to_s + body_str = body.to_s metadata = metadata.to_h rescue {} - return if body.empty? && metadata.empty? + return if body_str.empty? && metadata.empty? stub_logger.debug do |entry| entry.set "requestId", request_id - entry.set "request", body + entry.set "request", abridge_request_body(body_str) entry.set "headers", metadata entry.message = "(request payload as JSON)" end end + def abridge_request_body body_str + utf8_body = body_str.dup.force_encoding Encoding::UTF_8 + if body_str.bytesize > 1024 || !utf8_body.valid_encoding? + prefix_hex = body_str.byteslice(0, 32).unpack1 "H*" + "<#{body_str.bytesize} bytes, first 32: #{prefix_hex}>" + else + utf8_body + end + end + def log_response method_name, request_id, try_number, response, is_server_streaming return unless stub_logger&.enabled? stub_logger.info do |entry| diff --git a/gapic-common/lib/gapic/rest/error.rb b/gapic-common/lib/gapic/rest/error.rb index 910604b..278ba66 100644 --- a/gapic-common/lib/gapic/rest/error.rb +++ b/gapic-common/lib/gapic/rest/error.rb @@ -22,6 +22,9 @@ module Gapic module Rest # Gapic REST exception class class Error < ::Gapic::Common::Error + # @private + REST_ERROR_PREFIX = "An error has occurred when making a REST request".freeze + # @return [Integer, nil] the http status code for the error attr_reader :status_code # @return [Object, nil] the text representation of status as parsed from the response body @@ -79,7 +82,7 @@ def parse_faraday_error err if err.response_body msg, code, status, details = try_parse_from_body err.response_body - message = "An error has occurred when making a REST request: #{msg}" unless msg.nil? + message = "#{REST_ERROR_PREFIX}: #{msg}" unless msg.nil? status_code = code unless code.nil? end diff --git a/gapic-common/lib/gapic/rest/resumable_upload.rb b/gapic-common/lib/gapic/rest/resumable_upload.rb new file mode 100644 index 0000000..a7530e1 --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/rest/resumable_upload/errors" +require "gapic/rest/resumable_upload/data_types" +require "gapic/rest/resumable_upload/events" +require "gapic/rest/resumable_upload/instructions" +require "gapic/rest/resumable_upload/retry_policies" +require "gapic/rest/resumable_upload/driver/abridge" +require "gapic/rest/resumable_upload/driver/upload_log" +require "gapic/rest/resumable_upload/rules" +require "gapic/rest/resumable_upload/core" +require "gapic/rest/resumable_upload/driver" +require "gapic/rest/resumable_upload/session" + +module Gapic + module Rest + ## + # Resumable Upload Protocol implementation for REST transport. + # + # {Session} is the primary public entry point for initiating and resuming uploads. + # It manages session initiation, chunked streaming, automatic retries, progress + # callbacks via {Progress}, and cross-session resumption via {ResumeHandle}. + # + # ### Error Types + # * {RequestFailedError} - Transport connection failure, timeout, or retries exhausted (includes {HasResumeHandle}). + # * {DeadlineExceededError} - Global upload timeout exceeded (includes {HasResumeHandle}). + # * {BadResponseError} - Unexpected or malformed HTTP response (includes {HasResumeHandle}). + # * {UnseekableStreamError} - Stream rewinding required on an unseekable stream (includes {HasResumeHandle}). + # * {StreamMismatchError} - Stream content or length does not match resumed upload (includes {HasResumeHandle}). + # * {InvalidTransitionError} - Unmatched event for the current protocol state (includes {HasResumeHandle}). + # * {UploadRejectedError} - Server explicitly rejected the upload session (final). + # * {SessionStateError} - Session lifecycle rule violation, e.g., calling `#start` twice (final). + # + # @example Initiating an upload, rescuing an error, and resuming from a fresh session + # session = Gapic::Rest::ResumableUpload::Session.new( + # client_stub: client_stub, + # stream: stream + # ) + # + # begin + # response = session.start( + # initial_url: "https://example.googleapis.com/resumable/upload/v1/example/upload:new" + # ) + # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e + # handle = e.resume_handle + # raise unless handle + # + # stream.rewind + # resumed_session = Gapic::Rest::ResumableUpload::Session.new( + # client_stub: client_stub, + # stream: stream + # ) + # response = resumed_session.resume resume_handle: handle + # end + # + module ResumableUpload + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/core.rb b/gapic-common/lib/gapic/rest/resumable_upload/core.rb new file mode 100644 index 0000000..24b010a --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/core.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/rest/resumable_upload/data_types" +require "gapic/rest/resumable_upload/rules" + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # State machine container holding the immutable State snapshot. + # Contains zero protocol branching logic and zero side-effects. + # + # The middle tier of the three-tier design: `Driver` executes side effects, {Rules} decides transitions, + # and Core holds the {State} between the two. See {Rules} for the protocol narrative and state graph, and + # `design/resumable_upload/implementation-guide.md` section 1 for the tier boundaries. + # + class Core + # @private + # @return [State] Current immutable state snapshot + attr_reader :state + + # @private + # @return [Decision, nil] Decision emitted during the last dispatch + attr_reader :last_decision + + ## + # @private + # Initializes a Core state machine container. + # + # @param config [StartUploadConfig, ResumeUploadConfig] Upload session configuration + # + def initialize config + @config = config + @last_decision = nil + @state = State.new( + status: :initializing, + upload_url: nil, + offset: 0, + chunk_size: config.chunk_size || Rules::DEFAULT_CHUNK_SIZE, + chunk_granularity: nil, + in_flight_length: 0, + last_error: nil + ) + end + + ## + # @private + # Dispatches event to Rules and updates internal state snapshot. + # + # @param event [Object] Input event + # @return [Array] Driver instructions + # + def dispatch event + decision = Rules.decide @state, event, @config + @state = decision.next_state + @last_decision = decision + decision.instructions + end + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb b/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb new file mode 100644 index 0000000..d08c00d --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb @@ -0,0 +1,415 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Gapic + module Rest + # rubocop:disable Metrics/ModuleLength + module ResumableUpload + ## + # @private + # Configuration members shared by {StartUploadConfig} and {ResumeUploadConfig}, in the order both + # definitions splat them. + # + # The two config types are deliberately *flat*: {Core}, {Rules} and {Driver} read every member + # straight off `config`. Nesting the shared members inside a common object would turn every + # `config.upload_size` into `config.common.upload_size` at some thirty call sites for no behavioural + # gain, so they are spliced into each `Data.define` instead. + # + # * `stream` [IO] Binary input stream to upload. Required. + # * `upload_size` [Integer, nil] Total upload bytes if known upfront. + # * `content_type` [String, nil] MIME type of uploaded media. + # * `timeout` [Numeric, nil] Total upload timeout in seconds (zero or negative is treated as nil). + # * `control_plane_retry_policy` [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for control + # commands (query, cancel). + # * `data_plane_retry_policy` [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for data commands + # (upload, finalize). + # * `on_progress` [Proc, nil] Callback invoked as `->(progress)` with a {Progress} instance. + # + # Every retry policy member, here and in the per-run configs, follows the same convention: a + # {Gapic::Common::RetryPolicy} replaces the default policy outright, while a Hash overrides only the + # settings it names and leaves the remaining defaults — including retry codes and predicates — in place. + # + COMMON_MEMBERS = [ + :stream, + :upload_size, + :content_type, + :timeout, + :control_plane_retry_policy, + :data_plane_retry_policy, + :on_progress + ].freeze + + ## + # @private + # Header names a caller may not use in `initial_headers`, lowercased for comparison. + # + # These five headers are protocol machinery the driver owns: the protocol identifier, the command + # verb, the byte offset, and the content descriptors derived from `content_type` and `upload_size`. + # `x-goog-upload-offset` is included for completeness even though initiation never sets an offset: + # supplying an offset at initiation is meaningless and indicates a confused caller. Pass-through + # headers such as `X-Goog-Upload-Header-Content-Disposition` remain permitted. + # + # See `Driver#start_headers`, which builds the initiation headers this list protects. + # + # @return [Array] + RESERVED_INITIAL_HEADERS = [ + "x-goog-upload-protocol", + "x-goog-upload-command", + "x-goog-upload-offset", + "x-goog-upload-header-content-type", + "x-goog-upload-header-content-length" + ].freeze + + ## + # @private + # Immutable configuration for a run that initiates a new upload session, i.e. {Session#start}. + # + # Carries {COMMON_MEMBERS} plus the members only an initiating run uses. + # + # @!attribute [r] initial_url + # @return [String] Initial endpoint URI for session initiation + # @!attribute [r] initial_body + # @return [String, nil] Request payload for session initiation + # @!attribute [r] initial_headers + # @return [Hash] Additional headers for initiation, merged over the driver's + # own headers. Keys in {RESERVED_INITIAL_HEADERS} are rejected in any casing; use + # `content_type` and `upload_size` to shape those. + # @!attribute [r] chunk_size + # @return [Integer, nil] Requested chunk size in bytes, aligned to the granularity the server + # reports during initiation. A resumed run takes its chunk size from {ResumeUploadConfig}. + # @!attribute [r] start_retry_policy + # @return [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for session initiation + # + StartUploadConfig = Data.define( + *COMMON_MEMBERS, + :initial_url, + :initial_body, + :initial_headers, + :chunk_size, + :start_retry_policy + ) do + ## + # @private + # Initializes a new upload configuration. + # + # @param initial_url [String] Initial endpoint URI for session initiation + # @param stream [IO] Binary input stream to upload + # @param initial_body [String, nil] Request payload for session initiation + # @param initial_headers [Hash] Additional headers for initiation. Keys in + # {RESERVED_INITIAL_HEADERS} are rejected in any casing; use `content_type` and `upload_size` + # to shape those. + # @param upload_size [Integer, nil] Total upload bytes if known upfront + # @param chunk_size [Integer, nil] Requested chunk size in bytes + # @param content_type [String, nil] MIME type of uploaded media + # @param timeout [Numeric, nil] Total upload timeout in seconds (zero/negative values treated as nil) + # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for session initiation + # @param control_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for control commands + # @param data_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for data commands + # @param on_progress [Proc, nil] Callback invoked as `->(progress)` with a {Progress} instance + # @raise [ArgumentError] If required arguments are missing or invalid + # + def initialize initial_url:, + stream:, + initial_body: nil, + initial_headers: {}, + upload_size: nil, + chunk_size: nil, + content_type: nil, + timeout: nil, + start_retry_policy: nil, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + on_progress: nil + raise ArgumentError, "initial_url is required" if initial_url.nil? || initial_url.to_s.strip.empty? + raise ArgumentError, "stream is required" if stream.nil? + reserved = (initial_headers || {}).keys.find do |key| + RESERVED_INITIAL_HEADERS.include? key.to_s.downcase + end + if reserved + raise ArgumentError, + "initial_headers must not set protocol header #{reserved.inspect}; " \ + "use content_type and upload_size instead" + end + + super( + initial_url: initial_url, + initial_body: initial_body, + initial_headers: initial_headers || {}, + stream: stream, + upload_size: upload_size, + chunk_size: chunk_size, + content_type: content_type, + timeout: timeout, + start_retry_policy: start_retry_policy, + control_plane_retry_policy: control_plane_retry_policy, + data_plane_retry_policy: data_plane_retry_policy, + on_progress: on_progress + ) + end + end + + ## + # @private + # Immutable configuration for a run that resumes an existing upload session, i.e. {Session#resume}. + # + # Carries {COMMON_MEMBERS} plus the upload URL and chunk size the earlier run established. There is + # no `start_retry_policy` here: a resumed run issues no initiation request, so the member would + # always be dead. + # + # @!attribute [r] upload_url + # @return [String] Session upload URL returned by the upload backend + # @!attribute [r] chunk_size + # @return [Integer] Explicit chunk size in bytes (must be a positive integer). Server granularity is + # reported only during initiation, which a resumed run skips, so the size is carried forward from + # the earlier run rather than re-negotiated. + # + ResumeUploadConfig = Data.define( + *COMMON_MEMBERS, + :upload_url, + :chunk_size + ) do + ## + # @private + # Initializes a new upload resume configuration. + # + # @param upload_url [String] Session upload URL + # @param chunk_size [Integer] Explicit chunk size in bytes (must be a positive integer) + # @param stream [IO] Binary input stream to upload + # @param upload_size [Integer, nil] Total upload bytes if known upfront + # @param content_type [String, nil] MIME type of uploaded media + # @param timeout [Numeric, nil] Total upload timeout in seconds (zero/negative values treated as nil) + # @param control_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for control commands + # @param data_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for data commands + # @param on_progress [Proc, nil] Callback invoked as `->(progress)` with a {Progress} instance + # @raise [ArgumentError] If required arguments are missing or invalid + # + def initialize upload_url:, + chunk_size:, + stream:, + upload_size: nil, + content_type: nil, + timeout: nil, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + on_progress: nil + raise ArgumentError, "upload_url is required" if upload_url.nil? || upload_url.to_s.strip.empty? + unless chunk_size.is_a?(Integer) && chunk_size.positive? + raise ArgumentError, "chunk_size must be a positive integer" + end + raise ArgumentError, "stream is required" if stream.nil? + + super( + upload_url: upload_url, + chunk_size: chunk_size, + stream: stream, + upload_size: upload_size, + content_type: content_type, + timeout: timeout, + control_plane_retry_policy: control_plane_retry_policy, + data_plane_retry_policy: data_plane_retry_policy, + on_progress: on_progress + ) + end + end + + ## + # Immutable progress snapshot passed to the `on_progress` callback. + # + # The `on_progress` callback runs synchronously on the same thread as the upload protocol + # and must not block. Any exception raised inside the callback aborts the upload session + # and propagates out of {Session#start} or {Session#resume}. + # + # @!attribute [r] phase + # @return [Symbol] Current upload phase, one of {Progress::PHASES} + # @!attribute [r] bytes_uploaded + # @return [Integer] Cumulative bytes acknowledged by the server. Note that this is the + # server-confirmed offset and is not guaranteed to be monotonic — a server rewind during + # recovery can decrease this value. + # @!attribute [r] total_bytes + # @return [Integer, nil] Total upload size in bytes if known, or `nil`. Always set on the + # `:completed` phase — the total is known once the transfer finishes, even when `upload_size` + # was not supplied upfront. + # + Progress = Data.define( + :phase, + :bytes_uploaded, + :total_bytes + ) do + ## + # Initializes a new progress snapshot. + # + # @param phase [Symbol] Current upload phase, one of {Progress::PHASES} + # @param bytes_uploaded [Integer] Cumulative bytes acknowledged by the server + # @param total_bytes [Integer, nil] Total upload size in bytes if known, or nil + # @raise [ArgumentError] If the phase is not one of {Progress::PHASES} + # + def initialize phase:, bytes_uploaded:, total_bytes: nil + # Must use `self.class::` to access constants from the class scope + unless self.class::PHASES.include? phase + raise ArgumentError, "Invalid phase: #{phase.inspect}. Expected one of #{self.class::PHASES.inspect}" + end + + super( + phase: phase, + bytes_uploaded: bytes_uploaded, + total_bytes: total_bytes + ) + end + end + + ## + # Allowed lifecycle phases for an upload session. + # + # A callback observes `:initiating`, `:uploading`, `:recovering`, `:finalizing` and `:completed`. + # `:cancelling` is reserved: cancellation is not exposed on {Session}, so no phase with that value is + # currently emitted. + # + # @return [Array] + Progress::PHASES = [:initiating, :uploading, :recovering, :finalizing, :cancelling, :completed].freeze + + ## + # Immutable handle containing parameters necessary to resume an in-progress upload session. + # These parameters are provided by the server and can be persisted to resume the upload + # at a later time. + # + # @!attribute [r] upload_url + # @return [String] Upload session URL provided by the server + # @!attribute [r] chunk_size + # @return [Integer] Effective chunk size in bytes + # + ResumeHandle = Data.define( + :upload_url, + :chunk_size + ) do + ## + # Initializes a new resume handle. + # + # @param upload_url [String] Upload session URL provided by the server + # @param chunk_size [Integer] Effective chunk size in bytes + # + def initialize upload_url:, chunk_size: + super( + upload_url: upload_url, + chunk_size: chunk_size + ) + end + end + + ## + # @private + # Immutable state snapshot representing the current protocol progression. + # + # @!attribute [r] status + # @return [Symbol] Protocol lifecycle status, one of {Rules::STATUSES} + # @!attribute [r] upload_url + # @return [String, nil] Session upload URL returned by the upload backend + # @!attribute [r] offset + # @return [Integer] Contiguous bytes acknowledged by server + # @!attribute [r] chunk_size + # @return [Integer] Resolved effective chunk size in bytes + # @!attribute [r] chunk_granularity + # @return [Integer, nil] Alignment modulus returned by server + # @!attribute [r] in_flight_length + # @return [Integer] Byte length of in-flight chunk currently being transmitted + # @!attribute [r] last_error + # @return [StandardError, nil] Terminal exception if in an error or rejected status + # + State = Data.define( + :status, + :upload_url, + :offset, + :chunk_size, + :chunk_granularity, + :in_flight_length, + :last_error + ) do + ## + # @private + # Initializes a protocol state snapshot. + # + # @param status [Symbol] Protocol lifecycle status, one of {Rules::STATUSES} + # @param upload_url [String, nil] Session upload URL + # @param offset [Integer] Contiguous bytes acknowledged by server + # @param chunk_size [Integer] Resolved effective chunk size in bytes + # @param chunk_granularity [Integer, nil] Alignment modulus returned by server + # @param in_flight_length [Integer] Byte length of in-flight chunk + # @param last_error [StandardError, nil] Terminal exception + # + def initialize status: :initializing, + upload_url: nil, + offset: 0, + chunk_size: Rules::DEFAULT_CHUNK_SIZE, + chunk_granularity: nil, + in_flight_length: 0, + last_error: nil + super( + status: status, + upload_url: upload_url, + offset: offset, + chunk_size: chunk_size, + chunk_granularity: chunk_granularity, + in_flight_length: in_flight_length, + last_error: last_error + ) + end + end + + ## + # @private + # Immutable decision snapshot emitted by Rules.decide. + # + # @!attribute [r] from_status + # @return [Symbol] The protocol status before the transition + # @!attribute [r] shape + # @return [Symbol] The canonical event shape + # @!attribute [r] recipe + # @return [Symbol] Selected transition recipe method name + # @!attribute [r] next_state + # @return [State] The new protocol state snapshot after transition + # @!attribute [r] instructions + # @return [Array] Emitted instructions for the Driver + # + Decision = Data.define( + :from_status, + :shape, + :recipe, + :next_state, + :instructions + ) do + ## + # @private + # Initializes a decision snapshot. + # + # @param from_status [Symbol] The protocol status before the transition + # @param shape [Symbol] The canonical event shape + # @param recipe [Symbol] Selected transition recipe method name + # @param next_state [State] Resulting protocol state snapshot + # @param instructions [Array] Emitted instructions for the Driver + # + def initialize from_status:, shape:, recipe:, next_state:, instructions: [] + super( + from_status: from_status, + shape: shape, + recipe: recipe, + next_state: next_state, + instructions: instructions + ) + end + end + end + # rubocop:enable Metrics/ModuleLength + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb new file mode 100644 index 0000000..563171d --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb @@ -0,0 +1,780 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "uri" +require "gapic/logging_concerns" +require "gapic/rest/error" +require "gapic/rest/resumable_upload/core" +require "gapic/rest/resumable_upload/data_types" +require "gapic/rest/resumable_upload/errors" +require "gapic/rest/resumable_upload/events" +require "gapic/rest/resumable_upload/instructions" +require "gapic/rest/resumable_upload/retry_policies" +require "gapic/rest/resumable_upload/driver/upload_log" + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # Synchronous execution engine for the Resumable Upload Protocol. + # Coordinates HTTP network operations, stream buffering, monotonic deadlines, + # and delegates state transitions to Core. + # + # The outer tier of the three-tier design. All side effects live here; all protocol decisions live in + # {Rules}, which carries the state graph and the error category taxonomy. Category 1 transient retries + # are absorbed here by `Gapic::Common::RetryPolicy` and never reach {Core}. See + # `design/resumable_upload/implementation-guide.md` section 2.5 for the buffer and stream position + # invariants, and section 6.3 for the deadline model. + # + # rubocop:disable Metrics/ClassLength + class Driver + include Gapic::LoggingConcerns + + ## + # @private + # Minimum assumed upload throughput in bytes per second (1 MB/s). + # @return [Integer] + MIN_ASSUMED_THROUGHPUT = 1_048_576 + + ## + # @private + # Default base timeout in seconds (1 hour). + # @return [Integer] + BASE_TIMEOUT = 3_600 + + # @private + # @return [Core] + attr_reader :core + + ## + # @private + # Returns a {ResumeHandle} representing the current upload session parameters. + # Reading this property mid-run provides a best-effort snapshot of the current session state. + # Completed uploads (`:success`), rejected uploads (`:rejected`), and cancelled uploads + # (`:cancelled`) are finalized and not resumable, returning `nil`. + # + # @return [ResumeHandle, nil] Resume handle if upload URL is established and resumable, or nil + def resume_handle + Rules.resume_handle_from @core.state + end + + ## + # @private + # Returns the raw upload session URL from protocol state, regardless of lifecycle status. + # + # @return [String, nil] Session upload URL if established, or nil + def upload_url + @core.state.upload_url + end + + ## + # @private + # Initializes a new Resumable Upload Driver. + # + # @param client_stub [Gapic::Rest::ClientStub] Underlying REST client stub + # @param config [StartUploadConfig, ResumeUploadConfig] Configuration for this upload session + # @param core [Core, nil] Optional Core state machine (defaults to new Core with config) + # @param logger [Logger, nil] Optional logger override + def initialize client_stub:, config:, core: nil, logger: nil + @client_stub = client_stub + @config = config + @core = core || Core.new(config) + @buffer = "".b + @buffer_start_offset = 0 + + endpoint = client_stub.respond_to?(:endpoint) ? client_stub.endpoint : nil + setup_logging logger: logger || (client_stub.respond_to?(:logger) ? client_stub.logger : nil), + system_name: "gapic-common", + service: "ResumableUpload", + endpoint: endpoint, + client_id: client_stub.object_id + @upload_log = UploadLog.new stub_logger, upload_id: "unstarted" + + # Only an initiating run carries a start policy; a resumed run issues no initiation request. + configured_start_policy = config.is_a?(StartUploadConfig) ? config.start_retry_policy : nil + @start_retry_policy = resolve_retry_policy configured_start_policy, RetryPolicies::START_DEFAULTS + + @control_plane_retry_policy = resolve_retry_policy config.control_plane_retry_policy, + RetryPolicies::CONTROL_PLANE_DEFAULTS + @data_plane_retry_policy = resolve_retry_policy config.data_plane_retry_policy, + RetryPolicies::DATA_PLANE_DEFAULTS + end + + ## + # @private + # Default retry policy for session initiation requests (start). + # + # @return [Gapic::Common::RetryPolicy] + def self.default_start_retry_policy + RetryPolicies.default_start + end + + ## + # @private + # Default retry policy for control plane requests (query, cancel). + # + # @return [Gapic::Common::RetryPolicy] + def self.default_control_plane_retry_policy + RetryPolicies.default_control_plane + end + + ## + # @private + # Default retry policy for data plane requests (upload, finalize). + # + # @return [Gapic::Common::RetryPolicy] + def self.default_data_plane_retry_policy + RetryPolicies.default_data_plane + end + + ## + # @private + # Executes event loop until terminal state. + # Establishes a guaranteed monotonic deadline at the start of execution + # so the upload cannot stall indefinitely. + # + # Enforces the trampoline loop invariant: each dispatched instruction batch + # is validated by {#validate_batch} before any instruction executes, ensuring it + # produces either a single continuation event or terminates the session + # (via {Instruction::TerminateSuccess} or {Instruction::TerminateFailure}). + # Side-effect instructions ({Instruction::NotifyProgress}, + # {Instruction::RealignBuffer}) explicitly return `nil` by construction, + # so only {Instruction::FillBuffer} and `Send*` instructions produce + # continuation events. + # + # @return [String, nil] Final response body + def run + @upload_log = UploadLog.new stub_logger, upload_id: LoggingConcerns.random_uuid4 + @deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + resolve_timeout + pending_event = initial_event + + loop do + instructions = dispatch_event pending_event + + if deadline_exceeded? && !terminal_instructions?(instructions) + instructions = dispatch_event Event::GlobalDeadlineExceeded.new + end + + pending_event, terminal_result = execute_batch instructions + return terminal_result if pending_event.nil? + end + end + + private + + ## + # @private + # Executes an instruction batch and enforces the single-continuation-event invariant. + # + # @param instructions [Array] Emitted instructions + # @return [Array] Tuple of [pending_event, terminal_result] + # + def execute_batch instructions + recipe = @core.last_decision&.recipe + validate_batch instructions, recipe + + pending_event = nil + instructions.each do |instruction| + result = dispatch_instruction instruction + return [nil, result] if instruction.is_a? Instruction::TerminateSuccess + pending_event = result if Instruction::CONTINUATION.any? { |klass| instruction.is_a? klass } + end + + unless pending_event_type? pending_event + raise InternalError, + "Resumable upload internal error: recipe :#{recipe} continuation instruction " \ + "returned #{pending_event.class} instead of an event" + end + [pending_event, nil] + end + + ## + # @private + # Validates that an instruction batch satisfies the trampoline invariant before execution. + # + # @param instructions [Array] Emitted instructions + # @param recipe [Symbol, nil] Recipe symbol from last decision + # @return [void] + # @raise [InternalError] If the batch is malformed or contains an unclassified instruction + # + def validate_batch instructions, recipe + continuation = 0 + terminal = 0 + instructions.each do |instruction| + case instruction + when *Instruction::CONTINUATION then continuation += 1 + when *Instruction::TERMINAL then terminal += 1 + when *Instruction::SIDE_EFFECT then nil + else + raise InternalError, + "Resumable upload internal error: recipe :#{recipe} emitted " \ + "unclassified instruction #{instruction.class}" + end + end + return if continuation + terminal == 1 + + raise InternalError, batch_shape_message(recipe, continuation, terminal) + end + + ## + # @private + # Formats diagnostic error message for a malformed instruction batch. + # + # @param recipe [Symbol, nil] Recipe symbol from last decision + # @param continuation [Integer] Number of continuation instructions + # @param terminal [Integer] Number of terminal instructions + # @return [String] Error message + # + def batch_shape_message recipe, continuation, terminal + reason = if continuation.zero? && terminal.zero? + "produced no continuation event and did not terminate" + elsif continuation > 1 && terminal.zero? + "produced multiple continuation events" + elsif continuation.zero? && terminal > 1 + "produced multiple terminal instructions" + else + "produced both a continuation event and a terminal instruction" + end + "Resumable upload internal error: recipe :#{recipe} #{reason}" + end + + ## + # @private + # Dispatches an event to Core, logging decisions and transitions. + # + # @param event [Object] Input event + # @return [Array] Emitted instructions + # + def dispatch_event event + instructions = begin + @core.dispatch event + rescue InvalidTransitionError => e + @upload_log.unmatched_transition @core.state, event, e + raise + end + @upload_log.decision @core.last_decision + @upload_log.lifecycle @core.last_decision, @config + instructions + end + + ## + # @private + # Checks whether an instruction execution result represents a pending event. + # + # @param obj [Object] Execution result + # @return [Boolean] + # + def pending_event_type? obj + obj.is_a?(Event::ChunkRead) || obj.is_a?(Event::HttpResponse) || + obj.is_a?(Event::RequestFailed) || obj.is_a?(Event::GlobalDeadlineExceeded) + end + + ## + # @private + # Executes an instruction emitted by the state machine. + # + # @param instruction [Object] Instruction to execute + # @return [Object, nil] Resulting event or terminal response + # + def dispatch_instruction instruction + case instruction + when Instruction::NotifyProgress then execute_notify_progress instruction + when Instruction::RealignBuffer then execute_realign_buffer instruction + when Instruction::FillBuffer then execute_fill_buffer instruction + when Instruction::SendStart then execute_send_start instruction + when Instruction::SendChunk then execute_send_chunk instruction + when Instruction::SendFinalize then execute_send_finalize instruction + when Instruction::SendQuery then execute_send_query instruction + when Instruction::SendCancel then execute_send_cancel instruction + when Instruction::TerminateSuccess + instruction.response.body + when Instruction::TerminateFailure then raise instruction.error + end + end + + ## + # @private + # Resolves a configured retry policy or applies defaults. + # + # @param value [Gapic::Common::RetryPolicy, Hash, nil] Configured policy or overrides + # @param defaults [Hash] Default policy configuration + # @return [Gapic::Common::RetryPolicy] + # + def resolve_retry_policy value, defaults + case value + when Gapic::Common::RetryPolicy + value + when Hash + Gapic::Common::RetryPolicy.new(**value).apply_defaults(defaults) + when nil + Gapic::Common::RetryPolicy.new(**defaults) + else + raise ArgumentError, "Expected RetryPolicy, Hash, or nil, got #{value.class}" + end + end + + ## + # @private + # Determines the initial event to dispatch based on configuration class. + # + # @return [Event::StartUpload, Event::ResumeUpload] + def initial_event + if @config.is_a? ResumeUploadConfig + Event::ResumeUpload.new + else + Event::StartUpload.new + end + end + + ## + # @private + # Resolves the total upload deadline timeout in seconds. + # + # @return [Numeric] Timeout in seconds + # + def resolve_timeout + return @config.timeout if @config.timeout&.positive? + + # When timeout is unset, BASE_TIMEOUT (1 hour) acts as a floor so small uploads still get + # a full hour while large uploads scale past it at MIN_ASSUMED_THROUGHPUT (1 MiB/s). + if @config.upload_size + [@config.upload_size.fdiv(MIN_ASSUMED_THROUGHPUT), BASE_TIMEOUT].max + else + BASE_TIMEOUT + end + end + + ## + # @private + # Computes the per-request timeout bounded by the global monotonic deadline. + # + # @param retry_policy [Gapic::Common::RetryPolicy, nil] Target command retry policy + # @return [Numeric] Effective per-request timeout + # + def request_timeout retry_policy + remaining = if @deadline + [@deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max + else + resolve_timeout + end + return [remaining, retry_policy.timeout].min if retry_policy&.timeout + + remaining + end + + ## + # @private + # Checks whether the monotonic clock has exceeded the session deadline. + # + # @return [Boolean] + # + def deadline_exceeded? + return false unless @deadline + Process.clock_gettime(Process::CLOCK_MONOTONIC) > @deadline + end + + ## + # @private + # Determines whether the instruction list contains a terminal instruction. + # + # @param instructions [Array] Instruction list + # @return [Boolean] + # + def terminal_instructions? instructions + instructions.any? do |i| + i.is_a?(Instruction::TerminateSuccess) || i.is_a?(Instruction::TerminateFailure) + end + end + + ## + # @private + # Invokes caller progress callback with snapshot. + # + # @param instruction [Instruction::NotifyProgress] Progress instruction + # + def execute_notify_progress instruction + @config.on_progress&.call instruction.progress + nil + end + + ## + # @private + # Realigns in-memory buffer and underlying stream to match server offset. + # + # @param instruction [Instruction::RealignBuffer] Realign instruction + # + def execute_realign_buffer instruction + server_offset = instruction.server_offset + if @config.upload_size && server_offset > @config.upload_size + raise StreamMismatchError.new( + "Server reported offset #{server_offset} exceeds total upload size #{@config.upload_size}", + resume_handle: resume_handle + ) + end + + buffer_start = @buffer_start_offset + buffer_end = @buffer_start_offset + @buffer.bytesize + + realign_case = if server_offset >= buffer_start && server_offset <= buffer_end + "within_buffer" + elsif server_offset < buffer_start + "rewind" + else + "fast_forward" + end + + unseekable = realign_case == "rewind" && !@config.stream.respond_to?(:seek) + @upload_log.buffer_realign realign_case, server_offset: server_offset, + current_offset: buffer_start, + unseekable: unseekable + + if server_offset >= buffer_start && server_offset <= buffer_end + realign_within_buffer server_offset + elsif server_offset < buffer_start + realign_rewind_stream server_offset + else + realign_fast_forward_stream server_offset, buffer_end + end + + nil + end + + ## + # @private + # Slices the in-memory buffer when server offset falls within current buffer range. + # + # @param server_offset [Integer] Target server offset + # + def realign_within_buffer server_offset + slice_index = server_offset - @buffer_start_offset + @buffer = @buffer.byteslice(slice_index..-1) || "".b + @buffer_start_offset = server_offset + end + + ## + # @private + # Rewinds seekable stream when server offset is before current buffer window. + # + # @param server_offset [Integer] Target server offset + # @raise [UnseekableStreamError] If stream does not respond to #seek + # + def realign_rewind_stream server_offset + unless @config.stream.respond_to? :seek + raise UnseekableStreamError.new( + "Cannot rewind unseekable stream to offset #{server_offset} (buffered from #{@buffer_start_offset})", + resume_handle: resume_handle + ) + end + + if @config.upload_size.nil? && @config.stream.respond_to?(:size) && server_offset > @config.stream.size + raise StreamMismatchError.new( + "Server reported offset #{server_offset} exceeds stream size #{@config.stream.size}", + resume_handle: resume_handle + ) + end + + @config.stream.seek server_offset + @buffer = "".b + @buffer_start_offset = server_offset + end + + ## + # @private + # Fast-forwards stream by seeking or discarding bytes. + # + # @param server_offset [Integer] Target server offset + # @param buffer_end [Integer] Current end offset of buffered data + # + def realign_fast_forward_stream server_offset, buffer_end + @buffer = "".b + if @config.stream.respond_to? :seek + if @config.upload_size.nil? && @config.stream.respond_to?(:size) && server_offset > @config.stream.size + raise StreamMismatchError.new( + "Server reported offset #{server_offset} exceeds stream size #{@config.stream.size}", + resume_handle: resume_handle + ) + end + @config.stream.seek server_offset + else + needed_discard = server_offset - buffer_end + while needed_discard.positive? + chunk = @config.stream.read [needed_discard, 65_536].min + if chunk.nil? || chunk.empty? + raise StreamMismatchError.new( + "Stream encountered unexpected EOF during fast-forward to offset #{server_offset} " \ + "(expected at least #{needed_discard} more bytes)", + resume_handle: resume_handle + ) + end + + needed_discard -= chunk.bytesize + end + end + @buffer_start_offset = server_offset + end + + ## + # @private + # Fills internal buffer from stream up to target byte size or EOF. + # + # @param instruction [Instruction::FillBuffer] FillBuffer instruction + # @return [Event::ChunkRead] Chunk read event + # + def execute_fill_buffer instruction + target = instruction.target_bytesize + eof = false + + while @buffer.bytesize < target + bytes_needed = target - @buffer.bytesize + chunk = @config.stream.read bytes_needed + if chunk.nil? || chunk.empty? + eof = true + break + end + @buffer << chunk.b + end + + Event::ChunkRead.new bytes_buffered: @buffer.bytesize, eof: eof + end + + ## + # @private + # Executes session initiation HTTP request. + # + # @param instruction [Instruction::SendStart] SendStart instruction + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def execute_send_start instruction + policy = @start_retry_policy.dup.start! + headers = start_headers instruction + attempt = 1 + + loop do + return Event::GlobalDeadlineExceeded.new if deadline_exceeded? + + event = make_post_request instruction.url, headers: headers, body: instruction.body, + retry_policy: policy, method_name: "ResumableUpload.start", + start_attempt: attempt + return event unless event.is_a? Event::HttpResponse + + status_hdr = Rules.header_value event.headers, "x-goog-upload-status" + return event unless status_hdr.nil? || status_hdr.empty? + return event if Rules::FATAL_STATUS_CODES.include? event.status + + err = BadResponseError.new "Missing X-Goog-Upload-Status header in start response", + event.status, + headers: event.headers + # `retry_with_deadline?` is public; its `@private` tag hides it from docs, not from callers. + can_retry = policy.retry_with_deadline? && policy.call(event) + unless can_retry + if event.status == 200 + failed_event = Event::RequestFailed.new( + kind: :retries_exhausted, message: err.message, source_error: err + ) + @upload_log.wire_failure failed_event + return failed_event + end + return event + end + attempt += 1 + end + end + + ## + # @private + # Builds initiation HTTP headers from instruction and config. + # + # Every header derived here is listed in {RESERVED_INITIAL_HEADERS}, and caller headers in + # that list are rejected when the config is built. The two sets are disjoint, so a plain merge + # cannot drop a driver header or duplicate one under a different casing. + # + # @param instruction [Instruction::SendStart] Start instruction + # @return [Hash] HTTP request headers + # + def start_headers instruction + headers = { "X-Goog-Upload-Protocol" => "resumable", "X-Goog-Upload-Command" => "start" } + headers["X-Goog-Upload-Header-Content-Type"] = @config.content_type if @config.content_type + headers["X-Goog-Upload-Header-Content-Length"] = @config.upload_size.to_s if @config.upload_size + headers.merge instruction.headers || {} + end + + ## + # @private + # Transmits a buffered chunk over HTTP. + # + # @param instruction [Instruction::SendChunk] SendChunk instruction + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def execute_send_chunk instruction + headers = { + "X-Goog-Upload-Command" => instruction.finalize ? "upload, finalize" : "upload", + "X-Goog-Upload-Offset" => instruction.offset.to_s, + "Content-Type" => @config.content_type || "application/octet-stream", + "Content-Length" => instruction.length.to_s + } + slice_index = instruction.offset - @buffer_start_offset + body = @buffer.byteslice slice_index, instruction.length + + make_post_request instruction.url, headers: headers, body: body, + retry_policy: @data_plane_retry_policy.dup.start!, + method_name: "ResumableUpload.upload" + end + + ## + # @private + # Sends a standalone finalize command over HTTP. + # + # @param instruction [Instruction::SendFinalize] SendFinalize instruction + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def execute_send_finalize instruction + headers = { + "X-Goog-Upload-Command" => "finalize", + "X-Goog-Upload-Offset" => @core.state.offset.to_s, + "Content-Length" => "0" + } + make_post_request instruction.url, headers: headers, body: "", + retry_policy: @data_plane_retry_policy.dup.start!, + method_name: "ResumableUpload.finalize" + end + + ## + # @private + # Sends an offset query command over HTTP. + # + # @param instruction [Instruction::SendQuery] SendQuery instruction + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def execute_send_query instruction + headers = { "X-Goog-Upload-Command" => "query", "Content-Length" => "0" } + make_post_request instruction.url, headers: headers, body: "", + retry_policy: @control_plane_retry_policy.dup.start!, + method_name: "ResumableUpload.query" + end + + ## + # @private + # Sends a cancellation command over HTTP. + # + # @param instruction [Instruction::SendCancel] SendCancel instruction + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def execute_send_cancel instruction + headers = { "X-Goog-Upload-Command" => "cancel", "Content-Length" => "0" } + make_post_request instruction.url, headers: headers, body: "", + retry_policy: @control_plane_retry_policy.dup.start!, + method_name: "ResumableUpload.cancel" + end + + ## + # @private + # Dispatches an HTTP POST request through client stub. + # + # @param url [String] Target URL + # @param headers [Hash] Request headers + # @param body [String] Request body + # @param retry_policy [Gapic::Common::RetryPolicy] Command retry policy + # @param method_name [String, nil] RPC method name for logging + # @param start_attempt [Integer] Attempt counter + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def make_post_request url, headers:, body:, retry_policy:, method_name: nil, start_attempt: 1 + return Event::GlobalDeadlineExceeded.new if deadline_exceeded? + + options = { + metadata: headers, + retry_policy: retry_policy, + timeout: request_timeout(retry_policy) + } + @upload_log.wire_send method: "POST", url: url, headers: headers, + start_attempt: start_attempt, body_size: body.to_s.bytesize, body: body + + response = @client_stub.make_post_request uri: url, body: body, params: {}, + options: options, method_name: method_name + event = Event::HttpResponse.new status: response.status, headers: response.headers || {}, body: response.body + @upload_log.wire_receive event + event + rescue StandardError => e + # If the global deadline expired during the HTTP call (e.g. Net::HTTP connection or read timeout + # triggered by request_timeout reaching 0 at @deadline), emit GlobalDeadlineExceeded rather than + # Event::RequestFailed. Otherwise, in states like Recovery where Event::RequestFailed is immediately + # terminal, the state machine would raise the underlying transport error instead of DeadlineExceededError. + return Event::GlobalDeadlineExceeded.new if deadline_exceeded? + + event = rescue_request_error e + if event.is_a? Event::HttpResponse + @upload_log.wire_receive event + else + @upload_log.wire_failure event + end + event + end + + ## + # @private + # Converts client stub transport exceptions into canonical events. + # + # @param err [StandardError] Rescued transport error + # @return [Event::HttpResponse, Event::RequestFailed] + # + def rescue_request_error err + case err + when Gapic::Rest::DeadlineExceededError + Event::RequestFailed.new kind: :timeout, message: err.message, source_error: err + when Gapic::Rest::Error + if err.status_code + Event::HttpResponse.new status: err.status_code, headers: err.headers || {}, body: err.message, + error: err + else + Event::RequestFailed.new kind: :connection_failed, message: err.message, source_error: err + end + when Faraday::Error + rescue_faraday_error err + else + Event::RequestFailed.new kind: :connection_failed, message: err.message, source_error: err + end + end + + ## + # @private + # Converts Faraday client exceptions into canonical events. + # + # @param err [Faraday::Error] Rescued Faraday error + # @return [Event::HttpResponse, Event::RequestFailed] + # + def rescue_faraday_error err + if err.response && err.response[:status] + rest_err = Gapic::Rest::Error.wrap_faraday_error err + Event::HttpResponse.new( + status: err.response[:status], + headers: err.response[:headers] || {}, + body: err.response[:body], + error: rest_err + ) + elsif err.is_a? Faraday::TimeoutError + Event::RequestFailed.new kind: :timeout, message: err.message, source_error: err + elsif err.is_a? Faraday::ConnectionFailed + Event::RequestFailed.new kind: :connection_failed, message: err.message, source_error: err + else + Event::RequestFailed.new kind: :retries_exhausted, message: err.message, source_error: err + end + end + end + # rubocop:enable Metrics/ClassLength + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb new file mode 100644 index 0000000..36cae29 --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "uri" + +module Gapic + module Rest + module ResumableUpload + class Driver + ## + # @private + # Pure functions for redacting and abridging sensitive data and large payloads in logs. + # + module Abridge + module_function + + ## + # @private + # Formats binary payload into truncated hex representation. + # + # @param data [Object, nil] Binary or string payload + # @return [String, nil] Truncated hex representation or nil + # + def bytes data + return nil if data.nil? + + str = data.to_s + if str.bytesize >= 64 + "#{str.byteslice(0, 32).unpack1('H*')}... <#{str.bytesize} bytes>" + else + str.unpack1 "H*" + end + end + + ## + # @private + # Truncates error body to a safe log length. + # + # @param data [Object, nil] Error body payload + # @return [String, nil] UTF-8 scrubbed and truncated string + # + def error_body data + return nil if data.nil? + + data.to_s.dup.force_encoding(Encoding::UTF_8).scrub[0, 512] + end + + ## + # @private + # Redacts query parameter values in URLs for safe logging. + # + # @param url [Object, nil] URL string or URI + # @return [String, nil] URL with query values elided + # + def url url + return nil if url.nil? + + uri = URI.parse url.to_s + if uri.query && !uri.query.empty? + elided = uri.query.split("&").map do |pair| + key, _val = pair.split "=", 2 + "#{key}=<...>" + end.join "&" + uri.query = nil + return "#{uri}?#{elided}" + end + uri.to_s + rescue URI::InvalidURIError + url.to_s + end + + ## + # @private + # Redacts non-protocol headers for safe logging. + # + # @param headers [Object] Headers hash + # @return [Hash] Redacted headers + # + def headers headers + return {} unless headers.is_a? Hash + + headers.each_with_object({}) do |(k, v), acc| + key_str = k.to_s + acc[key_str] = if key_str.downcase == "x-goog-upload-url" + url v + elsif key_str.downcase.start_with? "x-goog-upload-" + v + else + "<...>" + end + end + end + + ## + # @private + # Converts a list of instructions into log-safe representation hashes. + # + # @param instructions [Array] List of instructions + # @return [Array] Log-safe instruction summaries + # + def instructions instructions + instructions.map { |i| instruction i } + end + + # rubocop:disable Metrics/MethodLength + ## + # @private + # Converts an instruction into a log-safe representation hash. + # + # @param instruction [Object] Instruction object + # @return [Hash] Log-safe instruction summary + # + def instruction instruction + case instruction + when Instruction::SendStart + { "type" => "SendStart", "url" => url(instruction.url) } + when Instruction::SendChunk + { + "type" => "SendChunk", + "url" => url(instruction.url), + "offset" => instruction.offset, + "length" => instruction.length, + "finalize" => instruction.finalize + } + when Instruction::SendFinalize + { "type" => "SendFinalize", "url" => url(instruction.url) } + when Instruction::SendQuery + { "type" => "SendQuery", "url" => url(instruction.url) } + when Instruction::SendCancel + { "type" => "SendCancel", "url" => url(instruction.url) } + when Instruction::RealignBuffer + { "type" => "RealignBuffer", "serverOffset" => instruction.server_offset } + when Instruction::FillBuffer + { "type" => "FillBuffer", "targetBytesize" => instruction.target_bytesize } + when Instruction::NotifyProgress + { + "type" => "NotifyProgress", + "phase" => instruction.progress.phase.to_s, + "bytesUploaded" => instruction.progress.bytes_uploaded, + "totalBytes" => instruction.progress.total_bytes + } + when Instruction::TerminateSuccess + { "type" => "TerminateSuccess" } + when Instruction::TerminateFailure + { "type" => "TerminateFailure", "error" => instruction.error.to_s } + else + { "type" => instruction.class.name } + end + end + # rubocop:enable Metrics/MethodLength + end + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb new file mode 100644 index 0000000..d7864b6 --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb @@ -0,0 +1,326 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "google/logging/message" +require "gapic/rest/resumable_upload/driver/abridge" + +module Gapic + module Rest + module ResumableUpload + class Driver + ## + # @private + # Structured logging helper for a single Resumable Upload run. + # + class UploadLog + ## + # @private + # Recipes omitted from INFO lifecycle logging. + # @return [Array] + SILENT_RECIPES = [ + :ack_chunk, # per-chunk transition, doesn't belong at INFO + :fail_with_unmatched_transition # raises before Decision exists, logged by #unmatched_transition + ].freeze + + ## + # @private + # Severity and message mapping for lifecycle transitions. + # @return [Hash] + LIFECYCLE = { + start_session: [:info, "Initiating resumable upload"], + resume_session: [:info, "Resuming upload session"], + begin_transmission: [:info, "Upload session established"], + send_chunk: [:debug, "Sending upload chunk"], + send_upload_finalize: [:info, "Sending final upload chunk"], + send_finalize: [:info, "Sending finalize command"], + enter_recovery: [:info, "Entering upload recovery"], + retry_recovery: [:info, "Retrying upload recovery query"], + realign_from_recovery: [:info, "Resuming upload from server offset"], + complete_upload_with_data: [:info, "Resumable upload completed"], + complete_upload_finalized: [:info, "Resumable upload completed"], + cancel_session: [:info, "Canceling resumable upload"], + complete_cancellation: [:info, "Resumable upload canceled"], + fail_with_deadline_exceeded: [:warn, "Resumable upload failed"], + fail_with_rejected: [:warn, "Resumable upload failed"], + fail_with_bad_response: [:warn, "Resumable upload failed"], + fail_with_request_error: [:warn, "Resumable upload failed"] + }.freeze + + # @private + # @return [String] + attr_reader :upload_id + + ## + # @private + # Initializes a new UploadLog logger wrapper. + # + # @param stub_logger [Logger, Object] Underlying structured logger + # @param upload_id [String] Unique session identifier + # + def initialize stub_logger, upload_id: + @stub_logger = stub_logger + @upload_id = upload_id + end + + ## + # @private + # Logs state machine transition decision at DEBUG level. + # + # @param decision [Decision] Decision snapshot + # + def decision decision + msg = "Rules: #{decision.from_status} + #{decision.shape} -> " \ + "#{decision.recipe} -> #{decision.next_state.status}" + entry( + :debug, + msg, + fromStatus: decision.from_status, + shape: decision.shape, + recipe: decision.recipe, + toStatus: decision.next_state.status, + offset: decision.next_state.offset, + inFlightLength: decision.next_state.in_flight_length, + instructions: Abridge.instructions(decision.instructions) + ) + end + + ## + # @private + # Logs high-level protocol lifecycle milestone if configured. + # + # @param decision [Decision] Decision snapshot + # @param config [StartUploadConfig, ResumeUploadConfig] Upload configuration + # + def lifecycle decision, config + return if SILENT_RECIPES.include? decision.recipe + + severity, message = LIFECYCLE[decision.recipe] + return unless severity + + extra_fields = lifecycle_fields decision, config + entry severity, message, recipe: decision.recipe, **extra_fields + end + + ## + # @private + # Logs an outgoing HTTP request at DEBUG level. + # + # @param method [String] HTTP method + # @param url [String] Request URL + # @param headers [Hash] Request headers + # @param start_attempt [Integer] Attempt index for start command + # @param body_size [Integer, nil] Byte size of request payload + # @param body [Object, nil] Request payload + # @param body_is_error [Boolean] Whether body contains an error payload + # + def wire_send method:, url:, headers:, start_attempt:, body_size: nil, body: nil, body_is_error: false + command = Rules.header_value headers, "x-goog-upload-command" + offset = Rules.header_value headers, "x-goog-upload-offset" + fields = { + method: method, + url: Abridge.url(url), + headers: Abridge.headers(headers), + startAttempt: start_attempt + } + fields[:command] = command if command + fields[:offset] = offset.to_i if offset + fields[:bodySize] = body_size if body_size + fields[:body] = body_is_error ? Abridge.error_body(body) : Abridge.bytes(body) if body + + entry :debug, "Sending #{method} request", **fields + end + + ## + # @private + # Logs a received HTTP response at DEBUG level. + # + # @param event [Event::HttpResponse] Received HTTP event + # + def wire_receive event + upload_status = Rules.header_value event.headers, "x-goog-upload-status" + size_recv = Rules.header_value event.headers, "x-goog-upload-size-received" + gran = Rules.header_value event.headers, "x-goog-upload-chunk-granularity" + err = event.error if event.respond_to? :error + fields = { + status: event.status, + headers: Abridge.headers(event.headers), + body: wire_receive_body(event, err) + } + fields[:uploadStatus] = upload_status if upload_status + fields[:errorStatus] = err.status if err&.status + fields[:sizeReceived] = size_recv.to_i if size_recv + fields[:granularity] = gran.to_i if gran + + entry :debug, "Received HTTP #{event.status}", **fields + end + + ## + # @private + # Logs a network or transport failure at DEBUG level. + # + # @param event [Event::RequestFailed] Failure event + # + def wire_failure event + entry( + :debug, + "Request failed: #{event.kind}", + kind: event.kind, + error: event.message + ) + end + + ## + # @private + # Logs buffer realignment action. + # + # @param action [String] Realignment action description + # @param server_offset [Integer] Target server offset + # @param current_offset [Integer] Current buffer start offset + # @param unseekable [Boolean] Whether rewind was attempted on an unseekable stream + # + def buffer_realign action, server_offset:, current_offset:, unseekable: false + if unseekable + entry( + :warn, + "Server offset rewind on unseekable stream", + action: action, + serverOffset: server_offset, + currentOffset: current_offset + ) + end + + entry( + :debug, + "Buffer realignment: #{action}", + action: action, + serverOffset: server_offset, + currentOffset: current_offset + ) + end + + ## + # @private + # Logs an invalid or unmatched state machine transition at WARN level. + # + # @param state [State] Current state + # @param event [Object] Triggering event + # @param error [StandardError] Resulting error + # + def unmatched_transition state, event, error + entry( + :warn, + "Unmatched transition", + status: state.status, + shape: Rules.shape_of(event), + error: error.message + ) + end + + private + + ## + # @private + # Formats response body for wire log entry. + # + # @param event [Event::HttpResponse] Response event + # @param err [Gapic::Rest::Error, nil] Error instance + # @return [String, nil] Formatted body + # + def wire_receive_body event, err + return Abridge.bytes event.body if event.status < 400 + + err&.message ? Abridge.error_body(err.message) : Abridge.error_body(event.body) + end + + ## + # @private + # Extracts relevant state fields for lifecycle logging. + # + # @param decision [Decision] Decision snapshot + # @param config [StartUploadConfig, ResumeUploadConfig] Upload configuration + # @return [Hash] Metadata fields for log entry + # + def lifecycle_fields decision, config + state = decision.next_state + case decision.recipe + when :start_session + { uploadSize: config.upload_size, requestedChunkSize: config.chunk_size } + when :resume_session + { + uploadUrl: Abridge.url(state.upload_url), + chunkSize: state.chunk_size + } + when :begin_transmission + { + effectiveChunkSize: state.chunk_size, + granularity: state.chunk_granularity, + uploadUrl: Abridge.url(state.upload_url) + } + when :send_chunk, :send_upload_finalize + { offset: state.offset, inFlightLength: state.in_flight_length } + when :send_finalize, :enter_recovery, :retry_recovery, + :realign_from_recovery, :complete_upload_with_data, :complete_upload_finalized + { offset: state.offset } + when :cancel_session + { uploadUrl: Abridge.url(state.upload_url) } + when :fail_with_deadline_exceeded, :fail_with_rejected, :fail_with_bad_response, + :fail_with_request_error + failure_fields state + else + {} + end + end + + ## + # @private + # Extracts error and response details for failure lifecycle logs. + # + # @param state [State] Current protocol state + # @return [Hash] Failure metadata fields + # + def failure_fields state + err = state.last_error + fields = { error: err&.message || err.to_s } + if err.respond_to?(:response_body) && err.response_body + fields[:responseBody] = Abridge.error_body err.response_body + end + fields + end + + ## + # @private + # Dispatches structured log entry to stub logger. + # + # @param severity [Symbol] Log severity level + # @param log_msg [String] Primary log message + # @param fields [Hash] Structured key-value fields + # + def entry severity, log_msg, **fields + @stub_logger.public_send severity do |builder| + builder.set_system_name + builder.set_service + builder.set "uploadId", @upload_id + fields.each do |k, v| + builder.set k.to_s, v + end + builder.message = log_msg + end + end + end + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb new file mode 100644 index 0000000..b54e114 --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb @@ -0,0 +1,565 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/common/error" +require "gapic/rest/error" + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # HTTP status code to reason phrase mapping. + # @return [Hash] + HTTP_STATUS_PHRASES = { + 400 => "Bad Request", + 401 => "Unauthorized", + 403 => "Forbidden", + 404 => "Not Found", + 405 => "Method Not Allowed", + 408 => "Request Timeout", + 409 => "Conflict", + 410 => "Gone", + 411 => "Length Required", + 412 => "Precondition Failed", + 413 => "Payload Too Large", + 415 => "Unsupported Media Type", + 416 => "Range Not Satisfiable", + 429 => "Too Many Requests", + 499 => "Client Closed Request", + 500 => "Internal Server Error", + 502 => "Bad Gateway", + 503 => "Service Unavailable", + 504 => "Gateway Timeout" + }.freeze + + ## + # @private + # Internal formatting helper for terminal error message and attribute extraction. + # + module ErrorBuilder + class << self + ## + # @private + # Formats status representation. + # + # @param status [Object, nil] Status value + # @return [String, nil] + def format_status status + return nil if status.nil? || status.to_s.empty? + + status.to_s + end + + ## + # @private + # Strips REST error prefix from message string. + # + # @param raw_message [String, nil] Raw error message + # @return [String, nil] + def clean_message raw_message + return nil if raw_message.nil? || raw_message.empty? + + prefix = Gapic::Rest::Error::REST_ERROR_PREFIX + msg = raw_message.to_s + msg = msg.sub(/\A#{Regexp.escape prefix}:\s*/, "") if msg.start_with? prefix + msg = msg.sub(/\A:\s*/, "").strip + msg.empty? ? nil : msg + end + + ## + # @private + # Builds error attributes tuple from an HTTP event or wrapped error. + # + # @param event [Object] HTTP response event or failure event + # @param prefix [String] Error message prefix + # @return [Array] Tuple of [message, status_code, status, details, headers] + def build_attributes event, prefix: "Resumable upload failed" + if event.respond_to?(:error) && event.error + build_from_wrapped_error event, prefix: prefix + else + build_from_http_event event, prefix: prefix + end + end + + private + + ## + # @private + # Builds error attributes when a wrapped REST error is available. + # + # @param event [Object] HTTP response event containing wrapped error + # @param prefix [String] Error message prefix + # @return [Array] Tuple of [message, status_code, status, details, headers] + def build_from_wrapped_error event, prefix: + err = event.error + status_code = err.status_code || (event.respond_to?(:status) ? event.status : nil) + status = err.status + status_name = format_status(status) || HTTP_STATUS_PHRASES[status_code] + status_part = status_name ? " #{status_name}" : "" + inner_msg = clean_message err.message + msg = if inner_msg + "#{prefix} with HTTP #{status_code}#{status_part}: #{inner_msg}" + else + "#{prefix} with HTTP #{status_code}#{status_part}" + end + headers = err.headers || (event.respond_to?(:headers) ? event.headers : nil) + [msg, status_code, status, err.details, headers] + end + + ## + # @private + # Builds error attributes directly from raw HTTP response event. + # + # @param event [Object] HTTP response event + # @param prefix [String] Error message prefix + # @return [Array] Tuple of [message, status_code, status, details, headers] + def build_from_http_event event, prefix: + status_code = event.status + headers = event.respond_to?(:headers) && event.headers ? event.headers : {} + upload_status = headers["x-goog-upload-status"] || headers["X-Goog-Upload-Status"] + status_desc = upload_status ? "'#{upload_status}'" : "missing" + status_name = HTTP_STATUS_PHRASES[status_code] + status_part = status_name ? " #{status_name}" : "" + msg = "#{prefix} with HTTP #{status_code}#{status_part} " \ + "(X-Goog-Upload-Status: #{status_desc})" + [msg, status_code, nil, nil, headers] + end + end + end + + ## + # Mixin providing {ResumeHandle} access and uniform formatting for resumable errors. + # + # Every error that may carry a resume handle includes this module, so it doubles as the rescue target + # for "this upload failed but can be retried from where it stopped": + # + # @example + # begin + # session.start initial_url: url + # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e + # retry_later e.resume_handle if e.resume_handle + # raise + # end + # + # Included by {RequestFailedError}, {DeadlineExceededError}, {BadResponseError}, + # {UnseekableStreamError}, {StreamMismatchError} and {InvalidTransitionError}. + # + # Deliberately **not** included by {UploadRejectedError} or {SessionStateError}: the first means the + # server refused the upload outright and the second is a caller misuse, so neither is retryable. Note + # also that `resume_handle` may still be `nil` on an including error, for instance when the failure + # happened before initiation established an upload URL. + # + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated upload session resume handle + # + module HasResumeHandle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] + attr_reader :resume_handle + + ## + # Suffix appended to error message when a resume handle is present. + # @return [String] + RESUMABLE_SUFFIX = " (upload session is resumable: see #resume_handle)" + + ## + # Appends the uniform resumable suffix if resume_handle is non-nil. + # + # @param message [String, nil] Error message + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Resume handle + # @return [String, nil] + def self.append_suffix message, resume_handle + return message if resume_handle.nil? + return RESUMABLE_SUFFIX.strip if message.nil? || message.to_s.strip.empty? + return message if message.end_with? RESUMABLE_SUFFIX + + "#{message}#{RESUMABLE_SUFFIX}" + end + end + + ## + # Raised when an invalid or unmatched event is dispatched for the current protocol state. + # + # @!attribute [r] response + # @return [Gapic::Rest::ResumableUpload::Event::HttpResponse, Object, nil] Associated HTTP response + # @!attribute [r] state + # @return [Symbol, nil] Current protocol state + # @!attribute [r] event + # @return [Object, nil] Received event + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # + class InvalidTransitionError < Gapic::Common::Error + include HasResumeHandle + + # @return [Gapic::Rest::ResumableUpload::Event::HttpResponse, Object, nil] + attr_reader :response + + # @return [Symbol, nil] Current protocol state + attr_reader :state + + # @return [Object, nil] Received event + attr_reader :event + + ## + # Initializes a new InvalidTransitionError. + # + # @param message [String] Descriptive error message + # @param state [Symbol, nil] Current protocol state + # @param event [Object, nil] Received event + # @param response [Gapic::Rest::ResumableUpload::Event::HttpResponse, Object, nil] Associated HTTP response + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + def initialize message, state: nil, event: nil, response: nil, resume_handle: nil + @state = state + @event = event + @response = response || (event if defined?(Event::HttpResponse) && event.is_a?(Event::HttpResponse)) + @resume_handle = resume_handle + super HasResumeHandle.append_suffix(message, resume_handle) + end + + ## + # Creates an InvalidTransitionError from an event. + # + # @param event [Object] Received event + # @param state [Symbol, nil] Current protocol state + # @param message [String, nil] Descriptive error message + # @param response [Object, nil] Associated HTTP response + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @return [InvalidTransitionError] + def self.from event, state: nil, message: nil, response: nil, resume_handle: nil + new( + message || "Invalid transition for event #{event.inspect}", + state: state, + event: event, + response: response, + resume_handle: resume_handle + ) + end + end + + ## + # Raised when stream rewinding is required but the stream does not support seeking. + # + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # + class UnseekableStreamError < Gapic::Common::Error + include HasResumeHandle + + ## + # Initializes a new UnseekableStreamError. + # + # @param message [String, nil] Descriptive error message + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + def initialize message = nil, resume_handle: nil + @resume_handle = resume_handle + super HasResumeHandle.append_suffix(message, resume_handle) + end + + ## + # Creates an UnseekableStreamError with optional resume handle. + # + # @param message [String, nil] Descriptive error message + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @return [UnseekableStreamError] + def self.from message = nil, resume_handle: nil + new message, resume_handle: resume_handle + end + end + + ## + # Raised when stream content or length does not match resumed upload specifications. + # + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # + class StreamMismatchError < Gapic::Common::Error + include HasResumeHandle + + ## + # Initializes a new StreamMismatchError. + # + # @param message [String] Error message + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + def initialize message = "Stream content or length does not match resumed upload", resume_handle: nil + @resume_handle = resume_handle + super HasResumeHandle.append_suffix(message, resume_handle) + end + + ## + # Creates a StreamMismatchError with optional resume handle. + # + # @param message [String, nil] Error message + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @return [StreamMismatchError] + def self.from message = nil, resume_handle: nil + msg = message || "Stream content or length does not match resumed upload" + new msg, resume_handle: resume_handle + end + end + + ## + # Raised when an unrecoverable HTTP response is received. + # + # @!attribute [r] response_body + # @return [String, nil] Response body from backend + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # + class BadResponseError < Gapic::Rest::Error + include HasResumeHandle + + # @return [String, nil] Response body from backend + attr_reader :response_body + + ## + # Initializes a new BadResponseError. + # + # @param message [String, nil] Error message + # @param status_code [Integer, nil] HTTP status code + # @param status [String, nil] Status description + # @param details [Object, nil] Error details + # @param headers [Object, nil] Response headers + # @param response_body [String, nil] Response body + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + def initialize message = nil, status_code = nil, status: nil, details: nil, headers: nil, + response_body: nil, resume_handle: nil + @response_body = response_body + @resume_handle = resume_handle + super HasResumeHandle.append_suffix(message, resume_handle), + status_code, status: status, details: details, headers: headers + end + + ## + # Creates a BadResponseError from an HTTP response event. + # + # @param event [Object] HTTP response event + # @param response_body [String, nil] Optional response body override + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Optional resume handle + # @return [BadResponseError] + def self.from event, response_body: nil, resume_handle: nil + body = response_body || (event.respond_to?(:body) ? event.body : nil) + message, status_code, status, details, headers = ErrorBuilder.build_attributes event + new message, status_code, status: status, details: details, headers: headers, + response_body: body, resume_handle: resume_handle + end + end + + ## + # Raised when the resumable upload backend explicitly rejects the + # upload session (returns non-2xx with X-Goog-Upload-Status: final). + # + # @!attribute [r] response_body + # @return [String, nil] Response body from backend + # + class UploadRejectedError < Gapic::Rest::Error + # @return [String, nil] Response body from backend + attr_reader :response_body + + ## + # Initializes a new UploadRejectedError. + # + # @param message [String, nil] Error message + # @param status_code [Integer, nil] HTTP status code + # @param status [String, nil] Status description + # @param details [Object, nil] Error details + # @param headers [Object, nil] Response headers + # @param response_body [String, nil] Response body + def initialize message = nil, status_code = nil, status: nil, details: nil, headers: nil, response_body: nil + @response_body = response_body + super message, status_code, status: status, details: details, headers: headers + end + + ## + # Creates an UploadRejectedError from an HTTP response event. + # + # @param event [Object] HTTP response event + # @param response_body [String, nil] Optional response body override + # @return [UploadRejectedError] + def self.from event, response_body: nil + body = response_body || (event.respond_to?(:body) ? event.body : nil) + message, status_code, status, details, headers = + ErrorBuilder.build_attributes event, prefix: "Upload rejected by server" + new message, status_code, status: status, details: details, headers: headers, response_body: body + end + end + + ## + # @private + # Raised when the upload session is cancelled. + # Cancellation is not public yet. + # + class UploadCancelledError < Gapic::Common::Error + ## + # Initializes a new UploadCancelledError. + # + # @param message [String] Cancellation message + def initialize message = "Upload session was cancelled" + super message + end + + ## + # Creates an UploadCancelledError from a source event or message string. + # + # @param source [Object, String, nil] Source event or message + # @return [UploadCancelledError] + def self.from source = nil + if source.is_a?(String) && !source.empty? + new source + else + new + end + end + end + + ## + # Raised when an upload exceeds its global monotonic deadline. + # + # @!attribute [r] root_cause + # @return [Object, nil] Root cause exception if deadline exceeded during a retry loop + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # + class DeadlineExceededError < Gapic::Common::Error + include HasResumeHandle + + # @return [Object, nil] Root cause exception if deadline exceeded during a retry loop + attr_reader :root_cause + + ## + # Initializes a new DeadlineExceededError. + # + # @param message [String] Deadline exceeded message + # @param root_cause [Object, nil] Root cause exception + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + def initialize message = "Upload deadline exceeded", root_cause: nil, resume_handle: nil + super HasResumeHandle.append_suffix(message, resume_handle) + @root_cause = root_cause + @resume_handle = resume_handle + end + + ## + # Creates a DeadlineExceededError with optional resume handle. + # + # @param message [String, nil] Deadline exceeded message + # @param root_cause [Object, nil] Root cause exception + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @return [DeadlineExceededError] + def self.from message = "Upload deadline exceeded", root_cause: nil, resume_handle: nil + new message, root_cause: root_cause, resume_handle: resume_handle + end + end + + ## + # Raised when an HTTP request fails (e.g. transport connection failure, request timeout, or retries exhausted). + # + # @!attribute [r] cause + # @return [StandardError, nil] Underlying cause exception + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @!attribute [r] status_code + # @return [Integer, nil] HTTP status code if cause was a REST error + # @!attribute [r] status + # @return [String, nil] Status description if cause was a REST error + # @!attribute [r] details + # @return [Object, nil] Error details if cause was a REST error + # @!attribute [r] headers + # @return [Object, nil] Response headers if cause was a REST error + # + class RequestFailedError < Gapic::Common::Error + include HasResumeHandle + + # @return [Integer, nil] + attr_reader :status_code + + # @return [String, nil] + attr_reader :status + + # @return [Object, nil] + attr_reader :details + + # @return [Object, nil] + attr_reader :headers + + ## + # Initializes a new RequestFailedError. + # + # @param message [String, nil] Error message + # @param cause [StandardError, nil] Underlying cause exception + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @param status_code [Integer, nil] HTTP status code + # @param status [String, nil] Status description + # @param details [Object, nil] Error details + # @param headers [Object, nil] Response headers + def initialize message = nil, cause: nil, resume_handle: nil, + status_code: nil, status: nil, details: nil, headers: nil + @cause = cause + @resume_handle = resume_handle + @status_code = status_code || (cause.respond_to?(:status_code) ? cause.status_code : nil) + @status = status || (cause.respond_to?(:status) ? cause.status : nil) + @details = details || (cause.respond_to?(:details) ? cause.details : nil) + @headers = headers || (cause.respond_to?(:headers) ? cause.headers : nil) + msg = message || cause&.message || "Request failed" + super HasResumeHandle.append_suffix(msg, resume_handle) + end + + ## + # Returns the underlying cause exception. + # + # @return [StandardError, nil] + def cause + @cause || super + end + + ## + # Creates a RequestFailedError from a failure event or error. + # + # @param event_or_error [Event::RequestFailed, StandardError] Source event or error + # @param message [String, nil] Optional message override + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @return [RequestFailedError] + def self.from event_or_error, message: nil, resume_handle: nil + if event_or_error.respond_to? :source_error + cause = event_or_error.source_error + msg = message || event_or_error.message || cause&.message || "Request failed" + new msg, cause: cause, resume_handle: resume_handle + elsif event_or_error.is_a? Exception + msg = message || event_or_error.message || "Request failed" + new msg, cause: event_or_error, resume_handle: resume_handle + else + new message || event_or_error.to_s, resume_handle: resume_handle + end + end + end + + ## + # Raised when an operation violates the Session lifecycle rules + # (e.g. calling a `start` method more than once). + # + class SessionStateError < Gapic::Common::Error + end + + ## + # @private + # Raised when an internal state machine or driver invariant is violated. + # Produced by {Rules} when an unlisted shape or recipe is encountered, and by + # {Driver} when a recipe emits a malformed instruction batch. + # + class InternalError < Gapic::Common::Error + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/events.rb b/gapic-common/lib/gapic/rest/resumable_upload/events.rb new file mode 100644 index 0000000..6a8df2c --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/events.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # Event vocabulary emitted by the Driver and dispatched to Core/Rules. + # Events are `outside-in` signaling. Something happened, e.g. a chunk of data + # was successfully read, and the Driver is reporting that to Core/Rules. + # + module Event + ## + # @private + # Signals the start of the upload session. + # + StartUpload = Data.define + + ## + # @private + # Signals the resumption of an existing upload session. + # + ResumeUpload = Data.define + + ## + # @private + # Signals that binary data was read from the stream into the Driver's buffer. + # + # @!attribute [r] bytes_buffered + # @return [Integer] Number of bytes currently held in the Driver buffer + # @!attribute [r] eof + # @return [Boolean] Whether stream EOF was encountered during the read + # + ChunkRead = Data.define :bytes_buffered, :eof do + ## + # @private + # Initializes a ChunkRead event. + # + # @param bytes_buffered [Integer] Number of bytes currently held in buffer + # @param eof [Boolean] Whether stream EOF was encountered + # + def initialize bytes_buffered: 0, eof: false + super bytes_buffered: bytes_buffered, eof: eof + end + end + + ## + # @private + # Signals a completed HTTP exchange over the wire (status, headers, body, error). + # + # @!attribute [r] status + # @return [Integer] HTTP status code + # @!attribute [r] headers + # @return [Hash] Response headers + # @!attribute [r] body + # @return [String, nil] Response body + # @!attribute [r] error + # @return [Gapic::Rest::Error, nil] Wrapped REST error if status >= 400 + # + HttpResponse = Data.define :status, :headers, :body, :error do + ## + # @private + # Initializes an HttpResponse event. + # + # @param status [Integer] HTTP status code + # @param headers [Hash] Response headers + # @param body [String, nil] Response body + # @param error [Gapic::Rest::Error, nil] Wrapped REST error + # + def initialize status:, headers: {}, body: nil, error: nil + super status: status, headers: headers || {}, body: body, error: error + end + end + + ## + # @private + # Signals an HTTP request failure (e.g. request timeout, transport connection failure, or retries exhausted). + # + # @!attribute [r] kind + # @return [Symbol] Failure kind: `:timeout`, `:connection_failed`, or `:retries_exhausted` + # @!attribute [r] message + # @return [String, nil] Human-readable failure summary + # @!attribute [r] source_error + # @return [StandardError, nil] Original underlying exception + # + RequestFailed = Data.define :kind, :message, :source_error do + ## + # @private + # Initializes a RequestFailed event. + # + # @param kind [Symbol] Failure kind (`:timeout`, `:connection_failed`, `:retries_exhausted`) + # @param message [String, nil] Human-readable failure summary + # @param source_error [StandardError, nil] Original underlying exception + # + def initialize kind:, message: nil, source_error: nil + super kind: kind, message: message, source_error: source_error + end + end + + ## + # @private + # Signals a caller-requested session cancellation. + # + Cancel = Data.define + + ## + # @private + # Signals that the global monotonic clock exceeded the configured deadline. + # + GlobalDeadlineExceeded = Data.define + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/instructions.rb b/gapic-common/lib/gapic/rest/resumable_upload/instructions.rb new file mode 100644 index 0000000..57d02bd --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/instructions.rb @@ -0,0 +1,265 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # Instruction vocabulary emitted by Rules/Core to be executed by Driver. + # The vocabulary is partitioned three ways ({CONTINUATION}, {TERMINAL}, {SIDE_EFFECT}), + # and every instruction class must join exactly one list. + # + module Instruction + ## + # @private + # Execute initiation request to establish upload session. + # + # @!attribute [r] url + # @return [String] Initial endpoint URI + # @!attribute [r] headers + # @return [Hash] Additional headers for initiation request + # @!attribute [r] body + # @return [String, nil] Request payload for session initiation + # + SendStart = Data.define :url, :headers, :body do + ## + # @private + # Initializes a SendStart instruction. + # + # @param url [String] Initial endpoint URI + # @param headers [Hash] Additional headers + # @param body [String, nil] Request payload + # + def initialize url:, headers: {}, body: nil + super url: url, headers: headers || {}, body: body + end + end + + ## + # @private + # Transmit buffered chunk starting at offset for length bytes. + # + # @!attribute [r] url + # @return [String] Session upload URL + # @!attribute [r] offset + # @return [Integer] Byte offset within the full upload stream + # @!attribute [r] length + # @return [Integer] Number of bytes to transmit from buffer + # @!attribute [r] finalize + # @return [Boolean] Whether to append finalize command to upload request + # + SendChunk = Data.define :url, :offset, :length, :finalize do + ## + # @private + # Initializes a SendChunk instruction. + # + # @param url [String] Session upload URL + # @param offset [Integer] Byte offset within upload stream + # @param length [Integer] Number of bytes to transmit + # @param finalize [Boolean] Whether to combine upload and finalize commands + # + def initialize url:, offset:, length:, finalize: false + super url: url, offset: offset, length: length, finalize: finalize + end + end + + ## + # @private + # Send standalone finalize command when all data bytes were already uploaded. + # + # @!attribute [r] url + # @return [String] Session upload URL + # + SendFinalize = Data.define :url do + ## + # @private + # Initializes a SendFinalize instruction. + # + # @param url [String] Session upload URL + # + def initialize url: + super url: url + end + end + + ## + # @private + # Query backend for current acknowledged offset. + # + # @!attribute [r] url + # @return [String] Session upload URL + # + SendQuery = Data.define :url do + ## + # @private + # Initializes a SendQuery instruction. + # + # @param url [String] Session upload URL + # + def initialize url: + super url: url + end + end + + ## + # @private + # Cancel upload session on backend. + # + # @!attribute [r] url + # @return [String] Session upload URL + # + SendCancel = Data.define :url do + ## + # @private + # Initializes a SendCancel instruction. + # + # @param url [String] Session upload URL + # + def initialize url: + super url: url + end + end + + ## + # @private + # Realign Driver in-memory buffer and stream position to match server_offset. + # + # @!attribute [r] server_offset + # @return [Integer] Acknowledged byte offset reported by server + # + RealignBuffer = Data.define :server_offset do + ## + # @private + # Initializes a RealignBuffer instruction. + # + # @param server_offset [Integer] Target server byte offset + # + def initialize server_offset: + super server_offset: server_offset + end + end + + ## + # @private + # Read from stream until in-memory buffer reaches target_bytesize or stream hits EOF. + # + # @!attribute [r] target_bytesize + # @return [Integer] Target buffer size in bytes + # + FillBuffer = Data.define :target_bytesize do + ## + # @private + # Initializes a FillBuffer instruction. + # + # @param target_bytesize [Integer] Target buffer size in bytes + # + def initialize target_bytesize: + super target_bytesize: target_bytesize + end + end + + ## + # @private + # Invoke user progress callback with a Progress instance. + # + # @!attribute [r] progress + # @return [Gapic::Rest::ResumableUpload::Progress] Progress notification snapshot + # + NotifyProgress = Data.define :progress do + ## + # @private + # Initializes a NotifyProgress instruction. + # + # @param progress [Gapic::Rest::ResumableUpload::Progress] Progress notification snapshot + # + def initialize progress: + super progress: progress + end + end + + ## + # @private + # Upload finalized cleanly; return response. + # + # @!attribute [r] response + # @return [Gapic::Rest::ResumableUpload::Event::HttpResponse] Final response object + # + TerminateSuccess = Data.define :response do + ## + # @private + # Initializes a TerminateSuccess instruction. + # + # @param response [Gapic::Rest::ResumableUpload::Event::HttpResponse] Final response object + # + def initialize response: + super response: response + end + end + + ## + # @private + # Terminate upload with error. + # + # @!attribute [r] error + # @return [StandardError] Terminal exception to raise + # + TerminateFailure = Data.define :error do + ## + # @private + # Initializes a TerminateFailure instruction. + # + # @param error [StandardError] Terminal exception to raise + # + def initialize error: + super error: error + end + end + + ## + # @private + # Instruction classes that produce a continuation event for the next step of the trampoline loop. + # @return [Array] + CONTINUATION = [ + FillBuffer, + SendStart, + SendChunk, + SendFinalize, + SendQuery, + SendCancel + ].freeze + + ## + # @private + # Instruction classes that terminate the upload run. + # @return [Array] + TERMINAL = [ + TerminateSuccess, + TerminateFailure + ].freeze + + ## + # @private + # Instruction classes that perform side effects without producing continuation events or terminating. + # @return [Array] + SIDE_EFFECT = [ + NotifyProgress, + RealignBuffer + ].freeze + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb new file mode 100644 index 0000000..deb92bf --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb @@ -0,0 +1,165 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/common/retry_policy" +require "gapic/rest/resumable_upload/rules" + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # Default retry policy generators for control plane and data plane requests. + # + module RetryPolicies + ## + # @private + # Retry predicate determining retriability for session initiation requests. + # Retries missing status header on non-fatal codes. + # @return [Proc] + START_PREDICATE = lambda do |error_or_response| + status = extract_status_code error_or_response + return false if Rules::FATAL_STATUS_CODES.include? status + + headers = extract_headers error_or_response + if headers + status_hdr = headers["x-goog-upload-status"] || headers["X-Goog-Upload-Status"] + return true if status_hdr.nil? || status_hdr.empty? + end + nil + end + + ## + # @private + # Retry predicate determining retriability for data plane requests. + # Disallows retry when upload status header is missing. + # @return [Proc] + DATA_PLANE_PREDICATE = lambda do |error_or_response| + headers = extract_headers error_or_response + if headers + status_hdr = headers["x-goog-upload-status"] || headers["X-Goog-Upload-Status"] + return false if status_hdr.nil? || status_hdr.empty? + end + nil + end + + ## + # @private + # Default options for start command retry policy. + # Keep in sync with the "Retry Policies" section of Session's class doc. + # @return [Hash] + START_DEFAULTS = { + retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, + initial_delay: 1.0, + max_delay: 15.0, + multiplier: 1.3, + retry_predicate: START_PREDICATE + }.freeze + + ## + # @private + # Default options for query and cancel commands retry policy. + # Keep in sync with the "Retry Policies" section of Session's class doc. + # @return [Hash] + CONTROL_PLANE_DEFAULTS = { + retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, + initial_delay: 1.0, + max_delay: 15.0, + multiplier: 1.3 + }.freeze + + ## + # @private + # Default options for upload and finalize commands retry policy. + # Keep in sync with the "Retry Policies" section of Session's class doc. + # @return [Hash] + DATA_PLANE_DEFAULTS = { + retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, + initial_delay: 1.0, + max_delay: 15.0, + multiplier: 1.3, + retry_predicate: DATA_PLANE_PREDICATE + }.freeze + + ## + # @private + # Default retry policy for session initiation requests (start). + # Missing X-Goog-Upload-Status header is retriable across any response code, + # including 200 (predicate returns true). + # + # @return [Gapic::Common::RetryPolicy] + def self.default_start + Gapic::Common::RetryPolicy.new(**START_DEFAULTS) + end + + ## + # @private + # Default retry policy for session control requests (query, cancel). + # Does not retry on missing X-Goog-Upload-Status header. + # + # @return [Gapic::Common::RetryPolicy] + def self.default_control_plane + Gapic::Common::RetryPolicy.new(**CONTROL_PLANE_DEFAULTS) + end + + ## + # @private + # Default retry policy for data plane requests (upload, finalize). + # Missing X-Goog-Upload-Status header is unretriable (predicate returns false). + # + # @return [Gapic::Common::RetryPolicy] + def self.default_data_plane + Gapic::Common::RetryPolicy.new(**DATA_PLANE_DEFAULTS) + end + + ## + # @private + # Extracts headers hash from Faraday response or error object. + # + # @param error_or_response [Object] Response, error, or hash object + # @return [Hash, nil] + def self.extract_headers error_or_response + if error_or_response.respond_to? :headers + error_or_response.headers + elsif error_or_response.respond_to? :response_headers + error_or_response.response_headers + elsif error_or_response.respond_to?(:response) && error_or_response.response.is_a?(Hash) + error_or_response.response[:headers] + end + end + + ## + # @private + # Extracts HTTP status code from Faraday response, error, or event object. + # + # @param error_or_response [Object] Response, error, or event object + # @return [Integer, nil] + def self.extract_status_code error_or_response + if error_or_response.respond_to?(:status_code) && error_or_response.status_code.is_a?(Integer) + error_or_response.status_code + elsif error_or_response.respond_to?(:response) && error_or_response.response.is_a?(Hash) && + error_or_response.response[:status].is_a?(Integer) + error_or_response.response[:status] + elsif error_or_response.respond_to?(:response_status) && error_or_response.response_status.is_a?(Integer) + error_or_response.response_status + elsif error_or_response.respond_to?(:status) && error_or_response.status.is_a?(Integer) + error_or_response.status + end + end + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb new file mode 100644 index 0000000..90cc14f --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb @@ -0,0 +1,1016 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/common/error" +require "gapic/rest/resumable_upload/errors" +require "gapic/rest/resumable_upload/data_types" +require "gapic/rest/resumable_upload/events" +require "gapic/rest/resumable_upload/instructions" + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # Pure functional transition engine for the Resumable Upload Protocol. + # Contains zero side-effects and zero persistent state. + # + # ### Model + # + # {Rules.decide} is the protocol. It is a total function of `[state.status, shape_of(event)]` returning a + # {Decision} that carries the next {State} and the instructions for the Driver to execute. Three + # vocabularies define it, each published as a frozen constant: + # + # * {STATUSES} - protocol lifecycle statuses a {State} may hold. + # * {SHAPES} - canonical event shapes that {Rules.shape_of} reduces raw events to. + # * {RECIPES} - transition handlers that {Rules.decide} may select. + # + # Every router arm maps one (status, shape) pair to exactly one recipe, and every recipe returns + # `[next_state, instructions]`. Adding a protocol behaviour means adding a shape, a recipe and an arm. + # It never means adding branching to the Driver. + # + # ### Trampoline invariant + # + # {Driver#run} executes as a synchronous trampoline loop, so every recipe in {RECIPES} must return an + # instruction batch that yields either: + # + # 1. **Exactly one** event-producing instruction (`FillBuffer` or `Send*`) and zero terminal instructions, or + # 2. **Exactly one** terminal instruction (`TerminateSuccess` or `TerminateFailure`) and zero event-producing + # instructions. + # + # A recipe returning zero event-producing instructions without terminating stalls the loop, and a recipe + # returning multiple event-producing instructions discards continuation events. The Driver validates each + # batch against {Instruction::CONTINUATION}, {Instruction::TERMINAL} and {Instruction::SIDE_EFFECT} before + # executing any instruction, rejecting malformed batches up front. Side-effect instructions + # ({Instruction::NotifyProgress}, {Instruction::RealignBuffer}) explicitly return `nil` in the Driver by + # construction, so only {Instruction::FillBuffer} and `Send*` instructions produce continuation events. + # + # ### State transition graph + # + # ```mermaid + # stateDiagram-v2 + # [*] --> initializing + # initializing --> starting : start_upload + # initializing --> recovery : resume_upload + # starting --> transmission_reading : response_active + # transmission_reading --> transmission_sending : chunk_read_full + # transmission_sending --> transmission_reading : response_active + # transmission_reading --> finalizing_sending_upload : chunk_read_eof_with_data + # transmission_reading --> finalizing_sending_finalize : chunk_read_eof_empty + # finalizing_sending_upload --> success : response_final + # finalizing_sending_finalize --> success : response_final + # transmission_sending --> recovery : response_cat2 / connection_failed / timeout + # finalizing_sending_upload --> recovery : response_cat2 / connection_failed / timeout + # finalizing_sending_finalize --> recovery : response_cat2 / connection_failed / timeout + # recovery --> recovery : response_cat2 + # recovery --> transmission_reading : response_active + # recovery --> success : response_final + # starting --> error : response_cat2 / response_fatal_bad_response / request_* + # recovery --> error : request_* + # transmission_sending --> rejected : response_rejected + # recovery --> rejected : response_rejected + # cancelling --> cancelled : response_cancelled + # success --> [*] + # rejected --> [*] + # cancelled --> [*] + # error --> [*] + # ``` + # + # The graph shows the protocol's intended path and its recoverable detours. Failure edges are largely omitted + # to keep it readable: every non-terminal status can also reach `error` (on `:global_deadline_exceeded`, on an + # unretriable request failure, on a fatally bad response, or on any unmatched event) and `rejected` (on + # `:response_rejected`), and every status listed in the `:user_cancel` arm can reach `cancelling`. `cancelling` + # in particular has only its success edge drawn; it fails like any other in-flight state. {Rules.decide} is the + # authoritative enumeration. + # + # ### Router ordering + # + # Arms are evaluated top to bottom, so their order encodes precedence and is load-bearing: + # + # * `enter_recovery` and `fail_with_request_error` both match + # `[:transmission_sending | :finalizing_sending_upload | :finalizing_sending_finalize,` + # `:request_connection_failed | :request_timeout]`. Recovery wins purely because its arm precedes + # `fail_with_request_error`. + # * `[:starting, :response_cat2]` fails instead of recovering, unlike the same shape during transmission + # and finalizing. There is no upload to recover to until initiation yields an upload URL. + # * `recovery` re-queries on `:response_cat2` with no attempt cap. Termination is guaranteed only by the + # global deadline the Driver enforces, not by anything in this module. + # + # See `design/resumable_upload/implementation-guide.md` section 4 for the transition specification and + # section 6.1 for the error category taxonomy this module implements. + # + # rubocop:disable Metrics/ModuleLength + module Rules + ## + # @private + # Default chunk size in bytes (8 MB). + # @return [Integer] + DEFAULT_CHUNK_SIZE = 8_388_608 # 8 MB + + # Failures are classified into three categories, which the rest of this module is written in terms of: + # + # * **Category 1 (transient transport)** - connection resets, DNS failures, load shedding. Handled + # entirely inside the Driver by `Gapic::Common::RetryPolicy`; Core never sees them. Only their + # exhaustion reaches this module, as `:request_retries_exhausted`. + # * **Category 2 (recoverable protocol)** - the client offset may be misaligned with the server, or a + # proxy stripped the protocol headers. Resolved by querying the server for its acknowledged offset + # and realigning, never by blindly retransmitting. Shape: `:response_cat2`. + # * **Category 3 (terminal)** - structurally invalid, unauthorized, rejected, or out of budget. + # Resolved by transitioning to `:error` or `:rejected` and emitting `Instruction::TerminateFailure`. + # + # See `design/resumable_upload/implementation-guide.md` section 6.1 for the full classification. + + ## + # @private + # HTTP status codes eligible for Category 2 (recovery) handling. + # + # Descriptive rather than load-bearing: {Rules.classify_http_response} routes any non-fatal status with a + # missing or empty `X-Goog-Upload-Status` to `:response_cat2`, so this list does not gate the decision. + # It records the codes the upload backend is expected to produce in that situation, and is asserted against + # {Rules.classify_http_response} by the classification tests. + # + # @return [Array] + CAT2_STATUS_CODES = [400, 408, 409, 412, 416, 429, 499].freeze + + ## + # @private + # HTTP status codes that are immediately fatal and non-retriable (Category 3). + # + # Unlike {CAT2_STATUS_CODES} this list is load-bearing: {Rules.classify_http_response} consults it to decide + # between `:response_fatal_bad_response` and `:response_cat2` when the upload status header is absent, + # and {RetryPolicies::START_PREDICATE} consults it to refuse retries outright. + # + # @return [Array] + FATAL_STATUS_CODES = [401, 403, 404, 405, 410, 413, 415].freeze + + ## + # @private + # Human-readable state descriptions for error reporting. + # @return [Hash] + STATE_DESCRIPTIONS = { + initializing: "initializing upload", + starting: "initiating upload session", + transmission_reading: "reading chunk from stream", + transmission_sending: "sending a chunk of data", + finalizing_sending_upload: "sending final data chunk", + finalizing_sending_finalize: "sending finalize command", + recovery: "querying upload offset for recovery", + cancelling: "cancelling upload session", + success: "in completed upload state", + cancelled: "in cancelled upload state", + error: "in error state", + rejected: "in rejected upload state" + }.freeze + + ## + # @private + # Canonical list of protocol lifecycle statuses a {State} may hold. Guaranteed to match the keys of + # {STATE_DESCRIPTIONS} by the classification test suite. + # + # * `:initializing` - nothing dispatched yet; awaits `:start_upload` or `:resume_upload`. + # * `:starting` - initiation request in flight; no upload URL yet. + # * `:transmission_reading` - filling the buffer from the stream. + # * `:transmission_sending` - a non-final chunk is in flight. + # * `:finalizing_sending_upload` - the last chunk is in flight, combined with the finalize command. + # * `:finalizing_sending_finalize` - a standalone finalize is in flight; all data bytes were already sent. + # * `:recovery` - offset query in flight, either after a recoverable failure or as the first step of a + # resume. + # * `:cancelling` - cancel command in flight. Not reachable from the public API. + # * `:success` - terminal; the upload finalized. + # * `:cancelled` - terminal; the server acknowledged cancellation. + # * `:rejected` - terminal; the server refused the upload. + # * `:error` - terminal for this run; `last_error` holds the exception. + # + # `:success`, `:cancelled` and `:rejected` are finalized and yield no {ResumeHandle}. `:error` ends the + # run but may still be resumable from a fresh session; see {Rules.resume_handle_from}. + # + # @return [Array] + STATUSES = [ + :initializing, + :starting, + :transmission_reading, + :transmission_sending, + :finalizing_sending_upload, + :finalizing_sending_finalize, + :recovery, + :cancelling, + :success, + :cancelled, + :rejected, + :error + ].freeze + + ## + # @private + # Terminal protocol lifecycle statuses in {STATUSES}. + # @return [Array] + TERMINAL_STATUSES = [:success, :cancelled, :rejected, :error].freeze + + ## + # @private + # Canonical list of event shapes produced by {Rules.shape_of} and matched by {Rules.decide}, + # grouped by the event family each is reduced from. + # + # Lifecycle signals, one shape each from {Event::StartUpload}, {Event::ResumeUpload}, {Event::Cancel} + # and {Event::GlobalDeadlineExceeded}: `:start_upload`, `:resume_upload`, `:user_cancel`, + # `:global_deadline_exceeded`. + # + # Stream reads, from {Event::ChunkRead} split by EOF and buffer occupancy. The three-way split is what + # lets a zero-length tail finalize without sending an empty chunk: `:chunk_read_full`, + # `:chunk_read_eof_with_data`, `:chunk_read_eof_empty`. + # + # Request failures, from {Event::RequestFailed} split by `kind`: `:request_timeout`, + # `:request_retries_exhausted`, `:request_connection_failed`, `:request_failed_unknown`. + # + # HTTP responses, from {Event::HttpResponse} split by `X-Goog-Upload-Status` and HTTP status: + # `:response_active`, `:response_final`, `:response_cancelled`, `:response_rejected`, `:response_cat2`, + # `:response_fatal_bad_response`. + # + # `:unknown` is a live shape rather than an error sentinel. It is what {Rules.shape_of} returns for anything + # it does not recognise, and it routes to {Rules.fail_with_unmatched_transition}. + # + # @return [Array] + SHAPES = [ + :start_upload, + :resume_upload, + :user_cancel, + :global_deadline_exceeded, + :chunk_read_full, + :chunk_read_eof_with_data, + :chunk_read_eof_empty, + :request_timeout, + :request_retries_exhausted, + :request_connection_failed, + :request_failed_unknown, + :response_active, + :response_final, + :response_cancelled, + :response_rejected, + :response_cat2, + :response_fatal_bad_response, + :unknown + ].freeze + + ## + # @private + # Canonical list of recipe symbols emitted by {Rules.decide}. + # @return [Array] + RECIPES = [ + :start_session, + :resume_session, + :begin_transmission, + :send_chunk, + :send_upload_finalize, + :send_finalize, + :ack_chunk, + :enter_recovery, + :retry_recovery, + :realign_from_recovery, + :complete_upload_with_data, + :complete_upload_finalized, + :cancel_session, + :complete_cancellation, + :fail_with_deadline_exceeded, + :fail_with_rejected, + :fail_with_bad_response, + :fail_with_request_error, + :fail_with_unmatched_transition + ].freeze + + ## + # @private + # Mapping of notifying recipes to their emitted {Progress} phase. + # @return [Hash] + RECIPE_PHASES = { + start_session: :initiating, + resume_session: :initiating, + begin_transmission: :uploading, + ack_chunk: :uploading, + realign_from_recovery: :uploading, + enter_recovery: :recovering, + send_upload_finalize: :finalizing, + send_finalize: :finalizing, + complete_upload_with_data: :completed, + complete_upload_finalized: :completed, + cancel_session: :cancelling + }.freeze + + ## + # @private + # Recipes that do not emit {Instruction::NotifyProgress}. + # @return [Array] + NON_NOTIFYING_RECIPES = [ + :send_chunk, + :retry_recovery, + :complete_cancellation, + :fail_with_deadline_exceeded, + :fail_with_rejected, + :fail_with_bad_response, + :fail_with_request_error, + :fail_with_unmatched_transition + ].freeze + + ## + # @private + # Classifies incoming event into a canonical shape symbol. + # + # @param event [Object] Input event + # @return [Symbol] Canonical event shape + def self.shape_of event + case event + when Event::StartUpload, Event::StartUpload.singleton_class + :start_upload + when Event::ResumeUpload, Event::ResumeUpload.singleton_class + :resume_upload + when Event::ChunkRead + classify_chunk_read event + when Event::Cancel, Event::Cancel.singleton_class + :user_cancel + when Event::GlobalDeadlineExceeded, Event::GlobalDeadlineExceeded.singleton_class + :global_deadline_exceeded + when Event::RequestFailed + classify_request_failed event + when Event::HttpResponse + classify_http_response event + when Class + classify_event_class event + else + :unknown + end + end + + ## + # @private + # Top-level transition decision engine. Matches [state.status, shape]. + # + # @param state [State] Current state + # @param event [Object] Input event + # @param config [StartUploadConfig, ResumeUploadConfig] Static configuration + # @return [Decision] Decision snapshot + # + # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength + def self.decide state, event, config + shape = shape_of event + unless SHAPES.include? shape + raise InternalError, "Resumable upload internal error: shape_of returned unknown shape #{shape.inspect}" + end + + recipe = case [state.status, shape] + in [:initializing, :start_upload] + :start_session + in [:initializing, :resume_upload] + :resume_session + in [:starting, :response_active] + :begin_transmission + in [:transmission_reading, :chunk_read_full] + :send_chunk + in [:transmission_reading, :chunk_read_eof_with_data] + :send_upload_finalize + in [:transmission_reading, :chunk_read_eof_empty] + :send_finalize + in [:transmission_sending, :response_active] + :ack_chunk + # Order matters: `enter_recovery` and `fail_with_request_error` below both match + # `[:transmission_sending | :finalizing_sending_upload | :finalizing_sending_finalize, + # :request_connection_failed | :request_timeout]`. Recovery wins purely because this arm comes first. + in [:transmission_sending | :finalizing_sending_upload | :finalizing_sending_finalize, + :response_cat2 | :request_connection_failed | :request_timeout] + :enter_recovery + in [:finalizing_sending_upload, :response_final] + :complete_upload_with_data + in [:finalizing_sending_finalize | :recovery, :response_final] + :complete_upload_finalized + in [:recovery, :response_active] + :realign_from_recovery + # Re-query with no attempt cap. Only the Driver's global deadline guarantees termination. + in [:recovery, :response_cat2] + :retry_recovery + in [:cancelling, :response_cancelled] + :complete_cancellation + in [_, :global_deadline_exceeded] + :fail_with_deadline_exceeded + in [:transmission_reading | :transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery, :user_cancel] + :cancel_session + in [:starting | :transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery | :cancelling, :response_rejected] + :fail_with_rejected + # `:starting` fails on `:response_cat2` rather than entering recovery, unlike the + # transmission and finalizing states above: there is no upload to recover to until + # initiation has returned an upload URL. + in [:starting | :cancelling, :response_cat2] | + [:starting | :transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery | :cancelling, :response_fatal_bad_response] + :fail_with_bad_response + in [:starting | :transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery | :cancelling, + :request_retries_exhausted | :request_connection_failed | :request_timeout | + :request_failed_unknown] + :fail_with_request_error + else + :fail_with_unmatched_transition + end + + unless RECIPES.include? recipe + raise InternalError, "Resumable upload internal error: decide selected unknown recipe #{recipe.inspect}" + end + + next_state, instructions = public_send recipe, state, event, config + Decision.new( + from_status: state.status, + shape: shape, + recipe: recipe, + next_state: next_state, + instructions: instructions + ) + end + # rubocop:enable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength + + ## + # @private + # Top-level transition router. Matches [state.status, shape]. + # + # @param state [State] Current state + # @param event [Object] Input event + # @param config [StartUploadConfig, ResumeUploadConfig] Static configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.step state, event, config + decision = decide state, event, config + [decision.next_state, decision.instructions] + end + + ## + # @private + # Initiates the upload session. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [StartUploadConfig] Session configuration; only an initiating run reaches this recipe + # @return [Array>] Tuple of [next_state, instructions] + def self.start_session state, _event, config + next_state = state.with status: :starting + progress = Progress.new phase: :initiating, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendStart.new( + url: config.initial_url, + headers: config.initial_headers, + body: config.initial_body + ) + ] + [next_state, instructions] + end + + ## + # @private + # Resumes an existing upload session by transitioning to recovery and querying backend offset. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [ResumeUploadConfig] Resume session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.resume_session state, _event, config + next_state = state.with( + status: :recovery, + upload_url: config.upload_url, + chunk_size: config.chunk_size, + offset: 0 + ) + progress = Progress.new( + phase: :initiating, + bytes_uploaded: 0, + total_bytes: config.upload_size + ) + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendQuery.new(url: config.upload_url) + ] + [next_state, instructions] + end + + ## + # @private + # Processes initiation response and begins data reading. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Initiation response + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.begin_transmission state, event, config + granularity_str = header_value event.headers, "x-goog-upload-chunk-granularity" + granularity = granularity_str&.to_i + chunk_size = resolve_chunk_size config.chunk_size, granularity + upload_url = header_value event.headers, "x-goog-upload-url" + next_state = state.with( + status: :transmission_reading, + upload_url: upload_url, + chunk_granularity: granularity, + chunk_size: chunk_size, + offset: 0, + in_flight_length: 0 + ) + progress = Progress.new phase: :uploading, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::FillBuffer.new(target_bytesize: chunk_size) + ] + [next_state, instructions] + end + + ## + # @private + # Emits instruction to transmit a filled data chunk. + # + # @param state [State] Current state + # @param event [Event::ChunkRead] Chunk read event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.send_chunk state, event, _config + next_state = state.with( + status: :transmission_sending, + in_flight_length: event.bytes_buffered + ) + instructions = [ + Instruction::SendChunk.new( + url: state.upload_url, + offset: state.offset, + length: event.bytes_buffered, + finalize: false + ) + ] + [next_state, instructions] + end + + ## + # @private + # Emits instruction to transmit the final data chunk with finalize. + # + # @param state [State] Current state + # @param event [Event::ChunkRead] Chunk read event with EOF + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.send_upload_finalize state, event, config + next_state = state.with( + status: :finalizing_sending_upload, + in_flight_length: event.bytes_buffered + ) + progress = Progress.new phase: :finalizing, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendChunk.new( + url: state.upload_url, + offset: state.offset, + length: event.bytes_buffered, + finalize: true + ) + ] + [next_state, instructions] + end + + ## + # @private + # Emits instruction to send a zero-length finalize command. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.send_finalize state, _event, config + next_state = state.with( + status: :finalizing_sending_finalize, + in_flight_length: 0 + ) + progress = Progress.new phase: :finalizing, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendFinalize.new(url: state.upload_url) + ] + [next_state, instructions] + end + + ## + # @private + # Acknowledges transmitted chunk and advances offset. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.ack_chunk state, _event, config + new_offset = state.offset + state.in_flight_length + next_state = state.with( + status: :transmission_reading, + offset: new_offset, + in_flight_length: 0 + ) + progress = Progress.new phase: :uploading, bytes_uploaded: new_offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::RealignBuffer.new(server_offset: new_offset), + Instruction::FillBuffer.new(target_bytesize: state.chunk_size) + ] + [next_state, instructions] + end + + ## + # @private + # Transitions to recovery state to query backend byte offset. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.enter_recovery state, _event, config + next_state = state.with( + status: :recovery, + in_flight_length: 0 + ) + progress = Progress.new phase: :recovering, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendQuery.new(url: state.upload_url) + ] + [next_state, instructions] + end + + ## + # @private + # Retries offset query during recovery. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.retry_recovery state, _event, _config + next_state = state.with( + status: :recovery, + in_flight_length: 0 + ) + [next_state, [Instruction::SendQuery.new(url: state.upload_url)]] + end + + ## + # @private + # Completes upload when final chunk transmission succeeds. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Final HTTP response + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.complete_upload_with_data state, event, _config + new_offset = state.offset + state.in_flight_length + next_state = state.with( + status: :success, + offset: new_offset, + in_flight_length: 0 + ) + # `total_bytes` is set from `new_offset` even when `config.upload_size` is nil (see `Progress#total_bytes`). + progress = Progress.new phase: :completed, bytes_uploaded: new_offset, total_bytes: new_offset + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::TerminateSuccess.new(response: event) + ] + [next_state, instructions] + end + + ## + # @private + # Completes upload when standalone finalize succeeds. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Final HTTP response + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.complete_upload_finalized state, event, _config + next_state = state.with( + status: :success, + in_flight_length: 0 + ) + # `total_bytes` is set from `next_state.offset` even when `config.upload_size` is nil + # (see `Progress#total_bytes`). + progress = Progress.new phase: :completed, bytes_uploaded: next_state.offset, total_bytes: next_state.offset + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::TerminateSuccess.new(response: event) + ] + [next_state, instructions] + end + + ## + # @private + # Realigns buffer and resumes transmission from recovered offset. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Query response containing acknowledged offset + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.realign_from_recovery state, event, config + server_offset_str = header_value event.headers, "x-goog-upload-size-received" + server_offset = server_offset_str.to_i + next_state = state.with( + status: :transmission_reading, + offset: server_offset, + in_flight_length: 0 + ) + progress = Progress.new phase: :uploading, bytes_uploaded: server_offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::RealignBuffer.new(server_offset: server_offset), + Instruction::FillBuffer.new(target_bytesize: state.chunk_size) + ] + [next_state, instructions] + end + + ## + # @private + # Completes session cancellation and emits failure instruction. + # + # @param state [State] Current state + # @param event [Object] Cancellation response event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.complete_cancellation state, event, _config + err = UploadCancelledError.from event + next_state = state.with status: :cancelled, in_flight_length: 0, last_error: err + [next_state, [Instruction::TerminateFailure.new(error: err)]] + end + + ## + # @private + # Initiates session cancellation request. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.cancel_session state, _event, config + next_state = state.with status: :cancelling + progress = Progress.new phase: :cancelling, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendCancel.new(url: state.upload_url) + ] + [next_state, instructions] + end + + ## + # @private + # Extracts a {ResumeHandle} from current protocol state. + # Completed uploads (`:success`), rejected uploads (`:rejected`), and cancelled uploads + # (`:cancelled`) are finalized and not resumable, returning `nil`. + # + # @param state [State] Protocol state + # @return [ResumeHandle, nil] Resume handle if upload URL is established and resumable, or nil + def self.resume_handle_from state + return nil if state.nil? || state.upload_url.nil? || [:rejected, :cancelled, :success].include?(state.status) + + ResumeHandle.new upload_url: state.upload_url, chunk_size: state.chunk_size + end + + ## + # @private + # Fails upload due to exceeded execution deadline. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.fail_with_deadline_exceeded state, _event, _config + handle = resume_handle_from state + err = DeadlineExceededError.new resume_handle: handle + next_state = state.with( + status: :error, + in_flight_length: 0, + last_error: err + ) + [next_state, [Instruction::TerminateFailure.new(error: err)]] + end + + ## + # @private + # Fails upload when backend explicitly rejects session. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Rejected HTTP response + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.fail_with_rejected state, event, _config + err = UploadRejectedError.from event + next_state = state.with( + status: :rejected, + in_flight_length: 0, + last_error: err + ) + [next_state, [Instruction::TerminateFailure.new(error: err)]] + end + + ## + # @private + # Fails upload when an unrecoverable HTTP response is encountered. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Fatal HTTP response + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.fail_with_bad_response state, event, _config + handle = resume_handle_from state + err = BadResponseError.from event, resume_handle: handle + next_state = state.with( + status: :error, + in_flight_length: 0, + last_error: err + ) + [next_state, [Instruction::TerminateFailure.new(error: err)]] + end + + ## + # @private + # Fails upload when an unrecoverable network or request error occurs. + # + # @param state [State] Current state + # @param event [Event::RequestFailed] Request failure event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.fail_with_request_error state, event, _config + handle = resume_handle_from state + err = RequestFailedError.from event, resume_handle: handle + next_state = state.with( + status: :error, + in_flight_length: 0, + last_error: err + ) + [next_state, [Instruction::TerminateFailure.new(error: err)]] + end + + ## + # @private + # Raises InvalidTransitionError for unmatched state and event pair. + # + # @param state [State] Current state + # @param event [Object] Dispatched event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @raise [InvalidTransitionError] + def self.fail_with_unmatched_transition state, event, _config + shape = shape_of event + action = STATE_DESCRIPTIONS[state.status] || "processing #{state.status}" + happened = describe_event event, shape + message = "Resumable upload failed while #{action}: #{happened}." + response = event.is_a?(Event::HttpResponse) ? event : nil + handle = resume_handle_from state + raise InvalidTransitionError.new( + message, + state: state.status, + event: event, + response: response, + resume_handle: handle + ) + end + + ## + # @private + # Formats human-readable summary of an event. + # + # @param event [Object] Event instance + # @param shape [Symbol] Event shape symbol + # @return [String] Formatted description + def self.describe_event event, shape + case event + when Event::HttpResponse + upload_status = event.headers["x-goog-upload-status"] || event.headers["X-Goog-Upload-Status"] + status_desc = upload_status ? "'#{upload_status}'" : "missing" + "received an unexpected HTTP #{event.status} response (X-Goog-Upload-Status: #{status_desc})" + when Event::ChunkRead + "received unexpected stream chunk read (#{event.bytes_buffered} bytes, eof: #{event.eof})" + when Event::RequestFailed + "encountered unexpected request failure (#{event.kind}: #{event.message})" + else + "received unexpected event #{shape} (#{event.class.name})" + end + end + + ## + # @private + # Resolves effective chunk size given user specification and backend granularity. + # + # @param user_chunk_size [Integer, nil] Configured chunk size + # @param chunk_granularity [Integer, nil] Backend alignment granularity + # @return [Integer] Effective chunk size in bytes + def self.resolve_chunk_size user_chunk_size, chunk_granularity + base_size = user_chunk_size || DEFAULT_CHUNK_SIZE + return base_size if chunk_granularity.nil? || chunk_granularity <= 0 + return chunk_granularity if base_size <= chunk_granularity + + base_size - (base_size % chunk_granularity) + end + + ## + # @private + # Classifies an HTTP response into a canonical response shape. + # + # @param response [Event::HttpResponse] Response event + # @return [Symbol] Canonical response shape + def self.classify_http_response response + status_header = header_value(response.headers, "x-goog-upload-status")&.downcase + + case status_header + when "active" + response.status == 200 ? :response_active : :response_cat2 + when "final" + response.status == 200 ? :response_final : :response_rejected + when "cancelled" + response.status == 200 ? :response_cancelled : :response_fatal_bad_response + when nil, "" + if FATAL_STATUS_CODES.include? response.status + :response_fatal_bad_response + else + :response_cat2 + end + else + :response_fatal_bad_response + end + end + + ## + # @private + # Case-insensitive header lookup helper. + # + # @param headers [Hash, Object] Headers collection + # @param key [String] Target header key + # @return [String, nil] Header value + def self.header_value headers, key + return nil unless headers.is_a? Hash + return headers[key] if headers.key? key + + target = key.downcase + _, val = headers.find { |k, _| k.to_s.downcase == target } + val + end + + ## + # @private + # Classifies chunk read event by buffer size and EOF flag. + # + # @param event [Event::ChunkRead] Chunk read event + # @return [Symbol] Canonical chunk shape + def self.classify_chunk_read event + if !event.eof + :chunk_read_full + elsif event.bytes_buffered.positive? + :chunk_read_eof_with_data + else + :chunk_read_eof_empty + end + end + + ## + # @private + # Classifies request failure event by failure kind. + # + # @param event [Event::RequestFailed] Request failed event + # @return [Symbol] Canonical failure shape + def self.classify_request_failed event + case event.kind + when :timeout then :request_timeout + when :retries_exhausted then :request_retries_exhausted + when :connection_failed then :request_connection_failed + else :request_failed_unknown + end + end + + ## + # @private + # Classifies raw event class objects. + # + # @param event_class [Class] Event class + # @return [Symbol] Canonical shape + def self.classify_event_class event_class + if event_class == Event::StartUpload + :start_upload + elsif event_class == Event::ResumeUpload + :resume_upload + elsif event_class == Event::Cancel + :user_cancel + elsif event_class == Event::GlobalDeadlineExceeded + :global_deadline_exceeded + else + :unknown + end + end + end + # rubocop:enable Metrics/ModuleLength + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/session.rb b/gapic-common/lib/gapic/rest/resumable_upload/session.rb new file mode 100644 index 0000000..ac0e431 --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/session.rb @@ -0,0 +1,546 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/rest/resumable_upload/data_types" +require "gapic/rest/resumable_upload/driver" +require "gapic/rest/resumable_upload/errors" + +module Gapic + module Rest + module ResumableUpload + ## + # Coordinates a resumable upload across its lifecycle. + # + # A Session performs exactly one run (`start` or `resume`), never both, never twice. + # + # ### Two-State Model + # 1. **Unbound** (`!bound?`): Fresh session prior to execution. Permitted operations: `start` + # or `resume(...)`. + # 2. **Bound** (`bound?`): Session has executed or bound to an upload URL. Permitted operations: + # none (`start` and `resume` both raise {SessionStateError}). + # + # Calling {#resumable?} reports whether a new session can resume the upload (`!resume_handle.nil?`). + # Completed uploads (`:success`) and rejected uploads are finalized and not resumable + # (`resumable?` returns `false`, `resume_handle` returns `nil`). + # + # ### Execution Model + # + # {#start} and {#resume} are synchronous: they block the calling thread for the entire duration of the + # upload and return only on completion or failure. The `on_progress` callback runs on that same thread. + # + # The remaining readers ({#upload_url}, {#bound?}, {#resume_handle}, {#resumable?}, {#running?}) are + # guarded by an internal mutex and may be called from another thread while a run is in progress. Values + # read mid-run are a best-effort snapshot of a state the upload thread is still advancing. + # + # ### Where Arguments Live + # + # The constructor takes what both run types share: the client stub, the stream, `upload_size`, + # `content_type`, `timeout`, the control- and data-plane retry policies, `on_progress` and `logger`. + # Arguments that belong to one run live on the method performing it — `initial_url`, `initial_body`, + # `initial_headers`, `chunk_size` and `start_retry_policy` on {#start}; `upload_url` and `chunk_size`, + # or a {ResumeHandle}, on {#resume}. + # + # ### Recovering From a Failure + # + # A bound session never runs again, so recovery means constructing a new Session. Errors that carry a + # resume handle include the {HasResumeHandle} mixin, which can be rescued directly to catch all of them: + # + # @example Resuming after a recoverable failure + # begin + # session.start initial_url: url + # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e + # raise unless e.resume_handle + # Session.new(client_stub: client_stub, stream: File.open(path, "rb")) + # .resume(resume_handle: e.resume_handle) + # end + # + # The replacement session needs a stream positioned at byte 0 of the whole object, not at the server's + # acknowledged offset; {#resume} fast-forwards on its own. For an unseekable stream that means opening a + # fresh one, since it cannot be rewound. + # + # ### Defaults + # + # * `chunk_size` defaults to 8 MB, then rounds down to a multiple of any chunk granularity the server + # requires. + # * `timeout` defaults to `upload_size / 1 MB per second` when `upload_size` is known, floored at one + # hour, and to one hour flat when it is not. + # + # ### Retry Policies + # + # Retry behavior is partitioned across three policies: `start_retry_policy` on {#start}, and + # `control_plane_retry_policy` and `data_plane_retry_policy` on {#initialize}. + # + # Passing a {Gapic::Common::RetryPolicy} replaces the corresponding default policy outright. Passing a + # Hash overrides only the keys it names and leaves the remaining defaults — including `retry_codes` and + # any status-header predicates — in place. + # + # All three policies share the same default retry codes and exponential backoff settings: + # + # | Setting | Default | + # |---|---| + # | `retry_codes` | `UNAVAILABLE`, `DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED`, `INTERNAL` | + # | `initial_delay` | `1.0` s | + # | `max_delay` | `15.0` s | + # | `multiplier` | `1.3` | + # + # They differ in which requests they govern and how a missing or empty `X-Goog-Upload-Status` response + # header is treated: + # + # | Policy | Governs | Missing status header | + # |---|---|---| + # | `start_retry_policy` | session initiation | **Retriable** on any status (incl. `200`), unless fatal | + # | `control_plane_retry_policy` | `query` and `cancel` | No predicate; decided on `retry_codes` alone | + # | `data_plane_retry_policy` | `upload` and `finalize` | **Not** retriable | + # + # Initiation treats a response missing `X-Goog-Upload-Status` as gateway noise worth retrying; the data + # plane treats it as a response it cannot interpret and refuses to replay bytes against it. + # + class Session + # @return [Gapic::Rest::ClientStub] Underlying REST client stub + attr_reader :client_stub + + ## + # Binary input stream to upload. The stream is assumed to be positioned at byte 0 + # (it is not rewound prior to reading) and is not closed after use. + # + # @return [IO] + attr_reader :stream + + # @return [Integer, nil] Total upload bytes if known upfront + attr_reader :upload_size + + # @return [String, nil] MIME type of uploaded media + attr_reader :content_type + + ## + # Total upload timeout in seconds, covering the whole run rather than any single request. When `nil`, + # it resolves to `upload_size / 1 MB per second` floored at one hour if `upload_size` is known, and to + # one hour flat otherwise. Zero and negative values are treated as `nil`. + # + # @return [Numeric, nil] + attr_reader :timeout + + ## + # Retry policy for control commands (query, cancel). A {Gapic::Common::RetryPolicy} replaces the + # default policy outright; a Hash overrides only the settings it names. + # + # @return [Gapic::Common::RetryPolicy, Hash, nil] + attr_reader :control_plane_retry_policy + + ## + # Retry policy for data commands (upload, finalize). A {Gapic::Common::RetryPolicy} replaces the + # default policy outright; a Hash overrides only the settings it names. + # + # @return [Gapic::Common::RetryPolicy, Hash, nil] + attr_reader :data_plane_retry_policy + + ## + # Callback invoked with {Progress} snapshots during upload execution. + # Executed synchronously on the thread running the upload protocol; it must not block. + # Exceptions raised inside the callback immediately abort the upload session and + # propagate out of {#start} or {#resume}. + # + # @return [Proc, nil] + attr_reader :on_progress + + # @return [Logger, nil] Logger instance + attr_reader :logger + + ## + # Initializes a new Resumable Upload Session. + # + # The constructor takes only what both run types share. Arguments specific to a single run live on + # the method that performs it: initiation details on {#start}, the upload URL and chunk size on + # {#resume}. + # + # @param client_stub [Gapic::Rest::ClientStub] Underlying REST client stub + # @param stream [IO] Binary input stream to upload. Precondition: assumed to be positioned at byte 0 + # (not rewound prior to reading) and not closed after use. + # @param upload_size [Integer, nil] Total upload bytes if known upfront + # @param content_type [String, nil] MIME type of uploaded media + # @param timeout [Numeric, nil] Total upload timeout in seconds covering the whole run. When `nil`, + # resolves to `upload_size / 1 MB per second` floored at one hour if `upload_size` is known, and to + # one hour flat otherwise. + # @param control_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for control + # commands (`query` and `cancel`). See the "Retry Policies" section in the class documentation. + # @param data_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for data + # commands (`upload` and `finalize`). See the "Retry Policies" section in the class documentation. + # @param on_progress [Proc, nil] Progress callback invoked as `->(progress)` with a {Progress} instance. + # Executed synchronously on the upload protocol thread; it must not block. + # Exceptions raised inside the callback abort the session and propagate out of {#start} or {#resume}. + # @param logger [Logger, nil] Logger instance + # + def initialize client_stub:, + stream:, + upload_size: nil, + content_type: nil, + timeout: nil, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + on_progress: nil, + logger: nil + @client_stub = client_stub + @stream = stream + @upload_size = upload_size + @content_type = content_type + @timeout = timeout + @control_plane_retry_policy = control_plane_retry_policy + @data_plane_retry_policy = data_plane_retry_policy + @on_progress = on_progress + @logger = logger + + @mutex = Mutex.new + @running = false + @executed = false + @upload_url = nil + @last_driver = nil + end + + ## + # Returns the raw upload session URL if established. + # + # @return [String, nil] + def upload_url + @mutex.synchronize { upload_url_internal } + end + + ## + # Returns whether the session is bound to a server-side upload. + # + # @return [Boolean] + def bound? + @mutex.synchronize { bound_internal? } + end + + ## + # Returns the current {ResumeHandle} if the session is alive and resumable. + # Completed uploads are not resumable (returns nil). Rejected uploads and + # cancelled uploads are also finalized and not resumable, returning nil. + # + # @return [ResumeHandle, nil] + def resume_handle + @mutex.synchronize { resume_handle_internal } + end + + ## + # Returns whether a new session can resume the upload. + # Completed uploads are not resumable (returns false). Rejected uploads and + # cancelled uploads are also finalized and not resumable (returns false). + # + # @return [Boolean] + def resumable? + @mutex.synchronize { !resume_handle_internal.nil? } + end + + ## + # Returns whether a run is currently executing. + # + # @return [Boolean] + def running? + @mutex.synchronize { @running } + end + + ## + # Starts a new upload session on the server. + # + # A session performs exactly one run (`start` or `resume`). Calling `start` on an already-bound + # or executed session raises {SessionStateError}. Precondition: the stream is assumed to be + # positioned at byte 0 (the session does not rewind it before reading) and is not closed after use. + # + # Blocks the calling thread until the upload completes or fails. + # + # @example Uploading a file with progress reporting + # session = Gapic::Rest::ResumableUpload::Session.new( + # client_stub: client_stub, + # stream: File.open("movie.mp4", "rb"), + # upload_size: File.size("movie.mp4"), + # content_type: "video/mp4", + # on_progress: ->(progress) { puts "#{progress.phase}: #{progress.bytes_uploaded} bytes" } + # ) + # response = session.start initial_url: "https://example.googleapis.com/upload/v1/media" + # + # @param initial_url [String] Initial endpoint URI for session initiation + # @param initial_body [String, nil] Request payload for session initiation + # @param initial_headers [Hash] Additional headers for the initiation request. + # The five reserved protocol headers (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, + # `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, + # `X-Goog-Upload-Header-Content-Length`) are rejected with an `ArgumentError` in any casing — + # they carry protocol mechanics the session owns. Use the constructor's `content_type` and + # `upload_size` to shape the media descriptors. Pass-through headers such as + # `X-Goog-Upload-Header-Content-Disposition` are permitted. + # @param chunk_size [Integer, nil] Requested chunk size in bytes, defaulting to 8 MB. The effective + # size is rounded down to a multiple of any chunk granularity the server requires, or raised to that + # granularity if it exceeds the requested size. + # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for the initiation + # request (`start`). See the "Retry Policies" section in the class documentation. + # @return [String, nil] Raw, undecoded body of the finalizing HTTP response (or `nil` if the + # response carried no body), typically the JSON resource the backend created that the caller + # parses. A client stub carrying response-decoding middleware is outside the contract. + # @raise [ArgumentError] If `initial_url` is missing or blank, if `initial_headers` sets a + # reserved protocol header, or if a retry policy argument is neither a + # {Gapic::Common::RetryPolicy}, a Hash, nor `nil` + # @raise [SessionStateError] If already bound/executed or if a run is currently in progress + # @raise [RequestFailedError] If a transport error, timeout, or retry exhaustion occurs + # @raise [DeadlineExceededError] If the global upload timeout is exceeded + # @raise [BadResponseError] If an unexpected or malformed HTTP response is received + # @raise [UnseekableStreamError] If stream rewinding is required during recovery on an unseekable stream + # @raise [StreamMismatchError] If stream content or length does not match protocol expectations + # @raise [InvalidTransitionError] If an unmatched event occurs for the current protocol state + # @raise [UploadRejectedError] If the server explicitly rejects the upload session + # @raise [InternalError] If the library detects an internal invariant breach; this signals a bug + # in this library rather than a caller or server error + def start initial_url:, + initial_body: nil, + initial_headers: {}, + chunk_size: nil, + start_retry_policy: nil + config = build_start_config initial_url: initial_url, + initial_body: initial_body, + initial_headers: initial_headers, + chunk_size: chunk_size, + start_retry_policy: start_retry_policy + + driver = nil + @mutex.synchronize do + raise SessionStateError, "A run is already in progress for this session" if @running + raise SessionStateError, "Session has already executed a run" if bound_internal? + + driver = Driver.new client_stub: @client_stub, config: config, logger: @logger + @executed = true + @running = true + end + + execute_run driver + end + + ## + # Resumes an upload session using one of two explicit keyword forms: + # 1. `resume(upload_url:, chunk_size:)`: Resumes with explicit URL and chunk size. + # 2. `resume(resume_handle:)`: Resumes via {ResumeHandle}. + # + # A session performs exactly one run (`start` or `resume`). Resuming must be executed on a + # fresh, unexecuted session. Blocks the calling thread until the upload completes or fails. + # + # A resumed run targets an upload the server has already created, so it takes no initiation + # arguments; everything it needs beyond the constructor is on this method. + # + # ### Chunk size + # + # A chunk size must be given explicitly because the server reports chunk granularity during + # initiation, which a resumed run skips. {ResumeHandle} carries the effective value from the original + # run for exactly this reason. + # + # ### Stream position + # + # The stream must be positioned at byte 0 of the whole object, not at the server's acknowledged + # offset, and is not closed after use. The Driver fast-forwards on its own, by seeking on seekable + # streams or by reading and discarding on unseekable ones. An unseekable stream therefore has to be + # freshly opened rather than rewound. + # + # A completed upload is finalized: {#resume_handle} returns `nil` and {#resumable?} returns + # `false`, so there is no handle to resume from. Calling `#resume` on the session that completed + # the run raises {SessionStateError}, as it would after any run. Resuming a *fresh* session + # against a finalized `upload_url` is undefined behavior: it queries the server and might return + # the response body or raise an error, depending on the server response. + # + # @example Resuming from a handle persisted by an earlier process + # handle = Gapic::Rest::ResumableUpload::ResumeHandle.new( + # upload_url: row[:upload_url], + # chunk_size: row[:chunk_size] + # ) + # session = Gapic::Rest::ResumableUpload::Session.new( + # client_stub: client_stub, + # stream: File.open("movie.mp4", "rb"), + # upload_size: File.size("movie.mp4") + # ) + # response = session.resume resume_handle: handle + # + # @param upload_url [String, nil] Explicit upload URL + # @param chunk_size [Integer, nil] Explicit chunk size + # @param resume_handle [ResumeHandle, nil] Explicit resume handle + # @return [String, nil] Raw, undecoded body of the finalizing HTTP response (or `nil` if the + # response carried no body), typically the JSON resource the backend created that the caller + # parses. A client stub carrying response-decoding middleware is outside the contract. + # @raise [ArgumentError] If argument shape is invalid, target upload is missing, or stream.pos != 0 + # @raise [SessionStateError] If already bound/executed or if a run is currently in progress + # @raise [RequestFailedError] If a transport error, timeout, or retry exhaustion occurs + # @raise [DeadlineExceededError] If the global upload timeout is exceeded + # @raise [BadResponseError] If an unexpected or malformed HTTP response is received + # @raise [UnseekableStreamError] If stream rewinding is required during recovery on an unseekable stream + # @raise [StreamMismatchError] If stream content or length does not match the resumed upload + # @raise [InvalidTransitionError] If an unmatched event occurs for the current protocol state + # @raise [UploadRejectedError] If the server explicitly rejects the upload session + # @raise [InternalError] If the library detects an internal invariant breach; this signals a bug + # in this library rather than a caller or server error + def resume upload_url: nil, + chunk_size: nil, + resume_handle: nil + target_url, target_chunk_size = resolve_resume_args( + upload_url: upload_url, + chunk_size: chunk_size, + resume_handle: resume_handle + ) + + driver = nil + @mutex.synchronize do + raise SessionStateError, "A run is already in progress for this session" if @running + raise SessionStateError, "Session has already executed a run" if bound_internal? + + if @stream.respond_to?(:pos) && !@stream.pos.zero? + raise ArgumentError, "Stream must be positioned at byte 0 to resume an upload (got pos #{@stream.pos})" + end + + config = build_resume_config target_url, target_chunk_size + driver = Driver.new client_stub: @client_stub, config: config, logger: @logger + @executed = true + @running = true + @upload_url = target_url + end + + execute_run driver + end + + private + + ## + # @private + # Returns the established upload URL without locking. + # + # @return [String, nil] + def upload_url_internal + @upload_url || @last_driver&.upload_url + end + + ## + # @private + # Returns whether the session is bound without locking. + # + # @return [Boolean] + def bound_internal? + @executed || !upload_url_internal.nil? + end + + ## + # @private + # Returns the current resume handle from the driver without locking. + # + # @return [ResumeHandle, nil] + def resume_handle_internal + @last_driver&.resume_handle + end + + ## + # @private + # Returns the configuration members shared by both run types, mirroring + # {ResumableUpload::COMMON_MEMBERS}. + # + # @return [Hash{Symbol=>Object}] + def common_config_args + { + stream: @stream, + upload_size: @upload_size, + content_type: @content_type, + timeout: @timeout, + control_plane_retry_policy: @control_plane_retry_policy, + data_plane_retry_policy: @data_plane_retry_policy, + on_progress: @on_progress + } + end + + ## + # @private + # Builds configuration for a new upload session. + # + # @param initial_url [String] Initial endpoint URI for session initiation + # @param initial_body [String, nil] Request payload for session initiation + # @param initial_headers [Hash, nil] Additional headers for initiation + # @param chunk_size [Integer, nil] Requested chunk size in bytes + # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Initiation retry policy + # @return [StartUploadConfig] + def build_start_config initial_url:, initial_body:, initial_headers:, chunk_size:, start_retry_policy: + StartUploadConfig.new( + initial_url: initial_url, + initial_body: initial_body, + initial_headers: initial_headers || {}, + chunk_size: chunk_size, + start_retry_policy: start_retry_policy, + **common_config_args + ) + end + + ## + # @private + # Builds configuration for resuming an upload session. + # + # @param target_url [String] Target upload session URL + # @param target_chunk_size [Integer] Effective chunk size in bytes + # @return [ResumeUploadConfig] + def build_resume_config target_url, target_chunk_size + ResumeUploadConfig.new( + upload_url: target_url, + chunk_size: target_chunk_size, + **common_config_args + ) + end + + ## + # @private + # Validates and extracts target upload URL and chunk size from resume keyword arguments. + # + # @param upload_url [String, nil] Explicit upload URL + # @param chunk_size [Integer, nil] Explicit chunk size + # @param resume_handle [ResumeHandle, nil] Explicit resume handle + # @return [Array] Tuple of [upload_url, chunk_size] + # @raise [ArgumentError] If arguments are missing or mutually exclusive + def resolve_resume_args upload_url:, chunk_size:, resume_handle: + if resume_handle + raise ArgumentError, "Cannot pass both resume_handle and upload_url/chunk_size" if upload_url || chunk_size + [resume_handle.upload_url, resume_handle.chunk_size] + elsif upload_url + raise ArgumentError, "Must provide chunk_size with upload_url" if chunk_size.nil? + [upload_url, chunk_size] + elsif chunk_size + raise ArgumentError, "Cannot pass chunk_size without upload_url" + else + raise ArgumentError, "Must provide either resume_handle or upload_url and chunk_size" + end + end + + ## + # @private + # Executes the driver run and records the final upload URL and state. + # + # @param driver [Driver] Driver instance to run + # @return [String, nil] Final response body upon completion + def execute_run driver + @mutex.synchronize { @last_driver = driver } + result = driver.run + @mutex.synchronize do + @upload_url ||= driver.upload_url + @running = false + end + result + rescue StandardError + @mutex.synchronize do + @upload_url ||= driver.upload_url + @running = false + end + raise + end + end + end + end +end diff --git a/gapic-common/test/gapic/rest/client_stub_test.rb b/gapic-common/test/gapic/rest/client_stub_test.rb index f60c9aa..f6c2420 100644 --- a/gapic-common/test/gapic/rest/client_stub_test.rb +++ b/gapic-common/test/gapic/rest/client_stub_test.rb @@ -173,4 +173,45 @@ def test_universe_domain_credentials_mismatch credentials: creds end end + + def test_log_request_retains_valid_utf8_under_1kib + recording = RecordingLogger.new + client_stub = ::Gapic::Rest::ClientStub.new endpoint: "google.example.com", + credentials: :dummy_credentials, + logger: recording + payload = '{"hello":"world"}' + client_stub.send :log_request, "MyMethod", "req-1", 1, payload, { "x-test" => "val" } + + debug_entry = recording.entries.find { |e| e.severity == Logger::DEBUG } + refute_nil debug_entry + assert_equal payload, debug_entry.message.fields["request"] + end + + def test_log_request_abridges_body_over_1kib + recording = RecordingLogger.new + client_stub = ::Gapic::Rest::ClientStub.new endpoint: "google.example.com", + credentials: :dummy_credentials, + logger: recording + payload = "A" * 1025 + expected_prefix = ("A" * 32).unpack1 "H*" + client_stub.send :log_request, "MyMethod", "req-1", 1, payload, {} + + debug_entry = recording.entries.find { |e| e.severity == Logger::DEBUG } + refute_nil debug_entry + assert_equal "<1025 bytes, first 32: #{expected_prefix}>", debug_entry.message.fields["request"] + end + + def test_log_request_abridges_invalid_utf8_body + recording = RecordingLogger.new + client_stub = ::Gapic::Rest::ClientStub.new endpoint: "google.example.com", + credentials: :dummy_credentials, + logger: recording + payload = "\xFF\xFE\x00\x01binary".b + expected_prefix = payload.unpack1 "H*" + client_stub.send :log_request, "MyMethod", "req-1", 1, payload, {} + + debug_entry = recording.entries.find { |e| e.severity == Logger::DEBUG } + refute_nil debug_entry + assert_equal "<10 bytes, first 32: #{expected_prefix}>", debug_entry.message.fields["request"] + end end diff --git a/gapic-common/test/gapic/rest/error_test.rb b/gapic-common/test/gapic/rest/error_test.rb index 6593db1..96c16d9 100644 --- a/gapic-common/test/gapic/rest/error_test.rb +++ b/gapic-common/test/gapic/rest/error_test.rb @@ -323,4 +323,17 @@ def test_surface_absent_details assert_nil gapic_err.details end + + def test_rest_error_prefix_constant + assert_equal "An error has occurred when making a REST request", ::Gapic::Rest::Error::REST_ERROR_PREFIX + + faraday_err = OpenStruct.new( + message: "raw", + response_body: JSON.dump({ "error" => { "message" => "Quota exceeded", "code" => 429 } }), + response_headers: {}, + response_status: 429 + ) + gapic_err = ::Gapic::Rest::Error.wrap_faraday_error faraday_err + assert_equal "#{::Gapic::Rest::Error::REST_ERROR_PREFIX}: Quota exceeded", gapic_err.message + end end diff --git a/gapic-common/test/gapic/rest/resumable_upload/core_test.rb b/gapic-common/test/gapic/rest/resumable_upload/core_test.rb new file mode 100644 index 0000000..7a7a205 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/core_test.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +class CoreTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("test content"), + upload_size: 2048, + chunk_size: 1024 + ) + @core = Core.new @config + end + + def test_initial_state + state = @core.state + assert_equal :initializing, state.status + assert_nil state.upload_url + assert_equal 0, state.offset + assert_equal 1024, state.chunk_size + assert_nil state.chunk_granularity + assert_equal 0, state.in_flight_length + assert_nil state.last_error + assert_nil @core.last_decision + end + + def test_dispatch_updates_state_and_returns_instructions + instructions = @core.dispatch Event::StartUpload.new + assert_equal :starting, @core.state.status + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal :initiating, instructions[0].progress.phase + assert_instance_of Instruction::SendStart, instructions[1] + assert_instance_of Decision, @core.last_decision + assert_equal :initializing, @core.last_decision.from_status + assert_equal :start_upload, @core.last_decision.shape + assert_equal :start_session, @core.last_decision.recipe + assert_equal @core.state, @core.last_decision.next_state + assert_equal instructions, @core.last_decision.instructions + + resp = Event::HttpResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Chunk-Granularity" => "512", + "X-Goog-Upload-Status" => "active" + } + ) + instructions = @core.dispatch resp + assert_equal :transmission_reading, @core.state.status + assert_equal "https://example.com/session/1", @core.state.upload_url + assert_equal 512, @core.state.chunk_granularity + assert_equal 1024, @core.state.chunk_size + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal :uploading, instructions[0].progress.phase + assert_instance_of Instruction::FillBuffer, instructions[1] + assert_equal 1024, instructions[1].target_bytesize + assert_instance_of Decision, @core.last_decision + assert_equal :starting, @core.last_decision.from_status + assert_equal :response_active, @core.last_decision.shape + assert_equal :begin_transmission, @core.last_decision.recipe + assert_equal @core.state, @core.last_decision.next_state + assert_equal instructions, @core.last_decision.instructions + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb b/gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb new file mode 100644 index 0000000..e78d849 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb @@ -0,0 +1,231 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for data types in resumable upload. +# +class DataTypesTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def test_start_upload_config_defaults + stream = StringIO.new "content" + config = StartUploadConfig.new( + initial_url: "https://example.com", + stream: stream + ) + + assert_equal "https://example.com", config.initial_url + assert_same stream, config.stream + assert_nil config.initial_body + assert_equal({}, config.initial_headers) + assert_nil config.upload_size + assert_nil config.chunk_size + assert_nil config.content_type + assert_nil config.timeout + assert_nil config.start_retry_policy + assert_nil config.control_plane_retry_policy + assert_nil config.data_plane_retry_policy + assert_nil config.on_progress + end + + def test_start_upload_config_validations + stream = StringIO.new "content" + + assert_raises ArgumentError do + StartUploadConfig.new initial_url: nil, stream: stream + end + + assert_raises ArgumentError do + StartUploadConfig.new initial_url: " ", stream: stream + end + + assert_raises ArgumentError do + StartUploadConfig.new initial_url: "https://example.com", stream: nil + end + end + + def test_start_upload_config_rejects_reserved_initial_headers + stream = StringIO.new "content" + reserved = ["X-Goog-Upload-Command", "x-goog-upload-command", "X-GOOG-UPLOAD-COMMAND", + "X-Goog-Upload-Protocol", "x-goog-upload-offset", + "X-Goog-Upload-Header-Content-Type", "x-goog-upload-header-content-length"] + + reserved.each do |header| + error = assert_raises ArgumentError do + StartUploadConfig.new initial_url: "https://example.com", stream: stream, initial_headers: { header => "x" } + end + assert_match(/must not set protocol header/, error.message) + assert_includes error.message, header + end + end + + def test_reserved_initial_headers_constant_is_lowercase_and_duplicate_free + assert_equal RESERVED_INITIAL_HEADERS.map(&:downcase), RESERVED_INITIAL_HEADERS + assert_equal RESERVED_INITIAL_HEADERS.uniq, RESERVED_INITIAL_HEADERS + assert_predicate RESERVED_INITIAL_HEADERS, :frozen? + end + + def test_start_upload_config_allows_caller_owned_initial_headers + stream = StringIO.new "content" + headers = { + "X-Goog-Test-Scenario" => "chunk_granularity", + "X-Goog-Upload-Header-Content-Disposition" => 'attachment; filename="movie.mp4"', + "X-Custom" => "value" + } + + config = StartUploadConfig.new initial_url: "https://example.com", stream: stream, initial_headers: headers + + assert_equal headers, config.initial_headers + end + + def test_state_defaults_and_with + state = State.new + + assert_equal :initializing, state.status + assert_nil state.upload_url + assert_equal 0, state.offset + assert_equal 8_388_608, state.chunk_size + assert_nil state.chunk_granularity + assert_equal 0, state.in_flight_length + assert_nil state.last_error + + modified = state.with status: :starting, upload_url: "https://example.com/upload" + assert_equal :starting, modified.status + assert_equal "https://example.com/upload", modified.upload_url + assert_equal :initializing, state.status + end + + def test_event_instantiation + start_event = Event::StartUpload.new + assert_instance_of Event::StartUpload, start_event + + chunk = Event::ChunkRead.new bytes_buffered: 100, eof: true + assert_equal 100, chunk.bytes_buffered + assert chunk.eof + + http = Event::HttpResponse.new status: 200, headers: { "a" => "b" }, body: "body" + assert_equal 200, http.status + assert_equal({ "a" => "b" }, http.headers) + assert_equal "body", http.body + + req_fail = Event::RequestFailed.new kind: :connection_failed, message: "err" + assert_equal :connection_failed, req_fail.kind + assert_equal "err", req_fail.message + end + + def test_instruction_instantiation + start = Instruction::SendStart.new url: "https://example.com" + assert_equal "https://example.com", start.url + assert_equal({}, start.headers) + assert_nil start.body + + chunk = Instruction::SendChunk.new url: "https://example.com", offset: 0, length: 100 + assert_equal 0, chunk.offset + assert_equal 100, chunk.length + refute chunk.finalize + + realign = Instruction::RealignBuffer.new server_offset: 500 + assert_equal 500, realign.server_offset + end + + def test_progress_instantiation + Progress::PHASES.each do |phase| + progress = Progress.new phase: phase, bytes_uploaded: 512, total_bytes: 2048 + assert_equal phase, progress.phase + assert_equal 512, progress.bytes_uploaded + assert_equal 2048, progress.total_bytes + end + + progress_unknown = Progress.new phase: :uploading, bytes_uploaded: 1024 + assert_equal :uploading, progress_unknown.phase + assert_equal 1024, progress_unknown.bytes_uploaded + assert_nil progress_unknown.total_bytes + + assert_raises ArgumentError do + Progress.new bytes_uploaded: 512, total_bytes: 2048 + end + + assert_raises ArgumentError do + Progress.new phase: :invalid_phase, bytes_uploaded: 512, total_bytes: 2048 + end + end + + def test_resume_handle_instantiation + handle = ResumeHandle.new upload_url: "https://upload.example.com/session123", chunk_size: 1_048_576 + assert_equal "https://upload.example.com/session123", handle.upload_url + assert_equal 1_048_576, handle.chunk_size + + assert_raises ArgumentError do + ResumeHandle.new upload_url: "https://upload.example.com/session123" + end + + assert_raises NoMethodError do + handle.upload_url = "https://mutated.com" + end + end + + def test_resume_upload_config_defaults + stream = StringIO.new "content" + config = ResumeUploadConfig.new( + upload_url: "https://upload.example.com/session1", + chunk_size: 1024, + stream: stream + ) + + assert_equal "https://upload.example.com/session1", config.upload_url + assert_equal 1024, config.chunk_size + assert_same stream, config.stream + assert_nil config.upload_size + assert_nil config.content_type + assert_nil config.timeout + refute_respond_to config, :start_retry_policy + assert_nil config.control_plane_retry_policy + assert_nil config.data_plane_retry_policy + assert_nil config.on_progress + end + + def test_resume_upload_config_validations + stream = StringIO.new "content" + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: nil, chunk_size: 1024, stream: stream + end + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: " ", chunk_size: 1024, stream: stream + end + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: "https://example.com", chunk_size: 0, stream: stream + end + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: "https://example.com", chunk_size: -10, stream: stream + end + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: "https://example.com", chunk_size: "1024", stream: stream + end + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: "https://example.com", chunk_size: 1024, stream: nil + end + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver/abridge_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver/abridge_test.rb new file mode 100644 index 0000000..14e1e80 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver/abridge_test.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" + +## +# Unit tests for Gapic::Rest::ResumableUpload::Driver::Abridge. +# +class AbridgeTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def test_bytes_hex_encodes_short_payloads + assert_nil Driver::Abridge.bytes(nil) + + short = "hello" + assert_equal short.unpack1("H*"), Driver::Abridge.bytes(short) + + boundary = "A" * 63 + assert_equal boundary.unpack1("H*"), Driver::Abridge.bytes(boundary) + end + + def test_bytes_abridges_and_hex_encodes_large_payloads + large = "A" * 100 + expected_prefix = ("A" * 32).unpack1 "H*" + assert_equal "#{expected_prefix}... <100 bytes>", Driver::Abridge.bytes(large) + end + + def test_error_body_truncates_at_512_bytes_and_scrubs_invalid_utf8 + assert_nil Driver::Abridge.error_body(nil) + + long_err = "E" * 600 + assert_equal 512, Driver::Abridge.error_body(long_err).bytesize + + invalid_utf8 = "error \xFF\xFE message".b + scrubbed = Driver::Abridge.error_body invalid_utf8 + assert scrubbed.valid_encoding? + assert_includes scrubbed, "error " + assert_includes scrubbed, " message" + end + + def test_url_elides_query_parameter_values + assert_nil Driver::Abridge.url(nil) + + url = "https://storage.googleapis.com/upload/storage/v1/b/bucket/o?uploadType=resumable&sid=SECRET123" + assert_equal "https://storage.googleapis.com/upload/storage/v1/b/bucket/o?uploadType=<...>&sid=<...>", + Driver::Abridge.url(url) + end + + def test_headers_retains_x_goog_upload_and_redacts_others + headers = { + "X-Goog-Upload-Command" => "upload, finalize", + "X-Goog-Upload-Offset" => "0", + "Authorization" => "Bearer SECRET123", + "Content-Type" => "application/octet-stream" + } + + abridged = Driver::Abridge.headers headers + assert_equal "upload, finalize", abridged["X-Goog-Upload-Command"] + assert_equal "0", abridged["X-Goog-Upload-Offset"] + assert_equal "<...>", abridged["Authorization"] + assert_equal "<...>", abridged["Content-Type"] + end + + def test_instructions_summarizes_without_bodies + instructions = [ + Instruction::SendStart.new(url: "https://example.com/upload?key=SECRET", headers: {}, body: "secret_body"), + Instruction::SendChunk.new(url: "https://example.com/session?id=123", offset: 0, length: 64, finalize: true), + Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: 64, total_bytes: 1024)) + ] + + summary = Driver::Abridge.instructions instructions + assert_equal "SendStart", summary[0]["type"] + assert_equal "https://example.com/upload?key=<...>", summary[0]["url"] + refute summary[0].key?("body") + + assert_equal "SendChunk", summary[1]["type"] + assert_equal "https://example.com/session?id=<...>", summary[1]["url"] + assert_equal 0, summary[1]["offset"] + assert_equal 64, summary[1]["length"] + assert_equal true, summary[1]["finalize"] + + assert_equal "NotifyProgress", summary[2]["type"] + assert_equal "uploading", summary[2]["phase"] + assert_equal 64, summary[2]["bytesUploaded"] + assert_equal 1024, summary[2]["totalBytes"] + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver/upload_log_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver/upload_log_test.rb new file mode 100644 index 0000000..49a651b --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver/upload_log_test.rb @@ -0,0 +1,216 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" + +## +# Unit tests for Gapic::Rest::ResumableUpload::Driver::UploadLog. +# +class UploadLogTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @recording = RecordingLogger.new + stub_logger = Gapic::LoggingConcerns::StubLogger.new logger: @recording, service: "ResumableUpload" + @upload_log = Driver::UploadLog.new stub_logger, upload_id: "test-upload-id" + @config = StartUploadConfig.new initial_url: "https://example.com/upload", + initial_body: nil, + initial_headers: {}, + stream: StringIO.new("data"), + upload_size: 1024, + chunk_size: 256, + content_type: "text/plain", + timeout: nil, + start_retry_policy: nil, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + on_progress: nil + end + + def test_decision_logs_debug_with_fields_from_rules_decide + decision = Rules.decide State.new(status: :initializing), Event::StartUpload.new, @config + + @upload_log.decision decision + + assert_equal 1, @recording.entries.size + entry = @recording.entries.first + assert_equal Logger::DEBUG, entry.severity + fields = entry.message.fields + assert_equal "test-upload-id", fields["uploadId"] + assert_equal "initializing", fields["fromStatus"] + assert_equal "start_upload", fields["shape"] + assert_equal "start_session", fields["recipe"] + assert_equal "starting", fields["toStatus"] + assert_equal 0, fields["offset"] + assert_equal 0, fields["inFlightLength"] + assert_equal [ + { "type" => "NotifyProgress", "phase" => "initiating", "bytesUploaded" => 0, "totalBytes" => 1024 }, + { "type" => "SendStart", "url" => "https://example.com/upload" } + ], fields["instructions"] + end + + def test_lifecycle_start_session_logs_info + decision = Rules.decide State.new(status: :initializing), Event::StartUpload.new, @config + + @upload_log.lifecycle decision, @config + + entry = @recording.entries.first + assert_equal Logger::INFO, entry.severity + fields = entry.message.fields + assert_equal "start_session", fields["recipe"] + assert_equal 1024, fields["uploadSize"] + assert_equal 256, fields["requestedChunkSize"] + end + + def test_lifecycle_send_chunk_logs_debug + state = State.new status: :transmission_reading, upload_url: "https://example.com/session", offset: 0, chunk_size: 256 + event = Event::ChunkRead.new bytes_buffered: 256, eof: false + decision = Rules.decide state, event, @config + + @upload_log.lifecycle decision, @config + + entry = @recording.entries.first + assert_equal Logger::DEBUG, entry.severity + fields = entry.message.fields + assert_equal "send_chunk", fields["recipe"] + assert_equal 0, fields["offset"] + assert_equal 256, fields["inFlightLength"] + end + + def test_lifecycle_terminal_failure_logs_warn + state = State.new status: :starting + event = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" }, body: "Forbidden" + decision = Rules.decide state, event, @config + + @upload_log.lifecycle decision, @config + + entry = @recording.entries.first + assert_equal Logger::WARN, entry.severity + fields = entry.message.fields + assert_equal "fail_with_rejected", fields["recipe"] + assert_includes fields["error"], "Forbidden" + end + + def test_lifecycle_silent_recipes_emit_no_logs + state = State.new status: :transmission_sending + active = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "active" } + decision = Rules.decide state, active, @config + assert_equal :ack_chunk, decision.recipe + + @upload_log.lifecycle decision, @config + + assert_empty @recording.entries + end + + def test_lifecycle_table_matches_rules_recipes + lifecycle_keys = Driver::UploadLog::LIFECYCLE.keys + silent_keys = Driver::UploadLog::SILENT_RECIPES + all_upload_log_recipes = lifecycle_keys + silent_keys + + assert_empty Rules::RECIPES - all_upload_log_recipes, + "Rules recipes not covered by UploadLog::LIFECYCLE or SILENT_RECIPES" + assert_empty all_upload_log_recipes - Rules::RECIPES, + "Extra recipes in UploadLog::LIFECYCLE or SILENT_RECIPES not in Rules::RECIPES" + assert_empty lifecycle_keys & silent_keys, + "Recipes present in both UploadLog::LIFECYCLE and SILENT_RECIPES" + end + + def test_wire_send_logs_debug_with_start_attempt_and_hex_body + @upload_log.wire_send method: "POST", + url: "https://example.com/session?key=SECRET", + headers: { + "X-Goog-Upload-Command" => "upload", + "X-Goog-Upload-Offset" => "256", + "Authorization" => "Bearer SECRET" + }, + start_attempt: 2, + body_size: 4, + body: "test" + + entry = @recording.entries.first + assert_equal Logger::DEBUG, entry.severity + fields = entry.message.fields + assert_equal "POST", fields["method"] + assert_equal "upload", fields["command"] + assert_equal 256, fields["offset"] + assert_equal "https://example.com/session?key=<...>", fields["url"] + assert_equal 2, fields["startAttempt"] + assert_equal 4, fields["bodySize"] + assert_equal "74657374", fields["body"] + assert_equal "<...>", fields["headers"]["Authorization"] + assert_equal "upload", fields["headers"]["X-Goog-Upload-Command"] + end + + def test_wire_receive_logs_debug + event = Event::HttpResponse.new status: 200, + headers: { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "256", + "X-Goog-Upload-Chunk-Granularity" => "256" + }, + body: "ok" + + @upload_log.wire_receive event + + entry = @recording.entries.first + assert_equal Logger::DEBUG, entry.severity + fields = entry.message.fields + assert_equal 200, fields["status"] + assert_equal "active", fields["uploadStatus"] + assert_equal 256, fields["sizeReceived"] + assert_equal 256, fields["granularity"] + assert_equal "6f6b", fields["body"] + end + + def test_wire_failure_logs_debug + event = Event::RequestFailed.new kind: :timeout, message: "timed out" + + @upload_log.wire_failure event + + entry = @recording.entries.first + assert_equal Logger::DEBUG, entry.severity + fields = entry.message.fields + assert_equal "timeout", fields["kind"] + end + + def test_buffer_realign_logs_warn_and_debug_on_unseekable_rewind + @upload_log.buffer_realign "rewind", server_offset: 0, current_offset: 256, unseekable: true + + assert_equal 2, @recording.entries.size + warn_entry, debug_entry = @recording.entries + assert_equal Logger::WARN, warn_entry.severity + assert_equal "rewind", warn_entry.message.fields["action"] + assert_equal 0, warn_entry.message.fields["serverOffset"] + assert_equal 256, warn_entry.message.fields["currentOffset"] + + assert_equal Logger::DEBUG, debug_entry.severity + end + + def test_unmatched_transition_logs_warn + state = State.new status: :initializing + event = Event::HttpResponse.new status: 200, headers: {}, body: "" + err = InvalidTransitionError.new "no transition" + + @upload_log.unmatched_transition state, event, err + + entry = @recording.entries.first + assert_equal Logger::WARN, entry.severity + fields = entry.message.fields + assert_equal "initializing", fields["status"] + assert_equal "no transition", fields["error"] + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb new file mode 100644 index 0000000..a7e110d --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb @@ -0,0 +1,378 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for Driver stream reading and buffer realignment mechanics. +# +class DriverBufferTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + ## + # Stream double that returns at most max_chunk_size bytes per read call. + # + class ChunkedStream + def initialize data, max_chunk_size + @io = StringIO.new data + @max_chunk_size = max_chunk_size + end + + def read length = nil + return @io.read if length.nil? + + actual_length = [length, @max_chunk_size].min + @io.read actual_length + end + + def seek offset + @io.seek offset + end + + def pos + @io.pos + end + end + + ## + # Stream double that intentionally does not implement #seek. + # + class UnseekableStream + def initialize data + @io = StringIO.new data + end + + def read length = nil + @io.read length + end + + def pos + @io.pos + end + end + + def setup + @dummy_client = Object.new + end + + # ============================================================================ + # execute_fill_buffer tests + # ============================================================================ + + def test_fill_buffer_short_reads_accumulates_until_target + data = "abcdefghijklmnopqrstuvwxyz" * 4 # 104 bytes + stream = ChunkedStream.new data, 20 + driver = build_driver stream: stream + + event = driver.send :execute_fill_buffer, Instruction::FillBuffer.new(target_bytesize: 100) + + assert_instance_of Event::ChunkRead, event + assert_equal 100, event.bytes_buffered + refute event.eof + assert_equal data.byteslice(0, 100), driver.instance_variable_get(:@buffer) + end + + def test_fill_buffer_eof_exactly_at_target_boundary + data = "0123456789" * 10 # exactly 100 bytes + stream = StringIO.new data + driver = build_driver stream: stream + + event = driver.send :execute_fill_buffer, Instruction::FillBuffer.new(target_bytesize: 100) + + assert_instance_of Event::ChunkRead, event + assert_equal 100, event.bytes_buffered + # eof stays false until a subsequent read attempts to read past boundary + refute event.eof + assert_equal 100, stream.pos + + # Subsequent fill detects EOF + second_event = driver.send :execute_fill_buffer, Instruction::FillBuffer.new(target_bytesize: 101) + assert_equal 100, second_event.bytes_buffered + assert second_event.eof + end + + def test_fill_buffer_eof_mid_fill + data = "short data of 45 bytes......................." # 45 bytes + stream = StringIO.new data + driver = build_driver stream: stream + + event = driver.send :execute_fill_buffer, Instruction::FillBuffer.new(target_bytesize: 100) + + assert_instance_of Event::ChunkRead, event + assert_equal 45, event.bytes_buffered + assert event.eof + assert_equal data, driver.instance_variable_get(:@buffer) + end + + def test_fill_buffer_empty_stream + stream = StringIO.new "" + driver = build_driver stream: stream + + event = driver.send :execute_fill_buffer, Instruction::FillBuffer.new(target_bytesize: 100) + + assert_instance_of Event::ChunkRead, event + assert_equal 0, event.bytes_buffered + assert event.eof + assert_equal "".b, driver.instance_variable_get(:@buffer) + end + + # ============================================================================ + # execute_realign_buffer tests: trim within buffer + # ============================================================================ + + def test_realign_buffer_trim_exact_beginning + driver = build_driver stream: StringIO.new + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "0123456789".b + + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1000) + + assert_equal 1000, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "0123456789".b, driver.instance_variable_get(:@buffer) + end + + def test_realign_buffer_trim_middle + driver = build_driver stream: StringIO.new + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "0123456789".b + + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1004) + + assert_equal 1004, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "456789".b, driver.instance_variable_get(:@buffer) + end + + def test_realign_buffer_trim_exact_end + driver = build_driver stream: StringIO.new + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "0123456789".b + + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1010) + + assert_equal 1010, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "".b, driver.instance_variable_get(:@buffer) + end + + # ============================================================================ + # execute_realign_buffer tests: rewind stream + # ============================================================================ + + def test_realign_buffer_rewind_seekable_stream + stream = StringIO.new "0123456789" * 100 + stream.seek 1000 + driver = build_driver stream: stream + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "buffered".b + + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 500) + + assert_equal 500, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "".b, driver.instance_variable_get(:@buffer) + assert_equal 500, stream.pos + end + + def test_realign_buffer_rewind_unseekable_stream_raises_error + stream = UnseekableStream.new "0123456789" * 100 + driver = build_driver stream: stream + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "buffered".b + + err = assert_raises UnseekableStreamError do + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 500) + end + + assert_includes err.message, "offset 500" + assert_includes err.message, "buffered from 1000" + assert_nil err.resume_handle + refute_includes err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_realign_buffer_rewind_unseekable_stream_with_resume_handle + stream = UnseekableStream.new "0123456789" * 100 + driver = build_driver stream: stream + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :recovery, upload_url: "https://upload.example.com/session_1", chunk_size: 256) + ) + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "buffered".b + + err = assert_raises UnseekableStreamError do + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 500) + end + + refute_nil err.resume_handle + assert_equal "https://upload.example.com/session_1", err.resume_handle.upload_url + assert_equal 256, err.resume_handle.chunk_size + assert_includes err.message, "offset 500" + assert_includes err.message, "buffered from 1000" + assert_includes err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_driver_resume_handle_property + stream = StringIO.new "test" + driver = build_driver stream: stream + assert_nil driver.resume_handle + + driver.core.instance_variable_set( + :@state, + driver.core.state.with(upload_url: "https://upload.example.com/session_2", chunk_size: 512) + ) + handle = driver.resume_handle + refute_nil handle + assert_equal "https://upload.example.com/session_2", handle.upload_url + assert_equal 512, handle.chunk_size + + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :rejected, upload_url: "https://upload.example.com/session_2", chunk_size: 512) + ) + assert_nil driver.resume_handle + + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :cancelled, upload_url: "https://upload.example.com/session_2", chunk_size: 512) + ) + assert_nil driver.resume_handle + end + + # ============================================================================ + # execute_realign_buffer tests: fast forward stream + # ============================================================================ + + def test_realign_buffer_fast_forward_seekable_stream + stream = StringIO.new "0123456789" * 200 + stream.seek 1010 + driver = build_driver stream: stream + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "0123456789".b # buffer ends at 1010 + + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1050) + + assert_equal 1050, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "".b, driver.instance_variable_get(:@buffer) + assert_equal 1050, stream.pos + end + + def test_realign_buffer_fast_forward_unseekable_stream + # Stream contains 1000 bytes. Buffer has consumed up to 10 bytes (buffer_start=0, length=10 -> buffer_end=10). + # Unseekable stream pos is currently at 10. + stream = UnseekableStream.new "0123456789" * 100 + stream.read 10 # advance stream to match buffer_end + assert_equal 10, stream.pos + + driver = build_driver stream: stream + driver.instance_variable_set :@buffer_start_offset, 0 + driver.instance_variable_set :@buffer, "0123456789".b # ends at offset 10 + + # Fast forward to 50 (discards 40 bytes from stream) + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 50) + + assert_equal 50, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "".b, driver.instance_variable_get(:@buffer) + assert_equal 50, stream.pos + assert_equal "0123456789", stream.read(10) + end + + def test_fast_forward_unseekable_stream_raises_stream_mismatch_on_unexpected_eof + # Stream has only 20 bytes total + stream = UnseekableStream.new "01234567890123456789" + resume_config = ResumeUploadConfig.new( + upload_url: "https://upload.example.com/session_resume", + chunk_size: 256, + stream: stream + ) + driver = Driver.new client_stub: @dummy_client, config: resume_config + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :recovery, upload_url: "https://upload.example.com/session_resume", chunk_size: 256) + ) + driver.instance_variable_set :@buffer, "".b + driver.instance_variable_set :@buffer_start_offset, 0 + + # Server offset is 50, but stream only has 20 bytes -> EOF hit during discard + err = assert_raises StreamMismatchError do + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 50) + end + + refute_nil err.resume_handle + assert_equal "https://upload.example.com/session_resume", err.resume_handle.upload_url + assert_includes err.message, "unexpected EOF during fast-forward" + assert_includes err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_realign_buffer_raises_stream_mismatch_when_server_offset_exceeds_upload_size + stream = StringIO.new "data" + resume_config = ResumeUploadConfig.new( + upload_url: "https://upload.example.com/session_resume", + chunk_size: 256, + stream: stream, + upload_size: 500 + ) + driver = Driver.new client_stub: @dummy_client, config: resume_config + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :recovery, upload_url: "https://upload.example.com/session_resume", chunk_size: 256) + ) + + err = assert_raises StreamMismatchError do + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 600) + end + + refute_nil err.resume_handle + assert_equal "https://upload.example.com/session_resume", err.resume_handle.upload_url + assert_includes err.message, "Server reported offset 600 exceeds total upload size 500" + assert_includes err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_realign_buffer_fast_forward_seekable_stream_raises_stream_mismatch_when_exceeding_stream_size + stream = StringIO.new "hello" + resume_config = ResumeUploadConfig.new( + upload_url: "https://upload.example.com/session_resume", + chunk_size: 256, + stream: stream, + upload_size: nil + ) + driver = Driver.new client_stub: @dummy_client, config: resume_config + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :recovery, upload_url: "https://upload.example.com/session_resume", chunk_size: 256) + ) + + err = assert_raises StreamMismatchError do + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1000) + end + + refute_nil err.resume_handle + assert_equal "https://upload.example.com/session_resume", err.resume_handle.upload_url + assert_includes err.message, "Server reported offset 1000 exceeds stream size 5" + assert_includes err.message, "(upload session is resumable: see #resume_handle)" + end + + private + + def build_driver stream: + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: stream, + upload_size: 2000, + chunk_size: 100 + ) + Driver.new client_stub: @dummy_client, config: config + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb new file mode 100644 index 0000000..64f859e --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb @@ -0,0 +1,295 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Driver configuration and deadline resolution. +# +class DriverConfigTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + # Fake client stub recording calls and yielding scripted responses. + class FakeClientStub + attr_reader :requests + + def initialize responses = [], on_request: nil + @responses = responses + @requests = [] + @on_request = on_request + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } + @on_request&.call + raise "Unexpected request: no scripted response left" if @responses.empty? + + resp = @responses.shift + raise resp if resp.is_a? Exception + + resp.respond_to?(:call) ? resp.call : resp + end + end + + FakeResponse = Data.define :status, :headers, :body + + def test_resolve_timeout_prefers_positive_config_timeout + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 10 * 1_048_576, + timeout: 42 + ) + driver = Driver.new client_stub: stub, config: config + + assert_equal 42, driver.send(:resolve_timeout) + end + + def test_resolve_timeout_treats_zero_timeout_same_as_nil + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + timeout: 0 + ) + driver = Driver.new client_stub: stub, config: config + + assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) + end + + def test_resolve_timeout_treats_negative_timeout_same_as_nil + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + timeout: -10 + ) + driver = Driver.new client_stub: stub, config: config + + assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) + end + + def test_resolve_timeout_calculates_from_upload_size_above_base_timeout + stub = FakeClientStub.new + large_size = 7_200 * Driver::MIN_ASSUMED_THROUGHPUT # 7200 seconds at 1MB/s + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: large_size + ) + driver = Driver.new client_stub: stub, config: config + + assert_in_delta 7_200.0, driver.send(:resolve_timeout), 0.001 + end + + def test_resolve_timeout_uses_base_timeout_floor_for_small_upload_size + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 1_048_576 # 1 second at 1MB/s < 3600 + ) + driver = Driver.new client_stub: stub, config: config + + assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) + end + + def test_resolve_timeout_defaults_to_base_timeout_when_upload_size_nil + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123") + ) + driver = Driver.new client_stub: stub, config: config + + assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) + end + + def test_run_raises_deadline_exceeded_when_timeout_expires + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + timeout: 5 + ) + driver = Driver.new client_stub: stub, config: config + + # Stub monotonic clock so that initial check sets deadline at t=105, and subsequent checks read t=110 + clock_ticks = [100.0, 110.0, 110.0] + Process.stub :clock_gettime, ->(_clock_id) { clock_ticks.shift || 110.0 } do + assert_raises DeadlineExceededError do + driver.run + end + end + assert_empty stub.requests + end + + def test_run_raises_deadline_exceeded_when_clock_advances_past_deadline_mid_batch + current_time = 100.0 + responses = [ + FakeResponse.new( + status: 200, + headers: { "X-Goog-Upload-Status" => "active", "X-Goog-Upload-URL" => "https://example.com/session" }, + body: "" + ) + ] + stub = FakeClientStub.new responses + # Advance clock past deadline (105.0) mid-batch during NotifyProgress(:finalizing) before SendChunk + on_progress = lambda do |progress| + current_time = 110.0 if progress.phase == :finalizing + end + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + timeout: 5, + on_progress: on_progress + ) + driver = Driver.new client_stub: stub, config: config + + Process.stub :clock_gettime, ->(_clock_id) { current_time } do + assert_raises DeadlineExceededError do + driver.run + end + end + + # Only the start request was made; SendChunk hit deadline_exceeded? inside make_post_request + assert_equal 1, stub.requests.size + end + + def test_make_post_request_passes_timeout_close_to_remaining_budget_and_decreases_across_calls + current_time = 1000.0 + stub = FakeClientStub.new(scripted_recovery_responses, on_request: -> { current_time += 10.0 }) + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + timeout: 100.0, + data_plane_retry_policy: Gapic::Common::RetryPolicy.new(timeout: 85.0) + ) + driver = Driver.new client_stub: stub, config: config + + Process.stub :clock_gettime, ->(_clock_id) { current_time } do + assert_equal "done", driver.run + end + + timeouts = stub.requests.map { |req| req[:options][:timeout] } + assert_equal [100.0, 85.0, 80.0, 70.0], timeouts + timeouts.each_cons 2 do |prev_timeout, next_timeout| + assert_operator prev_timeout, :>, next_timeout + end + end + + def test_start_headers_derives_content_descriptors_from_config + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + content_type: "application/octet-stream" + ) + driver = Driver.new client_stub: FakeClientStub.new, config: config + instruction = Instruction::SendStart.new url: "https://example.com/upload" + + headers = driver.send :start_headers, instruction + + assert_equal "application/octet-stream", headers["X-Goog-Upload-Header-Content-Type"] + assert_equal "4", headers["X-Goog-Upload-Header-Content-Length"] + end + + def test_start_headers_passes_unrelated_caller_headers_through + config = StartUploadConfig.new initial_url: "https://example.com/upload", stream: StringIO.new("0123") + driver = Driver.new client_stub: FakeClientStub.new, config: config + instruction = Instruction::SendStart.new( + url: "https://example.com/upload", + headers: { "X-Custom" => "value" } + ) + + headers = driver.send :start_headers, instruction + + assert_equal "value", headers["X-Custom"] + assert_equal "resumable", headers["X-Goog-Upload-Protocol"] + assert_equal "start", headers["X-Goog-Upload-Command"] + end + + def test_start_headers_without_caller_headers_is_unchanged + config = StartUploadConfig.new initial_url: "https://example.com/upload", stream: StringIO.new("0123") + driver = Driver.new client_stub: FakeClientStub.new, config: config + instruction = Instruction::SendStart.new url: "https://example.com/upload" + + headers = driver.send :start_headers, instruction + + assert_equal({ "X-Goog-Upload-Protocol" => "resumable", "X-Goog-Upload-Command" => "start" }, headers) + end + + def test_start_headers_merges_caller_pass_through_upload_header + caller_headers = { + "X-Goog-Upload-Header-Content-Disposition" => 'attachment; filename="movie.mp4"', + "X-Custom" => "value" + } + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("content"), + initial_headers: caller_headers, + content_type: "video/mp4", + upload_size: 7 + ) + driver = Driver.new client_stub: FakeClientStub.new, config: config + instruction = Instruction::SendStart.new url: "https://example.com/upload", headers: caller_headers + + headers = driver.send :start_headers, instruction + + assert_equal 'attachment; filename="movie.mp4"', headers["X-Goog-Upload-Header-Content-Disposition"] + assert_equal "value", headers["X-Custom"] + assert_equal "resumable", headers["X-Goog-Upload-Protocol"] + assert_equal "start", headers["X-Goog-Upload-Command"] + assert_equal "video/mp4", headers["X-Goog-Upload-Header-Content-Type"] + assert_equal "7", headers["X-Goog-Upload-Header-Content-Length"] + end + + private + + def scripted_recovery_responses + [ + FakeResponse.new( + status: 200, + headers: { "X-Goog-Upload-Status" => "active", "X-Goog-Upload-URL" => "https://example.com/session" }, + body: "" + ), + Gapic::Rest::Error.new( + "Service Unavailable", + 503, + headers: { "X-Goog-Upload-Status" => "active" } + ), + FakeResponse.new( + status: 200, + headers: { "X-Goog-Upload-Status" => "active", "X-Goog-Upload-Size-Received" => "0" }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "X-Goog-Upload-Status" => "final" }, + body: "done" + ) + ] + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb new file mode 100644 index 0000000..b7b5e1b --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb @@ -0,0 +1,209 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" +require "faraday" + +## +# Tests for Driver network error mapping to protocol events. +# +class DriverErrorMappingTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + ## + # Integration fake client stub that raises configured errors on make_post_request. + # + class FailingClientStub + attr_accessor :error_to_raise + + def make_post_request uri:, body: nil, params: {}, options: {}, method_name: nil + raise @error_to_raise if @error_to_raise + + raise "No error configured" + end + end + + def setup + @client_stub = FailingClientStub.new + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4 + ) + @driver = Driver.new client_stub: @client_stub, config: @config + end + + # ============================================================================ + # SUT: rescue_request_error + # ============================================================================ + + def test_rescue_request_error_rest_deadline_exceeded + err = Gapic::Rest::DeadlineExceededError.new "RPC deadline exceeded", 504 + event = @driver.send :rescue_request_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :timeout, event.kind + assert_equal "RPC deadline exceeded", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :timeout, integration_event.kind + assert_same err, integration_event.source_error + end + + def test_rescue_request_error_rest_error_with_status_code + err = Gapic::Rest::Error.new "Service Unavailable", 503, headers: { "Retry-After" => "15" } + event = @driver.send :rescue_request_error, err + + assert_instance_of Event::HttpResponse, event + assert_equal 503, event.status + assert_equal({ "Retry-After" => "15" }, event.headers) + assert_equal "Service Unavailable", event.body + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::HttpResponse, integration_event + assert_equal 503, integration_event.status + assert_equal({ "Retry-After" => "15" }, integration_event.headers) + assert_equal "Service Unavailable", integration_event.body + end + + def test_rescue_request_error_rest_error_without_status_code + err = Gapic::Rest::Error.new "Client network error", nil + event = @driver.send :rescue_request_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :connection_failed, event.kind + assert_equal "Client network error", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :connection_failed, integration_event.kind + assert_same err, integration_event.source_error + end + + def test_rescue_request_error_standard_error + err = RuntimeError.new "Unexpected low-level runtime error" + event = @driver.send :rescue_request_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :connection_failed, event.kind + assert_equal "Unexpected low-level runtime error", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :connection_failed, integration_event.kind + assert_same err, integration_event.source_error + end + + # ============================================================================ + # SUT: rescue_faraday_error + # ============================================================================ + + def test_rescue_faraday_error_with_response + response_env = { + status: 400, + headers: { "x-goog-upload-status" => "final" }, + body: '{"error":{"message":"Bad Request"}}' + } + err = Faraday::ClientError.new "the server responded with status 400", response_env + event = @driver.send :rescue_faraday_error, err + + assert_instance_of Event::HttpResponse, event + assert_equal 400, event.status + assert_equal({ "x-goog-upload-status" => "final" }, event.headers) + assert_equal '{"error":{"message":"Bad Request"}}', event.body + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::HttpResponse, integration_event + assert_equal 400, integration_event.status + assert_equal '{"error":{"message":"Bad Request"}}', integration_event.body + end + + def test_rescue_faraday_error_timeout + err = Faraday::TimeoutError.new "Net::ReadTimeout with https://example.com" + event = @driver.send :rescue_faraday_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :timeout, event.kind + assert_equal "Net::ReadTimeout with https://example.com", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :timeout, integration_event.kind + assert_same err, integration_event.source_error + end + + def test_rescue_faraday_error_connection_failed + err = Faraday::ConnectionFailed.new "Connection refused - connect(2)" + event = @driver.send :rescue_faraday_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :connection_failed, event.kind + assert_equal "Connection refused - connect(2)", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :connection_failed, integration_event.kind + assert_same err, integration_event.source_error + end + + def test_rescue_faraday_error_generic_without_response + err = Faraday::Error.new "Generic transport error without response env" + event = @driver.send :rescue_faraday_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :retries_exhausted, event.kind + assert_equal "Generic transport error without response env", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :retries_exhausted, integration_event.kind + assert_same err, integration_event.source_error + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb new file mode 100644 index 0000000..4cf93d2 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb @@ -0,0 +1,574 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" +require "google/rpc/error_details_pb" + +## +# Integration and unit tests for Driver logging concerns. +# +class DriverLoggingTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body + + class FakeStub + attr_reader :method_names + + def initialize responses + @responses = responses + @method_names = [] + end + + def endpoint + "https://storage.googleapis.com" + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + _ = uri + _ = body + _ = params + _ = options + @method_names << method_name + resp = @responses.shift + raise resp if resp.is_a?(Exception) || (resp.is_a?(Class) && resp < Exception) + + resp + end + end + + def test_all_entries_share_upload_id_and_pass_method_names + recording = RecordingLogger.new + responses = [ + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-URL" => "https://storage.googleapis.com/session?id=123" + }, + "" + ), + FakeResponse.new( + 200, + { "X-Goog-Upload-Status" => "final" }, + "done" + ) + ] + + stub = FakeStub.new responses + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("hello world"), + upload_size: 11, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + driver.run + + refute_empty recording.entries + upload_ids = recording.entries.map { |e| e.message.fields["uploadId"] }.uniq + assert_equal 1, upload_ids.size + refute_nil upload_ids.first + + assert_equal ["ResumableUpload.start", "ResumableUpload.upload"], stub.method_names + + info_recipes = recording.entries.select { |e| e.severity == Logger::INFO }.map { |e| e.message.fields["recipe"] } + assert_includes ["complete_upload_with_data", "complete_upload_finalized"], info_recipes.last + end + + def test_multi_chunk_upload_logs_lifecycle_entries + recording = RecordingLogger.new + run_two_chunk_upload_with_secret recording + + info_recipes = recording.entries.select { |e| e.severity == Logger::INFO }.map { |e| e.message.fields["recipe"] } + refute_includes info_recipes, "ack_chunk" + assert_includes info_recipes, "start_session" + assert_includes info_recipes, "begin_transmission" + assert(info_recipes.any? { |r| ["complete_upload_with_data", "complete_upload_finalized"].include? r }) + end + + def test_recovery_scenario_logs_enter_recovery_and_realign + recording = RecordingLogger.new + responses = [ + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-URL" => "https://storage.googleapis.com/session?id=123" + }, + "" + ), + FakeResponse.new(503, {}, "Service Unavailable"), + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "0" + }, + "" + ), + FakeResponse.new( + 200, + { "X-Goog-Upload-Status" => "final" }, + "done" + ) + ] + + stub = FakeStub.new responses + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("hello world"), + upload_size: 11, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + driver.run + + info_recipes = recording.entries.select { |e| e.severity == Logger::INFO }.map { |e| e.message.fields["recipe"] } + assert_includes info_recipes, "enter_recovery" + assert_includes info_recipes, "realign_from_recovery" + end + + def test_resume_upload_logs_resume_session_entry + recording = RecordingLogger.new + responses = [ + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "0" + }, + "" + ), + FakeResponse.new( + 200, + { "X-Goog-Upload-Status" => "final" }, + "done" + ) + ] + + stub = FakeStub.new responses + config = ResumeUploadConfig.new( + upload_url: "https://storage.googleapis.com/session?id=123", + stream: StringIO.new("hello world"), + upload_size: 11, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + driver.run + + resume_entry = recording.entries.find do |e| + e.severity == Logger::INFO && e.message.fields["recipe"] == "resume_session" + end + refute_nil resume_entry + assert_equal Logger::INFO, resume_entry.severity + assert_equal "Resuming upload session", resume_entry.message.message + assert_equal 256, resume_entry.message.fields["chunkSize"] + assert_includes resume_entry.message.fields["uploadUrl"], "session?id=" + end + + def test_fatal_failure_logs_warn_with_fail_with_recipe + recording = RecordingLogger.new + responses = [ + FakeResponse.new( + 403, + { "X-Goog-Upload-Status" => "final" }, + "Forbidden" + ) + ] + + stub = FakeStub.new responses + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("hello world"), + upload_size: 11, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + assert_raises UploadRejectedError do + driver.run + end + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + refute_empty warn_entries + assert(warn_entries.any? { |e| e.message.fields["recipe"]&.start_with? "fail_with_" }) + end + + def test_unmatched_transition_logs_warn_and_reraises + recording = RecordingLogger.new + stub = FakeStub.new [] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("hello"), + upload_size: 5, + chunk_size: 256 + ) + + failing_core = Minitest::Mock.new + failing_core.expect :dispatch, nil do |_event| + raise InvalidTransitionError, "unmatched transition in state" + end + failing_core.expect :state, State.new(status: :initializing) + + driver = Driver.new client_stub: stub, config: config, core: failing_core, logger: recording + + assert_raises InvalidTransitionError do + driver.run + end + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + assert_equal 1, warn_entries.size + fields = warn_entries.first.message.fields + assert_equal "initializing", fields["status"] + assert_equal "unmatched transition in state", fields["error"] + end + + def test_full_log_corpus_redacts_secrets + recording = RecordingLogger.new + run_two_chunk_upload_with_secret recording + + corpus = log_corpus recording + refute_includes corpus, "SECRET-123456" + end + + def test_full_log_corpus_size_under_64kib + recording = RecordingLogger.new + run_two_chunk_upload_with_secret recording + + corpus = log_corpus recording + assert_operator corpus.bytesize, :<, 65_536 + end + + def test_wire_receive_logs_error_status_and_abridged_error_message + recording = RecordingLogger.new + stub_logger = Gapic::LoggingConcerns::StubLogger.new logger: recording, service: "ResumableUpload" + upload_log = Driver::UploadLog.new stub_logger, upload_id: "test-upload-1" + + err = Gapic::Rest::Error.new( + "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: Permission denied on resource", + 403, + status: "PERMISSION_DENIED" + ) + event = Event::HttpResponse.new( + status: 403, + headers: { "x-goog-upload-status" => "final" }, + body: '{"raw":"error"}', + error: err + ) + + upload_log.wire_receive event + + debug_entries = recording.entries.select { |e| e.severity == Logger::DEBUG && e.message.message.include?("403") } + refute_empty debug_entries + fields = debug_entries.first.message.fields + assert_equal 403, fields["status"] + assert_equal "PERMISSION_DENIED", fields["errorStatus"] + assert_equal "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: Permission denied on resource", fields["body"] + end + + def test_wire_receive_fallback_body_when_error_absent + recording = RecordingLogger.new + stub_logger = Gapic::LoggingConcerns::StubLogger.new logger: recording, service: "ResumableUpload" + upload_log = Driver::UploadLog.new stub_logger, upload_id: "test-upload-2" + + event = Event::HttpResponse.new( + status: 503, + headers: {}, + body: "Server unavailable", + error: nil + ) + + upload_log.wire_receive event + + debug_entries = recording.entries.select { |e| e.severity == Logger::DEBUG && e.message.message.include?("503") } + refute_empty debug_entries + fields = debug_entries.first.message.fields + assert_equal 503, fields["status"] + assert_nil fields["errorStatus"] + assert_equal "Server unavailable", fields["body"] + end + + def test_lifecycle_warn_carries_rich_message_when_driver_fails + recording = RecordingLogger.new + wrapped_err = Gapic::Rest::Error.new( + "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: Bucket access denied", + 403, + status: "PERMISSION_DENIED", + headers: { "x-goog-upload-status" => "final" } + ) + stub = FakeStub.new [wrapped_err] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + err = assert_raises UploadRejectedError do + driver.run + end + + assert_equal "Upload rejected by server with HTTP 403 PERMISSION_DENIED: Bucket access denied", err.message + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + refute_empty warn_entries + fail_warn = warn_entries.find { |e| e.message.fields["recipe"] == "fail_with_rejected" } + refute_nil fail_warn + assert_equal "Upload rejected by server with HTTP 403 PERMISSION_DENIED: Bucket access denied", + fail_warn.message.fields["error"] + + debug_entries = recording.entries.select { |e| e.severity == Logger::DEBUG && e.message.message.include?("403") } + refute_empty debug_entries + wire_recv = debug_entries.first + assert_equal "PERMISSION_DENIED", wire_recv.message.fields["errorStatus"] + end + + def test_driver_error_mapping_populates_error_on_rescue + stub = FakeStub.new [] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + driver = Driver.new client_stub: stub, config: config + + rest_err = Gapic::Rest::Error.new "Forbidden", 403, status: "PERMISSION_DENIED" + event = driver.send :rescue_request_error, rest_err + assert_instance_of Event::HttpResponse, event + assert_equal rest_err, event.error + + faraday_err = Faraday::ClientError.new "Client error", { + status: 400, + headers: { "content-type" => "application/json" }, + body: '{"error":{"message":"Bad input","code":400,"status":"INVALID_ARGUMENT"}}' + } + faraday_event = driver.send :rescue_faraday_error, faraday_err + assert_instance_of Event::HttpResponse, faraday_event + assert_instance_of Gapic::Rest::Error, faraday_event.error + assert_equal 400, faraday_event.error.status_code + assert_equal "INVALID_ARGUMENT", faraday_event.error.status + end + + def test_lifecycle_warn_includes_response_body_for_rejected_error + recording = RecordingLogger.new + raw_body = '{"error":{"code":403,"message":"Rejected by backend"}}' + faraday_err = Faraday::ClientError.new "Client error", { + status: 403, + headers: { "x-goog-upload-status" => "final" }, + body: raw_body + } + stub = FakeStub.new [faraday_err] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + err = assert_raises UploadRejectedError do + driver.run + end + + assert_equal raw_body, err.response_body + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + fail_warn = warn_entries.find { |e| e.message.fields["recipe"] == "fail_with_rejected" } + refute_nil fail_warn + assert_equal raw_body, fail_warn.message.fields["responseBody"] + end + + def test_lifecycle_warn_includes_response_body_for_bad_response_error + recording = RecordingLogger.new + raw_body = '{"error":{"message":"Invalid input","code":400}}' + faraday_err = Faraday::ClientError.new "Client error", { + status: 400, + headers: { "x-goog-upload-status" => "active" }, + body: raw_body + } + stub = FakeStub.new [faraday_err] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + err = assert_raises BadResponseError do + driver.run + end + + assert_equal raw_body, err.response_body + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + fail_warn = warn_entries.find { |e| e.message.fields["recipe"] == "fail_with_bad_response" } + refute_nil fail_warn + assert_equal raw_body, fail_warn.message.fields["responseBody"] + end + + def test_lifecycle_warn_omits_response_body_when_error_lacks_it + recording = RecordingLogger.new + stub_logger = Gapic::LoggingConcerns::StubLogger.new logger: recording, service: "ResumableUpload" + upload_log = Driver::UploadLog.new stub_logger, upload_id: "test-upload-no-body" + state = State.new( + status: :error, + last_error: DeadlineExceededError.new("Upload deadline exceeded") + ) + decision = Decision.new( + from_status: :transferring, + shape: :deadline_exceeded, + recipe: :fail_with_deadline_exceeded, + next_state: state, + instructions: [] + ) + + upload_log.lifecycle decision, StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + fail_warn = warn_entries.find { |e| e.message.fields["recipe"] == "fail_with_deadline_exceeded" } + refute_nil fail_warn + assert_equal "Upload deadline exceeded", fail_warn.message.fields["error"] + assert_nil fail_warn.message.fields["responseBody"] + end + + def test_error_info_reason_in_details_survives_in_error_and_logs + recording = RecordingLogger.new + error_info = Google::Rpc::ErrorInfo.new( + reason: "SERVICE_DISABLED", + domain: "googleapis.com", + metadata: { "consumer" => "projects/12345", "service" => "storage.googleapis.com" } + ) + error_info_any = Google::Protobuf::Any.pack error_info + raw_body = JSON.dump( + { + "error" => { + "code" => 403, + "message" => "Google Cloud Storage API has not been used in project 12345 or it is disabled.", + "status" => "PERMISSION_DENIED", + "details" => [JSON.parse(error_info_any.to_json)] + } + } + ) + faraday_err = Faraday::ClientError.new "Client error", { + status: 403, + headers: { "x-goog-upload-status" => "final" }, + body: raw_body + } + stub = FakeStub.new [faraday_err] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + err = assert_raises UploadRejectedError do + driver.run + end + + refute_nil err.details + unpacked_info = err.details.find { |d| d.is_a? Google::Rpc::ErrorInfo } + refute_nil unpacked_info + assert_equal "SERVICE_DISABLED", unpacked_info.reason + assert_equal "googleapis.com", unpacked_info.domain + assert_equal "projects/12345", unpacked_info.metadata["consumer"] + + expected_msg = "Upload rejected by server with HTTP 403 PERMISSION_DENIED: " \ + "Google Cloud Storage API has not been used in project 12345 or it is disabled." + assert_equal expected_msg, err.message + assert_equal 403, err.status_code + assert_equal "PERMISSION_DENIED", err.status + assert_equal raw_body, err.response_body + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + fail_warn = warn_entries.find { |e| e.message.fields["recipe"] == "fail_with_rejected" } + refute_nil fail_warn + assert_equal expected_msg, fail_warn.message.fields["error"] + assert_equal raw_body, fail_warn.message.fields["responseBody"] + end + + private + + def run_two_chunk_upload_with_secret recording + chunk_size = 8 * 1024 * 1024 + secret = "SECRET-123456" + binary_prefix = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09".b + + half = (chunk_size / 2) - 10 + chunk1 = binary_prefix + ("A" * half) + secret + ("A" * (chunk_size - 10 - half - secret.bytesize)) + chunk2 = binary_prefix + ("A" * (chunk_size - 10)) + stream_data = chunk1 + chunk2 + + responses = [ + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-URL" => "https://storage.googleapis.com/session?sid=#{secret}" + }, + "" + ), + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => chunk_size.to_s + }, + "" + ), + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => (chunk_size * 2).to_s + }, + "" + ), + FakeResponse.new( + 200, + { "X-Goog-Upload-Status" => "final" }, + "done" + ) + ] + + stub = FakeStub.new responses + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload?token=#{secret}", + initial_headers: { "Authorization" => "Bearer #{secret}" }, + stream: StringIO.new(stream_data), + upload_size: stream_data.bytesize, + chunk_size: chunk_size + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + driver.run + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb new file mode 100644 index 0000000..54cf579 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb @@ -0,0 +1,178 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for Driver progress notification dispatching and callback error propagation. +# +class DriverProgressTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + CustomCallbackError = Class.new StandardError + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + class ScriptedClientStub + def initialize responses + @responses = responses + end + + def make_post_request uri:, body: nil, params: {}, options: {}, method_name: nil + raise "No scripted response" if @responses.empty? + + @responses.shift + end + end + + def test_execute_notify_progress_without_callback_does_not_raise + driver = build_driver on_progress: nil + + instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 1024, total_bytes: 4096) + # Must not raise when callback is nil + driver.send :execute_notify_progress, instruction + end + + def test_execute_notify_progress_happy_path_invoked_once + calls = [] + callback = ->(progress) { calls << progress } + driver = build_driver on_progress: callback + + instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 500, total_bytes: 1000) + driver.send :execute_notify_progress, instruction + + assert_equal 1, calls.size + assert_equal Progress.new(phase: :uploading, bytes_uploaded: 500, total_bytes: 1000), calls.first + end + + def test_execute_notify_progress_total_bytes_nil_passes_through + calls = [] + callback = ->(progress) { calls << progress } + driver = build_driver on_progress: callback + + instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 250, total_bytes: nil) + driver.send :execute_notify_progress, instruction + + assert_equal 1, calls.size + assert_equal Progress.new(phase: :uploading, bytes_uploaded: 250, total_bytes: nil), calls.first + end + + def test_execute_notify_progress_raises_error_to_caller_when_callback_fails + callback = ->(_progress) { raise CustomCallbackError, "User UI crashed in progress callback" } + driver = build_driver on_progress: callback + + instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 100, total_bytes: 1000) + err = assert_raises CustomCallbackError do + driver.send :execute_notify_progress, instruction + end + + assert_equal "User UI crashed in progress callback", err.message + end + + def test_driver_run_propagates_callback_error_end_to_end + # Script responses: 1. start response -> 2. chunk response (triggers NotifyProgress) + responses = [ + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-url" => "https://example.com/upload/123", "x-goog-upload-status" => "active" }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active" }, + body: "" + ) + ] + stub = ScriptedClientStub.new responses + + callback = ->(_progress) { raise CustomCallbackError, "Terminal failure in user progress handler" } + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4, + on_progress: callback + ) + driver = Driver.new client_stub: stub, config: config + + err = assert_raises CustomCallbackError do + driver.run + end + assert_equal "Terminal failure in user progress handler", err.message + end + + def test_completed_progress_reports_total_bytes_when_upload_size_unknown + responses = [ + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-url" => "https://example.com/upload/123", "x-goog-upload-status" => "active" }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active" }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active" }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: "{\"done\":true}" + ) + ] + stub = ScriptedClientStub.new responses + + progress_events = [] + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: nil, + chunk_size: 4, + on_progress: ->(progress) { progress_events << progress } + ) + driver = Driver.new client_stub: stub, config: config + + result = driver.run + assert_equal "{\"done\":true}", result + + uploading_snapshots = progress_events.select { |p| p.phase == :uploading } + refute_empty uploading_snapshots + assert uploading_snapshots.all? { |p| p.total_bytes.nil? } + + completed_snapshot = progress_events.last + assert_equal :completed, completed_snapshot.phase + assert_equal 10, completed_snapshot.bytes_uploaded + assert_equal 10, completed_snapshot.total_bytes + end + + private + + def build_driver on_progress: nil + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4, + on_progress: on_progress + ) + Driver.new client_stub: Object.new, config: config + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb new file mode 100644 index 0000000..76af4c0 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Driver retry policy resolution and configuration overrides. +# +class DriverRetryPolicyTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123") + ) + @driver = Driver.new client_stub: Object.new, config: @config + end + + def test_resolve_retry_policy_with_nil_returns_default_policy + policy = @driver.send :resolve_retry_policy, nil, RetryPolicies::START_DEFAULTS + + assert_kind_of Gapic::Common::RetryPolicy, policy + assert_equal RetryPolicies.default_start.retry_codes, policy.retry_codes + assert_in_delta 1.0, policy.initial_delay + assert_in_delta 15.0, policy.max_delay + assert_in_delta 1.3, policy.multiplier + assert_same RetryPolicies::START_PREDICATE, policy.retry_predicate + end + + def test_resolve_retry_policy_with_policy_instance_returns_as_is + custom_policy = Gapic::Common::RetryPolicy.new initial_delay: 5.0 + resolved = @driver.send :resolve_retry_policy, custom_policy, RetryPolicies::START_DEFAULTS + + assert_same custom_policy, resolved + assert_nil resolved.retry_predicate + assert_empty resolved.retry_codes + end + + def test_resolve_retry_policy_with_hash_applies_defaults_and_preserves_codes_and_predicate + hash_override = { initial_delay: 0.25, max_delay: 2.0 } + resolved = @driver.send :resolve_retry_policy, hash_override, RetryPolicies::START_DEFAULTS + + assert_kind_of Gapic::Common::RetryPolicy, resolved + assert_in_delta 0.25, resolved.initial_delay + assert_in_delta 2.0, resolved.max_delay + assert_in_delta 1.3, resolved.multiplier + assert_equal RetryPolicies.default_start.retry_codes, resolved.retry_codes + assert_same RetryPolicies::START_PREDICATE, resolved.retry_predicate + end + + def test_resolve_retry_policy_with_data_plane_hash_preserves_data_plane_predicate + hash_override = { timeout: 60.0 } + resolved = @driver.send :resolve_retry_policy, hash_override, RetryPolicies::DATA_PLANE_DEFAULTS + + assert_kind_of Gapic::Common::RetryPolicy, resolved + assert_in_delta 60.0, resolved.timeout + assert_equal RetryPolicies.default_data_plane.retry_codes, resolved.retry_codes + assert_same RetryPolicies::DATA_PLANE_PREDICATE, resolved.retry_predicate + end + + def test_resolve_retry_policy_with_empty_retry_codes_array_honors_empty_array + hash_override = { retry_codes: [] } + resolved = @driver.send :resolve_retry_policy, hash_override, RetryPolicies::START_DEFAULTS + + assert_kind_of Gapic::Common::RetryPolicy, resolved + assert_empty resolved.retry_codes + assert_same RetryPolicies::START_PREDICATE, resolved.retry_predicate + end + + def test_resolve_retry_policy_with_unknown_hash_key_raises_argument_error + err = assert_raises ArgumentError do + @driver.send :resolve_retry_policy, { unknown_key: 123 }, RetryPolicies::START_DEFAULTS + end + assert_match(/unknown keyword: :unknown_key/, err.message) + end + + def test_resolve_retry_policy_with_invalid_type_raises_argument_error + err = assert_raises ArgumentError do + @driver.send :resolve_retry_policy, "invalid", RetryPolicies::START_DEFAULTS + end + assert_match(/Expected RetryPolicy, Hash, or nil/, err.message) + end + + def test_driver_initialize_resolves_hash_overrides_from_config + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + start_retry_policy: { initial_delay: 0.1 }, + control_plane_retry_policy: { max_delay: 5.0 }, + data_plane_retry_policy: { multiplier: 2.0 } + ) + driver = Driver.new client_stub: Object.new, config: config + + start_policy = driver.instance_variable_get :@start_retry_policy + control_policy = driver.instance_variable_get :@control_plane_retry_policy + data_policy = driver.instance_variable_get :@data_plane_retry_policy + + assert_in_delta 0.1, start_policy.initial_delay + assert_same RetryPolicies::START_PREDICATE, start_policy.retry_predicate + + assert_in_delta 5.0, control_policy.max_delay + assert_nil control_policy.retry_predicate + + assert_in_delta 2.0, data_policy.multiplier + assert_same RetryPolicies::DATA_PLANE_PREDICATE, data_policy.retry_predicate + end + + def test_start_predicate_refutes_fatal_status_codes + [401, 403, 404, 405, 410, 413, 415].each do |code| + response_double = OpenStruct.new status: code, headers: {} + refute RetryPolicies::START_PREDICATE.call(response_double), + "Expected START_PREDICATE to return false for fatal status #{code}" + + gapic_err = Gapic::Rest::Error.new "Error", code, status: "FATAL_ERROR", headers: {} + refute RetryPolicies::START_PREDICATE.call(gapic_err), + "Expected START_PREDICATE to return false for Gapic::Rest::Error with fatal status #{code}" + end + end + + def test_start_predicate_retries_missing_header_for_non_fatal_codes + [200, 400, 500, 503].each do |code| + response_double = OpenStruct.new status: code, headers: {} + assert RetryPolicies::START_PREDICATE.call(response_double), + "Expected START_PREDICATE to return true for non-fatal status #{code} with missing status header" + end + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb new file mode 100644 index 0000000..2fca882 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb @@ -0,0 +1,187 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Driver retry behavior during session start and queries. +# +# rubocop:disable Metrics/MethodLength +class DriverRetryTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + # Fake client stub recording calls and yielding scripted responses. + class FakeClientStub + attr_reader :requests + + def initialize responses + @responses = responses + @requests = [] + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } + raise "Unexpected request: no scripted response left" if @responses.empty? + + @responses.shift + end + end + + def test_start_retries_when_response_lacks_status_header_even_on_200 + responses = [ + FakeResponse.new(status: 200, headers: {}, body: ""), + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + start_retry_policy: { initial_delay: 0.001, max_delay: 0.002, timeout: 1.0 } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 3, stub.requests.size + assert_start_request stub.requests[0] + assert_start_request stub.requests[1] + assert_chunk_request stub.requests[2], offset: "0", length: "4", body: "0123", finalize: true + end + + def test_start_exhausts_retries_when_200_responses_continually_lack_status_header + responses = Array.new(10) { FakeResponse.new status: 200, headers: {}, body: "" } + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + start_retry_policy: { initial_delay: 0.001, max_delay: 0.002, timeout: 0.01 } + ) + + driver = Driver.new client_stub: stub, config: config + err = assert_raises RequestFailedError do + driver.run + end + + assert_match(/Missing X-Goog-Upload-Status/, err.message) + assert_equal 200, err.status_code + assert_instance_of BadResponseError, err.cause + assert stub.requests.size > 1 + end + + def test_start_exhausts_retries_when_non_200_responses_continually_lack_status_header + responses = Array.new(10) { FakeResponse.new status: 503, headers: {}, body: "Service Unavailable" } + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + start_retry_policy: { initial_delay: 0.001, max_delay: 0.002, timeout: 0.01 } + ) + + driver = Driver.new client_stub: stub, config: config + err = assert_raises BadResponseError do + driver.run + end + + assert_equal 503, err.status_code + assert_includes err.message, "503" + refute_match(/Missing X-Goog-Upload-Status/, err.message) + assert stub.requests.size > 1 + end + + def test_query_does_not_retry_on_missing_status_header_in_driver + responses = [ + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ), + FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable"), + FakeResponse.new(status: 200, headers: {}, body: ""), + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "0" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10 + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 5, stub.requests.size + assert_start_request stub.requests[0] + assert_chunk_request stub.requests[1], offset: "0", length: "4", body: "0123", finalize: true + assert_query_request stub.requests[2] + assert_query_request stub.requests[3] + assert_chunk_request stub.requests[4], offset: "0", length: "4", body: "0123", finalize: true + end + + private + + def assert_start_request req + assert_equal "https://example.com/upload", req[:uri] + assert_equal "start", req[:options][:metadata]["X-Goog-Upload-Command"] + end + + def assert_query_request req + assert_equal "https://example.com/session/1", req[:uri] + assert_equal "query", req[:options][:metadata]["X-Goog-Upload-Command"] + end + + def assert_chunk_request req, offset:, length:, body:, finalize: + expected_cmd = finalize ? "upload, finalize" : "upload" + metadata = req[:options][:metadata] + assert_equal "https://example.com/session/1", req[:uri] + assert_equal expected_cmd, metadata["X-Goog-Upload-Command"] + assert_equal offset, metadata["X-Goog-Upload-Offset"] + assert_equal length, metadata["Content-Length"] + assert_equal body, req[:body] + end +end +# rubocop:enable Metrics/MethodLength diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb new file mode 100644 index 0000000..b5d73df --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb @@ -0,0 +1,430 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Driver synchronous upload execution engine. +# +class DriverTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + # Fake client stub recording calls and yielding scripted responses. + class FakeClientStub + attr_reader :requests + + def initialize responses + @responses = responses + @requests = [] + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + @requests << { uri: uri, body: body, params: params, options: options } + raise "Unexpected request: no scripted response left" if @responses.empty? + + @responses.shift + end + end + + def test_multi_chunk_upload_with_active_responses + progress_records = [] + stub = FakeClientStub.new build_scripted_responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4, + on_progress: ->(p) { progress_records << p } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 4, stub.requests.size + assert_start_request stub.requests[0] + assert_chunk_request stub.requests[1], offset: "0", length: "4", body: "0123", finalize: false + assert_chunk_request stub.requests[2], offset: "4", length: "4", body: "4567", finalize: false + assert_chunk_request stub.requests[3], offset: "8", length: "2", body: "89", finalize: true + + assert_equal [ + Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 4, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :finalizing, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :completed, bytes_uploaded: 10, total_bytes: 10) + ], progress_records + end + + def test_upload_recovers_when_chunk_response_lacks_status_header + progress_records = [] + responses = build_recovery_responses + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4, + on_progress: ->(p) { progress_records << p } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 5, stub.requests.size + assert_start_request stub.requests[0] + assert_chunk_request stub.requests[1], offset: "0", length: "4", body: "0123", finalize: false + assert_query_request stub.requests[2] + assert_chunk_request stub.requests[3], offset: "4", length: "4", body: "4567", finalize: false + assert_chunk_request stub.requests[4], offset: "8", length: "2", body: "89", finalize: true + + assert_equal [ + Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :recovering, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 4, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :finalizing, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :completed, bytes_uploaded: 10, total_bytes: 10) + ], progress_records + end + + def test_resume_upload_success + progress_records = [] + responses = [ + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "4" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + stub = FakeClientStub.new responses + config = ResumeUploadConfig.new( + upload_url: "https://example.com/session/1", + chunk_size: 4, + stream: StringIO.new("0123456789"), + upload_size: 10, + on_progress: ->(p) { progress_records << p } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 3, stub.requests.size + assert_query_request stub.requests[0] + assert_chunk_request stub.requests[1], offset: "4", length: "4", body: "4567", finalize: false + assert_chunk_request stub.requests[2], offset: "8", length: "2", body: "89", finalize: true + + assert_equal [ + Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 4, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :finalizing, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :completed, bytes_uploaded: 10, total_bytes: 10) + ], progress_records + end + + def test_resume_upload_with_409_recovery_retry + progress_records = [] + responses = [ + FakeResponse.new( + status: 409, + headers: { "X-Goog-Upload-Status" => "active" }, + body: "Conflict" + ), + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "4" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + stub = FakeClientStub.new responses + config = ResumeUploadConfig.new( + upload_url: "https://example.com/session/1", + chunk_size: 4, + stream: StringIO.new("0123456789"), + upload_size: 10, + on_progress: ->(p) { progress_records << p } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 4, stub.requests.size + assert_query_request stub.requests[0] + assert_query_request stub.requests[1] + assert_chunk_request stub.requests[2], offset: "4", length: "4", body: "4567", finalize: false + assert_chunk_request stub.requests[3], offset: "8", length: "2", body: "89", finalize: true + + assert_equal [ + Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 4, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :finalizing, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :completed, bytes_uploaded: 10, total_bytes: 10) + ], progress_records + end + + def test_run_returns_nil_body_when_final_response_has_none + responses = [ + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "X-Goog-Upload-Status" => "final" }, + body: nil + ) + ] + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("ab"), + upload_size: 2, + chunk_size: 4 + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_nil result + assert_equal 2, stub.requests.size + end + + # Fake Core yielding a fixed Decision to test Driver#run invariant guards. + class FakeCore + attr_reader :state, :last_decision + + def initialize decision + @decision = decision + @state = decision.next_state + @last_decision = nil + end + + def dispatch _event + @last_decision = @decision + @decision.instructions + end + end + + def test_run_raises_internal_error_on_empty_batch + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4 + ) + decision = Decision.new( + from_status: :initializing, + shape: :start_upload, + recipe: :broken_empty, + next_state: State.new(status: :starting), + instructions: [] + ) + driver = Driver.new client_stub: FakeClientStub.new([]), config: config, core: FakeCore.new(decision) + + err = assert_raises InternalError do + driver.run + end + assert_equal "Resumable upload internal error: recipe :broken_empty " \ + "produced no continuation event and did not terminate", + err.message + end + + def test_run_raises_internal_error_on_multiple_continuation_events + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4 + ) + send_start = Instruction::SendStart.new url: "https://example.com/upload", headers: {}, body: "" + decision = Decision.new( + from_status: :initializing, + shape: :start_upload, + recipe: :broken_multi, + next_state: State.new(status: :starting), + instructions: [send_start, send_start] + ) + resp = FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ) + stub = FakeClientStub.new [resp, resp] + driver = Driver.new client_stub: stub, config: config, core: FakeCore.new(decision) + + err = assert_raises InternalError do + driver.run + end + assert_equal "Resumable upload internal error: recipe :broken_multi produced multiple continuation events", + err.message + assert_empty stub.requests + end + + def test_run_raises_internal_error_on_mixed_continuation_and_terminal + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4 + ) + send_start = Instruction::SendStart.new url: "https://example.com/upload", headers: {}, body: "" + term_success = Instruction::TerminateSuccess.new response: Event::HttpResponse.new(status: 200, headers: {}, body: "") + decision = Decision.new( + from_status: :initializing, + shape: :start_upload, + recipe: :broken_mixed, + next_state: State.new(status: :starting), + instructions: [send_start, term_success] + ) + stub = FakeClientStub.new [] + driver = Driver.new client_stub: stub, config: config, core: FakeCore.new(decision) + + err = assert_raises InternalError do + driver.run + end + assert_equal "Resumable upload internal error: recipe :broken_mixed " \ + "produced both a continuation event and a terminal instruction", + err.message + assert_empty stub.requests + end + + def test_run_raises_internal_error_on_unclassified_instruction + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4 + ) + decision = Decision.new( + from_status: :initializing, + shape: :start_upload, + recipe: :broken_unclassified, + next_state: State.new(status: :starting), + instructions: [Object.new] + ) + stub = FakeClientStub.new [] + driver = Driver.new client_stub: stub, config: config, core: FakeCore.new(decision) + + err = assert_raises InternalError do + driver.run + end + assert_equal "Resumable upload internal error: recipe :broken_unclassified emitted unclassified instruction Object", + err.message + assert_empty stub.requests + end + + def test_on_progress_return_value_does_not_leak_into_trampoline_invariant + stub = FakeClientStub.new build_scripted_responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4, + on_progress: ->(_p) { Event::HttpResponse.new status: 200, headers: {} } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + end + + private + + def build_scripted_responses + [ + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + end + + def build_recovery_responses + [ + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ), + FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable"), + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "4" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + end + + def assert_start_request req + assert_equal "https://example.com/upload", req[:uri] + assert_equal "start", req[:options][:metadata]["X-Goog-Upload-Command"] + end + + def assert_query_request req + assert_equal "https://example.com/session/1", req[:uri] + assert_equal "query", req[:options][:metadata]["X-Goog-Upload-Command"] + end + + def assert_chunk_request req, offset:, length:, body:, finalize: + expected_cmd = finalize ? "upload, finalize" : "upload" + metadata = req[:options][:metadata] + assert_equal "https://example.com/session/1", req[:uri] + assert_equal expected_cmd, metadata["X-Goog-Upload-Command"] + assert_equal offset, metadata["X-Goog-Upload-Offset"] + assert_equal length, metadata["Content-Length"] + assert_equal body, req[:body] + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb new file mode 100644 index 0000000..da1a692 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb @@ -0,0 +1,222 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "ostruct" +require "faraday" + +## +# Tests for ResumableUpload RetryPolicies and header extraction. +# +class RetryPoliciesTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + # ============================================================================ + # SUT: extract_headers + # ============================================================================ + + def test_extract_headers_from_headers_method + obj = OpenStruct.new headers: { "X-Test-Header" => "value1" } + assert_equal({ "X-Test-Header" => "value1" }, RetryPolicies.extract_headers(obj)) + end + + def test_extract_headers_from_response_headers_method + obj = OpenStruct.new response_headers: { "X-Test-Header" => "value2" } + assert_equal({ "X-Test-Header" => "value2" }, RetryPolicies.extract_headers(obj)) + end + + def test_extract_headers_from_faraday_response_hash + err = Faraday::ClientError.new "error message", { headers: { "X-Test-Header" => "value3" } } + assert_equal({ "X-Test-Header" => "value3" }, RetryPolicies.extract_headers(err)) + end + + def test_extract_headers_returns_nil_when_no_headers_present + assert_nil RetryPolicies.extract_headers(StandardError.new("error")) + assert_nil RetryPolicies.extract_headers(nil) + assert_nil RetryPolicies.extract_headers(Object.new) + assert_nil RetryPolicies.extract_headers("string") + assert_nil RetryPolicies.extract_headers({}) + end + + # ============================================================================ + # SUT: RetryPolicies.default_start + # ============================================================================ + + def test_default_start_missing_status_header_retries_unconditionally + policy = RetryPolicies.default_start + + # Retriable code (503) without status header + err_503 = OpenStruct.new response_status: 503, headers: { "Content-Type" => "text/plain" } + assert policy.retry_error?(err_503) + + # Non-retriable code (400) without status header (still retries because missing status header is retriable) + err_400 = OpenStruct.new response_status: 400, headers: { "Content-Type" => "text/plain" } + assert policy.retry_error?(err_400) + + # Status 200 OK without status header + resp_200 = OpenStruct.new response_status: 200, headers: { "Content-Type" => "text/plain" } + assert policy.retry_error?(resp_200) + + # No status code without status header + err_no_code = OpenStruct.new headers: { "Content-Type" => "text/plain" } + assert policy.retry_error?(err_no_code) + + # Empty status header string + err_empty_status = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "" } + assert policy.retry_error?(err_empty_status) + end + + def test_default_start_with_status_header_falls_back_to_codes + policy = RetryPolicies.default_start + + # Retriable code (503) with status header + err_503 = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "active" } + assert policy.retry_error?(err_503) + + # Non-retriable code (400) with status header + err_400 = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_400) + + # Without status code with status header + err_no_code = OpenStruct.new headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_no_code) + end + + def test_default_start_no_headers_falls_back_to_codes + policy = RetryPolicies.default_start + + # Retriable code (503) without headers + err_503 = OpenStruct.new response_status: 503 + assert policy.retry_error?(err_503) + + # Non-retriable code (400) without headers + err_400 = OpenStruct.new response_status: 400 + refute policy.retry_error?(err_400) + + # Without status code and without headers + err_no_code = RuntimeError.new "generic network error" + refute policy.retry_error?(err_no_code) + end + + # ============================================================================ + # SUT: RetryPolicies.default_control_plane + # ============================================================================ + + def test_default_control_plane_missing_status_header_falls_back_to_codes + policy = RetryPolicies.default_control_plane + + # Retriable code (503) without status header + err_503 = OpenStruct.new response_status: 503, headers: { "Content-Type" => "text/plain" } + assert policy.retry_error?(err_503) + + # Non-retriable code (400) without status header + err_400 = OpenStruct.new response_status: 400, headers: { "Content-Type" => "text/plain" } + refute policy.retry_error?(err_400) + + # Without status code without status header + err_no_code = OpenStruct.new headers: { "Content-Type" => "text/plain" } + refute policy.retry_error?(err_no_code) + end + + def test_default_control_plane_with_status_header_falls_back_to_codes + policy = RetryPolicies.default_control_plane + + # Retriable code (503) with status header + err_503 = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "active" } + assert policy.retry_error?(err_503) + + # Non-retriable code (400) with status header + err_400 = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_400) + + # Without status code with status header + err_no_code = OpenStruct.new headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_no_code) + end + + def test_default_control_plane_no_headers_falls_back_to_codes + policy = RetryPolicies.default_control_plane + + # Retriable code (503) without headers + err_503 = OpenStruct.new response_status: 503 + assert policy.retry_error?(err_503) + + # Non-retriable code (400) without headers + err_400 = OpenStruct.new response_status: 400 + refute policy.retry_error?(err_400) + + # Without status code and without headers + err_no_code = RuntimeError.new "generic network error" + refute policy.retry_error?(err_no_code) + end + + # ============================================================================ + # SUT: RetryPolicies.default_data_plane + # ============================================================================ + + def test_default_data_plane_missing_status_header_unretriable + policy = RetryPolicies.default_data_plane + + # Retriable code (503) without status header (predicate returns false -> unretriable) + err_503 = OpenStruct.new response_status: 503, headers: { "Content-Type" => "text/plain" } + refute policy.retry_error?(err_503) + + # Non-retriable code (400) without status header + err_400 = OpenStruct.new response_status: 400, headers: { "Content-Type" => "text/plain" } + refute policy.retry_error?(err_400) + + # Without status code without status header + err_no_code = OpenStruct.new headers: { "Content-Type" => "text/plain" } + refute policy.retry_error?(err_no_code) + + # Empty status header string + err_empty_status = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "" } + refute policy.retry_error?(err_empty_status) + end + + def test_default_data_plane_with_status_header_falls_back_to_codes + policy = RetryPolicies.default_data_plane + + # Retriable code (503) with status header + err_503 = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "active" } + assert policy.retry_error?(err_503) + + # Non-retriable code (400) with status header + err_400 = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_400) + + # Without status code with status header + err_no_code = OpenStruct.new headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_no_code) + end + + def test_default_data_plane_no_headers_falls_back_to_codes + policy = RetryPolicies.default_data_plane + + # Retriable code (503) without headers + err_503 = OpenStruct.new response_status: 503 + assert policy.retry_error?(err_503) + + # Non-retriable code (400) without headers + err_400 = OpenStruct.new response_status: 400 + refute policy.retry_error?(err_400) + + # Without status code and without headers + err_no_code = RuntimeError.new "generic network error" + refute policy.retry_error?(err_no_code) + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb new file mode 100644 index 0000000..d2b7c3d --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb @@ -0,0 +1,351 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" + +## +# Tests for classification and header extraction rules in the Resumable Upload protocol. +# +class RulesClassificationTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def test_header_value_exact_match + headers = { "X-Goog-Upload-Status" => "active" } + assert_equal "active", Rules.header_value(headers, "X-Goog-Upload-Status") + end + + def test_header_value_case_insensitivity + assert_equal "active", Rules.header_value({ "x-goog-upload-status" => "active" }, "X-Goog-Upload-Status") + assert_equal "active", Rules.header_value({ "X-GOOG-UPLOAD-STATUS" => "active" }, "X-Goog-Upload-Status") + assert_equal "active", Rules.header_value({ "x-Goog-UpLoad-Status" => "active" }, "X-Goog-Upload-Status") + end + + def test_header_value_with_symbol_keys + headers = { :"x-goog-upload-status" => "active" } + assert_equal "active", Rules.header_value(headers, "X-Goog-Upload-Status") + end + + def test_header_value_missing_or_non_hash + assert_nil Rules.header_value({ "Content-Type" => "text/plain" }, "X-Goog-Upload-Status") + assert_nil Rules.header_value(nil, "X-Goog-Upload-Status") + assert_nil Rules.header_value([], "X-Goog-Upload-Status") + assert_nil Rules.header_value("string", "X-Goog-Upload-Status") + end + + def test_classify_http_response_active + resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: "" + assert_equal :response_active, Rules.classify_http_response(resp_200) + + # Value case variations + ["Active", "ACTIVE", "aCtIvE"].each do |val| + resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => val }, body: "" + assert_equal :response_active, Rules.classify_http_response(resp) + end + + # Non-200 with active maps to Category 2 + [503, 500, 400, 408].each do |code| + resp_non_200 = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "active" }, body: "" + assert_equal :response_cat2, Rules.classify_http_response(resp_non_200) + end + end + + def test_classify_http_response_final + resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: "" + assert_equal :response_final, Rules.classify_http_response(resp_200) + + # Value case variations + ["Final", "FINAL"].each do |val| + resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => val }, body: "" + assert_equal :response_final, Rules.classify_http_response(resp) + end + + # Non-200 with final maps to response_rejected + [400, 404, 500].each do |code| + resp_non_200 = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "final" }, body: "" + assert_equal :response_rejected, Rules.classify_http_response(resp_non_200) + end + end + + def test_classify_http_response_cancelled + resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "cancelled" }, body: "" + assert_equal :response_cancelled, Rules.classify_http_response(resp_200) + + # Value case variations + ["Cancelled", "CANCELLED"].each do |val| + resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => val }, body: "" + assert_equal :response_cancelled, Rules.classify_http_response(resp) + end + + # Non-200 with cancelled maps to fatal bad response + [400, 500].each do |code| + resp_non_200 = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "cancelled" }, body: "" + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_non_200) + end + end + + def test_classify_http_response_missing_header_non_fatal + # HTTP 200 missing header + resp_200 = Event::HttpResponse.new status: 200, headers: {}, body: "" + assert_equal :response_cat2, Rules.classify_http_response(resp_200) + + # Recoverable 4xx missing header + Rules::CAT2_STATUS_CODES.each do |code| + resp = Event::HttpResponse.new status: code, headers: {}, body: "" + assert_equal :response_cat2, Rules.classify_http_response(resp), "Expected #{code} to classify as :response_cat2" + end + + # 5xx server/gateway errors missing header + [500, 502, 503, 504].each do |code| + resp = Event::HttpResponse.new status: code, headers: {}, body: "" + assert_equal :response_cat2, Rules.classify_http_response(resp), "Expected #{code} to classify as :response_cat2" + end + + # Empty string header + resp_empty = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "" }, body: "" + assert_equal :response_cat2, Rules.classify_http_response(resp_empty) + end + + def test_classify_http_response_missing_header_fatal_status_codes + Rules::FATAL_STATUS_CODES.each do |code| + resp = Event::HttpResponse.new status: code, headers: {}, body: "" + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp), + "Expected fatal code #{code} to classify as :response_fatal_bad_response" + + resp_empty = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "" }, body: "" + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_empty), + "Expected fatal code #{code} with empty header to classify as :response_fatal_bad_response" + end + end + + def test_classify_http_response_unknown_header_values + ["absconded", "pending", "in_progress", "error", "unknown"].each do |unknown_val| + resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => unknown_val }, body: "" + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_200) + + resp_400 = Event::HttpResponse.new status: 400, headers: { "X-Goog-Upload-Status" => unknown_val }, body: "" + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_400) + end + end + + def test_classify_http_response_header_key_casing + keys = ["x-goog-upload-status", "X-GOOG-UPLOAD-STATUS", "X-Goog-Upload-Status", :"x-goog-upload-status"] + keys.each do |key| + resp = Event::HttpResponse.new status: 200, headers: { key => "active" }, body: "" + assert_equal :response_active, Rules.classify_http_response(resp) + end + end + + def test_shape_of_control_events + assert_equal :start_upload, Rules.shape_of(Event::StartUpload.new) + assert_equal :start_upload, Rules.shape_of(Event::StartUpload) + assert_equal :resume_upload, Rules.shape_of(Event::ResumeUpload.new) + assert_equal :resume_upload, Rules.shape_of(Event::ResumeUpload) + assert_equal :user_cancel, Rules.shape_of(Event::Cancel.new) + assert_equal :user_cancel, Rules.shape_of(Event::Cancel) + assert_equal :global_deadline_exceeded, Rules.shape_of(Event::GlobalDeadlineExceeded.new) + assert_equal :global_deadline_exceeded, Rules.shape_of(Event::GlobalDeadlineExceeded) + end + + def test_shape_of_chunk_read + full_chunk = Event::ChunkRead.new bytes_buffered: 4096, eof: false + assert_equal :chunk_read_full, Rules.shape_of(full_chunk) + + eof_data = Event::ChunkRead.new bytes_buffered: 1024, eof: true + assert_equal :chunk_read_eof_with_data, Rules.shape_of(eof_data) + + eof_empty = Event::ChunkRead.new bytes_buffered: 0, eof: true + assert_equal :chunk_read_eof_empty, Rules.shape_of(eof_empty) + end + + def test_shape_of_request_failed + timeout = Event::RequestFailed.new kind: :timeout, message: "read timeout" + assert_equal :request_timeout, Rules.shape_of(timeout) + + exhausted = Event::RequestFailed.new kind: :retries_exhausted, message: "exhausted" + assert_equal :request_retries_exhausted, Rules.shape_of(exhausted) + + conn_failed = Event::RequestFailed.new kind: :connection_failed, message: "dropped" + assert_equal :request_connection_failed, Rules.shape_of(conn_failed) + + other = Event::RequestFailed.new kind: :other, message: "unknown error" + assert_equal :request_failed_unknown, Rules.shape_of(other) + end + + def test_shape_of_http_response_delegates_to_classify + resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: "" + assert_equal :response_active, Rules.shape_of(resp) + end + + def test_shape_of_unknown_event + assert_equal :unknown, Rules.shape_of(Object.new) + assert_equal :unknown, Rules.shape_of(nil) + assert_equal :unknown, Rules.shape_of("unrecognized_event") + end + + ## + # One event per shape Rules can produce. Used to check SHAPES in both directions, so that a new shape must + # be added to the constant and a retired shape must be removed from it. + # + def shape_corpus + active = { "X-Goog-Upload-Status" => "active" } + final = { "X-Goog-Upload-Status" => "final" } + cancelled = { "X-Goog-Upload-Status" => "cancelled" } + { + start_upload: Event::StartUpload.new, + resume_upload: Event::ResumeUpload.new, + user_cancel: Event::Cancel.new, + global_deadline_exceeded: Event::GlobalDeadlineExceeded.new, + chunk_read_full: Event::ChunkRead.new(bytes_buffered: 4096, eof: false), + chunk_read_eof_with_data: Event::ChunkRead.new(bytes_buffered: 1024, eof: true), + chunk_read_eof_empty: Event::ChunkRead.new(bytes_buffered: 0, eof: true), + request_timeout: Event::RequestFailed.new(kind: :timeout), + request_retries_exhausted: Event::RequestFailed.new(kind: :retries_exhausted), + request_connection_failed: Event::RequestFailed.new(kind: :connection_failed), + request_failed_unknown: Event::RequestFailed.new(kind: :something_else), + response_active: Event::HttpResponse.new(status: 200, headers: active), + response_final: Event::HttpResponse.new(status: 200, headers: final), + response_cancelled: Event::HttpResponse.new(status: 200, headers: cancelled), + response_rejected: Event::HttpResponse.new(status: 400, headers: final), + response_cat2: Event::HttpResponse.new(status: 200, headers: {}), + response_fatal_bad_response: Event::HttpResponse.new(status: 401, headers: {}), + unknown: Object.new + } + end + + def test_shapes_constant_is_exhaustive_and_minimal + corpus = shape_corpus + + corpus.each do |expected_shape, event| + assert_equal expected_shape, Rules.shape_of(event), + "Corpus event for #{expected_shape} no longer classifies as that shape" + end + + assert_empty Rules::SHAPES - corpus.keys, + "SHAPES members that no corpus event produces (phantom or untested shapes)" + assert_empty corpus.keys - Rules::SHAPES, + "shape_of produces shapes that are missing from SHAPES" + assert_predicate Rules::SHAPES, :frozen? + end + + def test_statuses_tracks_state_descriptions + assert_empty Rules::STATUSES - Rules::STATE_DESCRIPTIONS.keys, + "status missing a description" + assert_empty Rules::STATE_DESCRIPTIONS.keys - Rules::STATUSES, + "description for unknown status" + assert_equal Rules::STATUSES.uniq, Rules::STATUSES + assert_predicate Rules::STATUSES, :frozen? + + assert_empty Rules::TERMINAL_STATUSES - Rules::STATUSES, + "TERMINAL_STATUSES contains statuses outside STATUSES" + assert_predicate Rules::TERMINAL_STATUSES, :frozen? + end + + def test_resume_handle_from_returns_nil_for_all_terminal_statuses_except_error + base_state = State.new upload_url: "https://example.com/session/123", chunk_size: 262_144 + + Rules::TERMINAL_STATUSES.each do |terminal_status| + state = base_state.with status: terminal_status + handle = Rules.resume_handle_from state + if terminal_status == :error + refute_nil handle, "Expected resume_handle_from to return a ResumeHandle for :error status" + else + assert_nil handle, "Expected resume_handle_from to return nil for terminal status #{terminal_status.inspect}" + end + end + end + + def test_decide_rejects_a_shape_outside_the_vocabulary + state = State.new + config = StartUploadConfig.new initial_url: "https://example.com/upload", stream: StringIO.new("data") + + error = Rules.stub :shape_of, :not_a_real_shape do + assert_raises InternalError do + Rules.decide state, Event::StartUpload.new, config + end + end + + assert_match(/Resumable upload internal error: shape_of returned unknown shape :not_a_real_shape/, error.message) + end + + def test_decide_rejects_a_recipe_outside_the_vocabulary + state = State.new + config = StartUploadConfig.new initial_url: "https://example.com/upload", stream: StringIO.new("data") + original_recipes = Rules::RECIPES + + error = begin + Rules.send :remove_const, :RECIPES + Rules.const_set :RECIPES, [].freeze + assert_raises InternalError do + Rules.decide state, Event::StartUpload.new, config + end + ensure + Rules.send :remove_const, :RECIPES + Rules.const_set :RECIPES, original_recipes + end + + assert_match(/Resumable upload internal error: decide selected unknown recipe :start_session/, error.message) + end + + def test_resolve_chunk_size_with_nil_or_non_positive_granularity + # nil granularity + assert_equal 1024, Rules.resolve_chunk_size(1024, nil) + assert_equal Rules::DEFAULT_CHUNK_SIZE, Rules.resolve_chunk_size(nil, nil) + + # 0 granularity + assert_equal 1024, Rules.resolve_chunk_size(1024, 0) + assert_equal Rules::DEFAULT_CHUNK_SIZE, Rules.resolve_chunk_size(nil, 0) + + # Negative granularity + assert_equal 1024, Rules.resolve_chunk_size(1024, -256) + assert_equal Rules::DEFAULT_CHUNK_SIZE, Rules.resolve_chunk_size(nil, -1) + end + + def test_resolve_chunk_size_when_divisible + # User-specified evenly divides granularity + assert_equal 1024, Rules.resolve_chunk_size(1024, 256) + assert_equal 1_048_576, Rules.resolve_chunk_size(1_048_576, 262_144) + + # Default chunk size (8_388_608) evenly divides the standard 256 KB backend granularity + assert_equal 8_388_608, Rules.resolve_chunk_size(nil, 262_144) + assert_equal 8_388_608, Rules.resolve_chunk_size(nil, 524_288) + end + + def test_resolve_chunk_size_when_not_divisible_rounds_down + # User-specified rounds down to nearest multiple + assert_equal 768, Rules.resolve_chunk_size(1000, 256) + assert_equal 9_961_472, Rules.resolve_chunk_size(10_000_000, 262_144) + + # Default chunk size (8_388_608) rounds down with non-divisor granularity (500_000 * 16) + assert_equal 8_000_000, Rules.resolve_chunk_size(nil, 500_000) + assert_equal 8_192_000, Rules.resolve_chunk_size(nil, 1_024_000) + end + + def test_resolve_chunk_size_when_granularity_equal_to_chunk_size + assert_equal 256, Rules.resolve_chunk_size(256, 256) + assert_equal 262_144, Rules.resolve_chunk_size(262_144, 262_144) + assert_equal 8_388_608, Rules.resolve_chunk_size(nil, 8_388_608) + end + + def test_resolve_chunk_size_when_granularity_greater_than_chunk_size + # User chunk size strictly less than granularity promotes to granularity (avoids 0) + assert_equal 256, Rules.resolve_chunk_size(100, 256) + assert_equal 262_144, Rules.resolve_chunk_size(1, 262_144) + + # Default chunk size strictly less than large server granularity promotes to granularity + assert_equal 16_777_216, Rules.resolve_chunk_size(nil, 16_777_216) + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb new file mode 100644 index 0000000..9b3edf1 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb @@ -0,0 +1,334 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Rules.decide transitions per router row. +# +class RulesDecideTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + initial_headers: { "X-Custom" => "value" }, + initial_body: '{"name":"obj"}', + stream: StringIO.new("data"), + upload_size: 1024, + chunk_size: 512 + ) + end + + def test_recipe_phases_partition + notifying = Rules::RECIPE_PHASES.keys + non_notifying = Rules::NON_NOTIFYING_RECIPES + all_classified = notifying + non_notifying + + assert_empty Rules::RECIPES - all_classified, + "Recipes missing from RECIPE_PHASES or NON_NOTIFYING_RECIPES" + assert_empty all_classified - Rules::RECIPES, + "Phantom recipes in RECIPE_PHASES or NON_NOTIFYING_RECIPES" + assert_empty notifying & non_notifying, + "Recipes present in both RECIPE_PHASES and NON_NOTIFYING_RECIPES" + end + + def test_row_initializing_start_upload + decision = Rules.decide State.new(status: :initializing), Event::StartUpload.new, @config + assert_equal :initializing, decision.from_status + assert_equal :start_upload, decision.shape + assert_equal :start_session, decision.recipe + assert_equal :starting, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendStart, decision.instructions[1] + end + + def test_row_initializing_resume_upload + resume_config = ResumeUploadConfig.new( + upload_url: "https://example.com/upload/session1", + chunk_size: 512, + stream: StringIO.new("data"), + upload_size: 1024 + ) + decision = Rules.decide State.new(status: :initializing), Event::ResumeUpload.new, resume_config + assert_equal :initializing, decision.from_status + assert_equal :resume_upload, decision.shape + assert_equal :resume_session, decision.recipe + assert_equal :recovery, decision.next_state.status + assert_equal "https://example.com/upload/session1", decision.next_state.upload_url + assert_equal 512, decision.next_state.chunk_size + assert_equal 0, decision.next_state.offset + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendQuery, decision.instructions[1] + assert_equal "https://example.com/upload/session1", decision.instructions[1].url + end + + def test_row_starting_response_active + active_resp = Event::HttpResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active", "x-goog-upload-url" => "https://example.com/session" } + ) + decision = Rules.decide State.new(status: :starting), active_resp, @config + assert_equal :starting, decision.from_status + assert_equal :response_active, decision.shape + assert_equal :begin_transmission, decision.recipe + assert_equal :transmission_reading, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::FillBuffer, decision.instructions[1] + end + + def test_row_transmission_reading_chunk_read_full + decision = Rules.decide( + State.new(status: :transmission_reading, upload_url: "https://example.com/session"), + Event::ChunkRead.new(bytes_buffered: 512, eof: false), + @config + ) + assert_equal :transmission_reading, decision.from_status + assert_equal :chunk_read_full, decision.shape + assert_equal :send_chunk, decision.recipe + assert_equal :transmission_sending, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendChunk, decision.instructions.first + refute decision.instructions.first.finalize + end + + def test_row_transmission_reading_chunk_read_eof_with_data + decision = Rules.decide( + State.new(status: :transmission_reading, upload_url: "https://example.com/session"), + Event::ChunkRead.new(bytes_buffered: 256, eof: true), + @config + ) + assert_equal :transmission_reading, decision.from_status + assert_equal :chunk_read_eof_with_data, decision.shape + assert_equal :send_upload_finalize, decision.recipe + assert_equal :finalizing_sending_upload, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendChunk, decision.instructions[1] + assert decision.instructions[1].finalize + end + + def test_row_transmission_reading_chunk_read_eof_empty + decision = Rules.decide( + State.new(status: :transmission_reading, upload_url: "https://example.com/session"), + Event::ChunkRead.new(bytes_buffered: 0, eof: true), + @config + ) + assert_equal :transmission_reading, decision.from_status + assert_equal :chunk_read_eof_empty, decision.shape + assert_equal :send_finalize, decision.recipe + assert_equal :finalizing_sending_finalize, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendFinalize, decision.instructions[1] + end + + def test_row_transmission_sending_response_active + active_resp = Event::HttpResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active", "x-goog-upload-url" => "https://example.com/session" } + ) + decision = Rules.decide( + State.new(status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, in_flight_length: 512), + active_resp, + @config + ) + assert_equal :transmission_sending, decision.from_status + assert_equal :response_active, decision.shape + assert_equal :ack_chunk, decision.recipe + assert_equal :transmission_reading, decision.next_state.status + assert_recipe_progress_notification decision + assert_equal 3, decision.instructions.size + end + + def test_row_transmission_sending_enter_recovery + cat2_resp = Event::HttpResponse.new status: 503, headers: {} + decision = Rules.decide( + State.new(status: :transmission_sending, upload_url: "https://example.com/session"), + cat2_resp, + @config + ) + assert_equal :transmission_sending, decision.from_status + assert_equal :response_cat2, decision.shape + assert_equal :enter_recovery, decision.recipe + assert_equal :recovery, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendQuery, decision.instructions[1] + end + + def test_row_finalizing_sending_upload_response_final + final_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" } + decision = Rules.decide( + State.new(status: :finalizing_sending_upload, offset: 512, in_flight_length: 512), + final_resp, + @config + ) + assert_equal :finalizing_sending_upload, decision.from_status + assert_equal :response_final, decision.shape + assert_equal :complete_upload_with_data, decision.recipe + assert_equal :success, decision.next_state.status + assert_recipe_progress_notification decision + assert_equal 2, decision.instructions.size + end + + def test_row_finalizing_sending_finalize_response_final + final_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" } + decision = Rules.decide State.new(status: :finalizing_sending_finalize), final_resp, @config + assert_equal :finalizing_sending_finalize, decision.from_status + assert_equal :response_final, decision.shape + assert_equal :complete_upload_finalized, decision.recipe + assert_equal :success, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::TerminateSuccess, decision.instructions[1] + end + + def test_row_recovery_response_active + recovery_active_resp = Event::HttpResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active", "x-goog-upload-size-received" => "256" } + ) + decision = Rules.decide State.new(status: :recovery), recovery_active_resp, @config + assert_equal :recovery, decision.from_status + assert_equal :response_active, decision.shape + assert_equal :realign_from_recovery, decision.recipe + assert_equal :transmission_reading, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::RealignBuffer, decision.instructions[1] + end + + def test_row_recovery_response_cat2 + cat2_resp = Event::HttpResponse.new status: 503, headers: {} + decision = Rules.decide State.new(status: :recovery), cat2_resp, @config + assert_equal :recovery, decision.from_status + assert_equal :response_cat2, decision.shape + assert_equal :retry_recovery, decision.recipe + assert_equal :recovery, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendQuery, decision.instructions.first + end + + def test_row_cancelling_response_cancelled + cancelled_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + decision = Rules.decide State.new(status: :cancelling), cancelled_resp, @config + assert_equal :cancelling, decision.from_status + assert_equal :response_cancelled, decision.shape + assert_equal :complete_cancellation, decision.recipe + assert_equal :cancelled, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::TerminateFailure, decision.instructions.first + end + + def test_row_cancelling_user_cancel_raises_invalid_transition + err = assert_raises InvalidTransitionError do + Rules.decide State.new(status: :cancelling), Event::Cancel.new, @config + end + assert_equal :cancelling, err.state + end + + def test_row_global_deadline_exceeded + decision = Rules.decide State.new(status: :transmission_sending), Event::GlobalDeadlineExceeded.new, @config + assert_equal :transmission_sending, decision.from_status + assert_equal :global_deadline_exceeded, decision.shape + assert_equal :fail_with_deadline_exceeded, decision.recipe + assert_equal :error, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of DeadlineExceededError, decision.next_state.last_error + end + + def test_row_user_cancel + decision = Rules.decide State.new(status: :transmission_sending), Event::Cancel.new, @config + assert_equal :transmission_sending, decision.from_status + assert_equal :user_cancel, decision.shape + assert_equal :cancel_session, decision.recipe + assert_equal :cancelling, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendCancel, decision.instructions[1] + end + + def test_user_cancel_across_all_statuses + cancellable = [ + :transmission_reading, + :transmission_sending, + :finalizing_sending_upload, + :finalizing_sending_finalize, + :recovery + ] + + Rules::STATUSES.each do |status| + state = State.new status: status, upload_url: "https://example.com/upload/session-1" + if cancellable.include? status + decision = Rules.decide state, Event::Cancel.new, @config + assert_equal :cancel_session, decision.recipe, "Expected :cancel_session for status #{status.inspect}" + assert_equal :cancelling, decision.next_state.status + else + err = assert_raises InvalidTransitionError, "Expected InvalidTransitionError for status #{status.inspect}" do + Rules.decide state, Event::Cancel.new, @config + end + assert_equal status, err.state + end + end + end + + def test_row_response_rejected + rejected_resp = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" }, body: "Rejected" + decision = Rules.decide State.new(status: :starting), rejected_resp, @config + assert_equal :starting, decision.from_status + assert_equal :response_rejected, decision.shape + assert_equal :fail_with_rejected, decision.recipe + assert_equal :rejected, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of UploadRejectedError, decision.next_state.last_error + end + + def test_row_fail_with_bad_response + cat2_resp = Event::HttpResponse.new status: 503, headers: {} + decision = Rules.decide State.new(status: :starting), cat2_resp, @config + assert_equal :starting, decision.from_status + assert_equal :response_cat2, decision.shape + assert_equal :fail_with_bad_response, decision.recipe + assert_equal :error, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of BadResponseError, decision.next_state.last_error + end + + def test_row_fail_with_request_error + req_failed = Event::RequestFailed.new kind: :retries_exhausted, message: "Exhausted" + decision = Rules.decide State.new(status: :starting), req_failed, @config + assert_equal :starting, decision.from_status + assert_equal :request_retries_exhausted, decision.shape + assert_equal :fail_with_request_error, decision.recipe + assert_equal :error, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::TerminateFailure, decision.instructions.first + end + + private + + def assert_recipe_progress_notification decision + if Rules::RECIPE_PHASES.key? decision.recipe + expected_phase = Rules::RECIPE_PHASES[decision.recipe] + first_inst = decision.instructions.first + assert_instance_of Instruction::NotifyProgress, first_inst, + "Expected #{decision.recipe} to emit NotifyProgress as first instruction" + assert_equal expected_phase, first_inst.progress.phase, + "Expected #{decision.recipe} to emit phase #{expected_phase}" + else + assert_includes Rules::NON_NOTIFYING_RECIPES, decision.recipe + refute decision.instructions.any? { |i| i.is_a? Instruction::NotifyProgress }, + "Expected non-notifying recipe #{decision.recipe} to emit no NotifyProgress" + end + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb new file mode 100644 index 0000000..212deec --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb @@ -0,0 +1,474 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Rules terminal error transitions and actionable error formatting. +# +class RulesErrorTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + initial_headers: { "X-Custom" => "value" }, + initial_body: '{"name":"obj"}', + stream: StringIO.new("data"), + upload_size: 1024, + chunk_size: 512 + ) + end + + def test_transition_starting_rejected + state = State.new status: :starting + resp = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" }, body: "Forbidden" + next_state, instructions = Rules.step state, resp, @config + + assert_equal :rejected, next_state.status + assert_instance_of UploadRejectedError, next_state.last_error + assert_equal "Forbidden", next_state.last_error.response_body + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + assert_equal next_state.last_error, instructions.first.error + end + + def test_transition_starting_fatal_error + state = State.new status: :starting + resp = Event::HttpResponse.new status: 400, headers: {}, body: "Bad Request" + next_state, instructions = Rules.step state, resp, @config + + assert_equal :error, next_state.status + assert_instance_of BadResponseError, next_state.last_error + assert_equal 400, next_state.last_error.status_code + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + end + + def test_transition_starting_request_failed + state = State.new status: :starting + err = StandardError.new "DNS resolution failed" + failed = Event::RequestFailed.new kind: :connection_failed, message: "DNS resolution failed", source_error: err + next_state, instructions = Rules.step state, failed, @config + + assert_equal :error, next_state.status + assert_instance_of RequestFailedError, next_state.last_error + assert_equal err, next_state.last_error.cause + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + assert_equal next_state.last_error, instructions.first.error + end + + def test_transition_starting_timeout_terminates_failure + state = State.new status: :starting + err = StandardError.new "Read timeout" + req_failed = Event::RequestFailed.new kind: :timeout, message: "Read timeout", source_error: err + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :error, next_state.status + assert_equal 0, next_state.in_flight_length + assert_instance_of RequestFailedError, next_state.last_error + assert_equal err, next_state.last_error.cause + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + assert_equal next_state.last_error, instructions.first.error + end + + def test_transition_transmission_sending_retries_exhausted_terminates_failure + state = State.new status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, + in_flight_length: 512 + err = StandardError.new "Retries exhausted" + req_failed = Event::RequestFailed.new kind: :retries_exhausted, message: "Retries exhausted", source_error: err + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :error, next_state.status + assert_equal 0, next_state.in_flight_length + assert_instance_of RequestFailedError, next_state.last_error + assert_equal err, next_state.last_error.cause + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + assert_equal next_state.last_error, instructions.first.error + end + + def test_transition_recovery_timeout_terminates_failure + state = State.new status: :recovery, upload_url: "https://example.com/session" + err = StandardError.new "Query read timeout" + req_failed = Event::RequestFailed.new kind: :timeout, message: "Query read timeout", source_error: err + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :error, next_state.status + assert_equal 0, next_state.in_flight_length + assert_instance_of RequestFailedError, next_state.last_error + assert_equal err, next_state.last_error.cause + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + assert_equal next_state.last_error, instructions.first.error + end + + def test_transition_global_deadline_exceeded + state = State.new status: :transmission_sending, upload_url: "https://example.com/session" + next_state, instructions = Rules.step state, Event::GlobalDeadlineExceeded.new, @config + + assert_equal :error, next_state.status + assert_instance_of DeadlineExceededError, next_state.last_error + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + end + + def test_invalid_transition_raises_actionable_error_with_response_details_and_header + state = State.new status: :transmission_sending + resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}' + + err = assert_raises InvalidTransitionError do + Rules.step state, resp, @config + end + + expected_msg = "Resumable upload failed while sending a chunk of data: " \ + "received an unexpected HTTP 200 response (X-Goog-Upload-Status: 'final')." + assert_equal expected_msg, err.message + assert_equal :transmission_sending, err.state + assert_equal resp, err.event + assert_equal resp, err.response + end + + def test_invalid_transition_shows_missing_when_upload_status_header_absent + state = State.new status: :transmission_reading + resp = Event::HttpResponse.new status: 200, headers: {}, body: "" + + err = assert_raises InvalidTransitionError do + Rules.step state, resp, @config + end + + expected_msg = "Resumable upload failed while reading chunk from stream: " \ + "received an unexpected HTTP 200 response (X-Goog-Upload-Status: missing)." + assert_equal expected_msg, err.message + assert_equal :transmission_reading, err.state + assert_equal resp, err.response + end + + def test_invalid_transition_raises_error_for_non_http_event + state = State.new status: :starting + event = Event::ChunkRead.new bytes_buffered: 512, eof: false + + err = assert_raises InvalidTransitionError do + Rules.step state, event, @config + end + + expected_msg = "Resumable upload failed while initiating upload session: " \ + "received unexpected stream chunk read (512 bytes, eof: false)." + assert_equal expected_msg, err.message + assert_equal :starting, err.state + assert_equal event, err.event + assert_nil err.response + end + + def test_rejected_with_wrapped_error_deprefixes_message_and_preserves_metadata + details = [{ "reason" => "ACCESS_DENIED" }] + headers = { "x-goog-upload-status" => "final", "content-type" => "application/json" } + wrapped_err = Gapic::Rest::Error.new( + "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: The caller does not have permission", + 403, + status: "PERMISSION_DENIED", + details: details, + headers: headers + ) + resp = Event::HttpResponse.new( + status: 403, + headers: headers, + body: '{"error":{"message":"The caller does not have permission"}}', + error: wrapped_err + ) + + state = State.new status: :starting + next_state, instructions = Rules.step state, resp, @config + + assert_equal :rejected, next_state.status + err = next_state.last_error + assert_instance_of UploadRejectedError, err + assert_equal "Upload rejected by server with HTTP 403 PERMISSION_DENIED: The caller does not have permission", + err.message + assert_equal 403, err.status_code + assert_equal "PERMISSION_DENIED", err.status + assert_equal details, err.details + assert_equal details, err.status_details + assert_equal headers, err.headers + assert_equal headers, err.header + assert_equal '{"error":{"message":"The caller does not have permission"}}', err.response_body + assert_equal err, instructions.first.error + end + + def test_rejected_fallback_without_wrapped_error + resp = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" }, body: "Forbidden" + state = State.new status: :starting + next_state, _instructions = Rules.step state, resp, @config + + assert_equal :rejected, next_state.status + err = next_state.last_error + assert_instance_of UploadRejectedError, err + assert_equal "Upload rejected by server with HTTP 403 Forbidden (X-Goog-Upload-Status: 'final')", err.message + assert_equal 403, err.status_code + assert_equal "Forbidden", err.response_body + end + + def test_bad_response_with_wrapped_error_deprefixes_message_and_preserves_metadata + details = ["Quota limit details"] + headers = { "x-goog-upload-status" => "active" } + wrapped_err = Gapic::Rest::Error.new( + "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: Quota limit reached", + 429, + status: "RESOURCE_EXHAUSTED", + details: details, + headers: headers + ) + resp = Event::HttpResponse.new status: 429, headers: headers, body: "Too many requests", error: wrapped_err + + state = State.new status: :starting + next_state, _instructions = Rules.step state, resp, @config + + assert_equal :error, next_state.status + err = next_state.last_error + assert_instance_of BadResponseError, err + assert_equal "Resumable upload failed with HTTP 429 RESOURCE_EXHAUSTED: Quota limit reached", err.message + assert_equal 429, err.status_code + assert_equal "RESOURCE_EXHAUSTED", err.status + assert_equal details, err.status_details + assert_equal headers, err.headers + assert_equal "Too many requests", err.response_body + end + + def test_bad_response_fallback_without_wrapped_error + resp = Event::HttpResponse.new status: 503, headers: {}, body: "Service unavailable" + state = State.new status: :starting + next_state, _instructions = Rules.step state, resp, @config + + assert_equal :error, next_state.status + err = next_state.last_error + assert_instance_of BadResponseError, err + assert_equal "Resumable upload failed with HTTP 503 Service Unavailable (X-Goog-Upload-Status: missing)", err.message + assert_equal 503, err.status_code + assert_equal "Service unavailable", err.response_body + end + + def test_format_status_preserves_canonical_status_token + wrapped_err = Gapic::Rest::Error.new( + "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: Object not found", + 404, + status: "NOT_FOUND", + headers: { "x-goog-upload-status" => "final" } + ) + resp = Event::HttpResponse.new status: 404, headers: { "x-goog-upload-status" => "final" }, + body: "Not found", error: wrapped_err + err = UploadRejectedError.from resp + assert_equal "Upload rejected by server with HTTP 404 NOT_FOUND: Object not found", err.message + assert_equal "NOT_FOUND", err.status + end + + def test_error_class_inheritance_hierarchy + assert_operator UploadCancelledError, :<, Gapic::Common::Error + refute_operator UploadCancelledError, :<, Gapic::Rest::Error + + assert_operator DeadlineExceededError, :<, Gapic::Common::Error + refute_operator DeadlineExceededError, :<, Gapic::Rest::Error + + assert_operator UploadRejectedError, :<, Gapic::Rest::Error + assert_operator BadResponseError, :<, Gapic::Rest::Error + + assert_operator StreamMismatchError, :<, Gapic::Common::Error + assert_operator RequestFailedError, :<, Gapic::Common::Error + assert_operator HasResumeHandle, :===, BadResponseError.new + assert_operator HasResumeHandle, :===, DeadlineExceededError.new + assert_operator HasResumeHandle, :===, UnseekableStreamError.new + assert_operator HasResumeHandle, :===, InvalidTransitionError.new("invalid") + assert_operator HasResumeHandle, :===, StreamMismatchError.new + assert_operator HasResumeHandle, :===, RequestFailedError.new("failed") + refute_operator HasResumeHandle, :===, UploadRejectedError.new + refute_operator HasResumeHandle, :===, UploadCancelledError.new + end + + def test_rules_resume_handle_from + assert_nil Rules.resume_handle_from(nil) + assert_nil Rules.resume_handle_from(State.new(status: :starting, upload_url: nil)) + assert_nil Rules.resume_handle_from(State.new(status: :rejected, upload_url: "https://upload.example.com/id123")) + assert_nil Rules.resume_handle_from(State.new(status: :cancelled, upload_url: "https://upload.example.com/id123")) + assert_nil Rules.resume_handle_from(State.new(status: :success, upload_url: "https://upload.example.com/id123")) + + state = State.new status: :transmission_sending, upload_url: "https://upload.example.com/id123", chunk_size: 1024 + handle = Rules.resume_handle_from state + refute_nil handle + assert_equal "https://upload.example.com/id123", handle.upload_url + assert_equal 1024, handle.chunk_size + end + + def test_resume_handle_present_on_errors_when_upload_url_set + state = State.new( + status: :transmission_sending, + upload_url: "https://upload.example.com/session_abc", + chunk_size: 512 + ) + + # 1. Deadline exceeded + next_state, = Rules.step state, Event::GlobalDeadlineExceeded.new, @config + assert_equal :error, next_state.status + deadline_err = next_state.last_error + assert_instance_of DeadlineExceededError, deadline_err + refute_nil deadline_err.resume_handle + assert_equal "https://upload.example.com/session_abc", deadline_err.resume_handle.upload_url + assert_equal 512, deadline_err.resume_handle.chunk_size + assert_includes deadline_err.message, "(upload session is resumable: see #resume_handle)" + + # 2. Bad response + resp = Event::HttpResponse.new status: 401, headers: {}, body: "Fatal 401" + next_state, = Rules.step state, resp, @config + assert_equal :error, next_state.status + bad_resp_err = next_state.last_error + assert_instance_of BadResponseError, bad_resp_err + refute_nil bad_resp_err.resume_handle + assert_equal "https://upload.example.com/session_abc", bad_resp_err.resume_handle.upload_url + assert_equal 512, bad_resp_err.resume_handle.chunk_size + assert_includes bad_resp_err.message, "(upload session is resumable: see #resume_handle)" + + # 3. Unmatched transition + unmatched_err = assert_raises InvalidTransitionError do + Rules.step state, Object.new, @config + end + refute_nil unmatched_err.resume_handle + assert_equal "https://upload.example.com/session_abc", unmatched_err.resume_handle.upload_url + assert_equal 512, unmatched_err.resume_handle.chunk_size + assert_includes unmatched_err.message, "(upload session is resumable: see #resume_handle)" + + # 4. Request failed (retries exhausted) + req_failed = Event::RequestFailed.new( + kind: :retries_exhausted, + message: "Connection reset", + source_error: StandardError.new("reset") + ) + next_state, = Rules.step state, req_failed, @config + assert_equal :error, next_state.status + req_err = next_state.last_error + assert_instance_of RequestFailedError, req_err + refute_nil req_err.resume_handle + assert_equal "https://upload.example.com/session_abc", req_err.resume_handle.upload_url + assert_equal 512, req_err.resume_handle.chunk_size + assert_equal "reset", req_err.cause.message + assert_includes req_err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_resume_handle_nil_on_errors_before_session_created + state = State.new status: :starting, upload_url: nil + + # 1. Deadline exceeded before session creation + next_state, = Rules.step state, Event::GlobalDeadlineExceeded.new, @config + assert_equal :error, next_state.status + deadline_err = next_state.last_error + assert_nil deadline_err.resume_handle + refute_includes deadline_err.message, "(upload session is resumable: see #resume_handle)" + + # 2. Bad response before session creation + resp = Event::HttpResponse.new status: 503, headers: {}, body: "Init failed" + next_state, = Rules.step state, resp, @config + assert_equal :error, next_state.status + bad_resp_err = next_state.last_error + assert_nil bad_resp_err.resume_handle + refute_includes bad_resp_err.message, "(upload session is resumable: see #resume_handle)" + + # 3. Unmatched transition before session creation + unmatched_err = assert_raises InvalidTransitionError do + Rules.step state, Object.new, @config + end + assert_nil unmatched_err.resume_handle + refute_includes unmatched_err.message, "(upload session is resumable: see #resume_handle)" + + # 4. Request failed before session creation + req_failed = Event::RequestFailed.new( + kind: :connection_failed, + message: "Connection reset", + source_error: StandardError.new("reset") + ) + next_state, = Rules.step state, req_failed, @config + assert_equal :error, next_state.status + req_err = next_state.last_error + assert_instance_of RequestFailedError, req_err + assert_nil req_err.resume_handle + refute_includes req_err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_resume_handle_absent_on_rejected_and_cancelled + state = State.new( + status: :transmission_sending, + upload_url: "https://upload.example.com/session_abc", + chunk_size: 512 + ) + + # 1. Rejected error + rejected_resp = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" }, + body: "Access Denied" + next_state, = Rules.step state, rejected_resp, @config + assert_equal :rejected, next_state.status + rejected_err = next_state.last_error + assert_instance_of UploadRejectedError, rejected_err + refute_respond_to rejected_err, :resume_handle + refute_includes rejected_err.message, "(upload session is resumable: see #resume_handle)" + + # 2. Cancelled error + cancelling_state = state.with status: :cancelling + cancelled_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" }, + body: "" + next_state, = Rules.step cancelling_state, cancelled_resp, @config + assert_equal :cancelled, next_state.status + cancelled_err = next_state.last_error + assert_instance_of UploadCancelledError, cancelled_err + refute_respond_to cancelled_err, :resume_handle + refute_includes cancelled_err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_stream_mismatch_error_behavior + handle = ResumeHandle.new upload_url: "https://upload.example.com/resume", chunk_size: 256 + err_with_handle = StreamMismatchError.new "Stream too short", resume_handle: handle + + assert_instance_of StreamMismatchError, err_with_handle + assert_equal handle, err_with_handle.resume_handle + assert_equal "Stream too short (upload session is resumable: see #resume_handle)", err_with_handle.message + + err_from = StreamMismatchError.from "Stream corrupted", resume_handle: handle + assert_equal handle, err_from.resume_handle + assert_equal "Stream corrupted (upload session is resumable: see #resume_handle)", err_from.message + + err_without_handle = StreamMismatchError.new "No handle" + assert_nil err_without_handle.resume_handle + assert_equal "No handle", err_without_handle.message + end + + def test_request_failed_error_behavior + handle = ResumeHandle.new upload_url: "https://upload.example.com/resume", chunk_size: 256 + cause = Gapic::Rest::Error.new "Underlying Faraday error", 500, status: "INTERNAL", details: ["foo"], headers: { "k" => "v" } + + err_with_handle = RequestFailedError.from cause, resume_handle: handle + assert_instance_of RequestFailedError, err_with_handle + assert_equal cause, err_with_handle.cause + assert_equal handle, err_with_handle.resume_handle + assert_equal 500, err_with_handle.status_code + assert_equal "INTERNAL", err_with_handle.status + assert_equal ["foo"], err_with_handle.details + assert_equal({ "k" => "v" }, err_with_handle.headers) + assert_equal "Underlying Faraday error (upload session is resumable: see #resume_handle)", err_with_handle.message + + err_without_handle = RequestFailedError.from cause + assert_nil err_without_handle.resume_handle + assert_equal "Underlying Faraday error", err_without_handle.message + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_recovery_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_recovery_test.rb new file mode 100644 index 0000000..9c67f0d --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_recovery_test.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Rules protocol recovery state transitions. +# +class RulesRecoveryTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + initial_headers: { "X-Custom" => "value" }, + initial_body: '{"name":"obj"}', + stream: StringIO.new("data"), + upload_size: 1024, + chunk_size: 512 + ) + end + + def test_transition_transmission_sending_cat2_triggers_recovery + state = State.new status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, + in_flight_length: 512 + resp = Event::HttpResponse.new status: 503, headers: {} + next_state, instructions = Rules.step state, resp, @config + + assert_equal :recovery, next_state.status + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :recovering, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendQuery, instructions[1] + end + + def test_transition_transmission_sending_connection_failed_triggers_recovery + state = State.new status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, + in_flight_length: 512 + req_failed = Event::RequestFailed.new kind: :connection_failed, message: "Network unreachable" + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :recovery, next_state.status + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :recovering, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendQuery, instructions[1] + end + + def test_transition_transmission_sending_timeout_triggers_recovery + state = State.new status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, + in_flight_length: 512 + req_failed = Event::RequestFailed.new kind: :timeout, message: "Read timeout" + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :recovery, next_state.status + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :recovering, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendQuery, instructions[1] + end + + def test_transition_finalizing_sending_upload_timeout_triggers_recovery + state = State.new status: :finalizing_sending_upload, upload_url: "https://example.com/session", offset: 512, + in_flight_length: 512 + req_failed = Event::RequestFailed.new kind: :timeout, message: "Read timeout" + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :recovery, next_state.status + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :recovering, bytes_uploaded: 512, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendQuery, instructions[1] + end + + def test_transition_recovery_active_realigns_buffer + state = State.new status: :recovery, upload_url: "https://example.com/session", offset: 0, chunk_size: 512 + headers = { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "768" + } + resp = Event::HttpResponse.new status: 200, headers: headers + next_state, instructions = Rules.step state, resp, @config + + assert_equal :transmission_reading, next_state.status + assert_equal 768, next_state.offset + assert_equal 3, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :uploading, bytes_uploaded: 768, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::RealignBuffer, instructions[1] + assert_equal 768, instructions[1].server_offset + assert_instance_of Instruction::FillBuffer, instructions[2] + end + + def test_transition_recovery_final_completes_upload + state = State.new status: :recovery, upload_url: "https://example.com/session", offset: 512 + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" } + next_state, instructions = Rules.step state, resp, @config + + assert_equal :success, next_state.status + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :completed, bytes_uploaded: 512, total_bytes: 512), instructions[0].progress + assert_instance_of Instruction::TerminateSuccess, instructions[1] + end + + def test_transition_recovery_cat2_retries_query + state = State.new status: :recovery, upload_url: "https://example.com/session" + resp = Event::HttpResponse.new status: 416, headers: {} + next_state, instructions = Rules.step state, resp, @config + + assert_equal :recovery, next_state.status + assert_equal 1, instructions.size + assert_instance_of Instruction::SendQuery, instructions.first + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_test.rb new file mode 100644 index 0000000..0c8031f --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_test.rb @@ -0,0 +1,279 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Rules normal progression and session lifecycle state transitions. +# +class RulesTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + initial_headers: { "X-Custom" => "value" }, + initial_body: '{"name":"obj"}', + stream: StringIO.new("data"), + upload_size: 1024, + chunk_size: 512 + ) + end + + def test_transition_initializing_to_starting + state = State.new status: :initializing + next_state, instructions = Rules.step state, Event::StartUpload.new, @config + + assert_equal :starting, next_state.status + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendStart, instructions[1] + assert_equal "https://example.com/upload", instructions[1].url + assert_equal({ "X-Custom" => "value" }, instructions[1].headers) + assert_equal '{"name":"obj"}', instructions[1].body + end + + def test_transition_starting_to_transmission_reading + state = State.new status: :starting + headers = { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://example.com/session", + "x-goog-upload-chunk-granularity" => "256" + } + resp = Event::HttpResponse.new status: 200, headers: headers + next_state, instructions = Rules.step state, resp, @config + + assert_equal :transmission_reading, next_state.status + assert_equal "https://example.com/session", next_state.upload_url + assert_equal 256, next_state.chunk_granularity + assert_equal 512, next_state.chunk_size + assert_equal 0, next_state.offset + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::FillBuffer, instructions[1] + assert_equal 512, instructions[1].target_bytesize + end + + def test_transition_transmission_reading_full_chunk + state = State.new status: :transmission_reading, upload_url: "https://example.com/session", offset: 0, + chunk_size: 512 + event = Event::ChunkRead.new bytes_buffered: 512, eof: false + next_state, instructions = Rules.step state, event, @config + + assert_equal :transmission_sending, next_state.status + assert_equal 512, next_state.in_flight_length + assert_equal 1, instructions.size + assert_instance_of Instruction::SendChunk, instructions.first + assert_equal 0, instructions.first.offset + assert_equal 512, instructions.first.length + refute instructions.first.finalize + end + + def test_transition_transmission_reading_eof_with_data + state = State.new status: :transmission_reading, upload_url: "https://example.com/session", offset: 512, + chunk_size: 512 + event = Event::ChunkRead.new bytes_buffered: 200, eof: true + next_state, instructions = Rules.step state, event, @config + + assert_equal :finalizing_sending_upload, next_state.status + assert_equal 200, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :finalizing, bytes_uploaded: 512, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendChunk, instructions[1] + assert_equal 512, instructions[1].offset + assert_equal 200, instructions[1].length + assert instructions[1].finalize + end + + def test_transition_transmission_reading_eof_empty + state = State.new status: :transmission_reading, upload_url: "https://example.com/session", offset: 1024, + chunk_size: 512 + event = Event::ChunkRead.new bytes_buffered: 0, eof: true + next_state, instructions = Rules.step state, event, @config + + assert_equal :finalizing_sending_finalize, next_state.status + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :finalizing, bytes_uploaded: 1024, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendFinalize, instructions[1] + assert_equal "https://example.com/session", instructions[1].url + end + + def test_transition_transmission_sending_ack_chunk + state = State.new status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, + in_flight_length: 512, chunk_size: 512 + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "active" } + next_state, instructions = Rules.step state, resp, @config + + assert_equal :transmission_reading, next_state.status + assert_equal 512, next_state.offset + assert_equal 0, next_state.in_flight_length + assert_equal 3, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :uploading, bytes_uploaded: 512, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::RealignBuffer, instructions[1] + assert_equal 512, instructions[1].server_offset + assert_instance_of Instruction::FillBuffer, instructions[2] + assert_equal 512, instructions[2].target_bytesize + end + + def test_transition_finalizing_sending_upload_success + state = State.new status: :finalizing_sending_upload, offset: 512, in_flight_length: 512 + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"done":true}' + next_state, instructions = Rules.step state, resp, @config + + assert_equal :success, next_state.status + assert_equal 1024, next_state.offset + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :completed, bytes_uploaded: 1024, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::TerminateSuccess, instructions[1] + end + + def test_transition_finalizing_sending_finalize_success + state = State.new status: :finalizing_sending_finalize, offset: 1024, in_flight_length: 0 + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"done":true}' + next_state, instructions = Rules.step state, resp, @config + + assert_equal :success, next_state.status + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :completed, bytes_uploaded: 1024, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::TerminateSuccess, instructions[1] + end + + def test_transition_cancellation_flow + state = State.new status: :transmission_sending, upload_url: "https://example.com/session" + next_state, instructions = Rules.step state, Event::Cancel.new, @config + + assert_equal :cancelling, next_state.status + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :cancelling, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendCancel, instructions[1] + + # A duplicate cancel in :cancelling raises InvalidTransitionError + assert_raises InvalidTransitionError do + Rules.step next_state, Event::Cancel.new, @config + end + + # Cancellation confirmed + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + final_state, final_instructions = Rules.step next_state, resp, @config + assert_equal :cancelled, final_state.status + assert_instance_of UploadCancelledError, final_state.last_error + assert_equal 1, final_instructions.size + assert_instance_of Instruction::TerminateFailure, final_instructions.first + end + + def test_all_recipes_respond_to_rules_method + Rules::RECIPES.each do |recipe| + assert_respond_to Rules, recipe + end + end + + def test_all_recipes_satisfy_trampoline_invariant + resume_config = ResumeUploadConfig.new( + upload_url: "https://example.com/session", + chunk_size: 256, + stream: StringIO.new("abcd") + ) + active_resp = Event::HttpResponse.new status: 200, headers: { + "x-goog-upload-url" => "https://example.com/session", + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "256" + } + final_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" } + cancelled_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + cat2_resp = Event::HttpResponse.new status: 503, headers: {} + rejected_resp = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" } + bad_resp = Event::HttpResponse.new status: 401, headers: {} + req_err = Event::RequestFailed.new kind: :connection_failed, message: "connection lost" + chunk_full = Event::ChunkRead.new bytes_buffered: 256, eof: false + chunk_eof = Event::ChunkRead.new bytes_buffered: 256, eof: true + base_state = State.new( + status: :transmission_sending, + upload_url: "https://example.com/session", + chunk_size: 256, + in_flight_length: 256 + ) + + fixtures = { + start_session: [State.new(status: :initializing), Event::StartUpload.new, @config], + resume_session: [State.new(status: :initializing), Event::ResumeUpload.new, resume_config], + begin_transmission: [State.new(status: :starting), active_resp, @config], + send_chunk: [base_state.with(status: :transmission_reading), chunk_full, @config], + send_upload_finalize: [base_state.with(status: :transmission_reading), chunk_eof, @config], + send_finalize: [base_state.with(status: :transmission_reading), Event::ChunkRead.new(bytes_buffered: 0, eof: true), @config], + ack_chunk: [base_state, active_resp, @config], + enter_recovery: [base_state, cat2_resp, @config], + retry_recovery: [base_state.with(status: :recovery), cat2_resp, @config], + realign_from_recovery: [base_state.with(status: :recovery), active_resp, @config], + complete_upload_with_data: [base_state.with(status: :finalizing_sending_upload), final_resp, @config], + complete_upload_finalized: [base_state.with(status: :finalizing_sending_finalize), final_resp, @config], + cancel_session: [base_state, Event::Cancel.new, @config], + complete_cancellation: [base_state.with(status: :cancelling), cancelled_resp, @config], + fail_with_deadline_exceeded: [base_state, Event::GlobalDeadlineExceeded.new, @config], + fail_with_rejected: [base_state, rejected_resp, @config], + fail_with_bad_response: [base_state, bad_resp, @config], + fail_with_request_error: [base_state, req_err, @config], + fail_with_unmatched_transition: [State.new(status: :success), Event::StartUpload.new, @config] + } + + assert_equal Rules::RECIPES.sort, fixtures.keys.sort + + fixtures.each do |recipe, (state, event, cfg)| + if recipe == :fail_with_unmatched_transition + assert_raises InvalidTransitionError do + Rules.public_send recipe, state, event, cfg + end + next + end + + _next_state, instructions = Rules.public_send recipe, state, event, cfg + event_producing_count = instructions.count { |inst| Instruction::CONTINUATION.include? inst.class } + terminal_count = instructions.count { |inst| Instruction::TERMINAL.include? inst.class } + + valid = (event_producing_count == 1 && terminal_count.zero?) || + (event_producing_count.zero? && terminal_count == 1) + assert valid, "Recipe :#{recipe} produced #{event_producing_count} event-producing and #{terminal_count} terminal instructions" + end + end + + def test_instruction_constants_partition_all_instruction_classes + all_classes = Instruction.constants(false).map { |name| Instruction.const_get name }.grep(Class) + partition_union = Instruction::CONTINUATION + Instruction::TERMINAL + Instruction::SIDE_EFFECT + + assert_empty all_classes - partition_union, + "Instruction classes missing from CONTINUATION/TERMINAL/SIDE_EFFECT partition" + assert_empty partition_union - all_classes, + "Partition contains classes that are not Instruction classes" + assert_empty Instruction::CONTINUATION & Instruction::TERMINAL + assert_empty Instruction::CONTINUATION & Instruction::SIDE_EFFECT + assert_empty Instruction::TERMINAL & Instruction::SIDE_EFFECT + assert_predicate Instruction::CONTINUATION, :frozen? + assert_predicate Instruction::TERMINAL, :frozen? + assert_predicate Instruction::SIDE_EFFECT, :frozen? + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/session_test.rb b/gapic-common/test/gapic/rest/resumable_upload/session_test.rb new file mode 100644 index 0000000..fe6e782 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/session_test.rb @@ -0,0 +1,704 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +class SessionTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + class ScriptedClientStub + attr_reader :requests + + def initialize responses = [] + @responses = responses.dup + @requests = [] + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } + raise "Unexpected request: no scripted response left" if @responses.empty? + + res = @responses.shift + if res.is_a? Proc + res.call + elsif res.is_a? Exception + raise res + else + res + end + end + end + + class UnseekableStream + attr_reader :pos + + def initialize string + @io = StringIO.new string + @pos = 0 + end + + def read length = nil + chunk = @io.read length + @pos += chunk.bytesize if chunk + chunk + end + end + + class StreamWithoutPos + def initialize string + @io = StringIO.new string + end + + def read length = nil + @io.read length + end + end + + START_ONLY_KEYS = [:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy].freeze + + # Builds a session from the shared arguments and remembers the per-run arguments that #start needs, + # so tests can keep calling `start_session session`. + def build_session stub: nil, stream: nil, upload_size: 10, chunk_size: 4, **kwargs + stream ||= StringIO.new "0123456789" + stub ||= ScriptedClientStub.new + @start_args = { + initial_url: "https://example.com/initiate", + initial_body: '{"name":"test.txt"}', + chunk_size: chunk_size + }.merge(kwargs.slice(*START_ONLY_KEYS)) + + Session.new( + client_stub: stub, + stream: stream, + upload_size: upload_size, + **kwargs.except(*START_ONLY_KEYS) + ) + end + + def start_session session, **overrides + session.start(**@start_args, **overrides) + end + + # ============================================================================ + # 1. Initialization and argument validation + # ============================================================================ + + def test_initialize_mandatory_arguments + assert_raises ArgumentError do + Session.new stream: StringIO.new + end + + assert_raises ArgumentError do + Session.new client_stub: ScriptedClientStub.new + end + end + + def test_initialize_rejects_per_run_arguments + assert_raises ArgumentError do + Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new, initial_url: "http://x" + end + + assert_raises ArgumentError do + Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new, chunk_size: 4 + end + end + + def test_initialize_defaults + session = Session.new( + client_stub: ScriptedClientStub.new, + stream: StringIO.new("abc"), + upload_size: 300 + ) + + assert_equal 300, session.upload_size + assert_nil session.content_type + assert_nil session.timeout + assert_nil session.control_plane_retry_policy + assert_nil session.data_plane_retry_policy + assert_nil session.on_progress + assert_nil session.logger + end + + def test_start_without_initial_url_raises_argument_error + session = Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new("abc"), upload_size: 3 + + error = assert_raises ArgumentError do + session.start + end + assert_match(/initial_url/, error.message) + end + + def test_start_with_blank_initial_url_raises_argument_error + session = Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new("abc"), upload_size: 3 + + error = assert_raises ArgumentError do + session.start initial_url: " " + end + assert_match(/initial_url is required/, error.message) + end + + def test_start_with_malformed_retry_policy_raises_argument_error + session = build_session + + error = assert_raises ArgumentError do + start_session session, start_retry_policy: "nonsense" + end + assert_match(/Expected RetryPolicy, Hash, or nil/, error.message) + end + + def test_start_with_reserved_initial_header_raises_before_any_request + stub = ScriptedClientStub.new + session = build_session stub: stub + + error = assert_raises ArgumentError do + start_session session, initial_headers: { "X-Goog-Upload-Header-Content-Type" => "image/png" } + end + assert_match(/must not set protocol header/, error.message) + assert_empty stub.requests + refute session.bound? + end + + def test_failed_start_leaves_session_reusable + session = build_session + + assert_raises ArgumentError do + start_session session, start_retry_policy: "nonsense" + end + + refute session.running? + refute session.bound? + end + + def test_resume_needs_no_initiation_arguments + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"resumed":true}' + ) + ] + session = Session.new( + client_stub: ScriptedClientStub.new(responses), + stream: StringIO.new("01"), + upload_size: 2 + ) + handle = ResumeHandle.new upload_url: "https://upload.example.com/persisted", chunk_size: 4 + + result = session.resume resume_handle: handle + + assert_equal '{"resumed":true}', result + assert_equal "https://upload.example.com/persisted", session.upload_url + end + + # ============================================================================ + # 2. Observable States: Unbound & Bound + # ============================================================================ + + def test_initial_unbound_state + session = build_session + refute session.bound? + assert_nil session.upload_url + assert_nil session.resume_handle + refute session.resumable? + refute session.running? + end + + # ============================================================================ + # 3. Start Lifecycle & Single-Run Contract + # ============================================================================ + + def test_start_successful_upload_transitions_to_bound + responses = [ + # Initiation response + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://upload.example.com/session_1", + "x-goog-upload-chunk-granularity" => "4" + }, + body: "" + ), + # Chunk 1 (0-3) + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active" }, + body: "" + ), + # Chunk 2 (4-7) + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active" }, + body: "" + ), + # Final Chunk (8-9) + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"status":"completed"}' + ) + ] + + stub = ScriptedClientStub.new responses + session = build_session stub: stub, upload_size: 10, chunk_size: 4 + + result = start_session session + + assert_equal '{"status":"completed"}', result + assert session.bound? + assert_equal "https://upload.example.com/session_1", session.upload_url + refute session.running? + refute session.resumable? + assert_nil session.resume_handle + end + + def test_second_start_raises_session_state_error + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://upload.example.com/session_1", + "x-goog-upload-chunk-granularity" => "4" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"done":true}' + ) + ] + session = build_session( + stub: ScriptedClientStub.new(responses), + stream: StringIO.new("01"), + upload_size: 2, + chunk_size: 4 + ) + start_session session + + assert session.bound? + + err = assert_raises SessionStateError do + start_session session + end + assert_includes err.message, "Session has already executed a run" + end + + def test_resume_after_start_raises_session_state_error + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://upload.example.com/session_1", + "x-goog-upload-chunk-granularity" => "4" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"done":true}' + ) + ] + session = build_session( + stub: ScriptedClientStub.new(responses), + stream: StringIO.new("01"), + upload_size: 2, + chunk_size: 4 + ) + start_session session + + assert_nil session.resume_handle + refute session.resumable? + + handle = ResumeHandle.new upload_url: "https://upload.example.com/session_1", chunk_size: 4 + err = assert_raises SessionStateError do + session.resume resume_handle: handle + end + assert_includes err.message, "Session has already executed a run" + end + + # ============================================================================ + # 4. Resume Forms: Explicit URL or ResumeHandle + # ============================================================================ + + def test_resume_explicit_url_and_chunk_size_binds_and_executes + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"from_url":true}' + ) + ] + stub = ScriptedClientStub.new responses + session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 + + refute session.bound? + result = session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 + + assert_equal '{"from_url":true}', result + assert session.bound? + assert_equal "https://upload.example.com/direct", session.upload_url + end + + def test_resume_resume_handle_binds_and_executes + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"from_handle":true}' + ) + ] + stub = ScriptedClientStub.new responses + session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 + + handle = ResumeHandle.new upload_url: "https://upload.example.com/from_handle", chunk_size: 4 + + refute session.bound? + result = session.resume resume_handle: handle + + assert_equal '{"from_handle":true}', result + assert session.bound? + assert_equal "https://upload.example.com/from_handle", session.upload_url + end + + def test_start_after_resume_raises_session_state_error + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') + ] + stub = ScriptedClientStub.new responses + session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 + session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 + + assert session.bound? + err = assert_raises SessionStateError do + start_session session + end + assert_includes err.message, "Session has already executed a run" + end + + def test_second_resume_raises_session_state_error + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') + ] + stub = ScriptedClientStub.new responses + session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 + session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 + + assert session.bound? + err = assert_raises SessionStateError do + session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 + end + assert_includes err.message, "Session has already executed a run" + end + + # ============================================================================ + # 5. Argument Shape & Preconditions + # ============================================================================ + + def test_resume_without_arguments_raises_argument_error + session = build_session + err = assert_raises ArgumentError do + session.resume + end + assert_includes err.message, "Must provide either resume_handle or upload_url and chunk_size" + end + + def test_resume_mixing_arguments_raises_argument_error + session = build_session + handle = ResumeHandle.new upload_url: "https://example.com", chunk_size: 4 + + assert_raises ArgumentError do + session.resume resume_handle: handle, upload_url: "https://example.com" + end + + assert_raises ArgumentError do + session.resume resume_handle: handle, chunk_size: 4 + end + + assert_raises ArgumentError do + session.resume upload_url: "https://example.com" + end + + assert_raises ArgumentError do + session.resume chunk_size: 4 + end + + assert_raises ArgumentError do + session.resume handle + end + end + + def test_resume_with_non_zero_stream_pos_raises_argument_error + stream = StringIO.new "0123456789" + stream.seek 4 + + session = build_session stream: stream + handle = ResumeHandle.new upload_url: "https://upload.example.com/from_handle", chunk_size: 4 + + err = assert_raises ArgumentError do + session.resume resume_handle: handle + end + assert_includes err.message, "Stream must be positioned at byte 0 to resume an upload (got pos 4)" + end + + def test_resume_with_stream_without_pos_is_trusted + stream = StreamWithoutPos.new "01" + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') + ] + stub = ScriptedClientStub.new responses + session = build_session stub: stub, stream: stream, upload_size: 2, chunk_size: 4 + + result = session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 + assert_equal '{"ok":true}', result + end + + # ============================================================================ + # 6. Cross-Session Resumption + # ============================================================================ + + def test_cross_session_resumption_from_failed_run + stream = StringIO.new "0123456789" + stub1 = ScriptedClientStub.new [ + # Initiation succeeds + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://upload.example.com/session_cross", + "x-goog-upload-chunk-granularity" => "4" + }, + body: "" + ), + # Chunk 1 returns 503 + FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable"), + # Recovery query fails + Faraday::ConnectionFailed.new("network connection failed") + ] + + session1 = build_session stub: stub1, stream: stream, upload_size: 10, chunk_size: 4 + raised = assert_raises RequestFailedError do + start_session session1 + end + + assert session1.bound? + assert session1.resumable? + handle = session1.resume_handle + refute_nil handle + assert_equal handle, raised.resume_handle + assert_equal "https://upload.example.com/session_cross", handle.upload_url + assert_equal 4, handle.chunk_size + + # Prepare for session 2: rewind the stream to byte 0 + stream.rewind + assert_equal 0, stream.pos + + stub2 = ScriptedClientStub.new [ + # Recovery query on resume: server acknowledges 0 bytes received + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + # Chunk 1 + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "active" }, body: ""), + # Chunk 2 + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "active" }, body: ""), + # Chunk 3 (final) + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"resumed":true}') + ] + + session2 = build_session stub: stub2, stream: stream, upload_size: 10, chunk_size: 4 + refute session2.bound? + + result = session2.resume resume_handle: handle + assert_equal '{"resumed":true}', result + assert session2.bound? + refute session2.resumable? + assert_nil session2.resume_handle + end + + # ============================================================================ + # 7. Concurrency & Running Guard + # ============================================================================ + + def test_running_guard_prevents_concurrent_runs + started_q = Queue.new + unblock_q = Queue.new + + blocking_proc = proc do + started_q.push :started + unblock_q.pop # wait until test signals to proceed + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://upload.example.com/session_block", + "x-goog-upload-chunk-granularity" => "4" + }, + body: "" + ) + end + + stub = ScriptedClientStub.new [ + blocking_proc, + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"done":true}' + ) + ] + session = build_session( + stub: stub, + stream: StringIO.new("01"), + upload_size: 2, + chunk_size: 4 + ) + + worker = Thread.new do + start_session session + end + + started_q.pop # wait for worker thread to enter driver.run + assert session.running? + + # Concurrent call from another thread raises SessionStateError + handle = ResumeHandle.new upload_url: "https://upload.example.com/session_block", chunk_size: 4 + err = assert_raises SessionStateError do + session.resume resume_handle: handle + end + assert_includes err.message, "A run is already in progress for this session" + + err_start = assert_raises SessionStateError do + start_session session + end + assert_includes err_start.message, "A run is already in progress for this session" + + # Unblock worker thread + unblock_q.push :continue + result = worker.value + + assert_equal '{"done":true}', result + refute session.running? + end + + # ============================================================================ + # 8. Driver#upload_url Direct Verification + # ============================================================================ + + def test_driver_upload_url_across_statuses + dummy_client = ScriptedClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 4 + ) + driver = Driver.new client_stub: dummy_client, config: config + + assert_nil driver.upload_url + + # Active + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :transmission_sending, upload_url: "https://upload.example.com/sess1") + ) + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_equal "https://upload.example.com/sess1", driver.resume_handle.upload_url + + # Rejected (resume_handle is nil, but upload_url remains readable) + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :rejected, upload_url: "https://upload.example.com/sess1") + ) + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_nil driver.resume_handle + + # Cancelled (resume_handle is nil, but upload_url remains readable) + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :cancelled, upload_url: "https://upload.example.com/sess1") + ) + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_nil driver.resume_handle + + # Success (resume_handle is nil, but upload_url remains readable) + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :success, upload_url: "https://upload.example.com/sess1") + ) + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_nil driver.resume_handle + end +end diff --git a/gapic-common/test/test_helper.rb b/gapic-common/test/test_helper.rb index ea4cab9..fcf5283 100644 --- a/gapic-common/test/test_helper.rb +++ b/gapic-common/test/test_helper.rb @@ -15,6 +15,7 @@ gem "minitest" require "minitest/autorun" require "minitest/focus" +require "minitest/mock" require "minitest/rg" require "pp" @@ -127,3 +128,27 @@ def spoof_logging_env enabled: nil, cloud_run: false ensure ENV["GOOGLE_SDK_RUBY_LOGGING_GEMS"] = old_enabled end + +class RecordingLogger < Logger + Entry = Data.define :severity, :message + + attr_reader :entries + + def initialize + super nil + @entries = [] + end + + def add severity, message = nil, progname = nil + msg = block_given? ? yield : (message || progname) + @entries << Entry.new(severity: severity, message: msg) + end +end + +def log_corpus recording_logger + formatter = Google::Logging::StructuredFormatter.new + recording_logger.entries.map do |entry| + sev = Logger::SEV_LABEL[entry.severity] || "INFO" + formatter.call sev, Time.now, nil, entry.message + end.join +end From 1bb0c880cd153d1465afd80409140675e29c2e3e Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Tue, 15 Sep 2026 07:12:28 +0000 Subject: [PATCH 02/22] chore: fix toys (again) --- gapic-common/.toys.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gapic-common/.toys.rb b/gapic-common/.toys.rb index f16b2b0..4bb826c 100644 --- a/gapic-common/.toys.rb +++ b/gapic-common/.toys.rb @@ -30,10 +30,6 @@ end tool "yard", delegate_to: "yardoc" -expand :gem_build - -expand :gem_build, name: "install", install_gem: true - tool "ci" do include :exec, e: true include :terminal From 52949a745d8efc00b2b459febf0f0bfb3786d5b1 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Thu, 17 Sep 2026 19:37:27 +0000 Subject: [PATCH 03/22] feat: remove Session and replace with ::Gapic::ResumableUpload, and trim public surface --- .../resumable_upload/implementation-guide.md | 122 +-- .../resumable_upload/integration-test-plan.md | 16 +- .../integration/integration_helper.rb | 54 +- .../resumable_upload/resume_test.rb | 116 +-- gapic-common/lib/gapic/rest.rb | 1 + .../lib/gapic/rest/resumable_upload.rb | 109 ++- .../gapic/rest/resumable_upload/data_types.rb | 18 +- .../lib/gapic/rest/resumable_upload/driver.rb | 16 +- .../lib/gapic/rest/resumable_upload/errors.rb | 4 +- .../rest/resumable_upload/retry_policies.rb | 6 +- .../gapic/rest/resumable_upload/session.rb | 546 -------------- gapic-common/lib/gapic/resumable_upload.rb | 507 +++++++++++++ .../resumable_upload/driver_logging_test.rb | 28 + .../rest/resumable_upload/driver_test.rb | 31 + .../resumable_upload/retry_policies_test.rb | 64 ++ .../rest/resumable_upload/session_test.rb | 704 ------------------ .../test/gapic/resumable_upload_test.rb | 625 ++++++++++++++++ 17 files changed, 1528 insertions(+), 1439 deletions(-) delete mode 100644 gapic-common/lib/gapic/rest/resumable_upload/session.rb create mode 100644 gapic-common/lib/gapic/resumable_upload.rb delete mode 100644 gapic-common/test/gapic/rest/resumable_upload/session_test.rb create mode 100644 gapic-common/test/gapic/resumable_upload_test.rb diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md index 3a561a9..8ca3f1a 100644 --- a/gapic-common/design/resumable_upload/implementation-guide.md +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -20,9 +20,8 @@ graph TD ### 1.0 Domain Vocabulary * **Upload**: Server-side entity created by a successful session initiation (`start`), identified by `upload_url`. * **Resume Handle (`ResumeHandle`)**: An immutable snapshot (`upload_url`, `chunk_size`) identifying an upload for resumption. -* **Session (`Session`)**: Client-side transfer coordinator; performs exactly one run (`start` or `resume`), never both, never twice. It owns the input stream and configuration options. -* **Run**: One invocation of `Driver#run` (either a start or resume execution). -* **Bound**: The property that a session has executed a run or is bound to an upload (`session.bound?`). A session becomes bound when `start` or `resume` begins execution. A bound session never runs again. +* **Coordinator (`::Gapic::ResumableUpload`)**: Client-side transfer coordinator and the only public entry point. Reusable: it builds one `Driver` per run and retains the last one. Per-run arguments — the stream, sizes, the upload budget, the progress callback — are passed to the run, not to the coordinator. +* **Run**: One invocation of `Driver#run` (either a start or resume execution). At most one run per coordinator at a time. ### 1.1 Driver (Synchronous I/O Adapter) The `Driver` executes all operations with side-effects. It interacts with HTTP transport via `Gapic::Rest::ClientStub`, reads binary data from local input streams, tracks monotonic execution deadlines, and dispatches progress callbacks. @@ -42,60 +41,71 @@ The `Rules` module encapsulates the Resumable Upload Protocol state transitions ### 1.4 Stream Buffering Because arbitrary Ruby `IO` objects (network sockets, pipes, `STDIN`) do not support seeking (`#seek`), the Driver buffers the current in-flight chunk in memory (bounded by chunk size, default: 8MB). When `RetryPolicy` executes transport retries, or when `Core` triggers Category 2 recovery realignments within the buffered range, the Driver retransmits directly from memory. The buffer is discarded only after receiving a `200 OK` durably confirming receipt of the chunk. -### 1.5 Session (Transfer Coordinator) -The `Session` (`Gapic::Rest::ResumableUpload::Session`) provides a client-facing coordinator that encapsulates shared transfer configuration, owns the input stream, and manages upload execution across a strict single-run lifecycle. - -#### Single-Run Contract & Two-State Model -A session adheres to a two-state model with a strict single-run contract: a session performs exactly one run (`start` or `resume`), never both, never twice. - -1. **Unbound (`!session.bound?`)**: - * Initial state upon construction (`Session.new`). The session has not yet executed a run. - * Permitted operations: `start(...)` or `resume(...)`. -2. **Bound (`session.bound?`)**: - * Transitions to bound as soon as `start` or `resume` begins execution. - * The session has executed its run and cannot be reused. - * Both `start` and `resume` raise `SessionStateError` ("Session has already executed a run"). - -#### Constructor & Initiation Signatures -Configuration is split between transfer-wide options passed to `Session.new` (`COMMON_MEMBERS` plus `client_stub` and `logger`) and initiation-only arguments passed to `Session#start`: -* **Constructor (`Session#initialize`)**: - `Session.new(client_stub:, stream:, upload_size: nil, content_type: nil, timeout: nil, control_plane_retry_policy: nil, data_plane_retry_policy: nil, on_progress: nil, logger: nil)` -* **Initiation (`Session#start`)**: - `session.start(initial_url:, initial_body: nil, initial_headers: {}, chunk_size: nil, start_retry_policy: nil)` - * `initial_url` is required (`ArgumentError` if missing or blank). - * `initial_headers` accepts caller-supplied HTTP headers for the initiation request, merged over the driver's headers. Any key in `RESERVED_INITIAL_HEADERS` (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, `X-Goog-Upload-Header-Content-Length`, in any casing) raises an `ArgumentError`; callers influence `X-Goog-Upload-Header-Content-Type` and `X-Goog-Upload-Header-Content-Length` through `content_type:` and `upload_size:` on the constructor. Pass-through headers such as `X-Goog-Upload-Header-Content-Disposition` remain permitted. - -#### Resumability (`session.resumable?`) -* Reports whether a *new* session can resume the transfer (`!session.resume_handle.nil?`). -* Completed uploads are finalized: once a transfer succeeds, `resume_handle` returns `nil` and `resumable?` returns `false`, so there is no handle to resume from. Calling `#resume` on the session that completed the run raises `SessionStateError`. Resuming a *fresh* session against a finalized `upload_url` is undefined behavior: it queries the server and might return the response body or raise an error, depending on the server response. -* When a run fails with a recoverable error, `resume_handle` captures the upload parameters (`upload_url`, `chunk_size`) and `resumable?` returns `true`. +### 1.5 Coordinator (`::Gapic::ResumableUpload`) +The coordinator (`lib/gapic/resumable_upload.rb`) is the only public entry point above `Driver`, and the object a generated client method returns instead of a response. It lives outside the `Gapic::Rest::ResumableUpload` namespace on purpose: it is not part of the protocol implementation, it is the layer built on top of it. It builds the per-run configuration, constructs one `Driver` per run, retains it, and serves its readers from it. + +#### Construction +`::Gapic::ResumableUpload.new(client_stub_proc:, initial_request_proc:, response_type:, initial_headers: {}, start_retry_policy: nil, control_plane_retry_policy: nil, data_plane_retry_policy: nil, error_handler: nil, method_name: nil)` + +* **Procs only; there is no value form.** `client_stub_proc` returns the `Gapic::Rest::ClientStub` and is called at the top of every run, so a client that cannot perform REST calls can still hand back a working coordinator and fail only when an upload is attempted. `initial_request_proc` returns the `[url, body]` pair for initiation and is called by `#start` only, so a coordinator built without a request message is still fully functional for resuming. +* `response_type` is the protobuf message class the final body is decoded into. `nil` returns the raw body. +* `initial_headers` is stringified (keys and values) before reaching `StartUploadConfig`, which rejects any key in `RESERVED_INITIAL_HEADERS` (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, `X-Goog-Upload-Header-Content-Length`, in any casing). Callers shape the content descriptors with `content_type:` and `upload_size:` on the run; pass-through headers such as `X-Goog-Upload-Header-Content-Disposition` remain permitted. +* `method_name` is forwarded to `Driver`, which prefixes it onto the per-request logging names. +* **No `logger` argument.** `Driver#initialize` already falls back to `client_stub.logger`. + +#### Run Signatures +* `#start(stream:, content_type: nil, upload_size: nil, chunk_size: nil, upload_timeout: nil, on_progress: nil)` +* `#resume(stream:, resume_handle: nil, content_type: nil, upload_size: nil, upload_timeout: nil, on_progress: nil)` + +Everything a run owns is a per-run keyword, because the coordinator outlives the run. Two arguments are deliberately not: the control- and data-plane retry policies sit on the constructor (they describe the upload path, not one run), and `chunk_size` is a `#start` argument only — a resumed run takes its chunk size from the `ResumeHandle`, because the server reports its granularity during initiation and a resumed run skips initiation. + +The budget keyword is named `upload_timeout` even though the config member behind it is `timeout`: a generated client already has a per-call `timeout` in scope, which in an upload bounds the initiation request alone and reaches the protocol through `start_retry_policy`. The two are three orders of magnitude apart and must not be confusable. + +Order of operations in each run, before any byte is read from the stream: +1. Claim the run slot: raise `SessionStateError` if a run is already in flight. +2. `#resume` only: reject a stream that is not positioned at byte 0, then resolve the `ResumeHandle`. +3. `client_stub_proc.call` — raises here if REST is unavailable. +4. `#start` only: `initial_request_proc.call` -> `[url, body]`. +5. Build `StartUploadConfig` or `ResumeUploadConfig`, then `Driver.new`, and retain it. +6. Run it; decode and return, or wrap and raise. + +#### Reusability & Lifecycle +* One coordinator, many runs, one `Driver` per run. A run started while another is in flight raises `SessionStateError`; a coordinator whose run has finished may legitimately start another one. +* The run slot is released in an `ensure`, so every exit — a clean return, a protocol error, an `on_progress` callback raising, a `Thread#kill` — leaves the coordinator usable and the driver retained. +* A failure while building the configuration (a reserved header, a non-positive `chunk_size`, a malformed retry policy) retains no driver at all, so it cannot disturb the resume handle an earlier run left behind. +* Network execution occurs outside the mutex; the mutex guards only the lifecycle flag and the retained driver reference. + +#### Readers +All read from the retained driver under the coordinator's mutex, are safe to call from another thread mid-run, and return a best-effort snapshot. All are `nil`/`false` before the first run. +* `#resume_handle`: `Driver#resume_handle` — the upload URL and the resolved chunk size, or `nil` for a finalized upload. +* `#resumable?`: `!resume_handle.nil?`. +* `#running?`: the coordinator's own lifecycle flag. + +`#upload_url` and `#chunk_size` are **not** exposed: both are fields of the `ResumeHandle` this set already returns. + +#### Resume Forms +1. **Bare**: `upload.resume(stream: io)` takes the retained driver's `resume_handle`, and raises `ArgumentError` when there is none. That one rule covers a coordinator that has never run, a run that finished successfully, and a run that failed in a way the protocol considers unresumable. +2. **Explicit handle**: `upload.resume(stream: io, resume_handle: handle)` — what the protocol's own errors carry, and what a caller persists between processes. + +There is no explicit `upload_url:`/`chunk_size:` form: those are exactly the two fields of a `ResumeHandle`, so a caller holding them in a database row constructs one. Resuming against a finalized upload URL is undefined behavior: it queries the server and might return the response body or raise an error, depending on the server response. #### Precondition on Stream Position for Resume -* Before executing `resume`, the caller must ensure the input stream is positioned at byte 0. -* If `stream.respond_to?(:pos) && !stream.pos.zero?`, `Session#resume` raises `ArgumentError` ("Input stream must be at byte 0 to resume; rewind the stream before resuming"). -* For unseekable streams without `:pos` (or streams at `pos == 0`), `Session` trusts the stream is at byte 0 and delegates to `Driver`, which fast-forwards to the server-confirmed offset by seeking or reading and discarding bytes. - -#### Resume Invocations & Forms -The `Session#resume` method accepts strictly keyword-only arguments: `upload_url: nil, chunk_size: nil, resume_handle: nil`. -Resumption always requires an unbound session with one of two mutually exclusive parameter forms: -1. **Explicit URL & Chunk Size**: `session.resume(upload_url: url, chunk_size: size)` -2. **Resume Handle**: `session.resume(resume_handle: handle)` - -Calling `resume` without arguments (bare resume), calling `resume` with `upload_url` but omitting `chunk_size`, or mixing `resume_handle` with other parameters raises `ArgumentError`. - -#### Cross-Session Resumption Flow -Because a session performs only a single run, resuming an interrupted upload requires instantiating a fresh session: -1. Session 1 encounters a recoverable error. -2. Caller extracts `resume_handle = session1.resume_handle` (or from the error's `#resume_handle`). -3. Caller rewinds the stream to byte 0 (if seekable, or provides an equivalent stream starting at byte 0). -4. Caller instantiates Session 2 and invokes `session2.resume(resume_handle: resume_handle)`. - -#### Concurrency & Execution Model -* At most one run (`Driver#run`) may execute at any time. -* `@running` is checked and toggled exclusively inside a `Mutex`. -* Network execution (`driver.run`) occurs outside the mutex to prevent blocking reader threads. -* Invoking `start` or `resume` while `@running` is `true` raises `SessionStateError`. -* Errors propagate unchanged. The failed `Driver` remains referenced so `upload_url`, `resume_handle`, and `bound?` remain inspectable after an exception. +* The stream must be positioned at byte 0 of the whole object, not at the server's acknowledged offset. +* If `stream.respond_to?(:pos) && !stream.pos.zero?`, `#resume` raises `ArgumentError`. +* A stream that reports no position is trusted, and `Driver` fast-forwards to the server-confirmed offset by seeking or by reading and discarding bytes. + +#### Response Decoding +`response_type.decode_json body.to_s, ignore_unknown_fields: true`, identical to what a generated REST service stub does with a unary response. An empty or absent final body decodes to an empty message; malformed JSON raises `Google::Protobuf::ParseError`. A `nil` `response_type` returns the raw body — a `String`, or `nil` when the final response carried none. That is `@private` behavior, for this gem's own tests. + +#### Error Handling +`error_handler` is a lambda that **returns** the exception to raise; it must not raise. A `nil` return, or a return of the original error, re-raises the original. When the original carries `HasResumeHandle` and the replacement does not, the replacement is extended with the mixin and given the original's handle, so a library-specific error type cannot erase the fact that the upload is resumable. + +Only the run is wrapped. Argument and configuration errors are raised while the driver is still being built, and reach the caller as themselves. + +#### Retry Policy Placement +All three planes reach the coordinator on the **constructor**; none is per-run. `start_retry_policy` is the documented one and comes from the generated method's `CallOptions`. `control_plane_retry_policy` and `data_plane_retry_policy` are `@private`: no generated client passes them, but `Gapic::Common`'s own integration harness does, which keeps recovery and retry-exhaustion tests on the production code path rather than on a hand-built `Driver`. + +`Gapic::Rest::ResumableUpload.start_retry_policy_for(options)` (`@private`, beside `RetryPolicies::START_DEFAULTS`) converts per-call options into the initiation policy. It returns a **Hash**, never a policy object, because the protocol treats an object as a wholesale replacement — which would silently drop the initiation predicate that makes a missing `X-Goog-Upload-Status` retriable. It always sets `timeout:` from `options.timeout`, copies backoff settings and retry codes only where the caller set them (an empty `retry_codes` list counts as unset), and raises `ArgumentError` for a Proc retry policy. --- @@ -526,7 +536,7 @@ Terminal errors provide actionable context so downstream SDK callers can inspect * `InvalidTransitionError < Gapic::Common::Error`: Unexpected event dispatched for state; includes `HasResumeHandle`. * `StreamMismatchError < Gapic::Common::Error`: Stream content or length does not match resumed upload specifications; includes `HasResumeHandle`. * `RequestFailedError < Gapic::Common::Error`: Terminal HTTP request failure (e.g. transport connection failure, request timeout, or retries exhausted). Retains `attr_reader :cause` returning the underlying error, preserves REST error attributes (`status_code`, `status`, `details`, `headers`) when available, and includes `HasResumeHandle`. - * `SessionStateError < Gapic::Common::Error`: Raised when an operation violates Session lifecycle rules (e.g. attempting to start an already-bound session, resuming an unbound session without a target upload, re-binding to a different upload, resuming a finalized/dead session, or concurrent run invocations). Distinguished from `ArgumentError`, which is raised strictly for invalid argument shapes. + * `SessionStateError < Gapic::Common::Error`: Raised when an operation violates the upload session lifecycle rules, e.g. starting a second run on a coordinator while one is still in flight. Distinguished from `ArgumentError`, which is raised strictly for invalid argument shapes. * **Resume Handle Propagation (`HasResumeHandle`)**: * The `HasResumeHandle` mixin exposes `attr_reader :resume_handle` returning a `ResumeHandle` (or `nil` if session initiation was incomplete or if the session was `:rejected` or `:cancelled`). * Whenever `resume_handle` is non-nil, the uniform suffix `" (upload session is resumable: see #resume_handle)"` is automatically appended to the error message. diff --git a/gapic-common/design/resumable_upload/integration-test-plan.md b/gapic-common/design/resumable_upload/integration-test-plan.md index 46a5750..b3c5115 100644 --- a/gapic-common/design/resumable_upload/integration-test-plan.md +++ b/gapic-common/design/resumable_upload/integration-test-plan.md @@ -47,7 +47,9 @@ flowchart TD * **`ShowcaseIntegrationTest`**: Base class providing helper methods for test configuration: * `showcase_client_stub`: Instantiates a real `Gapic::Rest::ClientStub` targeting `SHOWCASE_ENDPOINT` with `raise_faraday_errors: false` and an attached `DEBUG` logger. * `build_config(scenario: nil, scenario_config: {}, **overrides)`: Creates a `StartUploadConfig` targeting `/resumable/upload/v1beta1/files:upload`. When `scenario` is provided, injects `X-Goog-Test-Scenario` and `X-Goog-Test-Scenario-Config` (with a generated `client_uuid` merged with `scenario_config`) into `initial_headers` (these test headers are unaffected by `RESERVED_INITIAL_HEADERS` since they are not in the five reserved protocol headers). Configures fast retry policies (`FAST_RETRY = { initial_delay: 0.01, max_delay: 0.05, multiplier: 1, timeout: 2 }`), a default 10-second session timeout, a default payload of `786_432` bytes (`3 * 262_144`), default chunk size of `262_144` bytes, and an `on_progress` callback appending each `Progress` struct to `@progress_records`. - * `build_session(scenario: nil, scenario_config: {}, **overrides)` & `start_session(session, **overrides)`: Partitions overrides using `START_ONLY_KEYS` (`[:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy]`). `build_session` instantiates a `Gapic::Rest::ResumableUpload::Session` with the common members (`client_stub`, `stream`, `upload_size`, `content_type`, `timeout`, `control_plane_retry_policy`, `data_plane_retry_policy`, `on_progress`, `logger`) and stores initiation arguments in `@start_args`, while `start_session` invokes `session.start(**@start_args, **overrides)`. + * `build_upload(scenario: nil, scenario_config: {}, initial_headers: {}, **overrides)`: Instantiates a `::Gapic::ResumableUpload` whose `client_stub_proc` returns a Showcase client stub and whose `initial_request_proc` returns `[UPLOAD_PATH, nil]`. Injects the same scenario headers as `build_config`, sets all three retry policies to `FAST_RETRY`, and passes `response_type: nil` so runs return the raw response body for the tests to parse. + * `start_args(**overrides)` & `resume_args(**overrides)`: Default per-run arguments — a `786_432`-byte payload and its `upload_size`, a 10-second `upload_timeout`, and an `on_progress` callback appending each `Progress` struct to `@progress_records`. `start_args` adds the `262_144`-byte `chunk_size`, which a resumed run takes from its `ResumeHandle` instead. + * `resume_handle_for(upload_url, chunk_size: DEFAULT_CHUNK_SIZE)`: Builds a `ResumeHandle` for an upload created out-of-band by `raw_start`. * `phases` & `offsets`: Convenience accessors returning `@progress_records.map(&:phase)` and `@progress_records.map(&:bytes_uploaded)`. * `payload(size)`: Generates deterministic binary strings of arbitrary byte length for stream uploads. * `UnseekableStream`: Stream wrapper around `StringIO` that exposes `#read` and `#pos` while omitting `#seek` (`respond_to?(:seek)` is `false`). @@ -256,10 +258,10 @@ Tests non-fatal transient retries, missing status headers, retry exhaustion, fat ### 2.5 Resumption Suite (`integration/resumable_upload/resume_test.rb`) -Tests `Gapic::Rest::ResumableUpload::Session` resumption capabilities against Showcase. +Tests `Gapic::ResumableUpload` resumption capabilities against Showcase. #### Case 1. Resume in-progress upload on seekable stream (`test_resume_in_progress_upload`) -* Uploads chunk 1 via `raw_upload`, then resumes with a fresh session and full stream. +* Uploads chunk 1 via `raw_upload`, then resumes a fresh upload handle with an explicit `resume_handle:` and the full stream. * Asserts `phases == [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed]` and offsets align correctly. #### Case 2. Resume already finalized upload (`test_resume_finalized_upload`) @@ -279,8 +281,8 @@ Tests `Gapic::Rest::ResumableUpload::Session` resumption capabilities against Sh * **Case 5b (`test_resume_wrong_stream_seekable_size_guard`)**: Seekable `StringIO` with `server_offset > stream.size` (and `upload_size: nil`) raises `StreamMismatchError` via stream size guard. #### Case 6. Golden user-style resume (`test_golden_user_style_resume_seekable`, `test_golden_user_style_resume_unseekable`) -* User raises exception in `on_progress` carrying `session.resume_handle` on first upload ack. -* Fresh session resumes via `resume_handle: handle` and completes the transfer. Tested on both seekable streams and fresh unseekable streams starting at byte 0. +* User raises exception in `on_progress` carrying `upload.resume_handle` on first upload ack. +* The same handle object then resumes — bare for the seekable case, with an explicit `resume_handle:` for the unseekable one — and completes the transfer. The unseekable variant uses a fresh stream starting at byte 0. -#### Case 7. Lifecycle and contract violations (`test_lifecycle_violations`) -* Verifies second `start` and `resume` on bound session raise `SessionStateError`, and bare `resume` raises `ArgumentError`. +#### Case 7. Lifecycle of a reused handle (`test_lifecycle_of_a_reused_handle`) +* Verifies a completed run leaves `resumable?` and `running?` false, a bare `resume` afterwards raises `ArgumentError`, a second `start` on the same handle is legal and initiates an unrelated upload, and a bare `resume` on a handle that has never run raises `ArgumentError`. diff --git a/gapic-common/integration/integration_helper.rb b/gapic-common/integration/integration_helper.rb index 0af5c0b..c33324f 100644 --- a/gapic-common/integration/integration_helper.rb +++ b/gapic-common/integration/integration_helper.rb @@ -123,13 +123,15 @@ def build_config scenario: nil, scenario_config: {}, **overrides Gapic::Rest::ResumableUpload::StartUploadConfig.new(**defaults, **overrides, initial_headers: headers) end - START_ONLY_KEYS = [:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy].freeze - - # Builds a session from the shared arguments and remembers the per-run arguments that #start needs, - # so callers can run it with `start_session session`. - def build_session scenario: nil, scenario_config: {}, **overrides + ## + # Builds a coordinator pointed at Showcase, with the short retry policies these tests rely on. + # + # The handle decodes nothing (`response_type: nil`), so runs return the raw response body and the tests + # can assert on what Showcase actually sent. + # + def build_upload scenario: nil, scenario_config: {}, initial_headers: {}, **overrides @progress_records = [] - headers = (overrides.delete(:initial_headers) || {}).dup + headers = initial_headers.dup if scenario headers["X-Goog-Test-Scenario"] = scenario headers["X-Goog-Test-Scenario-Config"] = JSON.generate( @@ -137,31 +139,39 @@ def build_session scenario: nil, scenario_config: {}, **overrides ) end - @start_args = { - initial_url: UPLOAD_PATH, - initial_headers: headers, - start_retry_policy: FAST_RETRY, - chunk_size: DEFAULT_CHUNK_SIZE - }.merge(overrides.slice(*START_ONLY_KEYS)) - - defaults = { - client_stub: showcase_client_stub, + Gapic::ResumableUpload.new( + client_stub_proc: -> { showcase_client_stub }, + initial_request_proc: -> { [UPLOAD_PATH, nil] }, + response_type: nil, + initial_headers: headers, + start_retry_policy: FAST_RETRY, control_plane_retry_policy: FAST_RETRY, data_plane_retry_policy: FAST_RETRY, - timeout: 10, - on_progress: ->(progress) { @progress_records << progress }, - logger: @logger + **overrides + ) + end + + # Default run arguments for `#start`. + def start_args **overrides + { chunk_size: DEFAULT_CHUNK_SIZE }.merge(resume_args(**overrides)) + end + + # Default run arguments for `#resume`, which takes no chunk size: a resumed run carries it on the + # resume handle. + def resume_args **overrides + defaults = { + upload_timeout: 10, + on_progress: ->(progress) { @progress_records << progress } } unless overrides.key? :stream defaults[:stream] = StringIO.new payload(DEFAULT_PAYLOAD_SIZE) defaults[:upload_size] = DEFAULT_PAYLOAD_SIZE end - - Gapic::Rest::ResumableUpload::Session.new(**defaults, **overrides.except(*START_ONLY_KEYS)) + defaults.merge overrides end - def start_session session, **overrides - session.start(**@start_args, **overrides) + def resume_handle_for upload_url, chunk_size: DEFAULT_CHUNK_SIZE + Gapic::Rest::ResumableUpload::ResumeHandle.new upload_url: upload_url, chunk_size: chunk_size end def raw_start scenario: nil, scenario_config: {}, upload_size: nil, headers: {} diff --git a/gapic-common/integration/resumable_upload/resume_test.rb b/gapic-common/integration/resumable_upload/resume_test.rb index 5c4dc5f..f751e14 100644 --- a/gapic-common/integration/resumable_upload/resume_test.rb +++ b/gapic-common/integration/resumable_upload/resume_test.rb @@ -19,7 +19,7 @@ require "stringio" ## -# Suite D: Integration tests for Resumable Upload Session resumption against Showcase. +# Suite D: Integration tests for resumption against Showcase, driven through ::Gapic::ResumableUpload. # class ResumeTest < ShowcaseIntegrationTest ## @@ -40,8 +40,8 @@ def test_resume_in_progress_upload chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false - session = build_session - result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload = build_upload + result = upload.resume(**resume_args(resume_handle: resume_handle_for(upload_url))) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] @@ -54,8 +54,10 @@ def test_resume_finalized_upload upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: true - session = build_session stream: StringIO.new(payload(DEFAULT_CHUNK_SIZE)), upload_size: DEFAULT_CHUNK_SIZE - result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload = build_upload + result = upload.resume(**resume_args(stream: StringIO.new(payload(DEFAULT_CHUNK_SIZE)), + upload_size: DEFAULT_CHUNK_SIZE, + resume_handle: resume_handle_for(upload_url))) parsed = JSON.parse result assert_equal DEFAULT_CHUNK_SIZE, parsed["size"] @@ -72,8 +74,8 @@ def test_resume_query_503_absorbed_by_retry chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false - session = build_session - result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload = build_upload + result = upload.resume(**resume_args(resume_handle: resume_handle_for(upload_url))) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] @@ -91,8 +93,8 @@ def test_resume_query_409_triggers_retry_recovery chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false - session = build_session - result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload = build_upload + result = upload.resume(**resume_args(resume_handle: resume_handle_for(upload_url))) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] @@ -107,9 +109,9 @@ def test_resume_unseekable_stream_fast_forwards chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false - stream = UnseekableStream.new payload(DEFAULT_PAYLOAD_SIZE) - session = build_session stream: stream - result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload = build_upload + result = upload.resume(**resume_args(stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)), + resume_handle: resume_handle_for(upload_url))) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] @@ -122,11 +124,12 @@ def test_resume_wrong_stream_unseekable_mismatch upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: false - stream = UnseekableStream.new payload(100) - session = build_session stream: stream, upload_size: nil + upload = build_upload assert_raises Gapic::Rest::ResumableUpload::StreamMismatchError do - session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload.resume(**resume_args(stream: UnseekableStream.new(payload(100)), + upload_size: nil, + resume_handle: resume_handle_for(upload_url))) end refute_includes phases, :finalizing end @@ -136,94 +139,93 @@ def test_resume_wrong_stream_seekable_size_guard upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: false - stream = StringIO.new payload(100) - session = build_session stream: stream, upload_size: nil + upload = build_upload assert_raises Gapic::Rest::ResumableUpload::StreamMismatchError do - session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload.resume(**resume_args(stream: StringIO.new(payload(100)), + upload_size: nil, + resume_handle: resume_handle_for(upload_url))) end refute_includes phases, :finalizing end - # D6a. Golden user-style resume on a seekable stream after user abort in on_progress. + # D6a. Golden user-style resume on a seekable stream after user abort in on_progress, reusing the handle + # and its retained resume handle rather than passing one back in. def test_golden_user_style_resume_seekable stream = StringIO.new payload(DEFAULT_PAYLOAD_SIZE) - session1 = nil + upload = nil on_progress = lambda do |progress| if progress.phase == :uploading && progress.bytes_uploaded == DEFAULT_CHUNK_SIZE - raise UserPauseError.new("user paused", session1.resume_handle) + raise UserPauseError.new("user paused", upload.resume_handle) end end - session1 = build_session stream: stream, on_progress: on_progress + upload = build_upload err = assert_raises UserPauseError do - start_session session1 + upload.start(**start_args(stream: stream, on_progress: on_progress)) end - assert session1.bound? - assert session1.resumable? - handle = err.resume_handle - refute_nil handle + assert upload.resumable? + refute_nil err.resume_handle + assert_equal err.resume_handle, upload.resume_handle stream.rewind - session2 = build_session stream: stream - result = session2.resume resume_handle: handle + result = upload.resume(**resume_args(stream: stream)) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] - assert session2.bound? - refute session2.resumable? + refute upload.resumable? end - # D6b. Golden user-style resume with a fresh unseekable stream starting at byte 0. + # D6b. Golden user-style resume with a fresh unseekable stream starting at byte 0, resuming from the + # handle the error carried. def test_golden_user_style_resume_unseekable - session1 = nil + upload = nil on_progress = lambda do |progress| if progress.phase == :uploading && progress.bytes_uploaded == DEFAULT_CHUNK_SIZE - raise UserPauseError.new("user paused", session1.resume_handle) + raise UserPauseError.new("user paused", upload.resume_handle) end end - session1 = build_session stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)), on_progress: on_progress + upload = build_upload err = assert_raises UserPauseError do - start_session session1 + upload.start(**start_args(stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)), + on_progress: on_progress)) end - assert session1.bound? - assert session1.resumable? + assert upload.resumable? handle = err.resume_handle refute_nil handle - session2 = build_session stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)) - result = session2.resume resume_handle: handle + result = upload.resume(**resume_args(stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)), + resume_handle: handle)) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] - assert session2.bound? - refute session2.resumable? + refute upload.resumable? end - # D7. Lifecycle and contract violations on session runs. - def test_lifecycle_violations - session = build_session stream: StringIO.new(payload(100)), upload_size: 100 - start_session session + # D7. Lifecycle of a reusable handle: a finished run leaves nothing to resume, but the handle itself + # stays usable. + def test_lifecycle_of_a_reused_handle + upload = build_upload + upload.start(**start_args(stream: StringIO.new(payload(100)), upload_size: 100)) - assert session.bound? + refute upload.resumable? + refute upload.running? - # Second start on executed session raises SessionStateError - assert_raises Gapic::Rest::ResumableUpload::SessionStateError do - start_session session + # A bare resume after a completed run has no handle to work from. + assert_raises ArgumentError do + upload.resume(**resume_args(stream: StringIO.new(payload(100)), upload_size: 100)) end - # Resume on already bound/executed session raises SessionStateError - assert_raises Gapic::Rest::ResumableUpload::SessionStateError do - session.resume upload_url: "https://example.com/test", chunk_size: DEFAULT_CHUNK_SIZE - end + # Starting again is legal, and initiates a second, unrelated upload. + result = upload.start(**start_args(stream: StringIO.new(payload(100)), upload_size: 100)) + assert_equal 100, JSON.parse(result)["size"] - # Resume without parameters on fresh session raises ArgumentError - fresh_session = build_session + # A bare resume on a handle that has never run has nothing to work from either. assert_raises ArgumentError do - fresh_session.resume + build_upload.resume(**resume_args) end end end diff --git a/gapic-common/lib/gapic/rest.rb b/gapic-common/lib/gapic/rest.rb index 693aba2..59be632 100644 --- a/gapic-common/lib/gapic/rest.rb +++ b/gapic-common/lib/gapic/rest.rb @@ -29,6 +29,7 @@ require "gapic/rest/operation" require "gapic/rest/paged_enumerable" require "gapic/rest/resumable_upload" +require "gapic/resumable_upload" require "gapic/rest/server_stream" require "gapic/rest/threaded_enumerator" require "gapic/rest/transport_operation" diff --git a/gapic-common/lib/gapic/rest/resumable_upload.rb b/gapic-common/lib/gapic/rest/resumable_upload.rb index a7530e1..2dfd37d 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload.rb @@ -24,50 +24,101 @@ require "gapic/rest/resumable_upload/rules" require "gapic/rest/resumable_upload/core" require "gapic/rest/resumable_upload/driver" -require "gapic/rest/resumable_upload/session" module Gapic module Rest ## - # Resumable Upload Protocol implementation for REST transport. + # Resumable Upload Protocol implementation for REST transport: session initiation, chunked + # streaming, automatic retries, progress reporting via {Progress}, and resumption via + # {ResumeHandle}. # - # {Session} is the primary public entry point for initiating and resuming uploads. - # It manages session initiation, chunked streaming, automatic retries, progress - # callbacks via {Progress}, and cross-session resumption via {ResumeHandle}. + # **This namespace has no callable surface.** Every method and class in it is internal machinery, + # documented as `@private` and excluded from these docs; what remains visible is the data a caller + # receives — {Progress}, {ResumeHandle}, {HasResumeHandle} — and the error classes listed below. + # Uploads are driven from {Gapic::ResumableUpload}, which sits above this namespace and coordinates + # runs against it. + # + # Errors raised from here carry a {ResumeHandle} where the upload can still be continued, so the + # usual shape of handling one is to rescue {HasResumeHandle} and hand the handle back to the + # coordinator: + # + # @example Uploading, then resuming after a recoverable failure + # upload = client.upload_media ... # returns a Gapic::ResumableUpload + # begin + # upload.start stream: File.open("movie.mp4", "rb"), upload_size: File.size("movie.mp4") + # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e + # handle = e.resume_handle + # upload.resume stream: File.open("movie.mp4", "rb"), resume_handle: handle + # end # # ### Error Types # * {RequestFailedError} - Transport connection failure, timeout, or retries exhausted (includes {HasResumeHandle}). - # * {DeadlineExceededError} - Global upload timeout exceeded (includes {HasResumeHandle}). + # * {DeadlineExceededError} - Whole-upload timeout exceeded (includes {HasResumeHandle}). # * {BadResponseError} - Unexpected or malformed HTTP response (includes {HasResumeHandle}). # * {UnseekableStreamError} - Stream rewinding required on an unseekable stream (includes {HasResumeHandle}). # * {StreamMismatchError} - Stream content or length does not match resumed upload (includes {HasResumeHandle}). # * {InvalidTransitionError} - Unmatched event for the current protocol state (includes {HasResumeHandle}). # * {UploadRejectedError} - Server explicitly rejected the upload session (final). - # * {SessionStateError} - Session lifecycle rule violation, e.g., calling `#start` twice (final). - # - # @example Initiating an upload, rescuing an error, and resuming from a fresh session - # session = Gapic::Rest::ResumableUpload::Session.new( - # client_stub: client_stub, - # stream: stream - # ) - # - # begin - # response = session.start( - # initial_url: "https://example.googleapis.com/resumable/upload/v1/example/upload:new" - # ) - # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e - # handle = e.resume_handle - # raise unless handle - # - # stream.rewind - # resumed_session = Gapic::Rest::ResumableUpload::Session.new( - # client_stub: client_stub, - # stream: stream - # ) - # response = resumed_session.resume resume_handle: handle - # end + # * {SessionStateError} - Upload session lifecycle rule violation, e.g. starting a second run while one + # is in flight (final). # module ResumableUpload + ## + # @private + # Backoff settings carried from a caller's retry policy into the initiation policy, mapped to the + # {Gapic::Common::RetryPolicy} value a reader returns when the setting was never set. + # + # `jitter`'s default is a private constant, so it is read off a default policy rather than named. + # + # @return [Hash{Symbol=>Numeric}] + BACKOFF_DEFAULTS = { + initial_delay: Gapic::Common::RetryPolicy::DEFAULT_INITIAL_DELAY, + max_delay: Gapic::Common::RetryPolicy::DEFAULT_MAX_DELAY, + multiplier: Gapic::Common::RetryPolicy::DEFAULT_MULTIPLIER, + jitter: Gapic::Common::RetryPolicy.new.jitter + }.freeze + + ## + # @private + # Converts the per-call options a generated client assembles into overrides for the initiation + # retry policy. + # + # Returns a **Hash**, never a policy object: the protocol treats a {Gapic::Common::RetryPolicy} as a + # wholesale replacement and a Hash as a per-key override. Initiation's default policy carries a + # predicate that treats a response missing `X-Goog-Upload-Status` as retriable gateway noise, and + # handing over an object would silently drop it. + # + # `timeout` is always set, and becomes the local deadline of the initiation request alone — the + # whole-upload budget is separate and is not derived here. Without it, initiation would inherit + # {Gapic::Common::RetryPolicy::DEFAULT_TIMEOUT} (one hour), because `Gapic::CallOptions::RetryPolicy` + # never populates `@timeout` even though it subclasses {Gapic::Common::RetryPolicy}. + # + # Backoff settings and retry codes are copied only where the caller set them, which is why each is + # compared against the corresponding default: a reader on an unset policy returns that default, and + # the initiation defaults are the values that should survive. An empty `retry_codes` list counts as + # unset. A `retry_predicate` is deliberately not copied; the initiation predicate stays in place. + # + # @param options [Gapic::CallOptions, nil] Per-call options from a generated client method + # @return [Hash] Overrides for the initiation retry policy + # @raise [ArgumentError] If the call options carry a Proc (or any other non-{Gapic::Common::RetryPolicy}) + # retry policy, which has no coherent meaning across the three retry planes of an upload + def self.start_retry_policy_for options + overrides = { timeout: options&.timeout } + policy = options&.retry_policy + return overrides if policy.nil? + unless policy.is_a? Gapic::Common::RetryPolicy + raise ArgumentError, + "Resumable upload cannot derive an initiation retry policy from a #{policy.class}; " \ + "use a Gapic::Common::RetryPolicy or a Hash of retry settings" + end + + overrides[:retry_codes] = policy.retry_codes unless policy.retry_codes.empty? + BACKOFF_DEFAULTS.each do |setting, default| + value = policy.public_send setting + overrides[setting] = value unless value == default + end + overrides + end end end end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb b/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb index d08c00d..5d90520 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# rubocop:disable Metrics/ModuleLength + module Gapic module Rest - # rubocop:disable Metrics/ModuleLength module ResumableUpload ## # @private @@ -75,7 +76,8 @@ module ResumableUpload ## # @private - # Immutable configuration for a run that initiates a new upload session, i.e. {Session#start}. + # Immutable configuration for a run that initiates a new upload session, i.e. + # {Gapic::ResumableUpload#start}. # # Carries {COMMON_MEMBERS} plus the members only an initiating run uses. # @@ -163,7 +165,8 @@ def initialize initial_url:, ## # @private - # Immutable configuration for a run that resumes an existing upload session, i.e. {Session#resume}. + # Immutable configuration for a run that resumes an existing upload session, i.e. + # {Gapic::ResumableUpload#resume}. # # Carries {COMMON_MEMBERS} plus the upload URL and chunk size the earlier run established. There is # no `start_retry_policy` here: a resumed run issues no initiation request, so the member would @@ -230,7 +233,7 @@ def initialize upload_url:, # # The `on_progress` callback runs synchronously on the same thread as the upload protocol # and must not block. Any exception raised inside the callback aborts the upload session - # and propagates out of {Session#start} or {Session#resume}. + # and propagates out of {Gapic::ResumableUpload#start} or {Gapic::ResumableUpload#resume}. # # @!attribute [r] phase # @return [Symbol] Current upload phase, one of {Progress::PHASES} @@ -274,8 +277,8 @@ def initialize phase:, bytes_uploaded:, total_bytes: nil # Allowed lifecycle phases for an upload session. # # A callback observes `:initiating`, `:uploading`, `:recovering`, `:finalizing` and `:completed`. - # `:cancelling` is reserved: cancellation is not exposed on {Session}, so no phase with that value is - # currently emitted. + # `:cancelling` is reserved: cancellation is not exposed on {Gapic::ResumableUpload}, so no phase + # with that value is currently emitted. # # @return [Array] Progress::PHASES = [:initiating, :uploading, :recovering, :finalizing, :cancelling, :completed].freeze @@ -410,6 +413,7 @@ def initialize from_status:, shape:, recipe:, next_state:, instructions: [] end end end - # rubocop:enable Metrics/ModuleLength end end + +# rubocop:enable Metrics/ModuleLength diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb index 563171d..8b0d7b7 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb @@ -89,12 +89,16 @@ def upload_url # @param config [StartUploadConfig, ResumeUploadConfig] Configuration for this upload session # @param core [Core, nil] Optional Core state machine (defaults to new Core with config) # @param logger [Logger, nil] Optional logger override - def initialize client_stub:, config:, core: nil, logger: nil + # @param method_name [String, nil] RPC name this upload was started from, prefixed onto the + # per-request logging names (`"create_media_upload.start"`, `"create_media_upload.upload"`, and + # so on). Defaults to `"ResumableUpload"`. + def initialize client_stub:, config:, core: nil, logger: nil, method_name: nil @client_stub = client_stub @config = config @core = core || Core.new(config) @buffer = "".b @buffer_start_offset = 0 + @method_name_prefix = method_name || "ResumableUpload" endpoint = client_stub.respond_to?(:endpoint) ? client_stub.endpoint : nil setup_logging logger: logger || (client_stub.respond_to?(:logger) ? client_stub.logger : nil), @@ -567,7 +571,7 @@ def execute_send_start instruction return Event::GlobalDeadlineExceeded.new if deadline_exceeded? event = make_post_request instruction.url, headers: headers, body: instruction.body, - retry_policy: policy, method_name: "ResumableUpload.start", + retry_policy: policy, method_name: "#{@method_name_prefix}.start", start_attempt: attempt return event unless event.is_a? Event::HttpResponse @@ -631,7 +635,7 @@ def execute_send_chunk instruction make_post_request instruction.url, headers: headers, body: body, retry_policy: @data_plane_retry_policy.dup.start!, - method_name: "ResumableUpload.upload" + method_name: "#{@method_name_prefix}.upload" end ## @@ -649,7 +653,7 @@ def execute_send_finalize instruction } make_post_request instruction.url, headers: headers, body: "", retry_policy: @data_plane_retry_policy.dup.start!, - method_name: "ResumableUpload.finalize" + method_name: "#{@method_name_prefix}.finalize" end ## @@ -663,7 +667,7 @@ def execute_send_query instruction headers = { "X-Goog-Upload-Command" => "query", "Content-Length" => "0" } make_post_request instruction.url, headers: headers, body: "", retry_policy: @control_plane_retry_policy.dup.start!, - method_name: "ResumableUpload.query" + method_name: "#{@method_name_prefix}.query" end ## @@ -677,7 +681,7 @@ def execute_send_cancel instruction headers = { "X-Goog-Upload-Command" => "cancel", "Content-Length" => "0" } make_post_request instruction.url, headers: headers, body: "", retry_policy: @control_plane_retry_policy.dup.start!, - method_name: "ResumableUpload.cancel" + method_name: "#{@method_name_prefix}.cancel" end ## diff --git a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb index b54e114..17141be 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb @@ -546,8 +546,8 @@ def self.from event_or_error, message: nil, resume_handle: nil end ## - # Raised when an operation violates the Session lifecycle rules - # (e.g. calling a `start` method more than once). + # Raised when an operation violates the upload session lifecycle rules, e.g. starting a second run + # on a coordinator while one is still in flight. # class SessionStateError < Gapic::Common::Error end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb index deb92bf..29d8341 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb @@ -59,7 +59,7 @@ module RetryPolicies ## # @private # Default options for start command retry policy. - # Keep in sync with the "Retry Policies" section of Session's class doc. + # Keep in sync with the "Retry Policies" section of the Gapic::ResumableUpload class doc. # @return [Hash] START_DEFAULTS = { retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, @@ -72,7 +72,7 @@ module RetryPolicies ## # @private # Default options for query and cancel commands retry policy. - # Keep in sync with the "Retry Policies" section of Session's class doc. + # Keep in sync with the "Retry Policies" section of the Gapic::ResumableUpload class doc. # @return [Hash] CONTROL_PLANE_DEFAULTS = { retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, @@ -84,7 +84,7 @@ module RetryPolicies ## # @private # Default options for upload and finalize commands retry policy. - # Keep in sync with the "Retry Policies" section of Session's class doc. + # Keep in sync with the "Retry Policies" section of the Gapic::ResumableUpload class doc. # @return [Hash] DATA_PLANE_DEFAULTS = { retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, diff --git a/gapic-common/lib/gapic/rest/resumable_upload/session.rb b/gapic-common/lib/gapic/rest/resumable_upload/session.rb deleted file mode 100644 index ac0e431..0000000 --- a/gapic-common/lib/gapic/rest/resumable_upload/session.rb +++ /dev/null @@ -1,546 +0,0 @@ -# frozen_string_literal: true - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require "gapic/rest/resumable_upload/data_types" -require "gapic/rest/resumable_upload/driver" -require "gapic/rest/resumable_upload/errors" - -module Gapic - module Rest - module ResumableUpload - ## - # Coordinates a resumable upload across its lifecycle. - # - # A Session performs exactly one run (`start` or `resume`), never both, never twice. - # - # ### Two-State Model - # 1. **Unbound** (`!bound?`): Fresh session prior to execution. Permitted operations: `start` - # or `resume(...)`. - # 2. **Bound** (`bound?`): Session has executed or bound to an upload URL. Permitted operations: - # none (`start` and `resume` both raise {SessionStateError}). - # - # Calling {#resumable?} reports whether a new session can resume the upload (`!resume_handle.nil?`). - # Completed uploads (`:success`) and rejected uploads are finalized and not resumable - # (`resumable?` returns `false`, `resume_handle` returns `nil`). - # - # ### Execution Model - # - # {#start} and {#resume} are synchronous: they block the calling thread for the entire duration of the - # upload and return only on completion or failure. The `on_progress` callback runs on that same thread. - # - # The remaining readers ({#upload_url}, {#bound?}, {#resume_handle}, {#resumable?}, {#running?}) are - # guarded by an internal mutex and may be called from another thread while a run is in progress. Values - # read mid-run are a best-effort snapshot of a state the upload thread is still advancing. - # - # ### Where Arguments Live - # - # The constructor takes what both run types share: the client stub, the stream, `upload_size`, - # `content_type`, `timeout`, the control- and data-plane retry policies, `on_progress` and `logger`. - # Arguments that belong to one run live on the method performing it — `initial_url`, `initial_body`, - # `initial_headers`, `chunk_size` and `start_retry_policy` on {#start}; `upload_url` and `chunk_size`, - # or a {ResumeHandle}, on {#resume}. - # - # ### Recovering From a Failure - # - # A bound session never runs again, so recovery means constructing a new Session. Errors that carry a - # resume handle include the {HasResumeHandle} mixin, which can be rescued directly to catch all of them: - # - # @example Resuming after a recoverable failure - # begin - # session.start initial_url: url - # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e - # raise unless e.resume_handle - # Session.new(client_stub: client_stub, stream: File.open(path, "rb")) - # .resume(resume_handle: e.resume_handle) - # end - # - # The replacement session needs a stream positioned at byte 0 of the whole object, not at the server's - # acknowledged offset; {#resume} fast-forwards on its own. For an unseekable stream that means opening a - # fresh one, since it cannot be rewound. - # - # ### Defaults - # - # * `chunk_size` defaults to 8 MB, then rounds down to a multiple of any chunk granularity the server - # requires. - # * `timeout` defaults to `upload_size / 1 MB per second` when `upload_size` is known, floored at one - # hour, and to one hour flat when it is not. - # - # ### Retry Policies - # - # Retry behavior is partitioned across three policies: `start_retry_policy` on {#start}, and - # `control_plane_retry_policy` and `data_plane_retry_policy` on {#initialize}. - # - # Passing a {Gapic::Common::RetryPolicy} replaces the corresponding default policy outright. Passing a - # Hash overrides only the keys it names and leaves the remaining defaults — including `retry_codes` and - # any status-header predicates — in place. - # - # All three policies share the same default retry codes and exponential backoff settings: - # - # | Setting | Default | - # |---|---| - # | `retry_codes` | `UNAVAILABLE`, `DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED`, `INTERNAL` | - # | `initial_delay` | `1.0` s | - # | `max_delay` | `15.0` s | - # | `multiplier` | `1.3` | - # - # They differ in which requests they govern and how a missing or empty `X-Goog-Upload-Status` response - # header is treated: - # - # | Policy | Governs | Missing status header | - # |---|---|---| - # | `start_retry_policy` | session initiation | **Retriable** on any status (incl. `200`), unless fatal | - # | `control_plane_retry_policy` | `query` and `cancel` | No predicate; decided on `retry_codes` alone | - # | `data_plane_retry_policy` | `upload` and `finalize` | **Not** retriable | - # - # Initiation treats a response missing `X-Goog-Upload-Status` as gateway noise worth retrying; the data - # plane treats it as a response it cannot interpret and refuses to replay bytes against it. - # - class Session - # @return [Gapic::Rest::ClientStub] Underlying REST client stub - attr_reader :client_stub - - ## - # Binary input stream to upload. The stream is assumed to be positioned at byte 0 - # (it is not rewound prior to reading) and is not closed after use. - # - # @return [IO] - attr_reader :stream - - # @return [Integer, nil] Total upload bytes if known upfront - attr_reader :upload_size - - # @return [String, nil] MIME type of uploaded media - attr_reader :content_type - - ## - # Total upload timeout in seconds, covering the whole run rather than any single request. When `nil`, - # it resolves to `upload_size / 1 MB per second` floored at one hour if `upload_size` is known, and to - # one hour flat otherwise. Zero and negative values are treated as `nil`. - # - # @return [Numeric, nil] - attr_reader :timeout - - ## - # Retry policy for control commands (query, cancel). A {Gapic::Common::RetryPolicy} replaces the - # default policy outright; a Hash overrides only the settings it names. - # - # @return [Gapic::Common::RetryPolicy, Hash, nil] - attr_reader :control_plane_retry_policy - - ## - # Retry policy for data commands (upload, finalize). A {Gapic::Common::RetryPolicy} replaces the - # default policy outright; a Hash overrides only the settings it names. - # - # @return [Gapic::Common::RetryPolicy, Hash, nil] - attr_reader :data_plane_retry_policy - - ## - # Callback invoked with {Progress} snapshots during upload execution. - # Executed synchronously on the thread running the upload protocol; it must not block. - # Exceptions raised inside the callback immediately abort the upload session and - # propagate out of {#start} or {#resume}. - # - # @return [Proc, nil] - attr_reader :on_progress - - # @return [Logger, nil] Logger instance - attr_reader :logger - - ## - # Initializes a new Resumable Upload Session. - # - # The constructor takes only what both run types share. Arguments specific to a single run live on - # the method that performs it: initiation details on {#start}, the upload URL and chunk size on - # {#resume}. - # - # @param client_stub [Gapic::Rest::ClientStub] Underlying REST client stub - # @param stream [IO] Binary input stream to upload. Precondition: assumed to be positioned at byte 0 - # (not rewound prior to reading) and not closed after use. - # @param upload_size [Integer, nil] Total upload bytes if known upfront - # @param content_type [String, nil] MIME type of uploaded media - # @param timeout [Numeric, nil] Total upload timeout in seconds covering the whole run. When `nil`, - # resolves to `upload_size / 1 MB per second` floored at one hour if `upload_size` is known, and to - # one hour flat otherwise. - # @param control_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for control - # commands (`query` and `cancel`). See the "Retry Policies" section in the class documentation. - # @param data_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for data - # commands (`upload` and `finalize`). See the "Retry Policies" section in the class documentation. - # @param on_progress [Proc, nil] Progress callback invoked as `->(progress)` with a {Progress} instance. - # Executed synchronously on the upload protocol thread; it must not block. - # Exceptions raised inside the callback abort the session and propagate out of {#start} or {#resume}. - # @param logger [Logger, nil] Logger instance - # - def initialize client_stub:, - stream:, - upload_size: nil, - content_type: nil, - timeout: nil, - control_plane_retry_policy: nil, - data_plane_retry_policy: nil, - on_progress: nil, - logger: nil - @client_stub = client_stub - @stream = stream - @upload_size = upload_size - @content_type = content_type - @timeout = timeout - @control_plane_retry_policy = control_plane_retry_policy - @data_plane_retry_policy = data_plane_retry_policy - @on_progress = on_progress - @logger = logger - - @mutex = Mutex.new - @running = false - @executed = false - @upload_url = nil - @last_driver = nil - end - - ## - # Returns the raw upload session URL if established. - # - # @return [String, nil] - def upload_url - @mutex.synchronize { upload_url_internal } - end - - ## - # Returns whether the session is bound to a server-side upload. - # - # @return [Boolean] - def bound? - @mutex.synchronize { bound_internal? } - end - - ## - # Returns the current {ResumeHandle} if the session is alive and resumable. - # Completed uploads are not resumable (returns nil). Rejected uploads and - # cancelled uploads are also finalized and not resumable, returning nil. - # - # @return [ResumeHandle, nil] - def resume_handle - @mutex.synchronize { resume_handle_internal } - end - - ## - # Returns whether a new session can resume the upload. - # Completed uploads are not resumable (returns false). Rejected uploads and - # cancelled uploads are also finalized and not resumable (returns false). - # - # @return [Boolean] - def resumable? - @mutex.synchronize { !resume_handle_internal.nil? } - end - - ## - # Returns whether a run is currently executing. - # - # @return [Boolean] - def running? - @mutex.synchronize { @running } - end - - ## - # Starts a new upload session on the server. - # - # A session performs exactly one run (`start` or `resume`). Calling `start` on an already-bound - # or executed session raises {SessionStateError}. Precondition: the stream is assumed to be - # positioned at byte 0 (the session does not rewind it before reading) and is not closed after use. - # - # Blocks the calling thread until the upload completes or fails. - # - # @example Uploading a file with progress reporting - # session = Gapic::Rest::ResumableUpload::Session.new( - # client_stub: client_stub, - # stream: File.open("movie.mp4", "rb"), - # upload_size: File.size("movie.mp4"), - # content_type: "video/mp4", - # on_progress: ->(progress) { puts "#{progress.phase}: #{progress.bytes_uploaded} bytes" } - # ) - # response = session.start initial_url: "https://example.googleapis.com/upload/v1/media" - # - # @param initial_url [String] Initial endpoint URI for session initiation - # @param initial_body [String, nil] Request payload for session initiation - # @param initial_headers [Hash] Additional headers for the initiation request. - # The five reserved protocol headers (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, - # `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, - # `X-Goog-Upload-Header-Content-Length`) are rejected with an `ArgumentError` in any casing — - # they carry protocol mechanics the session owns. Use the constructor's `content_type` and - # `upload_size` to shape the media descriptors. Pass-through headers such as - # `X-Goog-Upload-Header-Content-Disposition` are permitted. - # @param chunk_size [Integer, nil] Requested chunk size in bytes, defaulting to 8 MB. The effective - # size is rounded down to a multiple of any chunk granularity the server requires, or raised to that - # granularity if it exceeds the requested size. - # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for the initiation - # request (`start`). See the "Retry Policies" section in the class documentation. - # @return [String, nil] Raw, undecoded body of the finalizing HTTP response (or `nil` if the - # response carried no body), typically the JSON resource the backend created that the caller - # parses. A client stub carrying response-decoding middleware is outside the contract. - # @raise [ArgumentError] If `initial_url` is missing or blank, if `initial_headers` sets a - # reserved protocol header, or if a retry policy argument is neither a - # {Gapic::Common::RetryPolicy}, a Hash, nor `nil` - # @raise [SessionStateError] If already bound/executed or if a run is currently in progress - # @raise [RequestFailedError] If a transport error, timeout, or retry exhaustion occurs - # @raise [DeadlineExceededError] If the global upload timeout is exceeded - # @raise [BadResponseError] If an unexpected or malformed HTTP response is received - # @raise [UnseekableStreamError] If stream rewinding is required during recovery on an unseekable stream - # @raise [StreamMismatchError] If stream content or length does not match protocol expectations - # @raise [InvalidTransitionError] If an unmatched event occurs for the current protocol state - # @raise [UploadRejectedError] If the server explicitly rejects the upload session - # @raise [InternalError] If the library detects an internal invariant breach; this signals a bug - # in this library rather than a caller or server error - def start initial_url:, - initial_body: nil, - initial_headers: {}, - chunk_size: nil, - start_retry_policy: nil - config = build_start_config initial_url: initial_url, - initial_body: initial_body, - initial_headers: initial_headers, - chunk_size: chunk_size, - start_retry_policy: start_retry_policy - - driver = nil - @mutex.synchronize do - raise SessionStateError, "A run is already in progress for this session" if @running - raise SessionStateError, "Session has already executed a run" if bound_internal? - - driver = Driver.new client_stub: @client_stub, config: config, logger: @logger - @executed = true - @running = true - end - - execute_run driver - end - - ## - # Resumes an upload session using one of two explicit keyword forms: - # 1. `resume(upload_url:, chunk_size:)`: Resumes with explicit URL and chunk size. - # 2. `resume(resume_handle:)`: Resumes via {ResumeHandle}. - # - # A session performs exactly one run (`start` or `resume`). Resuming must be executed on a - # fresh, unexecuted session. Blocks the calling thread until the upload completes or fails. - # - # A resumed run targets an upload the server has already created, so it takes no initiation - # arguments; everything it needs beyond the constructor is on this method. - # - # ### Chunk size - # - # A chunk size must be given explicitly because the server reports chunk granularity during - # initiation, which a resumed run skips. {ResumeHandle} carries the effective value from the original - # run for exactly this reason. - # - # ### Stream position - # - # The stream must be positioned at byte 0 of the whole object, not at the server's acknowledged - # offset, and is not closed after use. The Driver fast-forwards on its own, by seeking on seekable - # streams or by reading and discarding on unseekable ones. An unseekable stream therefore has to be - # freshly opened rather than rewound. - # - # A completed upload is finalized: {#resume_handle} returns `nil` and {#resumable?} returns - # `false`, so there is no handle to resume from. Calling `#resume` on the session that completed - # the run raises {SessionStateError}, as it would after any run. Resuming a *fresh* session - # against a finalized `upload_url` is undefined behavior: it queries the server and might return - # the response body or raise an error, depending on the server response. - # - # @example Resuming from a handle persisted by an earlier process - # handle = Gapic::Rest::ResumableUpload::ResumeHandle.new( - # upload_url: row[:upload_url], - # chunk_size: row[:chunk_size] - # ) - # session = Gapic::Rest::ResumableUpload::Session.new( - # client_stub: client_stub, - # stream: File.open("movie.mp4", "rb"), - # upload_size: File.size("movie.mp4") - # ) - # response = session.resume resume_handle: handle - # - # @param upload_url [String, nil] Explicit upload URL - # @param chunk_size [Integer, nil] Explicit chunk size - # @param resume_handle [ResumeHandle, nil] Explicit resume handle - # @return [String, nil] Raw, undecoded body of the finalizing HTTP response (or `nil` if the - # response carried no body), typically the JSON resource the backend created that the caller - # parses. A client stub carrying response-decoding middleware is outside the contract. - # @raise [ArgumentError] If argument shape is invalid, target upload is missing, or stream.pos != 0 - # @raise [SessionStateError] If already bound/executed or if a run is currently in progress - # @raise [RequestFailedError] If a transport error, timeout, or retry exhaustion occurs - # @raise [DeadlineExceededError] If the global upload timeout is exceeded - # @raise [BadResponseError] If an unexpected or malformed HTTP response is received - # @raise [UnseekableStreamError] If stream rewinding is required during recovery on an unseekable stream - # @raise [StreamMismatchError] If stream content or length does not match the resumed upload - # @raise [InvalidTransitionError] If an unmatched event occurs for the current protocol state - # @raise [UploadRejectedError] If the server explicitly rejects the upload session - # @raise [InternalError] If the library detects an internal invariant breach; this signals a bug - # in this library rather than a caller or server error - def resume upload_url: nil, - chunk_size: nil, - resume_handle: nil - target_url, target_chunk_size = resolve_resume_args( - upload_url: upload_url, - chunk_size: chunk_size, - resume_handle: resume_handle - ) - - driver = nil - @mutex.synchronize do - raise SessionStateError, "A run is already in progress for this session" if @running - raise SessionStateError, "Session has already executed a run" if bound_internal? - - if @stream.respond_to?(:pos) && !@stream.pos.zero? - raise ArgumentError, "Stream must be positioned at byte 0 to resume an upload (got pos #{@stream.pos})" - end - - config = build_resume_config target_url, target_chunk_size - driver = Driver.new client_stub: @client_stub, config: config, logger: @logger - @executed = true - @running = true - @upload_url = target_url - end - - execute_run driver - end - - private - - ## - # @private - # Returns the established upload URL without locking. - # - # @return [String, nil] - def upload_url_internal - @upload_url || @last_driver&.upload_url - end - - ## - # @private - # Returns whether the session is bound without locking. - # - # @return [Boolean] - def bound_internal? - @executed || !upload_url_internal.nil? - end - - ## - # @private - # Returns the current resume handle from the driver without locking. - # - # @return [ResumeHandle, nil] - def resume_handle_internal - @last_driver&.resume_handle - end - - ## - # @private - # Returns the configuration members shared by both run types, mirroring - # {ResumableUpload::COMMON_MEMBERS}. - # - # @return [Hash{Symbol=>Object}] - def common_config_args - { - stream: @stream, - upload_size: @upload_size, - content_type: @content_type, - timeout: @timeout, - control_plane_retry_policy: @control_plane_retry_policy, - data_plane_retry_policy: @data_plane_retry_policy, - on_progress: @on_progress - } - end - - ## - # @private - # Builds configuration for a new upload session. - # - # @param initial_url [String] Initial endpoint URI for session initiation - # @param initial_body [String, nil] Request payload for session initiation - # @param initial_headers [Hash, nil] Additional headers for initiation - # @param chunk_size [Integer, nil] Requested chunk size in bytes - # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Initiation retry policy - # @return [StartUploadConfig] - def build_start_config initial_url:, initial_body:, initial_headers:, chunk_size:, start_retry_policy: - StartUploadConfig.new( - initial_url: initial_url, - initial_body: initial_body, - initial_headers: initial_headers || {}, - chunk_size: chunk_size, - start_retry_policy: start_retry_policy, - **common_config_args - ) - end - - ## - # @private - # Builds configuration for resuming an upload session. - # - # @param target_url [String] Target upload session URL - # @param target_chunk_size [Integer] Effective chunk size in bytes - # @return [ResumeUploadConfig] - def build_resume_config target_url, target_chunk_size - ResumeUploadConfig.new( - upload_url: target_url, - chunk_size: target_chunk_size, - **common_config_args - ) - end - - ## - # @private - # Validates and extracts target upload URL and chunk size from resume keyword arguments. - # - # @param upload_url [String, nil] Explicit upload URL - # @param chunk_size [Integer, nil] Explicit chunk size - # @param resume_handle [ResumeHandle, nil] Explicit resume handle - # @return [Array] Tuple of [upload_url, chunk_size] - # @raise [ArgumentError] If arguments are missing or mutually exclusive - def resolve_resume_args upload_url:, chunk_size:, resume_handle: - if resume_handle - raise ArgumentError, "Cannot pass both resume_handle and upload_url/chunk_size" if upload_url || chunk_size - [resume_handle.upload_url, resume_handle.chunk_size] - elsif upload_url - raise ArgumentError, "Must provide chunk_size with upload_url" if chunk_size.nil? - [upload_url, chunk_size] - elsif chunk_size - raise ArgumentError, "Cannot pass chunk_size without upload_url" - else - raise ArgumentError, "Must provide either resume_handle or upload_url and chunk_size" - end - end - - ## - # @private - # Executes the driver run and records the final upload URL and state. - # - # @param driver [Driver] Driver instance to run - # @return [String, nil] Final response body upon completion - def execute_run driver - @mutex.synchronize { @last_driver = driver } - result = driver.run - @mutex.synchronize do - @upload_url ||= driver.upload_url - @running = false - end - result - rescue StandardError - @mutex.synchronize do - @upload_url ||= driver.upload_url - @running = false - end - raise - end - end - end - end -end diff --git a/gapic-common/lib/gapic/resumable_upload.rb b/gapic-common/lib/gapic/resumable_upload.rb new file mode 100644 index 0000000..c65b717 --- /dev/null +++ b/gapic-common/lib/gapic/resumable_upload.rb @@ -0,0 +1,507 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/rest/resumable_upload" + +module Gapic + ## + # Coordinates resumable uploads for a client method that performs them. + # + # A client method that uploads media returns one of these handles instead of a response. No request is + # sent and no byte is read from the stream until {#start} or {#resume} is called on it. Both are + # synchronous: they block the calling thread for the whole upload and return the decoded response + # message. + # + # ### Reusable + # + # A handle is reusable, and a failed run is resumed on the same object: + # + # @example Uploading, then resuming after a recoverable failure + # upload = client.create_media_upload request + # begin + # upload.start stream: File.open("movie.mp4", "rb"), content_type: "video/mp4" + # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e + # raise unless upload.resumable? + # upload.resume stream: File.open("movie.mp4", "rb") + # end + # + # The stream handed to {#resume} must be positioned at byte 0 of the whole object, not at the server's + # acknowledged offset; the upload fast-forwards on its own, by seeking on a seekable stream or by + # reading and discarding on an unseekable one. An unseekable stream therefore has to be freshly opened + # rather than rewound. + # + # A run that failed in a way the protocol can recover from leaves a {Gapic::Rest::ResumableUpload::ResumeHandle} + # behind, readable from {#resume_handle} and also carried on the error. Persisting that handle lets a + # later process resume the same upload: + # + # @example Resuming an upload started by an earlier process + # upload = client.create_media_upload + # upload.resume stream: File.open("movie.mp4", "rb"), + # resume_handle: Gapic::Rest::ResumableUpload::ResumeHandle.new( + # upload_url: row[:upload_url], chunk_size: row[:chunk_size] + # ) + # + # A completed upload is finalized: {#resume_handle} returns `nil` and {#resumable?} returns `false`, so + # there is no handle to resume from. Calling {#start} again is permitted and begins a second, unrelated + # upload. + # + # ### The Two Timeouts + # + # An upload is bounded by two independent budgets, and they are three orders of magnitude apart: + # + # | Budget | Set by | Covers | + # |---|---|---| + # | whole upload | `upload_timeout:` on {#start} and {#resume} | every request, retry and byte of the run | + # | initiation request | per-call `timeout`, or `timeout:` in `start_retry_policy` | creating the session | + # + # The per-call `timeout` a client method takes reaches only the initiation request. An upload still + # transferring bytes an hour later has long outlived it, and that is expected. To bound the run as a + # whole, pass `upload_timeout:`. + # + # ### Threading + # + # {#start} and {#resume} block the calling thread, and the `on_progress` callback runs on that same + # thread. The readers ({#resume_handle}, {#resumable?}, {#running?}) are guarded by an internal mutex and + # may be called from another thread mid-run; values read that way are a best-effort snapshot of a state + # the upload thread is still advancing. + # + # ### Defaults + # + # * `chunk_size` defaults to 8 MB, then rounds down to a multiple of any chunk granularity the server + # requires. + # * `upload_timeout` defaults to `upload_size / 1 MB per second` when `upload_size` is known, floored at + # one hour, and to one hour flat when it is not. + # + # ### Retry Policies + # + # Retry behavior is partitioned across three policies. Only the initiation policy is caller-supplied; + # the other two are the protocol's own and govern the requests no call option describes. + # + # All three share the same default retry codes and exponential backoff settings: + # + # | Setting | Default | + # |---|---| + # | `retry_codes` | `UNAVAILABLE`, `DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED`, `INTERNAL` | + # | `initial_delay` | `1.0` s | + # | `max_delay` | `15.0` s | + # | `multiplier` | `1.3` | + # + # They differ in which requests they govern and how a missing or empty `X-Goog-Upload-Status` response + # header is treated: + # + # | Policy | Governs | Missing status header | + # |---|---|---| + # | initiation | session initiation | **Retriable** on any status (incl. `200`), unless fatal | + # | control plane | `query` and `cancel` | No predicate; decided on `retry_codes` alone | + # | data plane | `upload` and `finalize` | **Not** retriable | + # + # Initiation treats a response missing `X-Goog-Upload-Status` as gateway noise worth retrying; the data + # plane treats it as a response it cannot interpret and refuses to replay bytes against it. + # + class ResumableUpload + ## + # @private + # Builds a coordinator for one client method call. + # + # Instances come from generated client methods; the arguments below are what such a method has to + # hand over, not a surface a caller assembles. + # + # Both procs are deferred deliberately. `client_stub_proc` lets a client that cannot perform REST + # calls hand back a working handle and fail only when an upload is actually attempted. + # `initial_request_proc` means the initiation URL and body are computed on {#start} and never on + # {#resume}, so a handle built without a request message is still fully functional for resuming. + # + # @param client_stub_proc [Proc] Returns the {Gapic::Rest::ClientStub} to upload through. Called at + # the top of every run, and may raise if the client cannot perform REST calls. + # @param initial_request_proc [Proc] Returns the `[url, body]` pair for session initiation. Called by + # {#start} only. + # @param response_type [Class, nil] Protobuf message class the final response body is decoded into. + # `nil` returns the raw body; see {#start}. + # @param initial_headers [Hash] Headers for the initiation request. Keys and values are stringified. + # The five reserved protocol headers (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, + # `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, `X-Goog-Upload-Header-Content-Length`) + # are rejected in any casing; use `content_type` and `upload_size` on the run methods instead. + # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for the initiation + # request. A {Gapic::Common::RetryPolicy} replaces the default policy outright; a Hash overrides only + # the settings it names. See the "Retry Policies" section in the class documentation. + # @param control_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for `query` + # and `cancel`. `nil` uses the protocol default. + # @param data_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for `upload` and + # `finalize`. `nil` uses the protocol default. + # @param error_handler [Proc, nil] Called with a run failure and **returns** the exception to raise in + # its place. It must not raise. + # @param method_name [String, nil] RPC name used in log entries. + # + def initialize client_stub_proc:, + initial_request_proc:, + response_type:, + initial_headers: {}, + start_retry_policy: nil, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + error_handler: nil, + method_name: nil + @client_stub_proc = client_stub_proc + @initial_request_proc = initial_request_proc + @response_type = response_type + @initial_headers = stringify_headers initial_headers + @start_retry_policy = start_retry_policy + @control_plane_retry_policy = control_plane_retry_policy + @data_plane_retry_policy = data_plane_retry_policy + @error_handler = error_handler + @method_name = method_name + + @mutex = Mutex.new + @running = false + @driver = nil + end + + ## + # Creates an upload session on the server and transfers the stream into it. + # + # Blocks the calling thread until the upload completes or fails. The stream is assumed to be + # positioned at byte 0 (it is not rewound before reading) and is not closed after use. + # + # @example + # response = upload.start stream: File.open("movie.mp4", "rb"), + # content_type: "video/mp4", + # upload_size: File.size("movie.mp4"), + # upload_timeout: 4 * 3600, + # on_progress: ->(p) { puts "#{p.phase}: #{p.bytes_uploaded}" } + # + # @param stream [IO] Binary input stream to upload, positioned at byte 0. + # @param content_type [String, nil] MIME type of the uploaded media. + # @param upload_size [Integer, nil] Total upload bytes, if known upfront. + # @param chunk_size [Integer, nil] Requested chunk size in bytes, defaulting to 8 MB. The effective + # size is rounded down to a multiple of any chunk granularity the server requires, or raised to that + # granularity if it exceeds the requested size. A resumed run has no such argument: it takes its + # chunk size from the {Gapic::Rest::ResumableUpload::ResumeHandle}. + # @param upload_timeout [Numeric, nil] Budget in seconds for the **whole run** — every request, every + # retry, every byte — not for any single request. The per-call `timeout` a client method takes bounds + # the initiation request alone. When `nil`, resolves to `upload_size / 1 MB per second` floored at one + # hour if `upload_size` is known, and to one hour flat otherwise. + # @param on_progress [Proc, nil] Called as `->(progress)` with a {Gapic::Rest::ResumableUpload::Progress} + # instance. Runs synchronously on the upload thread and must not block; an exception raised inside it + # aborts the run and propagates out of this method. + # @return [Object] The final response decoded into the handle's response type, or the raw response body + # (a String, or `nil` when the final response carried none) when the handle has no response type. + # @raise [ArgumentError] If the initiation headers set a reserved protocol header, if a retry policy is + # neither a {Gapic::Common::RetryPolicy}, a Hash, nor `nil`, or if the client cannot perform REST calls + # @raise [Gapic::Rest::ResumableUpload::SessionStateError] If a run is already in progress + # @raise [Gapic::Rest::ResumableUpload::RequestFailedError] If a transport error, timeout, or retry + # exhaustion occurs + # @raise [Gapic::Rest::ResumableUpload::DeadlineExceededError] If `upload_timeout` is exceeded + # @raise [Gapic::Rest::ResumableUpload::BadResponseError] If an unexpected or malformed HTTP response + # is received + # @raise [Gapic::Rest::ResumableUpload::UnseekableStreamError] If stream rewinding is required during + # recovery on an unseekable stream + # @raise [Gapic::Rest::ResumableUpload::StreamMismatchError] If stream content or length does not match + # protocol expectations + # @raise [Gapic::Rest::ResumableUpload::InvalidTransitionError] If an unmatched event occurs for the + # current protocol state + # @raise [Gapic::Rest::ResumableUpload::UploadRejectedError] If the server explicitly rejects the upload + # @raise [Gapic::Rest::ResumableUpload::InternalError] If the library detects an internal invariant + # breach; this signals a bug in this library rather than a caller or server error + # + def start stream:, + content_type: nil, + upload_size: nil, + chunk_size: nil, + upload_timeout: nil, + on_progress: nil + execute_run do + client_stub = @client_stub_proc.call + initial_url, initial_body = @initial_request_proc.call + config = ::Gapic::Rest::ResumableUpload::StartUploadConfig.new( + initial_url: initial_url, + initial_body: initial_body, + initial_headers: @initial_headers, + chunk_size: chunk_size, + start_retry_policy: @start_retry_policy, + **run_config_args(stream: stream, content_type: content_type, upload_size: upload_size, + upload_timeout: upload_timeout, on_progress: on_progress) + ) + build_driver client_stub, config + end + end + + ## + # Resumes an upload session the server has already created, transferring whatever it has not yet + # acknowledged. + # + # Blocks the calling thread until the upload completes or fails. A resumed run sends no initiation + # request, so it takes no initiation arguments and needs no request message. + # + # The target is a {Gapic::Rest::ResumableUpload::ResumeHandle}: either the one passed in, or — when + # `resume_handle` is omitted — the one left behind by this handle's last run. The bare form raises + # `ArgumentError` when there is none, which covers a handle that has never run, a run that finished + # successfully, and a run that failed in a way the protocol considers unresumable. + # + # Resuming against a finalized upload URL is undefined behavior: it queries the server and might + # return the response body or raise an error, depending on the server response. + # + # @param stream [IO] Binary input stream to upload, positioned at byte 0 of the **whole object**, not + # at the server's acknowledged offset. The upload fast-forwards on its own, by seeking or by reading + # and discarding. Not closed after use. + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Upload to resume. Defaults to + # the one left behind by this handle's last run. + # @param content_type [String, nil] MIME type of the uploaded media. + # @param upload_size [Integer, nil] Total upload bytes, if known upfront. + # @param upload_timeout [Numeric, nil] Budget in seconds for the **whole run**. See {#start}. + # @param on_progress [Proc, nil] Called as `->(progress)` with a {Gapic::Rest::ResumableUpload::Progress} + # instance. Runs synchronously on the upload thread and must not block. + # @return [Object] The final response decoded into the handle's response type, or the raw response body + # (a String, or `nil` when the final response carried none) when the handle has no response type. + # @raise [ArgumentError] If there is no upload to resume, if the stream is not positioned at byte 0, if + # a retry policy is neither a {Gapic::Common::RetryPolicy}, a Hash, nor `nil`, or if the client cannot + # perform REST calls + # @raise [Gapic::Rest::ResumableUpload::SessionStateError] If a run is already in progress + # @raise [Gapic::Rest::ResumableUpload::RequestFailedError] If a transport error, timeout, or retry + # exhaustion occurs + # @raise [Gapic::Rest::ResumableUpload::DeadlineExceededError] If `upload_timeout` is exceeded + # @raise [Gapic::Rest::ResumableUpload::BadResponseError] If an unexpected or malformed HTTP response + # is received + # @raise [Gapic::Rest::ResumableUpload::UnseekableStreamError] If stream rewinding is required during + # recovery on an unseekable stream + # @raise [Gapic::Rest::ResumableUpload::StreamMismatchError] If stream content or length does not match + # the resumed upload + # @raise [Gapic::Rest::ResumableUpload::InvalidTransitionError] If an unmatched event occurs for the + # current protocol state + # @raise [Gapic::Rest::ResumableUpload::UploadRejectedError] If the server explicitly rejects the upload + # @raise [Gapic::Rest::ResumableUpload::InternalError] If the library detects an internal invariant + # breach; this signals a bug in this library rather than a caller or server error + # + def resume stream:, + resume_handle: nil, + content_type: nil, + upload_size: nil, + upload_timeout: nil, + on_progress: nil + execute_run do + verify_stream_at_origin stream + handle = resolve_resume_handle resume_handle + client_stub = @client_stub_proc.call + config = ::Gapic::Rest::ResumableUpload::ResumeUploadConfig.new( + upload_url: handle.upload_url, + chunk_size: handle.chunk_size, + **run_config_args(stream: stream, content_type: content_type, upload_size: upload_size, + upload_timeout: upload_timeout, on_progress: on_progress) + ) + build_driver client_stub, config + end + end + + ## + # Returns the handle needed to resume the last run, carrying its upload URL and resolved chunk size. + # + # `nil` before the first run, and after any run that left nothing to resume: a completed upload is + # finalized, and rejected and cancelled uploads are too. + # + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] + def resume_handle + @mutex.synchronize { @driver&.resume_handle } + end + + ## + # Returns whether the last run left an upload that can be resumed. + # + # @return [Boolean] + def resumable? + !resume_handle.nil? + end + + ## + # Returns whether a run is currently executing. + # + # @return [Boolean] + def running? + @mutex.synchronize { @running } + end + + private + + ## + # @private + # Claims the single run slot, runs the driver built by the block, and releases the slot. + # + # The slot is claimed before the block runs, so a second concurrent run is rejected while this one is + # still building its configuration. It is released in an `ensure`, so every exit — a clean return, a + # protocol error, an `on_progress` callback raising, a `Thread#kill` — leaves the handle usable and the + # driver retained. A failure inside the block leaves no driver retained at all, so a configuration + # `ArgumentError` cannot disturb the resume handle of an earlier run. + # + # @yieldreturn [Gapic::Rest::ResumableUpload::Driver] Driver to run + # @return [Object] Decoded final response + def execute_run + claim_run_slot + begin + driver = yield + @mutex.synchronize { @driver = driver } + run_driver driver + ensure + @mutex.synchronize { @running = false } + end + end + + ## + # @private + # Runs a driver and decodes its result, applying the caller's error handler to a failure. + # + # Only the run is wrapped. Argument and configuration errors are raised while the driver is still + # being built, and reach the caller as themselves: they describe a call that was never made. + # + # @param driver [Gapic::Rest::ResumableUpload::Driver] Driver to run + # @return [Object] Decoded final response + def run_driver driver + decode_response driver.run + rescue ::StandardError => e + raise wrap_error(e) + end + + ## + # @private + # Marks a run as in flight, rejecting a second concurrent one. + # + # @return [void] + # @raise [Gapic::Rest::ResumableUpload::SessionStateError] If a run is already in progress + def claim_run_slot + @mutex.synchronize do + if @running + raise ::Gapic::Rest::ResumableUpload::SessionStateError, + "A run is already in progress for this upload" + end + @running = true + end + end + + ## + # @private + # Builds the driver for a run. + # + # @param client_stub [Gapic::Rest::ClientStub] Stub returned by `client_stub_proc` + # @param config [Gapic::Rest::ResumableUpload::StartUploadConfig, + # Gapic::Rest::ResumableUpload::ResumeUploadConfig] Configuration for this run + # @return [Gapic::Rest::ResumableUpload::Driver] + def build_driver client_stub, config + ::Gapic::Rest::ResumableUpload::Driver.new client_stub: client_stub, + config: config, + method_name: @method_name + end + + ## + # @private + # Returns the configuration members both run types share, mirroring + # {Gapic::Rest::ResumableUpload::COMMON_MEMBERS}. + # + # @return [Hash{Symbol=>Object}] + def run_config_args stream:, content_type:, upload_size:, upload_timeout:, on_progress: + { + stream: stream, + upload_size: upload_size, + content_type: content_type, + timeout: upload_timeout, + control_plane_retry_policy: @control_plane_retry_policy, + data_plane_retry_policy: @data_plane_retry_policy, + on_progress: on_progress + } + end + + ## + # @private + # Resolves the upload a resumed run targets. + # + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Explicit handle, if given + # @return [Gapic::Rest::ResumableUpload::ResumeHandle] + # @raise [ArgumentError] If no handle was given and the last run left none + def resolve_resume_handle resume_handle + return resume_handle if resume_handle + + handle = @mutex.synchronize { @driver&.resume_handle } + if handle.nil? + raise ArgumentError, + "No upload to resume: this handle has not run, or its last run left nothing resumable. " \ + "Pass resume_handle: to resume an upload started elsewhere." + end + handle + end + + ## + # @private + # Rejects a stream that is not positioned at byte 0. Streams that do not report a position are + # trusted. + # + # @param stream [IO] Stream a resumed run will read + # @return [void] + # @raise [ArgumentError] If the stream reports a non-zero position + def verify_stream_at_origin stream + return unless stream.respond_to? :pos + return if stream.pos.zero? + + raise ArgumentError, "Stream must be positioned at byte 0 to resume an upload (got pos #{stream.pos})" + end + + ## + # @private + # Decodes the final response body into the handle's response type. + # + # A `nil` response type returns the raw body, exactly as the driver produced it. Generated call sites + # always pass a message class; the raw form exists so this gem's own tests can assert on what the + # server sent without decoding through a message type they do not have. + # + # @param body [String, nil] Raw body of the finalizing HTTP response + # @return [Object] Decoded message, or the raw body when there is no response type + def decode_response body + return body if @response_type.nil? + + @response_type.decode_json body.to_s, ignore_unknown_fields: true + end + + ## + # @private + # Applies the caller's error handler to a run failure. + # + # The handler returns the exception to raise. A replacement that loses the + # {Gapic::Rest::ResumableUpload::HasResumeHandle} mixin is re-extended with it and given the original's + # handle, so a library-specific error type cannot erase the fact that the upload is resumable. + # + # @param error [StandardError] Failure raised by the run + # @return [Exception] Exception to raise in its place + def wrap_error error + return error unless @error_handler + + wrapped = @error_handler.call error + return error if wrapped.nil? || wrapped.equal?(error) + + if error.is_a?(::Gapic::Rest::ResumableUpload::HasResumeHandle) && + !wrapped.is_a?(::Gapic::Rest::ResumableUpload::HasResumeHandle) + wrapped.extend ::Gapic::Rest::ResumableUpload::HasResumeHandle + wrapped.instance_variable_set :@resume_handle, error.resume_handle + end + wrapped + end + + ## + # @private + # Stringifies the keys and values of the initiation headers, which generated clients carry as a + # symbol-keyed metadata hash. + # + # @param headers [Hash, nil] Initiation headers + # @return [Hash{String=>String}] + def stringify_headers headers + (headers || {}).to_h { |key, value| [key.to_s, value.to_s] } + end + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb index 4cf93d2..9161f72 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb @@ -92,6 +92,34 @@ def test_all_entries_share_upload_id_and_pass_method_names assert_includes ["complete_upload_with_data", "complete_upload_finalized"], info_recipes.last end + def test_method_name_prefixes_per_request_log_names + responses = [ + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-URL" => "https://storage.googleapis.com/session?id=123" + }, + "" + ), + FakeResponse.new(200, { "X-Goog-Upload-Status" => "final" }, "done") + ] + + stub = FakeStub.new responses + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("hello world"), + upload_size: 11, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: RecordingLogger.new, + method_name: "create_media_upload" + driver.run + + assert_equal ["create_media_upload.start", "create_media_upload.upload"], stub.method_names + end + def test_multi_chunk_upload_logs_lifecycle_entries recording = RecordingLogger.new run_two_chunk_upload_with_secret recording diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb index b5d73df..2a99c6a 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb @@ -427,4 +427,35 @@ def assert_chunk_request req, offset:, length:, body:, finalize: assert_equal length, metadata["Content-Length"] assert_equal body, req[:body] end + + # `upload_url` reports the session URL whatever the lifecycle status, while `resume_handle` reports one + # only while the upload is still resumable. + def test_upload_url_and_resume_handle_across_statuses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 4 + ) + driver = Driver.new client_stub: FakeClientStub.new([]), config: config + + assert_nil driver.upload_url + + set_driver_status driver, :transmission_sending + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_equal "https://upload.example.com/sess1", driver.resume_handle.upload_url + + [:rejected, :cancelled, :success].each do |status| + set_driver_status driver, status + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_nil driver.resume_handle + end + end + + def set_driver_status driver, status + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: status, upload_url: "https://upload.example.com/sess1") + ) + end end diff --git a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb index da1a692..84b723a 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb @@ -219,4 +219,68 @@ def test_default_data_plane_no_headers_falls_back_to_codes err_no_code = RuntimeError.new "generic network error" refute policy.retry_error?(err_no_code) end + + # ============================================================================ + # SUT: start_retry_policy_for + # ============================================================================ + + def test_start_retry_policy_for_always_carries_the_call_timeout + options = Gapic::CallOptions.new timeout: 17 + + assert_equal({ timeout: 17 }, Gapic::Rest::ResumableUpload.start_retry_policy_for(options)) + end + + def test_start_retry_policy_for_carries_a_nil_timeout + assert_equal({ timeout: nil }, Gapic::Rest::ResumableUpload.start_retry_policy_for(Gapic::CallOptions.new)) + end + + def test_start_retry_policy_for_tolerates_nil_options + assert_equal({ timeout: nil }, Gapic::Rest::ResumableUpload.start_retry_policy_for(nil)) + end + + def test_start_retry_policy_for_copies_only_customized_backoff_settings + options = Gapic::CallOptions.new timeout: 5, retry_policy: { initial_delay: 0.5 } + + overrides = Gapic::Rest::ResumableUpload.start_retry_policy_for options + + assert_equal 0.5, overrides[:initial_delay] + refute overrides.key?(:max_delay) + refute overrides.key?(:multiplier) + end + + def test_start_retry_policy_for_treats_empty_retry_codes_as_unset + options = Gapic::CallOptions.new retry_policy: { initial_delay: 0.5 } + + refute Gapic::Rest::ResumableUpload.start_retry_policy_for(options).key?(:retry_codes) + end + + def test_start_retry_policy_for_copies_retry_codes_when_given + options = Gapic::CallOptions.new retry_policy: { retry_codes: ["UNAVAILABLE"] } + + overrides = Gapic::Rest::ResumableUpload.start_retry_policy_for options + + assert_equal [Gapic::Common::ErrorCodes::ERROR_STRING_MAPPING["UNAVAILABLE"]], overrides[:retry_codes] + end + + # Applying the overrides to the initiation defaults must leave the missing-status-header predicate in + # place: that is the whole reason the conversion returns a Hash rather than a policy object. + def test_start_retry_policy_for_overrides_leave_the_start_predicate_in_place + options = Gapic::CallOptions.new timeout: 5, retry_policy: { initial_delay: 0.5 } + overrides = Gapic::Rest::ResumableUpload.start_retry_policy_for options + + policy = Gapic::Common::RetryPolicy.new(**overrides).apply_defaults RetryPolicies::START_DEFAULTS + + assert_equal 0.5, policy.initial_delay + assert_equal 5, policy.timeout + assert_same RetryPolicies::START_PREDICATE, policy.retry_predicate + end + + def test_start_retry_policy_for_rejects_a_proc_retry_policy + options = Gapic::CallOptions.new retry_policy: ->(_error) { true } + + error = assert_raises ArgumentError do + Gapic::Rest::ResumableUpload.start_retry_policy_for options + end + assert_match(/cannot derive an initiation retry policy/, error.message) + end end diff --git a/gapic-common/test/gapic/rest/resumable_upload/session_test.rb b/gapic-common/test/gapic/rest/resumable_upload/session_test.rb deleted file mode 100644 index fe6e782..0000000 --- a/gapic-common/test/gapic/rest/resumable_upload/session_test.rb +++ /dev/null @@ -1,704 +0,0 @@ -# frozen_string_literal: true - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require "test_helper" -require "gapic/rest/resumable_upload" -require "stringio" - -class SessionTest < Minitest::Test - include Gapic::Rest::ResumableUpload - - FakeResponse = Struct.new :status, :headers, :body, keyword_init: true - - class ScriptedClientStub - attr_reader :requests - - def initialize responses = [] - @responses = responses.dup - @requests = [] - end - - def make_post_request uri:, body:, params:, options:, method_name: nil - @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } - raise "Unexpected request: no scripted response left" if @responses.empty? - - res = @responses.shift - if res.is_a? Proc - res.call - elsif res.is_a? Exception - raise res - else - res - end - end - end - - class UnseekableStream - attr_reader :pos - - def initialize string - @io = StringIO.new string - @pos = 0 - end - - def read length = nil - chunk = @io.read length - @pos += chunk.bytesize if chunk - chunk - end - end - - class StreamWithoutPos - def initialize string - @io = StringIO.new string - end - - def read length = nil - @io.read length - end - end - - START_ONLY_KEYS = [:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy].freeze - - # Builds a session from the shared arguments and remembers the per-run arguments that #start needs, - # so tests can keep calling `start_session session`. - def build_session stub: nil, stream: nil, upload_size: 10, chunk_size: 4, **kwargs - stream ||= StringIO.new "0123456789" - stub ||= ScriptedClientStub.new - @start_args = { - initial_url: "https://example.com/initiate", - initial_body: '{"name":"test.txt"}', - chunk_size: chunk_size - }.merge(kwargs.slice(*START_ONLY_KEYS)) - - Session.new( - client_stub: stub, - stream: stream, - upload_size: upload_size, - **kwargs.except(*START_ONLY_KEYS) - ) - end - - def start_session session, **overrides - session.start(**@start_args, **overrides) - end - - # ============================================================================ - # 1. Initialization and argument validation - # ============================================================================ - - def test_initialize_mandatory_arguments - assert_raises ArgumentError do - Session.new stream: StringIO.new - end - - assert_raises ArgumentError do - Session.new client_stub: ScriptedClientStub.new - end - end - - def test_initialize_rejects_per_run_arguments - assert_raises ArgumentError do - Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new, initial_url: "http://x" - end - - assert_raises ArgumentError do - Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new, chunk_size: 4 - end - end - - def test_initialize_defaults - session = Session.new( - client_stub: ScriptedClientStub.new, - stream: StringIO.new("abc"), - upload_size: 300 - ) - - assert_equal 300, session.upload_size - assert_nil session.content_type - assert_nil session.timeout - assert_nil session.control_plane_retry_policy - assert_nil session.data_plane_retry_policy - assert_nil session.on_progress - assert_nil session.logger - end - - def test_start_without_initial_url_raises_argument_error - session = Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new("abc"), upload_size: 3 - - error = assert_raises ArgumentError do - session.start - end - assert_match(/initial_url/, error.message) - end - - def test_start_with_blank_initial_url_raises_argument_error - session = Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new("abc"), upload_size: 3 - - error = assert_raises ArgumentError do - session.start initial_url: " " - end - assert_match(/initial_url is required/, error.message) - end - - def test_start_with_malformed_retry_policy_raises_argument_error - session = build_session - - error = assert_raises ArgumentError do - start_session session, start_retry_policy: "nonsense" - end - assert_match(/Expected RetryPolicy, Hash, or nil/, error.message) - end - - def test_start_with_reserved_initial_header_raises_before_any_request - stub = ScriptedClientStub.new - session = build_session stub: stub - - error = assert_raises ArgumentError do - start_session session, initial_headers: { "X-Goog-Upload-Header-Content-Type" => "image/png" } - end - assert_match(/must not set protocol header/, error.message) - assert_empty stub.requests - refute session.bound? - end - - def test_failed_start_leaves_session_reusable - session = build_session - - assert_raises ArgumentError do - start_session session, start_retry_policy: "nonsense" - end - - refute session.running? - refute session.bound? - end - - def test_resume_needs_no_initiation_arguments - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"resumed":true}' - ) - ] - session = Session.new( - client_stub: ScriptedClientStub.new(responses), - stream: StringIO.new("01"), - upload_size: 2 - ) - handle = ResumeHandle.new upload_url: "https://upload.example.com/persisted", chunk_size: 4 - - result = session.resume resume_handle: handle - - assert_equal '{"resumed":true}', result - assert_equal "https://upload.example.com/persisted", session.upload_url - end - - # ============================================================================ - # 2. Observable States: Unbound & Bound - # ============================================================================ - - def test_initial_unbound_state - session = build_session - refute session.bound? - assert_nil session.upload_url - assert_nil session.resume_handle - refute session.resumable? - refute session.running? - end - - # ============================================================================ - # 3. Start Lifecycle & Single-Run Contract - # ============================================================================ - - def test_start_successful_upload_transitions_to_bound - responses = [ - # Initiation response - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-url" => "https://upload.example.com/session_1", - "x-goog-upload-chunk-granularity" => "4" - }, - body: "" - ), - # Chunk 1 (0-3) - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "active" }, - body: "" - ), - # Chunk 2 (4-7) - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "active" }, - body: "" - ), - # Final Chunk (8-9) - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"status":"completed"}' - ) - ] - - stub = ScriptedClientStub.new responses - session = build_session stub: stub, upload_size: 10, chunk_size: 4 - - result = start_session session - - assert_equal '{"status":"completed"}', result - assert session.bound? - assert_equal "https://upload.example.com/session_1", session.upload_url - refute session.running? - refute session.resumable? - assert_nil session.resume_handle - end - - def test_second_start_raises_session_state_error - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-url" => "https://upload.example.com/session_1", - "x-goog-upload-chunk-granularity" => "4" - }, - body: "" - ), - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"done":true}' - ) - ] - session = build_session( - stub: ScriptedClientStub.new(responses), - stream: StringIO.new("01"), - upload_size: 2, - chunk_size: 4 - ) - start_session session - - assert session.bound? - - err = assert_raises SessionStateError do - start_session session - end - assert_includes err.message, "Session has already executed a run" - end - - def test_resume_after_start_raises_session_state_error - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-url" => "https://upload.example.com/session_1", - "x-goog-upload-chunk-granularity" => "4" - }, - body: "" - ), - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"done":true}' - ) - ] - session = build_session( - stub: ScriptedClientStub.new(responses), - stream: StringIO.new("01"), - upload_size: 2, - chunk_size: 4 - ) - start_session session - - assert_nil session.resume_handle - refute session.resumable? - - handle = ResumeHandle.new upload_url: "https://upload.example.com/session_1", chunk_size: 4 - err = assert_raises SessionStateError do - session.resume resume_handle: handle - end - assert_includes err.message, "Session has already executed a run" - end - - # ============================================================================ - # 4. Resume Forms: Explicit URL or ResumeHandle - # ============================================================================ - - def test_resume_explicit_url_and_chunk_size_binds_and_executes - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"from_url":true}' - ) - ] - stub = ScriptedClientStub.new responses - session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 - - refute session.bound? - result = session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 - - assert_equal '{"from_url":true}', result - assert session.bound? - assert_equal "https://upload.example.com/direct", session.upload_url - end - - def test_resume_resume_handle_binds_and_executes - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"from_handle":true}' - ) - ] - stub = ScriptedClientStub.new responses - session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 - - handle = ResumeHandle.new upload_url: "https://upload.example.com/from_handle", chunk_size: 4 - - refute session.bound? - result = session.resume resume_handle: handle - - assert_equal '{"from_handle":true}', result - assert session.bound? - assert_equal "https://upload.example.com/from_handle", session.upload_url - end - - def test_start_after_resume_raises_session_state_error - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') - ] - stub = ScriptedClientStub.new responses - session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 - session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 - - assert session.bound? - err = assert_raises SessionStateError do - start_session session - end - assert_includes err.message, "Session has already executed a run" - end - - def test_second_resume_raises_session_state_error - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') - ] - stub = ScriptedClientStub.new responses - session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 - session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 - - assert session.bound? - err = assert_raises SessionStateError do - session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 - end - assert_includes err.message, "Session has already executed a run" - end - - # ============================================================================ - # 5. Argument Shape & Preconditions - # ============================================================================ - - def test_resume_without_arguments_raises_argument_error - session = build_session - err = assert_raises ArgumentError do - session.resume - end - assert_includes err.message, "Must provide either resume_handle or upload_url and chunk_size" - end - - def test_resume_mixing_arguments_raises_argument_error - session = build_session - handle = ResumeHandle.new upload_url: "https://example.com", chunk_size: 4 - - assert_raises ArgumentError do - session.resume resume_handle: handle, upload_url: "https://example.com" - end - - assert_raises ArgumentError do - session.resume resume_handle: handle, chunk_size: 4 - end - - assert_raises ArgumentError do - session.resume upload_url: "https://example.com" - end - - assert_raises ArgumentError do - session.resume chunk_size: 4 - end - - assert_raises ArgumentError do - session.resume handle - end - end - - def test_resume_with_non_zero_stream_pos_raises_argument_error - stream = StringIO.new "0123456789" - stream.seek 4 - - session = build_session stream: stream - handle = ResumeHandle.new upload_url: "https://upload.example.com/from_handle", chunk_size: 4 - - err = assert_raises ArgumentError do - session.resume resume_handle: handle - end - assert_includes err.message, "Stream must be positioned at byte 0 to resume an upload (got pos 4)" - end - - def test_resume_with_stream_without_pos_is_trusted - stream = StreamWithoutPos.new "01" - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') - ] - stub = ScriptedClientStub.new responses - session = build_session stub: stub, stream: stream, upload_size: 2, chunk_size: 4 - - result = session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 - assert_equal '{"ok":true}', result - end - - # ============================================================================ - # 6. Cross-Session Resumption - # ============================================================================ - - def test_cross_session_resumption_from_failed_run - stream = StringIO.new "0123456789" - stub1 = ScriptedClientStub.new [ - # Initiation succeeds - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-url" => "https://upload.example.com/session_cross", - "x-goog-upload-chunk-granularity" => "4" - }, - body: "" - ), - # Chunk 1 returns 503 - FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable"), - # Recovery query fails - Faraday::ConnectionFailed.new("network connection failed") - ] - - session1 = build_session stub: stub1, stream: stream, upload_size: 10, chunk_size: 4 - raised = assert_raises RequestFailedError do - start_session session1 - end - - assert session1.bound? - assert session1.resumable? - handle = session1.resume_handle - refute_nil handle - assert_equal handle, raised.resume_handle - assert_equal "https://upload.example.com/session_cross", handle.upload_url - assert_equal 4, handle.chunk_size - - # Prepare for session 2: rewind the stream to byte 0 - stream.rewind - assert_equal 0, stream.pos - - stub2 = ScriptedClientStub.new [ - # Recovery query on resume: server acknowledges 0 bytes received - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - # Chunk 1 - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "active" }, body: ""), - # Chunk 2 - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "active" }, body: ""), - # Chunk 3 (final) - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"resumed":true}') - ] - - session2 = build_session stub: stub2, stream: stream, upload_size: 10, chunk_size: 4 - refute session2.bound? - - result = session2.resume resume_handle: handle - assert_equal '{"resumed":true}', result - assert session2.bound? - refute session2.resumable? - assert_nil session2.resume_handle - end - - # ============================================================================ - # 7. Concurrency & Running Guard - # ============================================================================ - - def test_running_guard_prevents_concurrent_runs - started_q = Queue.new - unblock_q = Queue.new - - blocking_proc = proc do - started_q.push :started - unblock_q.pop # wait until test signals to proceed - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-url" => "https://upload.example.com/session_block", - "x-goog-upload-chunk-granularity" => "4" - }, - body: "" - ) - end - - stub = ScriptedClientStub.new [ - blocking_proc, - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"done":true}' - ) - ] - session = build_session( - stub: stub, - stream: StringIO.new("01"), - upload_size: 2, - chunk_size: 4 - ) - - worker = Thread.new do - start_session session - end - - started_q.pop # wait for worker thread to enter driver.run - assert session.running? - - # Concurrent call from another thread raises SessionStateError - handle = ResumeHandle.new upload_url: "https://upload.example.com/session_block", chunk_size: 4 - err = assert_raises SessionStateError do - session.resume resume_handle: handle - end - assert_includes err.message, "A run is already in progress for this session" - - err_start = assert_raises SessionStateError do - start_session session - end - assert_includes err_start.message, "A run is already in progress for this session" - - # Unblock worker thread - unblock_q.push :continue - result = worker.value - - assert_equal '{"done":true}', result - refute session.running? - end - - # ============================================================================ - # 8. Driver#upload_url Direct Verification - # ============================================================================ - - def test_driver_upload_url_across_statuses - dummy_client = ScriptedClientStub.new - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("data"), - upload_size: 4, - chunk_size: 4 - ) - driver = Driver.new client_stub: dummy_client, config: config - - assert_nil driver.upload_url - - # Active - driver.core.instance_variable_set( - :@state, - driver.core.state.with(status: :transmission_sending, upload_url: "https://upload.example.com/sess1") - ) - assert_equal "https://upload.example.com/sess1", driver.upload_url - assert_equal "https://upload.example.com/sess1", driver.resume_handle.upload_url - - # Rejected (resume_handle is nil, but upload_url remains readable) - driver.core.instance_variable_set( - :@state, - driver.core.state.with(status: :rejected, upload_url: "https://upload.example.com/sess1") - ) - assert_equal "https://upload.example.com/sess1", driver.upload_url - assert_nil driver.resume_handle - - # Cancelled (resume_handle is nil, but upload_url remains readable) - driver.core.instance_variable_set( - :@state, - driver.core.state.with(status: :cancelled, upload_url: "https://upload.example.com/sess1") - ) - assert_equal "https://upload.example.com/sess1", driver.upload_url - assert_nil driver.resume_handle - - # Success (resume_handle is nil, but upload_url remains readable) - driver.core.instance_variable_set( - :@state, - driver.core.state.with(status: :success, upload_url: "https://upload.example.com/sess1") - ) - assert_equal "https://upload.example.com/sess1", driver.upload_url - assert_nil driver.resume_handle - end -end diff --git a/gapic-common/test/gapic/resumable_upload_test.rb b/gapic-common/test/gapic/resumable_upload_test.rb new file mode 100644 index 0000000..5adafe9 --- /dev/null +++ b/gapic-common/test/gapic/resumable_upload_test.rb @@ -0,0 +1,625 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest" +require "stringio" + +class ResumableUploadTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + INITIAL_URL = "https://example.com/initiate" + INITIAL_BODY = '{"name":"test.txt"}' + SESSION_URL = "https://upload.example.com/session_1" + + class ScriptedClientStub + attr_reader :requests + + def initialize responses = [] + @responses = responses.dup + @requests = [] + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } + raise "Unexpected request: no scripted response left" if @responses.empty? + + res = @responses.shift + if res.is_a? Proc + res.call + elsif res.is_a? Exception + raise res + else + res + end + end + end + + # Stream that reports no position at all, which a resumed run has to trust. + class StreamWithoutPos + def initialize string + @io = StringIO.new string + end + + def read length = nil + @io.read length + end + end + + # Stream that records whether anything read from it, to pin down when a run first touches it. + class WatchedStream + attr_reader :reads + + def initialize string + @io = StringIO.new string + @reads = 0 + end + + def pos + @io.pos + end + + def read length = nil + @reads += 1 + @io.read length + end + end + + def setup + @proc_calls = [] + end + + # ============================================================================ + # Helpers + # ============================================================================ + + def build_upload stub: nil, response_type: nil, initial_request: nil, **kwargs + @stub = stub || ScriptedClientStub.new + Gapic::ResumableUpload.new( + client_stub_proc: lambda { + @proc_calls << :client_stub + @stub + }, + initial_request_proc: lambda { + @proc_calls << :initial_request + initial_request || [INITIAL_URL, INITIAL_BODY] + }, + response_type: response_type, + **kwargs + ) + end + + def start_upload upload, stream: nil, upload_size: 10, chunk_size: 4, **overrides + upload.start stream: stream || StringIO.new("0123456789"), + upload_size: upload_size, + chunk_size: chunk_size, + **overrides + end + + def driver_of upload + upload.instance_variable_get :@driver + end + + def config_of upload + driver_of(upload).instance_variable_get :@config + end + + def initiation_response url: SESSION_URL, granularity: 4 + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => url, + "x-goog-upload-chunk-granularity" => granularity.to_s + }, + body: "" + ) + end + + def query_response received: 0 + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active", "x-goog-upload-size-received" => received.to_s }, + body: "" + ) + end + + def chunk_response + FakeResponse.new status: 200, headers: { "x-goog-upload-status" => "active" }, body: "" + end + + def final_response body = '{"text":"done"}' + FakeResponse.new status: 200, headers: { "x-goog-upload-status" => "final" }, body: body + end + + # Initiation plus a single finalizing chunk, enough for a two-byte stream. + def short_upload_responses body: '{"text":"done"}' + [initiation_response, final_response(body)] + end + + # Recovery query plus a single finalizing chunk, enough to resume a two-byte stream. + def short_resume_responses body: '{"text":"done"}' + [query_response, final_response(body)] + end + + def resume_handle_for url: SESSION_URL, chunk_size: 4 + ResumeHandle.new upload_url: url, chunk_size: chunk_size + end + + # ============================================================================ + # 1. Deferred procs + # ============================================================================ + + def test_start_calls_client_stub_proc_then_initial_request_proc + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal [:client_stub, :initial_request], @proc_calls + end + + def test_resume_never_calls_initial_request_proc + upload = build_upload stub: ScriptedClientStub.new(short_resume_responses) + + upload.resume stream: StringIO.new("01"), upload_size: 2, resume_handle: resume_handle_for + + assert_equal [:client_stub], @proc_calls + end + + def test_raising_client_stub_proc_surfaces_before_the_stream_is_read + stream = WatchedStream.new "0123456789" + upload = Gapic::ResumableUpload.new( + client_stub_proc: -> { raise ArgumentError, "REST is unavailable" }, + initial_request_proc: -> { [INITIAL_URL, INITIAL_BODY] }, + response_type: nil + ) + + error = assert_raises ArgumentError do + upload.start stream: stream, upload_size: 10 + end + + assert_equal "REST is unavailable", error.message + assert_equal 0, stream.reads + refute upload.running? + assert_nil driver_of(upload) + end + + def test_resume_rejects_a_non_zero_stream_before_calling_the_client_stub_proc + stream = StringIO.new "0123456789" + stream.seek 4 + upload = build_upload + + error = assert_raises ArgumentError do + upload.resume stream: stream, resume_handle: resume_handle_for + end + + assert_includes error.message, "Stream must be positioned at byte 0 to resume an upload (got pos 4)" + assert_empty @proc_calls + end + + # ============================================================================ + # 2. Configuration construction + # ============================================================================ + + def test_start_builds_a_start_config_from_constructor_and_run_arguments + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses), + initial_headers: { "X-Goog-Test" => "yes", :symbol_key => 7 }, + start_retry_policy: { initial_delay: 0.01 } + + start_upload upload, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4, + content_type: "text/plain", upload_timeout: 42 + + config = config_of upload + assert_instance_of StartUploadConfig, config + assert_equal INITIAL_URL, config.initial_url + assert_equal INITIAL_BODY, config.initial_body + assert_equal({ "X-Goog-Test" => "yes", "symbol_key" => "7" }, config.initial_headers) + assert_equal 4, config.chunk_size + assert_equal({ initial_delay: 0.01 }, config.start_retry_policy) + assert_equal 2, config.upload_size + assert_equal "text/plain", config.content_type + assert_equal 42, config.timeout + end + + def test_resume_builds_a_resume_config_from_the_handle + upload = build_upload stub: ScriptedClientStub.new(short_resume_responses) + handle = resume_handle_for url: "https://upload.example.com/persisted", chunk_size: 8 + + upload.resume stream: StringIO.new("01"), upload_size: 2, resume_handle: handle + + config = config_of upload + assert_instance_of ResumeUploadConfig, config + assert_equal "https://upload.example.com/persisted", config.upload_url + assert_equal 8, config.chunk_size + end + + def test_upload_timeout_is_omitted_when_unset + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_nil config_of(upload).timeout + end + + def test_plane_retry_policies_come_from_the_constructor + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses), + control_plane_retry_policy: { initial_delay: 0.02 }, + data_plane_retry_policy: { initial_delay: 0.03 } + + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + config = config_of upload + assert_equal({ initial_delay: 0.02 }, config.control_plane_retry_policy) + assert_equal({ initial_delay: 0.03 }, config.data_plane_retry_policy) + end + + def test_reserved_initial_header_raises_before_any_request + stub = ScriptedClientStub.new + upload = build_upload stub: stub, initial_headers: { "X-Goog-Upload-Header-Content-Type" => "image/png" } + + error = assert_raises ArgumentError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + assert_match(/must not set protocol header/, error.message) + assert_empty stub.requests + assert_nil driver_of(upload) + end + + # ============================================================================ + # 3. Lifecycle + # ============================================================================ + + def test_readers_before_the_first_run + upload = build_upload + + assert_nil upload.resume_handle + refute upload.resumable? + refute upload.running? + end + + def test_successful_run_leaves_nothing_to_resume + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + + result = start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal '{"text":"done"}', result + refute upload.running? + refute upload.resumable? + assert_nil upload.resume_handle + end + + def test_handle_is_reusable_across_runs + responses = short_upload_responses(body: '{"text":"first"}') + + short_upload_responses(body: '{"text":"second"}') + upload = build_upload stub: ScriptedClientStub.new(responses) + + assert_equal '{"text":"first"}', start_upload(upload, stream: StringIO.new("01"), upload_size: 2) + first_driver = driver_of upload + + assert_equal '{"text":"second"}', start_upload(upload, stream: StringIO.new("01"), upload_size: 2) + refute_same first_driver, driver_of(upload) + end + + def test_concurrent_run_raises_session_state_error + started_q = Queue.new + unblock_q = Queue.new + blocking_proc = proc do + started_q.push :started + unblock_q.pop + initiation_response + end + upload = build_upload stub: ScriptedClientStub.new([blocking_proc, final_response]) + + worker = Thread.new { start_upload upload, stream: StringIO.new("01"), upload_size: 2 } + started_q.pop + assert upload.running? + + error = assert_raises SessionStateError do + upload.resume stream: StringIO.new("01"), resume_handle: resume_handle_for + end + assert_includes error.message, "A run is already in progress for this upload" + + assert_raises SessionStateError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + unblock_q.push :continue + assert_equal '{"text":"done"}', worker.value + refute upload.running? + end + + def test_run_slot_is_released_after_a_failed_run + stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), + Faraday::ConnectionFailed.new("boom")] + upload = build_upload stub: stub + + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + refute upload.running? + assert upload.resumable? + end + + def test_run_slot_is_released_after_a_progress_callback_raises + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + on_progress = ->(_progress) { raise "callback exploded" } + + error = assert_raises RuntimeError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2, on_progress: on_progress + end + + assert_equal "callback exploded", error.message + refute upload.running? + end + + def test_run_slot_is_released_after_a_configuration_error + upload = build_upload start_retry_policy: "nonsense" + + assert_raises ArgumentError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + refute upload.running? + assert_nil driver_of(upload) + end + + def test_configuration_error_leaves_an_earlier_runs_resume_handle_intact + stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), + Faraday::ConnectionFailed.new("boom")] + upload = build_upload stub: stub + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + handle = upload.resume_handle + refute_nil handle + + assert_raises ArgumentError do + upload.resume stream: StringIO.new("01"), upload_size: 2, resume_handle: resume_handle_for(chunk_size: -1) + end + + assert_equal handle, upload.resume_handle + end + + # ============================================================================ + # 4. Resume forms + # ============================================================================ + + def test_resume_with_an_explicit_handle + upload = build_upload stub: ScriptedClientStub.new(short_resume_responses(body: '{"text":"resumed"}')) + handle = resume_handle_for url: "https://upload.example.com/persisted", chunk_size: 4 + + result = upload.resume stream: StringIO.new("01"), upload_size: 2, resume_handle: handle + + assert_equal '{"text":"resumed"}', result + assert_equal "https://upload.example.com/persisted", @stub.requests.first[:uri] + end + + def test_bare_resume_reuses_the_retained_drivers_handle + stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), + Faraday::ConnectionFailed.new("boom"), + query_response, final_response('{"text":"resumed"}')] + upload = build_upload stub: stub + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + assert upload.resumable? + + result = upload.resume stream: StringIO.new("01"), upload_size: 2 + + assert_equal '{"text":"resumed"}', result + assert_equal SESSION_URL, stub.requests.last[:uri] + end + + def test_bare_resume_without_a_previous_run_raises_argument_error + upload = build_upload + + error = assert_raises ArgumentError do + upload.resume stream: StringIO.new("01") + end + + assert_includes error.message, "No upload to resume" + end + + def test_bare_resume_after_a_successful_run_raises_argument_error + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_raises ArgumentError do + upload.resume stream: StringIO.new("01") + end + end + + def test_bare_resume_after_an_unresumable_failure_raises_argument_error + upload = build_upload stub: ScriptedClientStub.new([Faraday::ConnectionFailed.new("boom")]) + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + refute upload.resumable? + + assert_raises ArgumentError do + upload.resume stream: StringIO.new("01") + end + end + + def test_resume_trusts_a_stream_that_reports_no_position + upload = build_upload stub: ScriptedClientStub.new(short_resume_responses) + + result = upload.resume stream: StreamWithoutPos.new("01"), + upload_size: 2, + resume_handle: resume_handle_for + + assert_equal '{"text":"done"}', result + end + + # ============================================================================ + # 5. Response decoding + # ============================================================================ + + def test_decodes_the_final_body_into_the_response_type + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: '{"text":"decoded"}')), + response_type: Gapic::Examples::Post + + result = start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_instance_of Gapic::Examples::Post, result + assert_equal "decoded", result.text + end + + def test_decodes_an_empty_final_body_into_an_empty_message + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: "")), + response_type: Gapic::Examples::Post + + result = start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal Gapic::Examples::Post.new, result + end + + def test_decodes_an_absent_final_body_into_an_empty_message + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: nil)), + response_type: Gapic::Examples::Post + + result = start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal Gapic::Examples::Post.new, result + end + + def test_malformed_final_body_raises_a_parse_error + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: '{"text":')), + response_type: Gapic::Examples::Post + + assert_raises Google::Protobuf::ParseError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + end + + def test_ignores_unknown_fields_in_the_final_body + body = '{"text":"decoded","unknown_field":true}' + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: body)), + response_type: Gapic::Examples::Post + + assert_equal "decoded", start_upload(upload, stream: StringIO.new("01"), upload_size: 2).text + end + + def test_no_response_type_returns_the_raw_body + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: "not json at all")) + + assert_equal "not json at all", start_upload(upload, stream: StringIO.new("01"), upload_size: 2) + end + + def test_no_response_type_returns_nil_for_a_bodiless_response + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: nil)) + + assert_nil start_upload(upload, stream: StringIO.new("01"), upload_size: 2) + end + + # ============================================================================ + # 6. Error handling + # ============================================================================ + + class WrappedError < StandardError; end + + def failing_upload **kwargs + stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), + Faraday::ConnectionFailed.new("boom")] + build_upload stub: stub, **kwargs + end + + def test_error_handler_replaces_the_raised_error + upload = failing_upload error_handler: ->(e) { WrappedError.new "wrapped: #{e.message}" } + + error = assert_raises WrappedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + assert_includes error.message, "wrapped:" + end + + def test_wrapped_error_remains_rescuable_as_has_resume_handle + upload = failing_upload error_handler: ->(_e) { WrappedError.new "wrapped" } + + error = assert_raises HasResumeHandle do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + assert_instance_of WrappedError, error + assert_equal upload.resume_handle, error.resume_handle + assert_equal SESSION_URL, error.resume_handle.upload_url + end + + def test_error_handler_returning_nil_reraises_the_original + upload = failing_upload error_handler: ->(_e) { nil } + + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + end + + def test_error_handler_returning_the_original_reraises_it + upload = failing_upload error_handler: ->(e) { e } + + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + end + + def test_an_error_without_a_resume_handle_is_not_decorated + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses), + error_handler: ->(_e) { WrappedError.new "wrapped" } + on_progress = ->(_progress) { raise "callback exploded" } + + error = assert_raises WrappedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2, on_progress: on_progress + end + + refute error.is_a?(HasResumeHandle) + end + + def test_no_error_handler_propagates_the_protocol_error + upload = failing_upload + + error = assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + assert_equal upload.resume_handle, error.resume_handle + end + + # ============================================================================ + # 7. Logging + # ============================================================================ + + def test_method_name_reaches_the_client_stub + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses), method_name: "create_media_upload" + + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal ["create_media_upload.start", "create_media_upload.upload"], + @stub.requests.map { |request| request[:method_name] } + end + + def test_method_name_defaults_to_the_protocol_name + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal ["ResumableUpload.start", "ResumableUpload.upload"], + @stub.requests.map { |request| request[:method_name] } + end +end From 01f19fd9259d8e457bfee93eb60633aebebabcc7 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Fri, 18 Sep 2026 06:07:30 +0000 Subject: [PATCH 04/22] docs(resumable-upload): fix stale resume example and document handle replacement Show the coordinator alongside the driver in the architecture and test-plan diagrams. --- gapic-common/design/resumable_upload/implementation-guide.md | 4 +++- .../design/resumable_upload/integration-test-plan.md | 5 ++++- gapic-common/lib/gapic/rest/resumable_upload/errors.rb | 2 +- gapic-common/lib/gapic/resumable_upload.rb | 5 +++++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md index 8ca3f1a..e834395 100644 --- a/gapic-common/design/resumable_upload/implementation-guide.md +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -6,7 +6,9 @@ The Resumable Upload Protocol (RUP) implementation in `gapic-common` is structur ```mermaid graph TD - Client[Client Code] -->|StartUploadConfig| Driver + Client[Client Code] -->|start / resume| Coordinator["::Gapic::ResumableUpload
Coordinator"] + Coordinator -->|StartUploadConfig / ResumeUploadConfig| Driver + Driver -->|ResumeHandle, decoded response| Coordinator subgraph Gapic::Rest::ResumableUpload Driver[Driver
Synchronous I/O Adapter] -->|Events| Core[Core
State Container] Core -->|Instructions| Driver diff --git a/gapic-common/design/resumable_upload/integration-test-plan.md b/gapic-common/design/resumable_upload/integration-test-plan.md index b3c5115..7052026 100644 --- a/gapic-common/design/resumable_upload/integration-test-plan.md +++ b/gapic-common/design/resumable_upload/integration-test-plan.md @@ -20,6 +20,7 @@ flowchart TD end subgraph SUT["System Under Test"] + Upload["::Gapic::ResumableUpload
(Coordinator)"] Driver["Gapic::Rest::ResumableUpload::Driver"] Stub["Gapic::Rest::ClientStub
(raise_faraday_errors: false)"] end @@ -33,7 +34,9 @@ flowchart TD Toys --> BaseClass BaseClass --> PayloadGen BaseClass --> TraceLog - BaseClass --> Driver + BaseClass -->|"build_upload / start_args / resume_args"| Upload + BaseClass -->|"build_config (protocol-detail suites)"| Driver + Upload -->|"one per run"| Driver Driver --> Stub Stub <-->|"HTTP POST / PUT (REST)"| Showcase ``` diff --git a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb index 17141be..a6bb727 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb @@ -149,7 +149,7 @@ def build_from_http_event event, prefix: # # @example # begin - # session.start initial_url: url + # upload.start stream: io, upload_size: size # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e # retry_later e.resume_handle if e.resume_handle # raise diff --git a/gapic-common/lib/gapic/resumable_upload.rb b/gapic-common/lib/gapic/resumable_upload.rb index c65b717..863cb89 100644 --- a/gapic-common/lib/gapic/resumable_upload.rb +++ b/gapic-common/lib/gapic/resumable_upload.rb @@ -310,6 +310,11 @@ def resume stream:, # `nil` before the first run, and after any run that left nothing to resume: a completed upload is # finalized, and rejected and cancelled uploads are too. # + # Each run replaces this value rather than accumulating handles, so it always describes the most + # recent one. Starting a second upload therefore discards whatever the previous run left behind — + # persist the handle first if the earlier upload still matters. A call that fails while building its + # configuration does not count as a run: it never reaches a driver, so the earlier value survives. + # # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] def resume_handle @mutex.synchronize { @driver&.resume_handle } From 47d1acbdb62722e2bfa7d8801f8e143456b7e4cc Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Fri, 18 Sep 2026 06:07:54 +0000 Subject: [PATCH 05/22] test(resumable-upload): pin handle replacement and inject a state double in driver_test Replaces the core.instance_variable_set poke with a Core double passed through the designed test seam. --- .../rest/resumable_upload/driver_test.rb | 27 +++++++++++-------- .../test/gapic/resumable_upload_test.rb | 20 ++++++++++++++ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb index 2a99c6a..1e902c0 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb @@ -428,8 +428,19 @@ def assert_chunk_request req, offset:, length:, body:, finalize: assert_equal body, req[:body] end + # Core double holding a fixed State, for exercising the Driver readers without running the protocol. + # `Core` offers no way to set a state, so the readers are driven through `Driver.new(core:)` instead. + class StateCore + attr_accessor :state + + def initialize state + @state = state + end + end + # `upload_url` reports the session URL whatever the lifecycle status, while `resume_handle` reports one - # only while the upload is still resumable. + # only while the upload is still resumable. The status-by-status truth table lives in the Rules tests; + # this asserts that the Driver readers delegate to the current state. def test_upload_url_and_resume_handle_across_statuses config = StartUploadConfig.new( initial_url: "https://example.com/upload", @@ -437,25 +448,19 @@ def test_upload_url_and_resume_handle_across_statuses upload_size: 4, chunk_size: 4 ) - driver = Driver.new client_stub: FakeClientStub.new([]), config: config + core = StateCore.new Core.new(config).state + driver = Driver.new client_stub: FakeClientStub.new([]), config: config, core: core assert_nil driver.upload_url - set_driver_status driver, :transmission_sending + core.state = core.state.with status: :transmission_sending, upload_url: "https://upload.example.com/sess1" assert_equal "https://upload.example.com/sess1", driver.upload_url assert_equal "https://upload.example.com/sess1", driver.resume_handle.upload_url [:rejected, :cancelled, :success].each do |status| - set_driver_status driver, status + core.state = core.state.with status: status assert_equal "https://upload.example.com/sess1", driver.upload_url assert_nil driver.resume_handle end end - - def set_driver_status driver, status - driver.core.instance_variable_set( - :@state, - driver.core.state.with(status: status, upload_url: "https://upload.example.com/sess1") - ) - end end diff --git a/gapic-common/test/gapic/resumable_upload_test.rb b/gapic-common/test/gapic/resumable_upload_test.rb index 5adafe9..ac465de 100644 --- a/gapic-common/test/gapic/resumable_upload_test.rb +++ b/gapic-common/test/gapic/resumable_upload_test.rb @@ -397,6 +397,26 @@ def test_configuration_error_leaves_an_earlier_runs_resume_handle_intact assert_equal handle, upload.resume_handle end + def test_a_second_run_replaces_the_previous_runs_resume_handle + stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), + Faraday::ConnectionFailed.new("boom"), + Faraday::ConnectionFailed.new("boom")] + upload = build_upload stub: stub + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + refute_nil upload.resume_handle + + # The second run fails at initiation, so it establishes no upload of its own. Its driver still takes + # over the reader: the previous run's handle is replaced, not kept as a fallback. + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + assert_nil upload.resume_handle + refute upload.resumable? + end + # ============================================================================ # 4. Resume forms # ============================================================================ From e656e82d466bdd22df91600170ce0e9d27ed23cb Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Fri, 18 Sep 2026 06:08:14 +0000 Subject: [PATCH 06/22] refactor(common): add RetryPolicy#overrides and derive initiation retries from it Comparing readers against a pristine policy dropped a caller's setting whenever it happened to equal the library default. #overrides reports only what was explicitly supplied, so start_retry_policy_for collapses to a merge and BACKOFF_DEFAULTS goes away. --- gapic-common/lib/gapic/common/retry_policy.rb | 34 +++++++++++++++++++ .../lib/gapic/rest/resumable_upload.rb | 34 ++++--------------- .../test/gapic/common/retry_policy_test.rb | 30 ++++++++++++++++ .../resumable_upload/retry_policies_test.rb | 17 ++++++++++ 4 files changed, 87 insertions(+), 28 deletions(-) diff --git a/gapic-common/lib/gapic/common/retry_policy.rb b/gapic-common/lib/gapic/common/retry_policy.rb index e8420b5..eb31992 100644 --- a/gapic-common/lib/gapic/common/retry_policy.rb +++ b/gapic-common/lib/gapic/common/retry_policy.rb @@ -19,6 +19,12 @@ module Common ## # Gapic Common retry policy base class. # + # A policy distinguishes "set to this value" from "never set". Every setting is stored as `nil` + # until someone supplies it, and the reader for each substitutes the corresponding `DEFAULT_` + # constant on the way out, so a reader can never say which of the two happened. That distinction is + # what lets {#apply_defaults} fill in gaps without overwriting a caller's choices, and {#overrides} + # report what a caller actually asked for. + # class RetryPolicy # @return [Numeric] Default initial delay in seconds. DEFAULT_INITIAL_DELAY = 1 @@ -111,6 +117,34 @@ def retry_predicate @retry_predicate end + ## + # @private + # The settings this policy actually carries, as keyword arguments for {RetryPolicy.initialize}. + # + # A key is present only if that setting was explicitly supplied; a key that was never set is + # absent rather than `nil`. This is what the readers cannot tell you — {#max_delay} returns + # {DEFAULT_MAX_DELAY} whether the caller chose that number or said nothing — so it is the only + # safe way to carry one policy's settings onto another without dragging defaults along, and + # without mistaking a deliberate choice that happens to equal a default for silence. + # + # An empty `retry_codes` list counts as unset. It retries nothing, which is exactly what an + # unset list does, so there is nothing for it to carry. + # + # @return [Hash{Symbol=>Object}] Explicitly set settings only + def overrides + # Assigns nil, and so omits the key, when the list is absent or empty. + retry_codes = @retry_codes unless @retry_codes.nil? || @retry_codes.empty? + { + initial_delay: @initial_delay, + max_delay: @max_delay, + multiplier: @multiplier, + retry_codes: retry_codes, + timeout: @timeout, + jitter: @jitter, + retry_predicate: @retry_predicate + }.compact + end + ## # Returns a duplicate in a non-executing state, i.e. with the deadline # and current delay reset. diff --git a/gapic-common/lib/gapic/rest/resumable_upload.rb b/gapic-common/lib/gapic/rest/resumable_upload.rb index 2dfd37d..054a365 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload.rb @@ -63,21 +63,6 @@ module Rest # is in flight (final). # module ResumableUpload - ## - # @private - # Backoff settings carried from a caller's retry policy into the initiation policy, mapped to the - # {Gapic::Common::RetryPolicy} value a reader returns when the setting was never set. - # - # `jitter`'s default is a private constant, so it is read off a default policy rather than named. - # - # @return [Hash{Symbol=>Numeric}] - BACKOFF_DEFAULTS = { - initial_delay: Gapic::Common::RetryPolicy::DEFAULT_INITIAL_DELAY, - max_delay: Gapic::Common::RetryPolicy::DEFAULT_MAX_DELAY, - multiplier: Gapic::Common::RetryPolicy::DEFAULT_MULTIPLIER, - jitter: Gapic::Common::RetryPolicy.new.jitter - }.freeze - ## # @private # Converts the per-call options a generated client assembles into overrides for the initiation @@ -93,31 +78,24 @@ module ResumableUpload # {Gapic::Common::RetryPolicy::DEFAULT_TIMEOUT} (one hour), because `Gapic::CallOptions::RetryPolicy` # never populates `@timeout` even though it subclasses {Gapic::Common::RetryPolicy}. # - # Backoff settings and retry codes are copied only where the caller set them, which is why each is - # compared against the corresponding default: a reader on an unset policy returns that default, and - # the initiation defaults are the values that should survive. An empty `retry_codes` list counts as - # unset. A `retry_predicate` is deliberately not copied; the initiation predicate stays in place. + # Everything else the caller set is carried across as-is, by asking the policy what it carries + # ({Gapic::Common::RetryPolicy#overrides}) rather than inferring it from the readers. Two settings + # are dropped: the policy's own `timeout`, which is not the initiation deadline and would otherwise + # displace the call's, and `retry_predicate`, so the initiation predicate stays in place. # # @param options [Gapic::CallOptions, nil] Per-call options from a generated client method # @return [Hash] Overrides for the initiation retry policy # @raise [ArgumentError] If the call options carry a Proc (or any other non-{Gapic::Common::RetryPolicy}) # retry policy, which has no coherent meaning across the three retry planes of an upload def self.start_retry_policy_for options - overrides = { timeout: options&.timeout } policy = options&.retry_policy - return overrides if policy.nil? - unless policy.is_a? Gapic::Common::RetryPolicy + if policy && !policy.is_a?(Gapic::Common::RetryPolicy) raise ArgumentError, "Resumable upload cannot derive an initiation retry policy from a #{policy.class}; " \ "use a Gapic::Common::RetryPolicy or a Hash of retry settings" end - overrides[:retry_codes] = policy.retry_codes unless policy.retry_codes.empty? - BACKOFF_DEFAULTS.each do |setting, default| - value = policy.public_send setting - overrides[setting] = value unless value == default - end - overrides + { timeout: options&.timeout }.merge(policy&.overrides&.except(:timeout, :retry_predicate) || {}) end end end diff --git a/gapic-common/test/gapic/common/retry_policy_test.rb b/gapic-common/test/gapic/common/retry_policy_test.rb index 0455e94..56ade89 100644 --- a/gapic-common/test/gapic/common/retry_policy_test.rb +++ b/gapic-common/test/gapic/common/retry_policy_test.rb @@ -158,5 +158,35 @@ def test_retry_predicate_returns_nil_fallback retriable_error = GRPC::BadStatus.new GRPC::Core::StatusCodes::UNAVAILABLE, "unavailable" assert retry_policy.call(retriable_error) end + + def test_overrides_reports_every_setting_the_caller_supplied + predicate = ->(_error) { true } + retry_policy = Gapic::Common::RetryPolicy.new( + initial_delay: 2, max_delay: 20, multiplier: 1.7, + retry_codes: [GRPC::Core::StatusCodes::UNAVAILABLE], timeout: 600, jitter: 0, + retry_predicate: predicate + ) + + expected = { + initial_delay: 2, + max_delay: 20, + multiplier: 1.7, + retry_codes: [GRPC::Core::StatusCodes::UNAVAILABLE], + timeout: 600, + jitter: 0, + retry_predicate: predicate + } + assert_equal expected, retry_policy.overrides + end + + # An empty retry_codes list counts as unset: it retries nothing, exactly as silence does. + def test_overrides_omits_settings_the_caller_never_supplied + retry_policy = Gapic::Common::RetryPolicy.new retry_codes: [] + + assert_empty retry_policy.overrides + # The readers substitute defaults, which is precisely what overrides must not report. + assert_equal Gapic::Common::RetryPolicy::DEFAULT_MAX_DELAY, retry_policy.max_delay + assert_equal Gapic::Common::RetryPolicy::DEFAULT_RETRY_CODES, retry_policy.retry_codes + end end diff --git a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb index 84b723a..ebcb04a 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb @@ -283,4 +283,21 @@ def test_start_retry_policy_for_rejects_a_proc_retry_policy end assert_match(/cannot derive an initiation retry policy/, error.message) end + + # A setting the caller chose deliberately carries across even when it happens to equal the library + # default. Asking the policy what it holds, rather than comparing its readers against a default + # policy, is what makes that possible. + def test_start_retry_policy_for_carries_settings_that_equal_the_library_defaults + options = Gapic::CallOptions.new retry_policy: { + initial_delay: Gapic::Common::RetryPolicy::DEFAULT_INITIAL_DELAY, + max_delay: Gapic::Common::RetryPolicy::DEFAULT_MAX_DELAY, + multiplier: Gapic::Common::RetryPolicy::DEFAULT_MULTIPLIER + } + + overrides = Gapic::Rest::ResumableUpload.start_retry_policy_for options + + assert_equal Gapic::Common::RetryPolicy::DEFAULT_INITIAL_DELAY, overrides[:initial_delay] + assert_equal Gapic::Common::RetryPolicy::DEFAULT_MAX_DELAY, overrides[:max_delay] + assert_equal Gapic::Common::RetryPolicy::DEFAULT_MULTIPLIER, overrides[:multiplier] + end end From 2ec24b382c8f2ae70d33f9389b380694fc949463 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Fri, 18 Sep 2026 06:08:58 +0000 Subject: [PATCH 07/22] chore: ignore the temporary Gemfiles toys leaves behind --- gapic-common/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gapic-common/.gitignore b/gapic-common/.gitignore index 082d9f2..f70038d 100644 --- a/gapic-common/.gitignore +++ b/gapic-common/.gitignore @@ -15,3 +15,7 @@ vendor *~ .idea *.iml + +# Modified Gemfiles toys writes while running a step; bundler re-creates the lock +# just after toys deletes it, so a stray one is left behind by every `toys ci`. +.toys-tmp-gemfile-* From 4d1d83f5d9cb4a35654788c924be4f05b786c3c2 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Fri, 18 Sep 2026 06:11:24 +0000 Subject: [PATCH 08/22] refactor(resumable-upload): let call options override the initiation retry predicate Only the call timeout is now imposed unconditionally; every other setting the caller carries, including a retry predicate, reaches the initiation policy. --- .../lib/gapic/rest/resumable_upload.rb | 21 ++++++++++++------- .../resumable_upload/retry_policies_test.rb | 20 ++++++++++++++++++ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/gapic-common/lib/gapic/rest/resumable_upload.rb b/gapic-common/lib/gapic/rest/resumable_upload.rb index 054a365..8aa0b42 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload.rb @@ -73,15 +73,20 @@ module ResumableUpload # predicate that treats a response missing `X-Goog-Upload-Status` as retriable gateway noise, and # handing over an object would silently drop it. # - # `timeout` is always set, and becomes the local deadline of the initiation request alone — the - # whole-upload budget is separate and is not derived here. Without it, initiation would inherit - # {Gapic::Common::RetryPolicy::DEFAULT_TIMEOUT} (one hour), because `Gapic::CallOptions::RetryPolicy` - # never populates `@timeout` even though it subclasses {Gapic::Common::RetryPolicy}. + # `timeout` is the one setting the caller cannot express here: it is set unconditionally from the + # call's own timeout and becomes the local deadline of the initiation request alone — the + # whole-upload budget is separate and is not derived here. A `timeout` inside the caller's retry + # policy is inert at the call layer (`Gapic::CallOptions::RetryPolicy` never populates `@timeout`, + # and `RpcCall` deadlines on `CallOptions#timeout`), so honouring it here would invent a meaning it + # has nowhere else. With no call timeout, initiation falls back to + # {Gapic::Common::RetryPolicy::DEFAULT_TIMEOUT} — one hour. # # Everything else the caller set is carried across as-is, by asking the policy what it carries - # ({Gapic::Common::RetryPolicy#overrides}) rather than inferring it from the readers. Two settings - # are dropped: the policy's own `timeout`, which is not the initiation deadline and would otherwise - # displace the call's, and `retry_predicate`, so the initiation predicate stays in place. + # ({Gapic::Common::RetryPolicy#overrides}) rather than inferring it from the readers, so a choice + # that happens to equal a library default still lands. That includes `retry_predicate`: a caller + # who supplies one **replaces** the initiation predicate rather than composing with it, and so + # gives up the missing-`X-Goog-Upload-Status` handling. There is no chaining — a predicate that + # returns `nil` falls through to `retry_codes`, not to the predicate it displaced. # # @param options [Gapic::CallOptions, nil] Per-call options from a generated client method # @return [Hash] Overrides for the initiation retry policy @@ -95,7 +100,7 @@ def self.start_retry_policy_for options "use a Gapic::Common::RetryPolicy or a Hash of retry settings" end - { timeout: options&.timeout }.merge(policy&.overrides&.except(:timeout, :retry_predicate) || {}) + (policy&.overrides || {}).merge timeout: options&.timeout end end end diff --git a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb index ebcb04a..bd732fe 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb @@ -300,4 +300,24 @@ def test_start_retry_policy_for_carries_settings_that_equal_the_library_defaults assert_equal Gapic::Common::RetryPolicy::DEFAULT_MAX_DELAY, overrides[:max_delay] assert_equal Gapic::Common::RetryPolicy::DEFAULT_MULTIPLIER, overrides[:multiplier] end + + # `retry_predicate` and `timeout` cannot be expressed through a Hash: Gapic::CallOptions converts one + # into a Gapic::CallOptions::RetryPolicy, whose initializer takes neither. A policy object passed + # straight in is left alone, and is the only route by which either setting arrives here. + def test_start_retry_policy_for_carries_a_caller_retry_predicate + predicate = ->(_error) { true } + options = Gapic::CallOptions.new retry_policy: Gapic::Common::RetryPolicy.new(retry_predicate: predicate) + + overrides = Gapic::Rest::ResumableUpload.start_retry_policy_for options + policy = Gapic::Common::RetryPolicy.new(**overrides).apply_defaults RetryPolicies::START_DEFAULTS + + # The caller's predicate replaces the initiation one wholesale; the two are not composed. + assert_same predicate, policy.retry_predicate + end + + def test_start_retry_policy_for_call_timeout_displaces_a_policy_timeout + options = Gapic::CallOptions.new timeout: 5, retry_policy: Gapic::Common::RetryPolicy.new(timeout: 900) + + assert_equal 5, Gapic::Rest::ResumableUpload.start_retry_policy_for(options)[:timeout] + end end From 5755daee5ebbcf1cccc8cb69101c96a50ecb2e8e Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Fri, 18 Sep 2026 06:44:39 +0000 Subject: [PATCH 09/22] test(resumable-upload): run the integration suite through the coordinator A Showcase test that builds a Driver by hand exercises a path no generated client takes. All five suites now drive ::Gapic::ResumableUpload, build_config is gone, and the golden path gains a decode-into-a-message case. --- .../integration/integration_helper.rb | 53 ++++--------- .../chunk_granularity_test.rb | 24 +++--- .../resumable_upload/error_on_start_test.rb | 74 +++++++------------ .../resumable_upload/error_recovery_test.rb | 56 ++++---------- .../resumable_upload/golden_path_test.rb | 66 +++++++---------- 5 files changed, 95 insertions(+), 178 deletions(-) diff --git a/gapic-common/integration/integration_helper.rb b/gapic-common/integration/integration_helper.rb index c33324f..ab10591 100644 --- a/gapic-common/integration/integration_helper.rb +++ b/gapic-common/integration/integration_helper.rb @@ -26,6 +26,10 @@ require "gapic/rest" require "gapic/rest/resumable_upload" +# The protobuf fixtures live with the unit tests. The decoding test needs a real generated message class, +# and compiling a second one for the integration suite would buy nothing. +require File.expand_path("../test/fixtures/fixture_pb", __dir__) + ## # Base class for Showcase integration tests. # @@ -95,41 +99,16 @@ def showcase_client_stub ) end - def build_config scenario: nil, scenario_config: {}, **overrides - @progress_records = [] - headers = (overrides[:initial_headers] || {}).dup - if scenario - headers["X-Goog-Test-Scenario"] = scenario - headers["X-Goog-Test-Scenario-Config"] = JSON.generate( - { "client_uuid" => SecureRandom.uuid }.merge(scenario_config) - ) - end - - defaults = { - initial_url: UPLOAD_PATH, - initial_headers: headers, - start_retry_policy: FAST_RETRY, - control_plane_retry_policy: FAST_RETRY, - data_plane_retry_policy: FAST_RETRY, - timeout: 10, - chunk_size: DEFAULT_CHUNK_SIZE, - on_progress: ->(progress) { @progress_records << progress } - } - unless overrides.key? :stream - defaults[:stream] = StringIO.new payload(DEFAULT_PAYLOAD_SIZE) - defaults[:upload_size] = DEFAULT_PAYLOAD_SIZE - end - - Gapic::Rest::ResumableUpload::StartUploadConfig.new(**defaults, **overrides, initial_headers: headers) - end - ## # Builds a coordinator pointed at Showcase, with the short retry policies these tests rely on. # - # The handle decodes nothing (`response_type: nil`), so runs return the raw response body and the tests - # can assert on what Showcase actually sent. + # Runs return the raw response body by default (`response_type: nil`), so tests can assert on what + # Showcase actually sent; a test that cares about decoding passes a message class instead. + # + # Any constructor argument can be overridden, including with `nil` — the suites that measure real + # backoff pass `start_retry_policy: nil` to get the protocol's own policy. # - def build_upload scenario: nil, scenario_config: {}, initial_headers: {}, **overrides + def build_upload scenario: nil, scenario_config: {}, initial_headers: {}, initial_url: UPLOAD_PATH, **overrides @progress_records = [] headers = initial_headers.dup if scenario @@ -139,16 +118,16 @@ def build_upload scenario: nil, scenario_config: {}, initial_headers: {}, **over ) end - Gapic::ResumableUpload.new( + defaults = { client_stub_proc: -> { showcase_client_stub }, - initial_request_proc: -> { [UPLOAD_PATH, nil] }, + initial_request_proc: -> { [initial_url, nil] }, response_type: nil, - initial_headers: headers, start_retry_policy: FAST_RETRY, control_plane_retry_policy: FAST_RETRY, - data_plane_retry_policy: FAST_RETRY, - **overrides - ) + data_plane_retry_policy: FAST_RETRY + } + + Gapic::ResumableUpload.new(**defaults, **overrides, initial_headers: headers) end # Default run arguments for `#start`. diff --git a/gapic-common/integration/resumable_upload/chunk_granularity_test.rb b/gapic-common/integration/resumable_upload/chunk_granularity_test.rb index ea05965..2aef153 100644 --- a/gapic-common/integration/resumable_upload/chunk_granularity_test.rb +++ b/gapic-common/integration/resumable_upload/chunk_granularity_test.rb @@ -19,30 +19,26 @@ require "stringio" ## -# Integration tests for chunk granularity alignment against Showcase. +# Integration tests for chunk granularity alignment against Showcase, driven through +# ::Gapic::ResumableUpload. # class ChunkGranularityTest < ShowcaseIntegrationTest # Verifies chunk size alignment to server-specified granularity (300_000 -> 299_776) and progress notifications. def test_chunk_granularity_alignment size = 1_000_000 - config = build_config( - scenario: "chunk_granularity", - stream: StringIO.new(payload(size)), - upload_size: size, - chunk_size: 300_000, - timeout: 5 - ) - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config - ) - - result = driver.run + upload = build_upload scenario: "chunk_granularity" + result = upload.start(**start_args(stream: StringIO.new(payload(size)), + upload_size: size, + chunk_size: 300_000, + upload_timeout: 5)) parsed = JSON.parse result assert_equal size, parsed["size"] assert_equal [0, 0, 299_776, 599_552, 899_328, 899_328, 1_000_000], offsets assert_equal [:initiating, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed], phases + + # A finalized upload leaves nothing to resume, alignment or not. + assert_nil upload.resume_handle end end diff --git a/gapic-common/integration/resumable_upload/error_on_start_test.rb b/gapic-common/integration/resumable_upload/error_on_start_test.rb index f43d23d..a3632ae 100644 --- a/gapic-common/integration/resumable_upload/error_on_start_test.rb +++ b/gapic-common/integration/resumable_upload/error_on_start_test.rb @@ -16,43 +16,44 @@ require "integration_helper" require "json" -require "securerandom" require "stringio" ## -# Suite A: Integration tests for non-fatal and fatal errors on session initiation (`start`). +# Suite A: Integration tests for non-fatal and fatal errors on session initiation (`start`), driven +# through ::Gapic::ResumableUpload. # class ErrorOnStartTest < ShowcaseIntegrationTest PAYLOAD_SIZE = 100 - def build_start_error_config scenario:, scenario_config: {}, start_retry_policy: nil, **overrides - build_config( + ## + # Initiation is the subject here, so the plane policies are left at the protocol's own defaults and only + # the start policy varies. A `nil` start policy means the protocol's default backoff, which is what the + # timing assertions measure. + # + def build_start_error_upload scenario:, scenario_config: {}, start_retry_policy: nil, **overrides + build_upload( scenario: scenario, scenario_config: scenario_config, start_retry_policy: start_retry_policy, control_plane_retry_policy: nil, data_plane_retry_policy: nil, - stream: StringIO.new(payload(PAYLOAD_SIZE)), - upload_size: PAYLOAD_SIZE, **overrides ) end + def start_error_args **overrides + start_args(stream: StringIO.new(payload(PAYLOAD_SIZE)), upload_size: PAYLOAD_SIZE, **overrides) + end + # A1. Verifies non-fatal transient error (503) on start is retried and upload completes. def test_non_fatal_error_on_start_503 - config = build_start_error_config( + upload = build_start_error_upload( scenario: "non_fatal_error_on_start", scenario_config: { error_code: 503, failure_count: 1 }, start_retry_policy: FAST_RETRY ) - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config - ) - - result = driver.run - parsed = JSON.parse result + parsed = JSON.parse upload.start(**start_error_args) assert_equal PAYLOAD_SIZE, parsed["size"] assert_equal 1, phases.count(:initiating) @@ -61,19 +62,13 @@ def test_non_fatal_error_on_start_503 # A2. Verifies missing status header / 400 on start is retried and upload completes. def test_missing_header_retriable_on_start_400 - config = build_start_error_config( + upload = build_start_error_upload( scenario: "non_fatal_error_on_start", scenario_config: { error_code: 400, failure_count: 1 }, start_retry_policy: FAST_RETRY ) - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config - ) - - result = driver.run - parsed = JSON.parse result + parsed = JSON.parse upload.start(**start_error_args) assert_equal PAYLOAD_SIZE, parsed["size"] assert_equal 1, phases.count(:initiating) @@ -82,20 +77,14 @@ def test_missing_header_retriable_on_start_400 # A3. Verifies retry exhaustion on start with high failure count times out within ~3s without uploading. def test_retry_exhaustion_on_start_times_out - config = build_start_error_config( + upload = build_start_error_upload( scenario: "non_fatal_error_on_start", - scenario_config: { error_code: 503, failure_count: 10_000 }, - timeout: 3 - ) - - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config + scenario_config: { error_code: 503, failure_count: 10_000 } ) t0 = Process.clock_gettime Process::CLOCK_MONOTONIC err = assert_raises Gapic::Common::Error do - driver.run + upload.start(**start_error_args(upload_timeout: 3)) end t1 = Process.clock_gettime Process::CLOCK_MONOTONIC @@ -111,19 +100,14 @@ def test_retry_exhaustion_on_start_times_out # A4. Verifies fatal errors on start (403 and 404) immediately raise BadResponseError in < 0.5s without retrying. def test_fatal_error_on_start_raises_bad_response_immediately [403, 404].each do |code| - config = build_start_error_config( + upload = build_start_error_upload( scenario: "fatal_error_on_start", scenario_config: { error_code: code } ) - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config - ) - t0 = Process.clock_gettime Process::CLOCK_MONOTONIC err = assert_raises Gapic::Rest::ResumableUpload::BadResponseError do - driver.run + upload.start(**start_error_args) end t1 = Process.clock_gettime Process::CLOCK_MONOTONIC @@ -134,22 +118,18 @@ def test_fatal_error_on_start_raises_bad_response_immediately end end - # A5. Verifies sequential executions with fresh client UUIDs remain isolated. + # A5. Verifies sequential executions with fresh client UUIDs remain isolated. Each coordinator carries its + # own UUID in the scenario header, so this stays a test of server-side isolation rather than of reuse; + # reuse of a single coordinator is covered in the resume suite. def test_sequential_runs_session_isolation 2.times do - config = build_start_error_config( + upload = build_start_error_upload( scenario: "non_fatal_error_on_start", scenario_config: { error_code: 503, failure_count: 1 }, start_retry_policy: FAST_RETRY ) - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config - ) - - result = driver.run - parsed = JSON.parse result + parsed = JSON.parse upload.start(**start_error_args) assert_equal PAYLOAD_SIZE, parsed["size"] assert_equal 1, phases.count(:initiating) diff --git a/gapic-common/integration/resumable_upload/error_recovery_test.rb b/gapic-common/integration/resumable_upload/error_recovery_test.rb index fd0f948..b1ba1f4 100644 --- a/gapic-common/integration/resumable_upload/error_recovery_test.rb +++ b/gapic-common/integration/resumable_upload/error_recovery_test.rb @@ -19,24 +19,21 @@ require "stringio" ## -# Suite B: Integration tests for Category 1 transient retries and Category 2 error recovery against Showcase. +# Suite B: Integration tests for Category 1 transient retries and Category 2 error recovery against +# Showcase, driven through ::Gapic::ResumableUpload. The short data- and control-plane policies come from +# the coordinator's `@private` constructor arguments, so these run on the production code path. # class ErrorRecoveryTest < ShowcaseIntegrationTest SCENARIO = "non_fatal_error_on_chunk_upload" # B1. Verifies Category 1 transient error (503) is retried transparently by FAST_RETRY without entering recovery. def test_cat1_error_retried_transparently - config = build_config( + upload = build_upload( scenario: SCENARIO, scenario_config: { error_code: 503, failure_count: 1, after_offset: 0 } ) - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config - ) - - result = driver.run + result = upload.start(**start_args) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] @@ -47,17 +44,12 @@ def test_cat1_error_retried_transparently # B2. Verifies simple Category 2 error (409) at offset 0 triggers recovery query and resumes upload to completion. def test_simple_cat2_error_recovery - config = build_config( + upload = build_upload( scenario: SCENARIO, scenario_config: { error_code: 409, failure_count: 1, after_offset: 0 } ) - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config - ) - - result = driver.run + result = upload.start(**start_args) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] @@ -70,17 +62,12 @@ def test_simple_cat2_error_recovery # B3. Verifies two consecutive Category 2 recoveries (409) on chunk 2 at offset 262_144. def test_two_consecutive_cat2_recoveries_on_chunk_2 - config = build_config( + upload = build_upload( scenario: SCENARIO, scenario_config: { error_code: 409, failure_count: 2, after_offset: 262_144 } ) - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config - ) - - result = driver.run + result = upload.start(**start_args) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] @@ -99,19 +86,12 @@ def test_two_consecutive_cat2_recoveries_on_chunk_2 # B4. Verifies Category 2 error (409) on the finalizing chunk (upload, finalize) recovers and completes. def test_cat2_failure_on_finalizing_chunk size = (DEFAULT_CHUNK_SIZE * 3) - 100 - config = build_config( + upload = build_upload( scenario: SCENARIO, - scenario_config: { error_code: 409, failure_count: 1, after_offset: DEFAULT_CHUNK_SIZE * 2 }, - stream: StringIO.new(payload(size)), - upload_size: size - ) - - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config + scenario_config: { error_code: 409, failure_count: 1, after_offset: DEFAULT_CHUNK_SIZE * 2 } ) - result = driver.run + result = upload.start(**start_args(stream: StringIO.new(payload(size)), upload_size: size)) parsed = JSON.parse result assert_equal size, parsed["size"] @@ -124,19 +104,13 @@ def test_cat2_failure_on_finalizing_chunk # B5. Verifies unrecoverable 500 without status header triggers repeated recovery until DeadlineExceededError. def test_no_headers_failure_recovers_until_deadline_exceeded - config = build_config( + upload = build_upload( scenario: SCENARIO, - scenario_config: { failure_count: 0, action_after_failures: "terminate" }, - timeout: 1 - ) - - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config + scenario_config: { failure_count: 0, action_after_failures: "terminate" } ) assert_raises Gapic::Rest::ResumableUpload::DeadlineExceededError do - driver.run + upload.start(**start_args(upload_timeout: 1)) end assert_operator phases.count(:recovering), :>=, 2 diff --git a/gapic-common/integration/resumable_upload/golden_path_test.rb b/gapic-common/integration/resumable_upload/golden_path_test.rb index d40c31d..997f5b3 100644 --- a/gapic-common/integration/resumable_upload/golden_path_test.rb +++ b/gapic-common/integration/resumable_upload/golden_path_test.rb @@ -19,26 +19,17 @@ require "stringio" ## -# Golden path integration tests for ResumableUpload Driver against Showcase. +# Golden path integration tests against Showcase, driven through ::Gapic::ResumableUpload. # class GoldenPathTest < ShowcaseIntegrationTest def test_multi_chunk_known_size size = 1_500_000 chunk_size = 524_288 - stream = StringIO.new payload(size) - config = build_config( - stream: stream, - upload_size: size, - chunk_size: chunk_size - ) - - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config - ) - - result = driver.run + upload = build_upload + result = upload.start(**start_args(stream: StringIO.new(payload(size)), + upload_size: size, + chunk_size: chunk_size)) parsed = JSON.parse result assert_equal size, parsed["size"] @@ -54,20 +45,11 @@ def test_multi_chunk_known_size def test_small_upload_default_chunk_size size = 100_000 - stream = StringIO.new payload(size) - - config = build_config( - stream: stream, - upload_size: size, - chunk_size: nil # use default chunk size - ) - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config - ) - - result = driver.run + upload = build_upload + result = upload.start(**start_args(stream: StringIO.new(payload(size)), + upload_size: size, + chunk_size: nil)) # use default chunk size parsed = JSON.parse result assert_equal size, parsed["size"] @@ -82,19 +64,10 @@ def test_small_upload_default_chunk_size def test_standalone_finalize_unseekable_stream chunk_size = 262_144 size = 3 * chunk_size - stream = UnseekableStream.new payload(size) - - config = build_config( - stream: stream, - chunk_size: chunk_size - ) - - driver = Gapic::Rest::ResumableUpload::Driver.new( - client_stub: showcase_client_stub, - config: config - ) - result = driver.run + upload = build_upload + result = upload.start(**start_args(stream: UnseekableStream.new(payload(size)), + chunk_size: chunk_size)) parsed = JSON.parse result assert_equal size, parsed["size"] @@ -108,4 +81,19 @@ def test_standalone_finalize_unseekable_stream Gapic::Rest::ResumableUpload::Progress.new(phase: :completed, bytes_uploaded: 786_432, total_bytes: 786_432) ], progress_records end + + ## + # The decode path a generated client actually takes: a real message class, against a live response that + # carries a field the message does not declare. That field only survives because the coordinator decodes + # with `ignore_unknown_fields: true` — without it, `decode_json` would raise. + # + def test_decodes_the_final_body_into_the_response_type + size = 100_000 + + upload = build_upload response_type: Gapic::Examples::User + response = upload.start(**start_args(stream: StringIO.new(payload(size)), upload_size: size)) + + assert_instance_of Gapic::Examples::User, response + refute_empty response.name + end end From 1ebe40ab29e3ad79ed36de70ccf98fe59c521205 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Fri, 18 Sep 2026 06:51:32 +0000 Subject: [PATCH 10/22] chore: fixup comment --- gapic-common/lib/gapic/rest/resumable_upload/rules.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb index 90cc14f..c6a5a26 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb @@ -195,7 +195,7 @@ module Rules # * `:error` - terminal for this run; `last_error` holds the exception. # # `:success`, `:cancelled` and `:rejected` are finalized and yield no {ResumeHandle}. `:error` ends the - # run but may still be resumable from a fresh session; see {Rules.resume_handle_from}. + # run but may still be resumable from a fresh upload; see {Rules.resume_handle_from}. # # @return [Array] STATUSES = [ From 1dcbd016ceba16b62cf140b92455b93b109792a4 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Tue, 22 Sep 2026 17:48:43 +0000 Subject: [PATCH 11/22] chore: remove duplicated tests and collapse certain cases --- gapic-common/test/gapic/rest/error_test.rb | 13 -- .../rest/resumable_upload/data_types_test.rb | 66 ++--------- .../resumable_upload/driver_buffer_test.rb | 63 +--------- .../resumable_upload/driver_config_test.rb | 112 +++--------------- .../resumable_upload/driver_progress_test.rb | 57 --------- .../rules_classification_test.rb | 40 +------ 6 files changed, 33 insertions(+), 318 deletions(-) diff --git a/gapic-common/test/gapic/rest/error_test.rb b/gapic-common/test/gapic/rest/error_test.rb index 96c16d9..6593db1 100644 --- a/gapic-common/test/gapic/rest/error_test.rb +++ b/gapic-common/test/gapic/rest/error_test.rb @@ -323,17 +323,4 @@ def test_surface_absent_details assert_nil gapic_err.details end - - def test_rest_error_prefix_constant - assert_equal "An error has occurred when making a REST request", ::Gapic::Rest::Error::REST_ERROR_PREFIX - - faraday_err = OpenStruct.new( - message: "raw", - response_body: JSON.dump({ "error" => { "message" => "Quota exceeded", "code" => 429 } }), - response_headers: {}, - response_status: 429 - ) - gapic_err = ::Gapic::Rest::Error.wrap_faraday_error faraday_err - assert_equal "#{::Gapic::Rest::Error::REST_ERROR_PREFIX}: Quota exceeded", gapic_err.message - end end diff --git a/gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb b/gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb index e78d849..1222dfe 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb @@ -76,12 +76,6 @@ def test_start_upload_config_rejects_reserved_initial_headers end end - def test_reserved_initial_headers_constant_is_lowercase_and_duplicate_free - assert_equal RESERVED_INITIAL_HEADERS.map(&:downcase), RESERVED_INITIAL_HEADERS - assert_equal RESERVED_INITIAL_HEADERS.uniq, RESERVED_INITIAL_HEADERS - assert_predicate RESERVED_INITIAL_HEADERS, :frozen? - end - def test_start_upload_config_allows_caller_owned_initial_headers stream = StringIO.new "content" headers = { @@ -95,9 +89,8 @@ def test_start_upload_config_allows_caller_owned_initial_headers assert_equal headers, config.initial_headers end - def test_state_defaults_and_with + def test_state_and_instruction_defaults state = State.new - assert_equal :initializing, state.status assert_nil state.upload_url assert_equal 0, state.offset @@ -106,43 +99,16 @@ def test_state_defaults_and_with assert_equal 0, state.in_flight_length assert_nil state.last_error - modified = state.with status: :starting, upload_url: "https://example.com/upload" - assert_equal :starting, modified.status - assert_equal "https://example.com/upload", modified.upload_url - assert_equal :initializing, state.status - end - - def test_event_instantiation - start_event = Event::StartUpload.new - assert_instance_of Event::StartUpload, start_event - - chunk = Event::ChunkRead.new bytes_buffered: 100, eof: true - assert_equal 100, chunk.bytes_buffered - assert chunk.eof - - http = Event::HttpResponse.new status: 200, headers: { "a" => "b" }, body: "body" - assert_equal 200, http.status - assert_equal({ "a" => "b" }, http.headers) - assert_equal "body", http.body - - req_fail = Event::RequestFailed.new kind: :connection_failed, message: "err" - assert_equal :connection_failed, req_fail.kind - assert_equal "err", req_fail.message - end - - def test_instruction_instantiation - start = Instruction::SendStart.new url: "https://example.com" - assert_equal "https://example.com", start.url - assert_equal({}, start.headers) - assert_nil start.body + http_event = Event::HttpResponse.new status: 200 + assert_equal({}, http_event.headers) + assert_nil http_event.body - chunk = Instruction::SendChunk.new url: "https://example.com", offset: 0, length: 100 - assert_equal 0, chunk.offset - assert_equal 100, chunk.length - refute chunk.finalize + start_inst = Instruction::SendStart.new url: "https://example.com" + assert_equal({}, start_inst.headers) + assert_nil start_inst.body - realign = Instruction::RealignBuffer.new server_offset: 500 - assert_equal 500, realign.server_offset + chunk_inst = Instruction::SendChunk.new url: "https://example.com", offset: 0, length: 100 + refute chunk_inst.finalize end def test_progress_instantiation @@ -167,20 +133,6 @@ def test_progress_instantiation end end - def test_resume_handle_instantiation - handle = ResumeHandle.new upload_url: "https://upload.example.com/session123", chunk_size: 1_048_576 - assert_equal "https://upload.example.com/session123", handle.upload_url - assert_equal 1_048_576, handle.chunk_size - - assert_raises ArgumentError do - ResumeHandle.new upload_url: "https://upload.example.com/session123" - end - - assert_raises NoMethodError do - handle.upload_url = "https://mutated.com" - end - end - def test_resume_upload_config_defaults stream = StringIO.new "content" config = ResumeUploadConfig.new( diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb index a7e110d..36ef466 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb @@ -135,35 +135,23 @@ def test_fill_buffer_empty_stream # execute_realign_buffer tests: trim within buffer # ============================================================================ - def test_realign_buffer_trim_exact_beginning + def test_realign_buffer_within_window driver = build_driver stream: StringIO.new driver.instance_variable_set :@buffer_start_offset, 1000 driver.instance_variable_set :@buffer, "0123456789".b + # Exact left boundary (server_offset == buffer_start_offset): keeps full buffer driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1000) - assert_equal 1000, driver.instance_variable_get(:@buffer_start_offset) assert_equal "0123456789".b, driver.instance_variable_get(:@buffer) - end - - def test_realign_buffer_trim_middle - driver = build_driver stream: StringIO.new - driver.instance_variable_set :@buffer_start_offset, 1000 - driver.instance_variable_set :@buffer, "0123456789".b + # Interior of window: slices consumed prefix driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1004) - assert_equal 1004, driver.instance_variable_get(:@buffer_start_offset) assert_equal "456789".b, driver.instance_variable_get(:@buffer) - end - - def test_realign_buffer_trim_exact_end - driver = build_driver stream: StringIO.new - driver.instance_variable_set :@buffer_start_offset, 1000 - driver.instance_variable_set :@buffer, "0123456789".b + # Exact right boundary (server_offset == buffer_end_offset): empties buffer driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1010) - assert_equal 1010, driver.instance_variable_get(:@buffer_start_offset) assert_equal "".b, driver.instance_variable_get(:@buffer) end @@ -186,22 +174,6 @@ def test_realign_buffer_rewind_seekable_stream assert_equal 500, stream.pos end - def test_realign_buffer_rewind_unseekable_stream_raises_error - stream = UnseekableStream.new "0123456789" * 100 - driver = build_driver stream: stream - driver.instance_variable_set :@buffer_start_offset, 1000 - driver.instance_variable_set :@buffer, "buffered".b - - err = assert_raises UnseekableStreamError do - driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 500) - end - - assert_includes err.message, "offset 500" - assert_includes err.message, "buffered from 1000" - assert_nil err.resume_handle - refute_includes err.message, "(upload session is resumable: see #resume_handle)" - end - def test_realign_buffer_rewind_unseekable_stream_with_resume_handle stream = UnseekableStream.new "0123456789" * 100 driver = build_driver stream: stream @@ -224,33 +196,6 @@ def test_realign_buffer_rewind_unseekable_stream_with_resume_handle assert_includes err.message, "(upload session is resumable: see #resume_handle)" end - def test_driver_resume_handle_property - stream = StringIO.new "test" - driver = build_driver stream: stream - assert_nil driver.resume_handle - - driver.core.instance_variable_set( - :@state, - driver.core.state.with(upload_url: "https://upload.example.com/session_2", chunk_size: 512) - ) - handle = driver.resume_handle - refute_nil handle - assert_equal "https://upload.example.com/session_2", handle.upload_url - assert_equal 512, handle.chunk_size - - driver.core.instance_variable_set( - :@state, - driver.core.state.with(status: :rejected, upload_url: "https://upload.example.com/session_2", chunk_size: 512) - ) - assert_nil driver.resume_handle - - driver.core.instance_variable_set( - :@state, - driver.core.state.with(status: :cancelled, upload_url: "https://upload.example.com/session_2", chunk_size: 512) - ) - assert_nil driver.resume_handle - end - # ============================================================================ # execute_realign_buffer tests: fast forward stream # ============================================================================ diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb index 64f859e..691f0f6 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb @@ -48,77 +48,24 @@ def make_post_request uri:, body:, params:, options:, method_name: nil FakeResponse = Data.define :status, :headers, :body - def test_resolve_timeout_prefers_positive_config_timeout + def test_resolve_timeout stub = FakeClientStub.new - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - upload_size: 10 * 1_048_576, - timeout: 42 - ) - driver = Driver.new client_stub: stub, config: config - - assert_equal 42, driver.send(:resolve_timeout) - end - - def test_resolve_timeout_treats_zero_timeout_same_as_nil - stub = FakeClientStub.new - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - timeout: 0 - ) - driver = Driver.new client_stub: stub, config: config - - assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) - end - - def test_resolve_timeout_treats_negative_timeout_same_as_nil - stub = FakeClientStub.new - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - timeout: -10 - ) - driver = Driver.new client_stub: stub, config: config - - assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) - end - - def test_resolve_timeout_calculates_from_upload_size_above_base_timeout - stub = FakeClientStub.new - large_size = 7_200 * Driver::MIN_ASSUMED_THROUGHPUT # 7200 seconds at 1MB/s - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - upload_size: large_size - ) - driver = Driver.new client_stub: stub, config: config - - assert_in_delta 7_200.0, driver.send(:resolve_timeout), 0.001 - end + resolve_for = lambda do |**opts| + config = StartUploadConfig.new initial_url: "https://example.com/upload", stream: StringIO.new("0123"), **opts + Driver.new(client_stub: stub, config: config).send :resolve_timeout + end - def test_resolve_timeout_uses_base_timeout_floor_for_small_upload_size - stub = FakeClientStub.new - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - upload_size: 1_048_576 # 1 second at 1MB/s < 3600 - ) - driver = Driver.new client_stub: stub, config: config + # Explicit positive timeout takes precedence over upload_size + assert_equal 42, resolve_for.call(upload_size: 10 * 1_048_576, timeout: 42) - assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) - end + # Zero and negative timeouts are treated as unset (nil) + assert_equal Driver::BASE_TIMEOUT, resolve_for.call(timeout: 0) + assert_equal Driver::BASE_TIMEOUT, resolve_for.call(timeout: -10) - def test_resolve_timeout_defaults_to_base_timeout_when_upload_size_nil - stub = FakeClientStub.new - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123") - ) - driver = Driver.new client_stub: stub, config: config - - assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) + # Proportional to upload_size above BASE_TIMEOUT, floored at BASE_TIMEOUT for small or nil upload_size + assert_in_delta 7_200.0, resolve_for.call(upload_size: 7_200 * Driver::MIN_ASSUMED_THROUGHPUT), 0.001 + assert_equal Driver::BASE_TIMEOUT, resolve_for.call(upload_size: 1_048_576) + assert_equal Driver::BASE_TIMEOUT, resolve_for.call end def test_run_raises_deadline_exceeded_when_timeout_expires @@ -200,37 +147,6 @@ def test_make_post_request_passes_timeout_close_to_remaining_budget_and_decrease end end - def test_start_headers_derives_content_descriptors_from_config - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - upload_size: 4, - content_type: "application/octet-stream" - ) - driver = Driver.new client_stub: FakeClientStub.new, config: config - instruction = Instruction::SendStart.new url: "https://example.com/upload" - - headers = driver.send :start_headers, instruction - - assert_equal "application/octet-stream", headers["X-Goog-Upload-Header-Content-Type"] - assert_equal "4", headers["X-Goog-Upload-Header-Content-Length"] - end - - def test_start_headers_passes_unrelated_caller_headers_through - config = StartUploadConfig.new initial_url: "https://example.com/upload", stream: StringIO.new("0123") - driver = Driver.new client_stub: FakeClientStub.new, config: config - instruction = Instruction::SendStart.new( - url: "https://example.com/upload", - headers: { "X-Custom" => "value" } - ) - - headers = driver.send :start_headers, instruction - - assert_equal "value", headers["X-Custom"] - assert_equal "resumable", headers["X-Goog-Upload-Protocol"] - assert_equal "start", headers["X-Goog-Upload-Command"] - end - def test_start_headers_without_caller_headers_is_unchanged config = StartUploadConfig.new initial_url: "https://example.com/upload", stream: StringIO.new("0123") driver = Driver.new client_stub: FakeClientStub.new, config: config diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb index 54cf579..7abcddb 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb @@ -39,50 +39,6 @@ def make_post_request uri:, body: nil, params: {}, options: {}, method_name: nil end end - def test_execute_notify_progress_without_callback_does_not_raise - driver = build_driver on_progress: nil - - instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 1024, total_bytes: 4096) - # Must not raise when callback is nil - driver.send :execute_notify_progress, instruction - end - - def test_execute_notify_progress_happy_path_invoked_once - calls = [] - callback = ->(progress) { calls << progress } - driver = build_driver on_progress: callback - - instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 500, total_bytes: 1000) - driver.send :execute_notify_progress, instruction - - assert_equal 1, calls.size - assert_equal Progress.new(phase: :uploading, bytes_uploaded: 500, total_bytes: 1000), calls.first - end - - def test_execute_notify_progress_total_bytes_nil_passes_through - calls = [] - callback = ->(progress) { calls << progress } - driver = build_driver on_progress: callback - - instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 250, total_bytes: nil) - driver.send :execute_notify_progress, instruction - - assert_equal 1, calls.size - assert_equal Progress.new(phase: :uploading, bytes_uploaded: 250, total_bytes: nil), calls.first - end - - def test_execute_notify_progress_raises_error_to_caller_when_callback_fails - callback = ->(_progress) { raise CustomCallbackError, "User UI crashed in progress callback" } - driver = build_driver on_progress: callback - - instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 100, total_bytes: 1000) - err = assert_raises CustomCallbackError do - driver.send :execute_notify_progress, instruction - end - - assert_equal "User UI crashed in progress callback", err.message - end - def test_driver_run_propagates_callback_error_end_to_end # Script responses: 1. start response -> 2. chunk response (triggers NotifyProgress) responses = [ @@ -162,17 +118,4 @@ def test_completed_progress_reports_total_bytes_when_upload_size_unknown assert_equal 10, completed_snapshot.bytes_uploaded assert_equal 10, completed_snapshot.total_bytes end - - private - - def build_driver on_progress: nil - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123456789"), - upload_size: 10, - chunk_size: 4, - on_progress: on_progress - ) - Driver.new client_stub: Object.new, config: config - end end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb index d2b7c3d..7d9bfa7 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb @@ -300,52 +300,24 @@ def test_decide_rejects_a_recipe_outside_the_vocabulary assert_match(/Resumable upload internal error: decide selected unknown recipe :start_session/, error.message) end - def test_resolve_chunk_size_with_nil_or_non_positive_granularity - # nil granularity + def test_resolve_chunk_size + # nil or non-positive granularity keeps user or default chunk size assert_equal 1024, Rules.resolve_chunk_size(1024, nil) assert_equal Rules::DEFAULT_CHUNK_SIZE, Rules.resolve_chunk_size(nil, nil) - - # 0 granularity assert_equal 1024, Rules.resolve_chunk_size(1024, 0) - assert_equal Rules::DEFAULT_CHUNK_SIZE, Rules.resolve_chunk_size(nil, 0) - - # Negative granularity - assert_equal 1024, Rules.resolve_chunk_size(1024, -256) assert_equal Rules::DEFAULT_CHUNK_SIZE, Rules.resolve_chunk_size(nil, -1) - end - def test_resolve_chunk_size_when_divisible - # User-specified evenly divides granularity + # Exact multiple (including equality) is preserved assert_equal 1024, Rules.resolve_chunk_size(1024, 256) - assert_equal 1_048_576, Rules.resolve_chunk_size(1_048_576, 262_144) - - # Default chunk size (8_388_608) evenly divides the standard 256 KB backend granularity + assert_equal 256, Rules.resolve_chunk_size(256, 256) assert_equal 8_388_608, Rules.resolve_chunk_size(nil, 262_144) - assert_equal 8_388_608, Rules.resolve_chunk_size(nil, 524_288) - end - def test_resolve_chunk_size_when_not_divisible_rounds_down - # User-specified rounds down to nearest multiple + # Non-multiple rounds down to nearest multiple of granularity assert_equal 768, Rules.resolve_chunk_size(1000, 256) - assert_equal 9_961_472, Rules.resolve_chunk_size(10_000_000, 262_144) - - # Default chunk size (8_388_608) rounds down with non-divisor granularity (500_000 * 16) assert_equal 8_000_000, Rules.resolve_chunk_size(nil, 500_000) - assert_equal 8_192_000, Rules.resolve_chunk_size(nil, 1_024_000) - end - - def test_resolve_chunk_size_when_granularity_equal_to_chunk_size - assert_equal 256, Rules.resolve_chunk_size(256, 256) - assert_equal 262_144, Rules.resolve_chunk_size(262_144, 262_144) - assert_equal 8_388_608, Rules.resolve_chunk_size(nil, 8_388_608) - end - def test_resolve_chunk_size_when_granularity_greater_than_chunk_size - # User chunk size strictly less than granularity promotes to granularity (avoids 0) + # Chunk size smaller than granularity promotes to granularity assert_equal 256, Rules.resolve_chunk_size(100, 256) - assert_equal 262_144, Rules.resolve_chunk_size(1, 262_144) - - # Default chunk size strictly less than large server granularity promotes to granularity assert_equal 16_777_216, Rules.resolve_chunk_size(nil, 16_777_216) end end From 2b19444072760c98df480c0dd3dcbeb58a4f01fc Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Wed, 23 Sep 2026 19:22:35 +0000 Subject: [PATCH 12/22] fix: some 200 responses are invalid from the protocol view and should be BadResponseError --- .../resumable_upload/implementation-guide.md | 24 +- .../lib/gapic/rest/resumable_upload.rb | 5 +- .../resumable_upload/driver/upload_log.rb | 1 + .../lib/gapic/rest/resumable_upload/errors.rb | 66 +++-- .../lib/gapic/rest/resumable_upload/rules.rb | 226 ++++++++++++------ gapic-common/lib/gapic/resumable_upload.rb | 4 - .../resumable_upload/rules_decide_test.rb | 179 ++++++++++++++ .../rest/resumable_upload/rules_error_test.rb | 87 +++++-- .../gapic/rest/resumable_upload/rules_test.rb | 1 + 9 files changed, 467 insertions(+), 126 deletions(-) diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md index e834395..1fc6578 100644 --- a/gapic-common/design/resumable_upload/implementation-guide.md +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -390,8 +390,10 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` | **`Cancelling`** | `:response_cancelled` | `Event::HttpResponse(200, headers, _)` with `Status: cancelled` | `status = :cancelled` | `Cancelled` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadCancelledError.from(event))` | | **`Cancelling`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Cancelling`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:response_fatal_bad_response` | `Event::RequestFailed` or HTTP failure | `last_error = error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Transmission \| Sending chunk` / `Finalizing \| Sending with upload` / `Finalizing \| Sending finalize` / `Recovery`** | `:response_cancelled` | `Event::HttpResponse(200, headers, _)` with `Status: cancelled`, on a session this client never asked to cancel | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::UploadCancelledError.new("Resumable upload session was cancelled (detected while ...)")`
`status = :cancelled` | `Cancelled` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Starting` / `Transmission \| Sending chunk` / `Finalizing \| Sending with upload` / `Finalizing \| Sending finalize` / `Recovery` / `Cancelling`** | any HTTP response shape not claimed above | `Event::HttpResponse` whose `X-Goog-Upload-Status` does not match the phase of the request in flight (e.g. `final` during transmission, `active` while finalizing, `cancelled` during initiation) | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event, resume_handle:, prefix: "Resumable upload failed while ")`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **Any Non-Terminal** | `:global_deadline_exceeded` | `Event::GlobalDeadlineExceeded` | `last_error = Gapic::Rest::ResumableUpload::DeadlineExceededError.new`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | -| **Any State** | *Unmatched* | Any event not matched above | — | — | `fail_with_unmatched_transition(state, event)`: raises `InvalidTransitionError` stating in human terms what the protocol was doing (e.g. sending a chunk of data), what happened including HTTP status and `X-Goog-Upload-Status` header, and attaches the response. | +| **Any State** | *Unmatched* | Any event not matched above. Reachable only on a `Driver`/`Core` sequencing bug: every externally caused failure, including out-of-phase HTTP responses, is claimed by a row above. | — | — | `fail_with_unmatched_transition(state, event)`: raises the `@private` `InvalidTransitionError` stating in human terms what the protocol was doing (e.g. sending a chunk of data), what happened including HTTP status and `X-Goog-Upload-Status` header, and attaches the response. Carries no resume handle. | ### 4.3 State Transition Graph @@ -430,10 +432,22 @@ stateDiagram-v2 Starting --> Error : Event::RequestFailed / 4xx / 5xx Recovery --> Error : Event::RequestFailed - + + Transmission_Sending --> Cancelling : Event::Cancel + Finalizing_Sending_Upload --> Cancelling : Event::Cancel + Finalizing_Sending_Finalize --> Cancelling : Event::Cancel + Recovery --> Cancelling : Event::Cancel + Cancelling --> Cancelled : Event::HttpResponse(200, cancelled) + + Transmission_Sending --> Cancelled : Event::HttpResponse(200, cancelled) + Finalizing_Sending_Upload --> Cancelled : Event::HttpResponse(200, cancelled) + Finalizing_Sending_Finalize --> Cancelled : Event::HttpResponse(200, cancelled) + Recovery --> Cancelled : Event::HttpResponse(200, cancelled) + Success --> [*] Rejected --> [*] Error --> [*] + Cancelled --> [*] ``` --- @@ -532,10 +546,11 @@ Terminal errors provide actionable context so downstream SDK callers can inspect * **Error Classes**: * `BadResponseError < Gapic::Rest::Error`: Unrecoverable non-2xx HTTP responses or invalid payloads. Retains `attr_reader :response_body` returning `event.body`, and includes `HasResumeHandle`. * `UploadRejectedError < Gapic::Rest::Error`: Backend explicitly rejected the session with `X-Goog-Upload-Status: final`. Retains `attr_reader :response_body` returning `event.body`. Does NOT include `HasResumeHandle` (session is terminated permanently). - * `UploadCancelledError < Gapic::Common::Error`: Upload session cancelled by caller. Does NOT include `HasResumeHandle` (session is terminated permanently). + * `UploadCancelledError < Gapic::Common::Error`: Upload session was cancelled — either by the caller (via the coordinator's cancellation path) or out-of-band by the server/another client, detected when a response carries `X-Goog-Upload-Status: cancelled` while the session is still transmitting, finalizing, or recovering. Does NOT include `HasResumeHandle` (session is terminated permanently). Part of the public error surface. * `DeadlineExceededError < Gapic::Common::Error`: Upload deadline exceeded with optional root cause (`attr_reader :root_cause`), and includes `HasResumeHandle`. * `UnseekableStreamError < Gapic::Common::Error`: Stream rewind required on an unseekable stream; includes `HasResumeHandle`. - * `InvalidTransitionError < Gapic::Common::Error`: Unexpected event dispatched for state; includes `HasResumeHandle`. + * `InternalError < Gapic::Common::Error`: A defect in the upload implementation rather than a condition the caller can act on. Does NOT include `HasResumeHandle`. + * `InvalidTransitionError < InternalError`: Unexpected event dispatched for state. Marked `@private` — it is an implementation detail and is not part of the documented public surface; callers rescue `InternalError` (or `Gapic::Common::Error`). Does NOT include `HasResumeHandle`: an unmatched transition means the state machine's view of the session is untrustworthy, so advertising the session as resumable would be wrong. * `StreamMismatchError < Gapic::Common::Error`: Stream content or length does not match resumed upload specifications; includes `HasResumeHandle`. * `RequestFailedError < Gapic::Common::Error`: Terminal HTTP request failure (e.g. transport connection failure, request timeout, or retries exhausted). Retains `attr_reader :cause` returning the underlying error, preserves REST error attributes (`status_code`, `status`, `details`, `headers`) when available, and includes `HasResumeHandle`. * `SessionStateError < Gapic::Common::Error`: Raised when an operation violates the upload session lifecycle rules, e.g. starting a second run on a coordinator while one is still in flight. Distinguished from `ArgumentError`, which is raised strictly for invalid argument shapes. @@ -630,6 +645,7 @@ The `Driver` emits structured logs across three severity levels (`INFO`, `DEBUG` | `DEBUG` | Wire | Transport exception (`wire_failure`) | `Request failed: ` | | `DEBUG` | Buffer | Stream/buffer realignment (`buffer_realign`) | `Buffer realignment: ` | | `WARN` | Lifecycle | `:fail_with_deadline_exceeded`, `:fail_with_rejected`, `:fail_with_bad_response`, `:fail_with_request_error` | Resumable upload failed | +| `WARN` | Lifecycle | `:fail_with_cancelled` | Resumable upload canceled on the server | | `WARN` | Transition | `InvalidTransitionError` (`unmatched_transition`) | Unmatched transition | | `WARN` | Buffer | Backward server offset rewind on unseekable stream | Server offset rewind on unseekable stream | diff --git a/gapic-common/lib/gapic/rest/resumable_upload.rb b/gapic-common/lib/gapic/rest/resumable_upload.rb index 8aa0b42..cb86782 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload.rb @@ -54,11 +54,12 @@ module Rest # ### Error Types # * {RequestFailedError} - Transport connection failure, timeout, or retries exhausted (includes {HasResumeHandle}). # * {DeadlineExceededError} - Whole-upload timeout exceeded (includes {HasResumeHandle}). - # * {BadResponseError} - Unexpected or malformed HTTP response (includes {HasResumeHandle}). + # * {BadResponseError} - Unexpected, malformed, or out-of-phase HTTP response (includes {HasResumeHandle}). # * {UnseekableStreamError} - Stream rewinding required on an unseekable stream (includes {HasResumeHandle}). # * {StreamMismatchError} - Stream content or length does not match resumed upload (includes {HasResumeHandle}). - # * {InvalidTransitionError} - Unmatched event for the current protocol state (includes {HasResumeHandle}). # * {UploadRejectedError} - Server explicitly rejected the upload session (final). + # * {UploadCancelledError} - Upload session was cancelled, either by this client or on the server while the + # upload was in flight. A cancelled session cannot be resumed, so this error carries no {ResumeHandle} (final). # * {SessionStateError} - Upload session lifecycle rule violation, e.g. starting a second run while one # is in flight (final). # diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb index d7864b6..d08fbcc 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb @@ -53,6 +53,7 @@ class UploadLog complete_upload_finalized: [:info, "Resumable upload completed"], cancel_session: [:info, "Canceling resumable upload"], complete_cancellation: [:info, "Resumable upload canceled"], + fail_with_cancelled: [:warn, "Resumable upload canceled on the server"], fail_with_deadline_exceeded: [:warn, "Resumable upload failed"], fail_with_rejected: [:warn, "Resumable upload failed"], fail_with_bad_response: [:warn, "Resumable upload failed"], diff --git a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb index a6bb727..18dc971 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb @@ -23,8 +23,14 @@ module ResumableUpload ## # @private # HTTP status code to reason phrase mapping. + # + # Includes `200` because a well-formed success can still be a protocol failure: an + # `X-Goog-Upload-Status` that does not match the phase of the request in flight is reported as a + # {BadResponseError} carrying the 200 it arrived with. + # # @return [Hash] HTTP_STATUS_PHRASES = { + 200 => "OK", 400 => "Bad Request", 401 => "Unauthorized", 403 => "Forbidden", @@ -156,12 +162,12 @@ def build_from_http_event event, prefix: # end # # Included by {RequestFailedError}, {DeadlineExceededError}, {BadResponseError}, - # {UnseekableStreamError}, {StreamMismatchError} and {InvalidTransitionError}. + # {UnseekableStreamError} and {StreamMismatchError}. # - # Deliberately **not** included by {UploadRejectedError} or {SessionStateError}: the first means the - # server refused the upload outright and the second is a caller misuse, so neither is retryable. Note - # also that `resume_handle` may still be `nil` on an including error, for instance when the failure - # happened before initiation established an upload URL. + # Deliberately **not** included by {UploadRejectedError}, {UploadCancelledError} or {SessionStateError}: + # the first two mean the session is permanently terminated on the server and the third is a caller misuse, + # so none of them is retryable. Note also that `resume_handle` may still be `nil` on an including error, + # for instance when the failure happened before initiation established an upload URL. # # @!attribute [r] resume_handle # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated upload session resume handle @@ -191,6 +197,16 @@ def self.append_suffix message, resume_handle end ## + # @private + # Raised when an internal state machine or driver invariant is violated. + # Produced by {Rules} when an unlisted shape or recipe is encountered, and by + # {Driver} when a recipe emits a malformed instruction batch. + # + class InternalError < Gapic::Common::Error + end + + ## + # @private # Raised when an invalid or unmatched event is dispatched for the current protocol state. # # @!attribute [r] response @@ -199,12 +215,8 @@ def self.append_suffix message, resume_handle # @return [Symbol, nil] Current protocol state # @!attribute [r] event # @return [Object, nil] Received event - # @!attribute [r] resume_handle - # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle # - class InvalidTransitionError < Gapic::Common::Error - include HasResumeHandle - + class InvalidTransitionError < InternalError # @return [Gapic::Rest::ResumableUpload::Event::HttpResponse, Object, nil] attr_reader :response @@ -221,13 +233,11 @@ class InvalidTransitionError < Gapic::Common::Error # @param state [Symbol, nil] Current protocol state # @param event [Object, nil] Received event # @param response [Gapic::Rest::ResumableUpload::Event::HttpResponse, Object, nil] Associated HTTP response - # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle - def initialize message, state: nil, event: nil, response: nil, resume_handle: nil + def initialize message, state: nil, event: nil, response: nil @state = state @event = event @response = response || (event if defined?(Event::HttpResponse) && event.is_a?(Event::HttpResponse)) - @resume_handle = resume_handle - super HasResumeHandle.append_suffix(message, resume_handle) + super message end ## @@ -237,15 +247,13 @@ def initialize message, state: nil, event: nil, response: nil, resume_handle: ni # @param state [Symbol, nil] Current protocol state # @param message [String, nil] Descriptive error message # @param response [Object, nil] Associated HTTP response - # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle # @return [InvalidTransitionError] - def self.from event, state: nil, message: nil, response: nil, resume_handle: nil + def self.from event, state: nil, message: nil, response: nil new( message || "Invalid transition for event #{event.inspect}", - state: state, - event: event, - response: response, - resume_handle: resume_handle + state: state, + event: event, + response: response ) end end @@ -349,10 +357,12 @@ def initialize message = nil, status_code = nil, status: nil, details: nil, head # @param event [Object] HTTP response event # @param response_body [String, nil] Optional response body override # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Optional resume handle + # @param prefix [String] Leading clause of the error message, used to name the protocol phase the + # response arrived in # @return [BadResponseError] - def self.from event, response_body: nil, resume_handle: nil + def self.from event, response_body: nil, resume_handle: nil, prefix: "Resumable upload failed" body = response_body || (event.respond_to?(:body) ? event.body : nil) - message, status_code, status, details, headers = ErrorBuilder.build_attributes event + message, status_code, status, details, headers = ErrorBuilder.build_attributes event, prefix: prefix new message, status_code, status: status, details: details, headers: headers, response_body: body, resume_handle: resume_handle end @@ -398,9 +408,15 @@ def self.from event, response_body: nil end ## - # @private - # Raised when the upload session is cancelled. - # Cancellation is not public yet. + # Raised when the upload session was cancelled and will accept no further data. + # + # Reachable today only for a cancellation this client did not request: the server reports an + # established session as cancelled while a chunk, a finalize, or a recovery query is in flight, + # because another process, another client, or a server-side policy ended it. Client-initiated + # cancellation is not part of the public API yet, and this error is also what that will raise. + # + # Deliberately does not include {HasResumeHandle}. A cancelled session is gone server-side, so there + # is nothing to resume and retrying against it cannot succeed; a new upload must be started instead. # class UploadCancelledError < Gapic::Common::Error ## diff --git a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb index c6a5a26..b6ef961 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb @@ -83,6 +83,7 @@ module ResumableUpload # transmission_sending --> rejected : response_rejected # recovery --> rejected : response_rejected # cancelling --> cancelled : response_cancelled + # transmission_sending --> cancelled : response_cancelled # success --> [*] # rejected --> [*] # cancelled --> [*] @@ -91,19 +92,28 @@ module ResumableUpload # # The graph shows the protocol's intended path and its recoverable detours. Failure edges are largely omitted # to keep it readable: every non-terminal status can also reach `error` (on `:global_deadline_exceeded`, on an - # unretriable request failure, on a fatally bad response, or on any unmatched event) and `rejected` (on + # unretriable request failure, on a bad or out-of-phase response, or on any unmatched event) and `rejected` (on # `:response_rejected`), and every status listed in the `:user_cancel` arm can reach `cancelling`. `cancelling` - # in particular has only its success edge drawn; it fails like any other in-flight state. {Rules.decide} is the - # authoritative enumeration. + # in particular has only its success edge drawn; it fails like any other in-flight state. The + # `transmission_sending --> cancelled` edge stands in for all four statuses that reach `cancelled` on a + # server-reported out-of-band cancellation. {Rules.route} is the authoritative enumeration. # - # ### Router ordering + # ### Routing table # - # Arms are evaluated top to bottom, so their order encodes precedence and is load-bearing: + # {Rules.route} is the table: a pure function from `[status, shape]` to a recipe symbol, naming the handler + # without running it. {Rules.decide} runs whatever it names. Keeping selection separate from execution is what + # lets the tests enumerate the entire {STATUSES} x {SHAPES} product and assert every cell. + # + # Arms are written so that no arm matches a pair another arm also matches, which makes their order + # presentational. There is one deliberate exception: # # * `enter_recovery` and `fail_with_request_error` both match # `[:transmission_sending | :finalizing_sending_upload | :finalizing_sending_finalize,` # `:request_connection_failed | :request_timeout]`. Recovery wins purely because its arm precedes # `fail_with_request_error`. + # + # Two properties of the table are easy to misread and are called out at their arms: + # # * `[:starting, :response_cat2]` fails instead of recovering, unlike the same shape during transmission # and finalizing. There is no upload to recover to until initiation yields an upload URL. # * `recovery` re-queries on `:response_cat2` with no attempt cap. Termination is guaranteed only by the @@ -283,6 +293,7 @@ module Rules :complete_upload_finalized, :cancel_session, :complete_cancellation, + :fail_with_cancelled, :fail_with_deadline_exceeded, :fail_with_rejected, :fail_with_bad_response, @@ -316,6 +327,7 @@ module Rules :send_chunk, :retry_recovery, :complete_cancellation, + :fail_with_cancelled, :fail_with_deadline_exceeded, :fail_with_rejected, :fail_with_bad_response, @@ -354,76 +366,20 @@ def self.shape_of event ## # @private - # Top-level transition decision engine. Matches [state.status, shape]. + # Top-level transition decision engine. Routes `[state.status, shape_of(event)]` to a recipe and runs it. # # @param state [State] Current state # @param event [Object] Input event # @param config [StartUploadConfig, ResumeUploadConfig] Static configuration # @return [Decision] Decision snapshot - # - # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength + # @raise [InternalError] If {Rules.shape_of} or {Rules.route} produces an unlisted value def self.decide state, event, config shape = shape_of event unless SHAPES.include? shape raise InternalError, "Resumable upload internal error: shape_of returned unknown shape #{shape.inspect}" end - recipe = case [state.status, shape] - in [:initializing, :start_upload] - :start_session - in [:initializing, :resume_upload] - :resume_session - in [:starting, :response_active] - :begin_transmission - in [:transmission_reading, :chunk_read_full] - :send_chunk - in [:transmission_reading, :chunk_read_eof_with_data] - :send_upload_finalize - in [:transmission_reading, :chunk_read_eof_empty] - :send_finalize - in [:transmission_sending, :response_active] - :ack_chunk - # Order matters: `enter_recovery` and `fail_with_request_error` below both match - # `[:transmission_sending | :finalizing_sending_upload | :finalizing_sending_finalize, - # :request_connection_failed | :request_timeout]`. Recovery wins purely because this arm comes first. - in [:transmission_sending | :finalizing_sending_upload | :finalizing_sending_finalize, - :response_cat2 | :request_connection_failed | :request_timeout] - :enter_recovery - in [:finalizing_sending_upload, :response_final] - :complete_upload_with_data - in [:finalizing_sending_finalize | :recovery, :response_final] - :complete_upload_finalized - in [:recovery, :response_active] - :realign_from_recovery - # Re-query with no attempt cap. Only the Driver's global deadline guarantees termination. - in [:recovery, :response_cat2] - :retry_recovery - in [:cancelling, :response_cancelled] - :complete_cancellation - in [_, :global_deadline_exceeded] - :fail_with_deadline_exceeded - in [:transmission_reading | :transmission_sending | :finalizing_sending_upload | - :finalizing_sending_finalize | :recovery, :user_cancel] - :cancel_session - in [:starting | :transmission_sending | :finalizing_sending_upload | - :finalizing_sending_finalize | :recovery | :cancelling, :response_rejected] - :fail_with_rejected - # `:starting` fails on `:response_cat2` rather than entering recovery, unlike the - # transmission and finalizing states above: there is no upload to recover to until - # initiation has returned an upload URL. - in [:starting | :cancelling, :response_cat2] | - [:starting | :transmission_sending | :finalizing_sending_upload | - :finalizing_sending_finalize | :recovery | :cancelling, :response_fatal_bad_response] - :fail_with_bad_response - in [:starting | :transmission_sending | :finalizing_sending_upload | - :finalizing_sending_finalize | :recovery | :cancelling, - :request_retries_exhausted | :request_connection_failed | :request_timeout | - :request_failed_unknown] - :fail_with_request_error - else - :fail_with_unmatched_transition - end - + recipe = route state.status, shape unless RECIPES.include? recipe raise InternalError, "Resumable upload internal error: decide selected unknown recipe #{recipe.inspect}" end @@ -437,6 +393,99 @@ def self.decide state, event, config instructions: instructions ) end + + ## + # @private + # The routing table. Maps a `[status, shape]` pair to the recipe that handles it. + # + # Pure and side-effect free: it selects a recipe without running it, so the whole + # {STATUSES} x {SHAPES} product can be enumerated and asserted directly. {Rules.decide} is the only + # caller in production code. + # + # Arms are written so that no arm matches a pair another arm also matches, which makes their order + # presentational rather than load-bearing — with a single deliberate exception, flagged in place, + # where `enter_recovery` must precede `fail_with_request_error`. + # + # @param status [Symbol] Current protocol status, one of {STATUSES} + # @param shape [Symbol] Canonical event shape, one of {SHAPES} + # @return [Symbol] Recipe symbol, one of {RECIPES} + # + # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength + def self.route status, shape + case [status, shape] + in [:initializing, :start_upload] + :start_session + in [:initializing, :resume_upload] + :resume_session + in [:starting, :response_active] + :begin_transmission + in [:transmission_reading, :chunk_read_full] + :send_chunk + in [:transmission_reading, :chunk_read_eof_with_data] + :send_upload_finalize + in [:transmission_reading, :chunk_read_eof_empty] + :send_finalize + in [:transmission_sending, :response_active] + :ack_chunk + # Order matters: `enter_recovery` and `fail_with_request_error` below both match + # `[:transmission_sending | :finalizing_sending_upload | :finalizing_sending_finalize, + # :request_connection_failed | :request_timeout]`. Recovery wins purely because this arm comes first. + # This is the only overlap in the table. + in [:transmission_sending | :finalizing_sending_upload | :finalizing_sending_finalize, + :response_cat2 | :request_connection_failed | :request_timeout] + :enter_recovery + in [:finalizing_sending_upload, :response_final] + :complete_upload_with_data + in [:finalizing_sending_finalize | :recovery, :response_final] + :complete_upload_finalized + in [:recovery, :response_active] + :realign_from_recovery + # Re-query with no attempt cap. Only the Driver's global deadline guarantees termination. + in [:recovery, :response_cat2] + :retry_recovery + in [:cancelling, :response_cancelled] + :complete_cancellation + # A `cancelled` status on a session this client did not ask to cancel: the session was terminated + # out of band and no byte of it will ever be accepted again. Terminal, and deliberately not a bad + # response — the server answered correctly, so the caller gets no resume handle to loop on. + in [:transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery, :response_cancelled] + :fail_with_cancelled + in [_, :global_deadline_exceeded] + :fail_with_deadline_exceeded + in [:transmission_reading | :transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery, :user_cancel] + :cancel_session + in [:starting | :transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery | :cancelling, :response_rejected] + :fail_with_rejected + # Every HTTP response shape in an HTTP-awaiting status that no arm above claims. Spelled out per + # status rather than as a `[six statuses, five shapes]` cross product: the product would also cover + # `[:starting, :response_active]`, `[:transmission_sending, :response_active]` and + # `[:cancelling, :response_cancelled]`, which are handled above — the last of them a *successful* + # cancel acknowledgement. + # + # Note `[:starting, :response_cat2]`: initiation fails on a Category 2 response rather than + # recovering, unlike the transmission and finalizing statuses, because there is no upload to + # recover to until initiation has returned an upload URL. + in [:starting, :response_final | :response_cancelled | + :response_cat2 | :response_fatal_bad_response] | + [:transmission_sending, :response_final | :response_fatal_bad_response] | + [:finalizing_sending_upload | :finalizing_sending_finalize, + :response_active | :response_fatal_bad_response] | + [:recovery, :response_fatal_bad_response] | + [:cancelling, :response_active | :response_final | + :response_cat2 | :response_fatal_bad_response] + :fail_with_bad_response + in [:starting | :transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery | :cancelling, + :request_retries_exhausted | :request_connection_failed | :request_timeout | + :request_failed_unknown] + :fail_with_request_error + else + :fail_with_unmatched_transition + end + end # rubocop:enable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength ## @@ -748,6 +797,27 @@ def self.complete_cancellation state, event, _config [next_state, [Instruction::TerminateFailure.new(error: err)]] end + ## + # @private + # Terminates the run after the server reported a session this client did not cancel as cancelled. + # + # Distinct from {Rules.complete_cancellation}, which acknowledges a cancellation this client + # requested. Both land in `:cancelled` with an {UploadCancelledError}, so the caller sees one error + # type for "this session is gone" regardless of who ended it. No resume handle is attached: + # {UploadCancelledError} does not include {HasResumeHandle}, because a cancelled session will never + # accept another byte and retrying against it cannot succeed. + # + # @param state [State] Current state + # @param _event [Event::HttpResponse] Response carrying `X-Goog-Upload-Status: cancelled` + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.fail_with_cancelled state, _event, _config + action = STATE_DESCRIPTIONS[state.status] || "processing #{state.status}" + err = UploadCancelledError.new "Resumable upload session was cancelled (detected while #{action})" + next_state = state.with status: :cancelled, in_flight_length: 0, last_error: err + [next_state, [Instruction::TerminateFailure.new(error: err)]] + end + ## # @private # Initiates session cancellation request. @@ -821,13 +891,19 @@ def self.fail_with_rejected state, event, _config # @private # Fails upload when an unrecoverable HTTP response is encountered. # + # Covers both malformed responses and well-formed ones that arrived in the wrong protocol phase, so + # the message names the phase: an `X-Goog-Upload-Status` of `final` is unremarkable on its own and + # only makes sense as a failure once the reader knows a non-final chunk was in flight. + # # @param state [State] Current state - # @param event [Event::HttpResponse] Fatal HTTP response + # @param event [Event::HttpResponse] Fatal or out-of-phase HTTP response # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration # @return [Array>] Tuple of [next_state, instructions] def self.fail_with_bad_response state, event, _config handle = resume_handle_from state - err = BadResponseError.from event, resume_handle: handle + action = STATE_DESCRIPTIONS[state.status] || "processing #{state.status}" + err = BadResponseError.from event, resume_handle: handle, + prefix: "Resumable upload failed while #{action}" next_state = state.with( status: :error, in_flight_length: 0, @@ -857,7 +933,15 @@ def self.fail_with_request_error state, event, _config ## # @private - # Raises InvalidTransitionError for unmatched state and event pair. + # Raises {InvalidTransitionError} for an unmatched state and event pair. + # + # Only reachable on an internal sequencing bug. Every externally caused failure — including an HTTP + # response whose upload status does not match the phase of the request in flight — is claimed by an + # arm in {Rules.route} and terminates through a recipe instead of raising from here. + # + # No resume handle is attached, for the same reason: this reports a defect in the library, not a + # resumable upload condition. A caller that still wants the handle can read it from + # {Gapic::ResumableUpload#resume_handle}. # # @param state [State] Current state # @param event [Object] Dispatched event @@ -869,13 +953,11 @@ def self.fail_with_unmatched_transition state, event, _config happened = describe_event event, shape message = "Resumable upload failed while #{action}: #{happened}." response = event.is_a?(Event::HttpResponse) ? event : nil - handle = resume_handle_from state raise InvalidTransitionError.new( message, - state: state.status, - event: event, - response: response, - resume_handle: handle + state: state.status, + event: event, + response: response ) end diff --git a/gapic-common/lib/gapic/resumable_upload.rb b/gapic-common/lib/gapic/resumable_upload.rb index 863cb89..a7f8f27 100644 --- a/gapic-common/lib/gapic/resumable_upload.rb +++ b/gapic-common/lib/gapic/resumable_upload.rb @@ -210,8 +210,6 @@ def initialize client_stub_proc:, # recovery on an unseekable stream # @raise [Gapic::Rest::ResumableUpload::StreamMismatchError] If stream content or length does not match # protocol expectations - # @raise [Gapic::Rest::ResumableUpload::InvalidTransitionError] If an unmatched event occurs for the - # current protocol state # @raise [Gapic::Rest::ResumableUpload::UploadRejectedError] If the server explicitly rejects the upload # @raise [Gapic::Rest::ResumableUpload::InternalError] If the library detects an internal invariant # breach; this signals a bug in this library rather than a caller or server error @@ -278,8 +276,6 @@ def start stream:, # recovery on an unseekable stream # @raise [Gapic::Rest::ResumableUpload::StreamMismatchError] If stream content or length does not match # the resumed upload - # @raise [Gapic::Rest::ResumableUpload::InvalidTransitionError] If an unmatched event occurs for the - # current protocol state # @raise [Gapic::Rest::ResumableUpload::UploadRejectedError] If the server explicitly rejects the upload # @raise [Gapic::Rest::ResumableUpload::InternalError] If the library detects an internal invariant # breach; this signals a bug in this library rather than a caller or server error diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb index 9b3edf1..a5137a5 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb @@ -315,6 +315,185 @@ def test_row_fail_with_request_error assert_instance_of Instruction::TerminateFailure, decision.instructions.first end + def test_row_transmission_sending_response_cancelled + cancelled_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + state = State.new status: :transmission_sending, upload_url: "https://example.com/session" + decision = Rules.decide state, cancelled_resp, @config + assert_equal :transmission_sending, decision.from_status + assert_equal :response_cancelled, decision.shape + assert_equal :fail_with_cancelled, decision.recipe + assert_equal :cancelled, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of UploadCancelledError, decision.next_state.last_error + assert_instance_of Instruction::TerminateFailure, decision.instructions.first + end + + def test_row_recovery_response_cancelled + cancelled_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + state = State.new status: :recovery, upload_url: "https://example.com/session" + decision = Rules.decide state, cancelled_resp, @config + assert_equal :fail_with_cancelled, decision.recipe + assert_equal :cancelled, decision.next_state.status + assert_instance_of UploadCancelledError, decision.next_state.last_error + end + + def test_row_out_of_phase_responses_fail_as_bad_response + final_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" } + active_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "active" } + cancelled_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + + # Server finalizes a chunk that was not the last one. + decision = Rules.decide State.new(status: :transmission_sending), final_resp, @config + assert_equal :fail_with_bad_response, decision.recipe + assert_equal :error, decision.next_state.status + assert_instance_of BadResponseError, decision.next_state.last_error + + # Server refuses to finalize, answering a finalize command with `active`. + decision = Rules.decide State.new(status: :finalizing_sending_finalize), active_resp, @config + assert_equal :fail_with_bad_response, decision.recipe + assert_equal :error, decision.next_state.status + + # Server answers session initiation with `cancelled`, before any session exists to cancel. + decision = Rules.decide State.new(status: :starting), cancelled_resp, @config + assert_equal :fail_with_bad_response, decision.recipe + assert_equal :error, decision.next_state.status + end + + ## + # Pins every cell of the {Rules::STATUSES} x {Rules::SHAPES} product. + # + # `Rules.route` is pure, so this enumerates the entire table without building a State or running a + # recipe. Only cells that route somewhere other than `:fail_with_unmatched_transition` are listed below; + # everything else is asserted to fall through to it. Any routing change — a widened arm, a reordered + # arm, a new recipe — shows up here as a named cell rather than as a surprise in production. + # + def test_routing_table_is_pinned_for_every_status_and_shape + routed = { + initializing: { + start_upload: :start_session, + resume_upload: :resume_session + }, + starting: { + response_active: :begin_transmission, + response_final: :fail_with_bad_response, + response_cancelled: :fail_with_bad_response, + response_cat2: :fail_with_bad_response, + response_fatal_bad_response: :fail_with_bad_response, + response_rejected: :fail_with_rejected, + request_retries_exhausted: :fail_with_request_error, + request_connection_failed: :fail_with_request_error, + request_timeout: :fail_with_request_error, + request_failed_unknown: :fail_with_request_error + }, + transmission_reading: { + chunk_read_full: :send_chunk, + chunk_read_eof_with_data: :send_upload_finalize, + chunk_read_eof_empty: :send_finalize, + user_cancel: :cancel_session + }, + transmission_sending: { + response_active: :ack_chunk, + response_final: :fail_with_bad_response, + response_cancelled: :fail_with_cancelled, + response_cat2: :enter_recovery, + response_fatal_bad_response: :fail_with_bad_response, + response_rejected: :fail_with_rejected, + request_connection_failed: :enter_recovery, + request_timeout: :enter_recovery, + request_retries_exhausted: :fail_with_request_error, + request_failed_unknown: :fail_with_request_error, + user_cancel: :cancel_session + }, + finalizing_sending_upload: { + response_active: :fail_with_bad_response, + response_final: :complete_upload_with_data, + response_cancelled: :fail_with_cancelled, + response_cat2: :enter_recovery, + response_fatal_bad_response: :fail_with_bad_response, + response_rejected: :fail_with_rejected, + request_connection_failed: :enter_recovery, + request_timeout: :enter_recovery, + request_retries_exhausted: :fail_with_request_error, + request_failed_unknown: :fail_with_request_error, + user_cancel: :cancel_session + }, + finalizing_sending_finalize: { + response_active: :fail_with_bad_response, + response_final: :complete_upload_finalized, + response_cancelled: :fail_with_cancelled, + response_cat2: :enter_recovery, + response_fatal_bad_response: :fail_with_bad_response, + response_rejected: :fail_with_rejected, + request_connection_failed: :enter_recovery, + request_timeout: :enter_recovery, + request_retries_exhausted: :fail_with_request_error, + request_failed_unknown: :fail_with_request_error, + user_cancel: :cancel_session + }, + recovery: { + response_active: :realign_from_recovery, + response_final: :complete_upload_finalized, + response_cancelled: :fail_with_cancelled, + response_cat2: :retry_recovery, + response_fatal_bad_response: :fail_with_bad_response, + response_rejected: :fail_with_rejected, + request_retries_exhausted: :fail_with_request_error, + request_connection_failed: :fail_with_request_error, + request_timeout: :fail_with_request_error, + request_failed_unknown: :fail_with_request_error, + user_cancel: :cancel_session + }, + cancelling: { + response_active: :fail_with_bad_response, + response_final: :fail_with_bad_response, + response_cancelled: :complete_cancellation, + response_cat2: :fail_with_bad_response, + response_fatal_bad_response: :fail_with_bad_response, + response_rejected: :fail_with_rejected, + request_retries_exhausted: :fail_with_request_error, + request_connection_failed: :fail_with_request_error, + request_timeout: :fail_with_request_error, + request_failed_unknown: :fail_with_request_error + }, + # The four terminal statuses route nothing but the global-deadline wildcard, and the Driver stops + # the loop before it could dispatch into them anyway. + success: {}, + cancelled: {}, + rejected: {}, + error: {} + } + + # `[_, :global_deadline_exceeded]` is the table's one wildcard arm: it applies in every status. + expected = routed.transform_values do |row| + { global_deadline_exceeded: :fail_with_deadline_exceeded }.merge row + end + + assert_equal Rules::STATUSES.sort, expected.keys.sort, + "Routing table must name every status in Rules::STATUSES and no others" + expected.each do |status, row| + assert_empty row.keys - Rules::SHAPES, + "Routing table names shapes absent from Rules::SHAPES under #{status.inspect}" + end + + mismatches = [] + Rules::STATUSES.each do |status| + Rules::SHAPES.each do |shape| + want = expected[status][shape] || :fail_with_unmatched_transition + got = Rules.route status, shape + next if want == got + + mismatches << " [#{status.inspect}, #{shape.inspect}] expected #{want.inspect}, got #{got.inspect}" + end + end + cells = Rules::STATUSES.size * Rules::SHAPES.size + assert_empty mismatches, + "Routing changed in #{mismatches.size} of #{cells} cells:\n#{mismatches.join "\n"}" + + reachable = expected.values.flat_map(&:values).uniq + assert_empty Rules::RECIPES - reachable - [:fail_with_unmatched_transition], + "Recipes that no routing arm can select" + end + private def assert_recipe_progress_notification decision diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb index 212deec..212a3de 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb @@ -130,20 +130,65 @@ def test_transition_global_deadline_exceeded assert_instance_of Instruction::TerminateFailure, instructions.first end - def test_invalid_transition_raises_actionable_error_with_response_details_and_header - state = State.new status: :transmission_sending + def test_out_of_phase_final_response_during_transmission_fails_as_bad_response + state = State.new status: :transmission_sending, upload_url: "https://upload.example.com/session_abc" resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}' - err = assert_raises InvalidTransitionError do - Rules.step state, resp, @config - end + next_state, instructions = Rules.step state, resp, @config - expected_msg = "Resumable upload failed while sending a chunk of data: " \ - "received an unexpected HTTP 200 response (X-Goog-Upload-Status: 'final')." + assert_equal :error, next_state.status + err = next_state.last_error + assert_instance_of BadResponseError, err + expected_msg = "Resumable upload failed while sending a chunk of data with HTTP 200 OK " \ + "(X-Goog-Upload-Status: 'final')" \ + "#{HasResumeHandle::RESUMABLE_SUFFIX}" assert_equal expected_msg, err.message - assert_equal :transmission_sending, err.state - assert_equal resp, err.event - assert_equal resp, err.response + assert_equal 200, err.status_code + assert_equal '{"done":true}', err.response_body + assert_equal "https://upload.example.com/session_abc", err.resume_handle.upload_url + assert_instance_of Instruction::TerminateFailure, instructions.first + end + + def test_out_of_band_cancellation_during_transmission_terminates_as_cancelled + state = State.new status: :transmission_sending, upload_url: "https://upload.example.com/session_abc" + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + + next_state, instructions = Rules.step state, resp, @config + + assert_equal :cancelled, next_state.status + err = next_state.last_error + assert_instance_of UploadCancelledError, err + assert_equal "Resumable upload session was cancelled (detected while sending a chunk of data)", err.message + assert_instance_of Instruction::TerminateFailure, instructions.first + end + + def test_out_of_band_cancellation_is_not_resumable + state = State.new status: :recovery, upload_url: "https://upload.example.com/session_abc", chunk_size: 512 + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + + next_state, = Rules.step state, resp, @config + err = next_state.last_error + + assert_equal :cancelled, next_state.status + assert_instance_of UploadCancelledError, err + # The session is gone: rescuing HasResumeHandle must not catch this, or a caller following the + # documented "retry with the handle" idiom would loop against a session that can never accept a byte. + refute_operator HasResumeHandle, :===, err + refute_includes err.message, "(upload session is resumable: see #resume_handle)" + assert_equal "Resumable upload session was cancelled " \ + "(detected while querying upload offset for recovery)", + err.message + end + + def test_client_requested_cancellation_still_completes_normally + state = State.new status: :cancelling, upload_url: "https://upload.example.com/session_abc" + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + + next_state, = Rules.step state, resp, @config + + assert_equal :cancelled, next_state.status + assert_instance_of UploadCancelledError, next_state.last_error + assert_equal :complete_cancellation, Rules.route(:cancelling, :response_cancelled) end def test_invalid_transition_shows_missing_when_upload_status_header_absent @@ -243,7 +288,9 @@ def test_bad_response_with_wrapped_error_deprefixes_message_and_preserves_metada assert_equal :error, next_state.status err = next_state.last_error assert_instance_of BadResponseError, err - assert_equal "Resumable upload failed with HTTP 429 RESOURCE_EXHAUSTED: Quota limit reached", err.message + assert_equal "Resumable upload failed while initiating upload session with HTTP 429 RESOURCE_EXHAUSTED: " \ + "Quota limit reached", + err.message assert_equal 429, err.status_code assert_equal "RESOURCE_EXHAUSTED", err.status assert_equal details, err.status_details @@ -259,7 +306,9 @@ def test_bad_response_fallback_without_wrapped_error assert_equal :error, next_state.status err = next_state.last_error assert_instance_of BadResponseError, err - assert_equal "Resumable upload failed with HTTP 503 Service Unavailable (X-Goog-Upload-Status: missing)", err.message + assert_equal "Resumable upload failed while initiating upload session with HTTP 503 Service Unavailable " \ + "(X-Goog-Upload-Status: missing)", + err.message assert_equal 503, err.status_code assert_equal "Service unavailable", err.response_body end @@ -293,7 +342,9 @@ def test_error_class_inheritance_hierarchy assert_operator HasResumeHandle, :===, BadResponseError.new assert_operator HasResumeHandle, :===, DeadlineExceededError.new assert_operator HasResumeHandle, :===, UnseekableStreamError.new - assert_operator HasResumeHandle, :===, InvalidTransitionError.new("invalid") + refute_operator HasResumeHandle, :===, InvalidTransitionError.new("invalid") + assert_operator InvalidTransitionError, :<, InternalError + assert_operator InternalError, :===, InvalidTransitionError.new("invalid") assert_operator HasResumeHandle, :===, StreamMismatchError.new assert_operator HasResumeHandle, :===, RequestFailedError.new("failed") refute_operator HasResumeHandle, :===, UploadRejectedError.new @@ -342,14 +393,12 @@ def test_resume_handle_present_on_errors_when_upload_url_set assert_equal 512, bad_resp_err.resume_handle.chunk_size assert_includes bad_resp_err.message, "(upload session is resumable: see #resume_handle)" - # 3. Unmatched transition + # 3. Unmatched transition — an internal bug, never resumable, even with an established session unmatched_err = assert_raises InvalidTransitionError do Rules.step state, Object.new, @config end - refute_nil unmatched_err.resume_handle - assert_equal "https://upload.example.com/session_abc", unmatched_err.resume_handle.upload_url - assert_equal 512, unmatched_err.resume_handle.chunk_size - assert_includes unmatched_err.message, "(upload session is resumable: see #resume_handle)" + refute_operator HasResumeHandle, :===, unmatched_err + refute_includes unmatched_err.message, "(upload session is resumable: see #resume_handle)" # 4. Request failed (retries exhausted) req_failed = Event::RequestFailed.new( @@ -390,7 +439,7 @@ def test_resume_handle_nil_on_errors_before_session_created unmatched_err = assert_raises InvalidTransitionError do Rules.step state, Object.new, @config end - assert_nil unmatched_err.resume_handle + refute_operator HasResumeHandle, :===, unmatched_err refute_includes unmatched_err.message, "(upload session is resumable: see #resume_handle)" # 4. Request failed before session creation diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_test.rb index 0c8031f..654f073 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/rules_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_test.rb @@ -234,6 +234,7 @@ def test_all_recipes_satisfy_trampoline_invariant complete_upload_finalized: [base_state.with(status: :finalizing_sending_finalize), final_resp, @config], cancel_session: [base_state, Event::Cancel.new, @config], complete_cancellation: [base_state.with(status: :cancelling), cancelled_resp, @config], + fail_with_cancelled: [base_state, cancelled_resp, @config], fail_with_deadline_exceeded: [base_state, Event::GlobalDeadlineExceeded.new, @config], fail_with_rejected: [base_state, rejected_resp, @config], fail_with_bad_response: [base_state, bad_resp, @config], From 59a3c2787da7529bd32f13f290209a3061b9ec95 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Wed, 23 Sep 2026 19:46:50 +0000 Subject: [PATCH 13/22] fix: duplicate class --- gapic-common/lib/gapic/rest/resumable_upload/errors.rb | 9 --------- 1 file changed, 9 deletions(-) diff --git a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb index 18dc971..0d5afae 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb @@ -567,15 +567,6 @@ def self.from event_or_error, message: nil, resume_handle: nil # class SessionStateError < Gapic::Common::Error end - - ## - # @private - # Raised when an internal state machine or driver invariant is violated. - # Produced by {Rules} when an unlisted shape or recipe is encountered, and by - # {Driver} when a recipe emits a malformed instruction batch. - # - class InternalError < Gapic::Common::Error - end end end end From 3badfe7e5254938e5118f3e0c6a0d4f77e0079ce Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Thu, 24 Sep 2026 19:16:30 +0000 Subject: [PATCH 14/22] fix: don't pass internal events to retry policy --- .../resumable_upload/implementation-guide.md | 6 +- .../lib/gapic/rest/resumable_upload/driver.rb | 45 ++++++++----- .../rest/resumable_upload/retry_policies.rb | 5 +- .../lib/gapic/rest/resumable_upload/rules.rb | 4 +- .../resumable_upload/driver_retry_test.rb | 67 ++++++++++++++++++- 5 files changed, 100 insertions(+), 27 deletions(-) diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md index 1fc6578..0df42d6 100644 --- a/gapic-common/design/resumable_upload/implementation-guide.md +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -340,7 +340,7 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` 2. **Offset Extraction**: On `query` responses, the acknowledged byte count is extracted from `X-Goog-Upload-Size-Received` as an integer (`server_offset`). 3. **Request Modification on 4xx**: Retrying Category 2 errors requires querying the backend for `server_offset` first. 4. **Standard Retry Configuration & Distinct Policies**: The Driver manages distinct retry policy configurations for Category 1 transient errors: - * **Start Policy (`start_retry_policy`)**: Applies specifically to session initiation (`start`). Configured with standard retry codes (`["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"]`) and network errors (`[Faraday::ConnectionFailed, Faraday::TimeoutError, SocketError]`). A missing or empty `X-Goog-Upload-Status` header is treated as **retriable** (predicate returns `true`) across **any response code, including 200 OK**. + * **Start Policy (`start_retry_policy`)**: Applies specifically to session initiation (`start`). Configured with standard retry codes (`["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"]`) and `START_PREDICATE`. A missing or empty `X-Goog-Upload-Status` header is retriable across any non-fatal status code: non-fatal `4xx`/`5xx` responses are retried inside `ClientStub` via `START_PREDICATE` (which receives the `Faraday::Error`), while headerless `200 OK` responses (which Faraday does not raise on) are retried by `Driver#execute_send_start` via no-argument `policy.call` (consulting `start_retry_policy` only for its deadline and backoff schedule, without invoking `retry_predicate`). * **Control Plane Policy (`control_plane_retry_policy`)**: Applies to session control requests (`query`, `cancel`). Configured with standard retry codes and network errors. It does **not** retry on a missing `X-Goog-Upload-Status` header, allowing `Core` to evaluate responses immediately. * **Data Plane Policy (`data_plane_retry_policy`)**: Applies to data transmission requests (`upload`, `upload,finalize`, and standalone `finalize`). Shares the standard retry codes and network errors, but treats a missing `X-Goog-Upload-Status` header as **unretriable** (predicate returns `false`). This prevents blind chunk re-transmission and returns `Event::HttpResponse` immediately to `Core` so it can initiate Category 2 `Recovery`. * **Retry Policy Override Contract**: Each retry policy configuration field accepts a `Gapic::Common::RetryPolicy` instance, a `Hash`, or `nil`. Passing a `RetryPolicy` instance replaces the default policy entirely. Passing a `Hash` constructs a new `RetryPolicy` and applies the category's defaults (`RetryPolicy.new(**hash).apply_defaults(defaults)`), overriding the specified fields while preserving unspecified defaults such as `retry_codes` and `retry_predicate`. Passing `nil` constructs the default policy directly from the category defaults. @@ -515,7 +515,9 @@ The implementation distinguishes three categories of network and protocol-level 3. **Unretried Data Plane Connection Drops or Request Timeouts**: `Event::RequestFailed(kind: :connection_failed)` or `Event::RequestFailed(kind: :timeout)` (`:request_connection_failed`, `:request_timeout`) occurring during `Transmission` or `Finalizing`. * **Missing Header Handling & Retry Policy Contract**: * *Why Headers Go Missing*: Intermediate proxies, reverse-proxies, or Google Front End (GFE) edge proxies can strip the protocol response headers or return raw HTML/text error pages on failure. - * *Session Initiation (`start`)*: Missing `X-Goog-Upload-Status` is treated as **retriable** by `start_retry_policy` (retry predicate returns `true`) across **any response code, including 200 OK**. Driver retries transparently to smooth over transient gateway noise. If retries exhaust, `Starting` transitions to `:error` via `fail_with_request_error` or `fail_with_bad_response` (cannot recover a session before an upload URL is obtained). + * *Session Initiation (`start`)*: Missing `X-Goog-Upload-Status` is retriable across any non-fatal status code via two complementary layers: + * **Non-`200` (`4xx`/`5xx`)**: Retried inside `ClientStub` via `start_retry_policy`'s `START_PREDICATE` (which receives the `Faraday::Error` and returns `true` unless the status is in `FATAL_STATUS_CODES`). When `ClientStub` finishes and returns/raises, `Driver#execute_send_start` passes the non-`200` `Event::HttpResponse` immediately to `Core` (`fail_with_bad_response` → `BadResponseError`). + * **`200 OK`**: Faraday does not raise on `200`, so `ClientStub` never runs `start_retry_policy`. Instead, `Driver#execute_send_start` retries headerless `200` responses directly via no-argument `policy.call` (using `start_retry_policy` only for its deadline and backoff schedule, never passing the response to `retry_predicate`). If retries exhaust, `Driver#headerless_start_failure` emits `Event::RequestFailed(kind: :retries_exhausted, source_error: BadResponseError)` (`fail_with_request_error` → `RequestFailedError`). * *Session Control (`query`, `cancel`)*: `control_plane_retry_policy` does **not** treat missing status headers as retriable, returning the completed `Event::HttpResponse` immediately to `Core` so it can manage protocol recovery or fail fast. * *Data Plane (`upload`, `upload, finalize`, standalone `finalize`)*: Missing `X-Goog-Upload-Status` is treated as **unretriable** by `data_plane_retry_policy` (retry predicate returns `false`). The Driver immediately returns `Event::HttpResponse` to `Core` so it classifies as `:response_cat2` and initiates Category 2 `Recovery` via `Instruction::SendQuery` rather than blindly re-transmitting data. * **Resolution**: Core transitions to `Recovery` and emits `Instruction::SendQuery.new(url: state.upload_url)` to obtain `server_offset`. diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb index 8b0d7b7..41362b6 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb @@ -577,27 +577,38 @@ def execute_send_start instruction status_hdr = Rules.header_value event.headers, "x-goog-upload-status" return event unless status_hdr.nil? || status_hdr.empty? - return event if Rules::FATAL_STATUS_CODES.include? event.status - - err = BadResponseError.new "Missing X-Goog-Upload-Status header in start response", - event.status, - headers: event.headers - # `retry_with_deadline?` is public; its `@private` tag hides it from docs, not from callers. - can_retry = policy.retry_with_deadline? && policy.call(event) - unless can_retry - if event.status == 200 - failed_event = Event::RequestFailed.new( - kind: :retries_exhausted, message: err.message, source_error: err - ) - @upload_log.wire_failure failed_event - return failed_event - end - return event + + # Non-200 responses with missing protocol headers (e.g. a 503 from GFE) are retried + # inside ClientStub and should be classified by their status code rather than as a + # missing-header error. + return event unless event.status == 200 + + # Checking for policy here to avoid retrying into timeout + if policy.call + attempt += 1 + next end - attempt += 1 + return headerless_start_failure event end end + ## + # @private + # Builds and logs the `:retries_exhausted` failure event for a `200` initiation response + # that lacked `X-Goog-Upload-Status` once the start retry policy is out of budget. + # + # @param event [Event::HttpResponse] Final headerless `200` initiation response + # @return [Event::RequestFailed] + # + def headerless_start_failure event + err = BadResponseError.new "Missing X-Goog-Upload-Status header in start response", + event.status, headers: event.headers + failed_event = Event::RequestFailed.new kind: :retries_exhausted, message: err.message, + source_error: err + @upload_log.wire_failure failed_event + failed_event + end + ## # @private # Builds initiation HTTP headers from instruction and config. diff --git a/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb index 29d8341..d0cfaa8 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb @@ -97,8 +97,9 @@ module RetryPolicies ## # @private # Default retry policy for session initiation requests (start). - # Missing X-Goog-Upload-Status header is retriable across any response code, - # including 200 (predicate returns true). + # Its predicate treats a missing X-Goog-Upload-Status header as retriable across non-fatal + # 4xx/5xx responses in ClientStub; headerless 200 responses are retried by Driver#execute_send_start + # via the no-argument RetryPolicy#call. # # @return [Gapic::Common::RetryPolicy] def self.default_start diff --git a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb index b6ef961..e6c2ea9 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb @@ -461,9 +461,7 @@ def self.route status, shape :fail_with_rejected # Every HTTP response shape in an HTTP-awaiting status that no arm above claims. Spelled out per # status rather than as a `[six statuses, five shapes]` cross product: the product would also cover - # `[:starting, :response_active]`, `[:transmission_sending, :response_active]` and - # `[:cancelling, :response_cancelled]`, which are handled above — the last of them a *successful* - # cancel acknowledgement. + # combinations which are handled above. # # Note `[:starting, :response_cat2]`: initiation fails on a Category 2 response rather than # recovering, unlike the transmission and finalizing statuses, because there is no upload to diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb index 2fca882..f20cf52 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb @@ -98,8 +98,8 @@ def test_start_exhausts_retries_when_200_responses_continually_lack_status_heade assert stub.requests.size > 1 end - def test_start_exhausts_retries_when_non_200_responses_continually_lack_status_header - responses = Array.new(10) { FakeResponse.new status: 503, headers: {}, body: "Service Unavailable" } + def test_start_does_not_re_retry_non_200_responses_lacking_status_header_outside_client_stub + responses = [FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable")] stub = FakeClientStub.new responses config = StartUploadConfig.new( initial_url: "https://example.com/upload", @@ -117,7 +117,7 @@ def test_start_exhausts_retries_when_non_200_responses_continually_lack_status_h assert_equal 503, err.status_code assert_includes err.message, "503" refute_match(/Missing X-Goog-Upload-Status/, err.message) - assert stub.requests.size > 1 + assert_equal 1, stub.requests.size end def test_query_does_not_retry_on_missing_status_header_in_driver @@ -162,6 +162,67 @@ def test_query_does_not_retry_on_missing_status_header_in_driver assert_chunk_request stub.requests[4], offset: "0", length: "4", body: "0123", finalize: true end + # The initiation missing-header retry is a protocol requirement, so it must not depend on the retry + # policy carrying RetryPolicies::START_PREDICATE. A RetryPolicy instance is passed through verbatim + # by Driver#resolve_retry_policy, defaults and all, so this policy reaches the start loop with no + # predicate at all. Consulting it about the response would answer "no retry" on the first attempt, + # because Event::HttpResponse has no `response_status` for the retry_codes branch to read. + def test_start_retries_missing_status_header_under_a_policy_carrying_no_predicate + responses = Array.new(200) { FakeResponse.new status: 200, headers: {}, body: "" } + stub = FakeClientStub.new responses + policy = Gapic::Common::RetryPolicy.new initial_delay: 0.001, max_delay: 0.002, timeout: 0.05 + assert_nil policy.retry_predicate + + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + start_retry_policy: policy + ) + + driver = Driver.new client_stub: stub, config: config + err = assert_raises RequestFailedError do + driver.run + end + + assert_match(/Missing X-Goog-Upload-Status/, err.message) + assert stub.requests.size > 1, "Expected the missing-header retry to survive a predicate-less policy" + end + + # A caller-supplied predicate displaces START_PREDICATE and is written against the Faraday errors + # ClientStub rescues. The start loop must never hand it an Event, a BadResponseError, or anything + # else from the driver's private vocabulary. + def test_start_never_passes_driver_internals_to_a_caller_supplied_predicate + seen = [] + responses = Array.new(200) { FakeResponse.new status: 200, headers: {}, body: "" } + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + start_retry_policy: { + initial_delay: 0.001, + max_delay: 0.002, + timeout: 0.05, + retry_predicate: lambda { |arg| + seen << arg + nil + } + } + ) + + driver = Driver.new client_stub: stub, config: config + assert_raises RequestFailedError do + driver.run + end + + assert stub.requests.size > 1, "Expected retries, otherwise the predicate assertion proves nothing" + refute seen.any? { |arg| arg.is_a?(Event::HttpResponse) || arg.is_a?(BadResponseError) }, + "Predicate was handed driver internals: #{seen.map(&:class).uniq.inspect}" + end + private def assert_start_request req From df9da1274e0af541aea9556eaabb07243f74d0e7 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Fri, 25 Sep 2026 22:47:59 +0000 Subject: [PATCH 15/22] fix: retry transport errors on start and control plane --- .../resumable_upload/implementation-guide.md | 113 +++-- .../resumable_upload/error_on_start_test.rb | 15 +- .../resumable_upload/error_recovery_test.rb | 23 +- .../resumable_upload/resume_test.rb | 8 +- .../lib/gapic/rest/resumable_upload.rb | 14 +- .../lib/gapic/rest/resumable_upload/driver.rb | 222 +++++---- .../resumable_upload/driver/retry_decider.rb | 190 +++++++ .../lib/gapic/rest/resumable_upload/events.rb | 5 +- .../rest/resumable_upload/retry_policies.rb | 108 ++-- .../lib/gapic/rest/resumable_upload/rules.rb | 77 ++- gapic-common/lib/gapic/resumable_upload.rb | 33 +- .../driver/retry_decider_test.rb | 245 +++++++++ .../driver_error_mapping_test.rb | 23 +- .../driver_retry_policy_test.rb | 39 +- .../resumable_upload/driver_retry_test.rb | 467 +++++++++++------- .../resumable_upload/retry_policies_test.rb | 237 ++------- .../rules_classification_test.rb | 50 +- .../test/gapic/resumable_upload_test.rb | 17 +- 18 files changed, 1195 insertions(+), 691 deletions(-) create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/driver/retry_decider.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver/retry_decider_test.rb diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md index 0df42d6..4480a6f 100644 --- a/gapic-common/design/resumable_upload/implementation-guide.md +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -28,7 +28,7 @@ graph TD ### 1.1 Driver (Synchronous I/O Adapter) The `Driver` executes all operations with side-effects. It interacts with HTTP transport via `Gapic::Rest::ClientStub`, reads binary data from local input streams, tracks monotonic execution deadlines, and dispatches progress callbacks. -Crucially, the Driver delegates all **Category 1 (Transient)** transport retries directly to `Gapic::Common::RetryPolicy`. Transient retries occur entirely within the Driver's network execution wrapper. The `Core` state machine is never exposed to transient noise, receiving only verified successful HTTP responses or terminal transport exceptions. +Crucially, the Driver owns every **Category 1 (Transient)** retry decision. `ClientStub` is handed a never-retry policy, so each `ClientStub` call is exactly one attempt; the retry loop in `Driver#make_post_request` asks `Driver::RetryDecider` whether to re-send the same request, and `Gapic::Common::RetryPolicy` supplies only the budget, the backoff, and the caller-facing `retry_predicate` / `retry_codes`. The decider only ever chooses between re-sending and surfacing: an outcome it declines, and whatever is left when the budget runs out, is converted to an event unchanged, and `Rules` alone decides between recovery and a terminal outcome (Section 6.1). The Driver exposes: * `Driver#resume_handle`: Returns a `ResumeHandle` (or `nil` if initiation has not established an upload URL, or if the session is `:rejected`, `:cancelled`, or `:success`; completed uploads are not resumable). Reading this property mid-run provides a best-effort snapshot of current session parameters. @@ -234,8 +234,8 @@ end * `Event::ResumeUpload.new(upload_url:, chunk_size:, upload_size:)`: Resume an existing upload session with a known upload URL. * `Event::ChunkRead.new(bytes_buffered:, eof:)`: Binary data buffered in Driver memory; reports total bytes ready in buffer and whether the stream hit EOF. * `Event::HttpResponse.new(status:, headers:, body:, error: nil)`: Dispatched for any completed HTTP exchange over the wire (including 2xx, 4xx, 5xx, or responses with missing/unexpected headers). Carries optional parsed `error` (`Gapic::Rest::Error`) when rescued from transport errors. `Core` inspects status and headers to determine protocol progression or recovery. -* `Event::RequestFailed.new(kind:, message:, source_error:)`: Dispatched when an HTTP request fails to produce a usable HTTP response (e.g., request timeout, transport connection errors, or `RetryPolicy` exhaustion). - * `kind`: Normalized Symbol enum (`:timeout`, `:connection_failed`, `:retries_exhausted`). `Core` branches on `kind` and treats other fields as opaque. +* `Event::RequestFailed.new(kind:, message:, source_error:)`: Dispatched when the last attempt of an HTTP request fails to produce a usable HTTP response (e.g., request timeout, transport connection errors, or a non-transport failure before anything reached the wire). + * `kind`: Normalized Symbol enum: `:timeout`, `:connection_failed` (`Faraday::ConnectionFailed`, `Faraday::SSLError`, or a status-less `Gapic::Rest::Error`), `:retries_exhausted` (any other `Faraday::Error` without a response), or `:unknown` (a non-Faraday error, e.g. a credentials refresh failure). `Core` branches on `kind` and treats other fields as opaque. * `message`: Human-readable summary string. * `source_error`: Original underlying exception, preserved for terminal error propagation and logging. * `Event::Cancel`: Caller requested session cancellation. @@ -321,7 +321,7 @@ The Driver categorizes instructions into three execution types: * Executed immediately in-process. * Do not yield a new `Event` and do not break the batch loop. Exceptions raised within user callbacks (e.g. `on_progress`) are not swallowed and immediately propagate to the caller. 2. **I/O & Network Operations** (`FillBuffer`, `SendStart`, `SendChunk`, `SendFinalize`, `SendQuery`, `SendCancel`): - * Execute physical stream reads or HTTP requests (wrapped in `Gapic::Common::RetryPolicy` for Category 1 transient errors). + * Execute physical stream reads or HTTP requests (each request re-sent by the Driver's own retry loop for Category 1 transient errors; see Section 6.1.1). * Yield a single resulting `Event` (`ChunkRead`, `HttpResponse`, or `RequestFailed`) that becomes the input for the next cycle. 3. **Terminal Handlers** (`TerminateSuccess`, `TerminateFailure`): * Break the event loop and return the final response body string (`response.body`) or raise the terminal exception. @@ -339,11 +339,11 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` * Callers cannot supply either of these two headers via `initial_headers`; see the reserved-headers rule in Section 2 (doing so raises an `ArgumentError`). Other `X-Goog-Upload-Header-*` pass-through headers are permitted. 2. **Offset Extraction**: On `query` responses, the acknowledged byte count is extracted from `X-Goog-Upload-Size-Received` as an integer (`server_offset`). 3. **Request Modification on 4xx**: Retrying Category 2 errors requires querying the backend for `server_offset` first. -4. **Standard Retry Configuration & Distinct Policies**: The Driver manages distinct retry policy configurations for Category 1 transient errors: - * **Start Policy (`start_retry_policy`)**: Applies specifically to session initiation (`start`). Configured with standard retry codes (`["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"]`) and `START_PREDICATE`. A missing or empty `X-Goog-Upload-Status` header is retriable across any non-fatal status code: non-fatal `4xx`/`5xx` responses are retried inside `ClientStub` via `START_PREDICATE` (which receives the `Faraday::Error`), while headerless `200 OK` responses (which Faraday does not raise on) are retried by `Driver#execute_send_start` via no-argument `policy.call` (consulting `start_retry_policy` only for its deadline and backoff schedule, without invoking `retry_predicate`). - * **Control Plane Policy (`control_plane_retry_policy`)**: Applies to session control requests (`query`, `cancel`). Configured with standard retry codes and network errors. It does **not** retry on a missing `X-Goog-Upload-Status` header, allowing `Core` to evaluate responses immediately. - * **Data Plane Policy (`data_plane_retry_policy`)**: Applies to data transmission requests (`upload`, `upload,finalize`, and standalone `finalize`). Shares the standard retry codes and network errors, but treats a missing `X-Goog-Upload-Status` header as **unretriable** (predicate returns `false`). This prevents blind chunk re-transmission and returns `Event::HttpResponse` immediately to `Core` so it can initiate Category 2 `Recovery`. - * **Retry Policy Override Contract**: Each retry policy configuration field accepts a `Gapic::Common::RetryPolicy` instance, a `Hash`, or `nil`. Passing a `RetryPolicy` instance replaces the default policy entirely. Passing a `Hash` constructs a new `RetryPolicy` and applies the category's defaults (`RetryPolicy.new(**hash).apply_defaults(defaults)`), overriding the specified fields while preserving unspecified defaults such as `retry_codes` and `retry_predicate`. Passing `nil` constructs the default policy directly from the category defaults. +4. **Standard Retry Configuration & Distinct Policies**: The Driver resolves one `Gapic::Common::RetryPolicy` per plane. A policy supplies the retry budget (`timeout`), the backoff schedule, and the caller-facing `retry_predicate` / `retry_codes`; which outcomes are re-sent at all is decided by `Driver::RetryDecider` (Section 6.1.1). None of the defaults carries a `retry_predicate`, so a caller-supplied one is consulted as-is, ahead of `retry_codes`. `retry_codes` are derived from the HTTP status sets on `Rules` via `Gapic::Common::ErrorCodes.grpc_error_for`. + * **Start Policy (`start_retry_policy`)**: Applies to session initiation (`start`). Default `retry_codes`: `RETRIABLE_4XX_STATUS_CODES` + `RETRIABLE_5XX_STATUS_CODES` (HTTP `409`, `429`, `499`, `500`, `503`, `504` → `ALREADY_EXISTS`, `RESOURCE_EXHAUSTED`, `CANCELLED`, `INTERNAL`, `UNAVAILABLE`, `DEADLINE_EXCEEDED`). HTTP `408` and `502` map to `UNKNOWN` and are deliberately not retried. + * **Control Plane Policy (`control_plane_retry_policy`)**: Applies to session control requests (`query`, `cancel`). Same defaults as the start policy; the Driver treats both planes identically. + * **Data Plane Policy (`data_plane_retry_policy`)**: Applies to data transmission requests (`upload`, `upload, finalize`, and standalone `finalize`). Default `retry_codes`: `RETRIABLE_5XX_STATUS_CODES` only (`500`, `503`, `504`). The Driver never re-sends a data plane request after an outcome that leaves the server offset unknown, or after any `4xx`, whatever the policy says; those go to Category 2 `Recovery`. + * **Retry Policy Override Contract**: Each retry policy configuration field accepts a `Gapic::Common::RetryPolicy` instance, a `Hash`, or `nil`. Passing a `RetryPolicy` instance replaces the default policy entirely. Passing a `Hash` constructs a new `RetryPolicy` and applies the category's defaults (`RetryPolicy.new(**hash).apply_defaults(defaults)`), overriding the specified fields while preserving unspecified defaults such as `retry_codes`. Passing `nil` constructs the default policy directly from the category defaults. ### 4.2 State Transition & Data Mutation Specification @@ -357,39 +357,39 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` | **`Initializing`** | `:resume_upload` | `Event::ResumeUpload` | `upload_url = event.upload_url`
`chunk_size = event.chunk_size`
`offset = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: event.upload_size))`
`Instruction::SendQuery.new(url: event.upload_url)` | | **`Starting`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `upload_url = headers['X-Goog-Upload-URL']`
`chunk_granularity = headers['...-Granularity']&.to_i`
`chunk_size = resolve(config, chunk_granularity)`
`offset = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | | **`Starting`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | -| **`Starting`** | `:response_cat2` / `:response_fatal_bad_response` | `Event::HttpResponse` (Non-200; see Section 6.1) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | -| **`Starting`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind:, message:, source_error:)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Starting`** | `:response_cat2` / `:response_fatal_bad_response` | `Event::HttpResponse` (Non-200, or a headerless `200` after the start budget is spent; see Section 6.1) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Starting`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:request_failed_unknown` | `Event::RequestFailed(kind:, message:, source_error:)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Transmission \| Reading from stream`** | `:chunk_read_full` | `Event::ChunkRead(bytes_buffered, eof: false)` | `in_flight_length = event.bytes_buffered`
`status = :transmission_sending` | `Transmission \| Sending` | `Instruction::SendChunk.new(url: state.upload_url, offset: state.offset, length: event.bytes_buffered, finalize: false)` | | **`Transmission \| Reading from stream`** | `:chunk_read_eof_with_data` | `Event::ChunkRead(bytes_buffered, eof: true)` where `bytes_buffered > 0` | `in_flight_length = event.bytes_buffered`
`status = :finalizing_sending_upload` | `Finalizing \| Sending with upload` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :finalizing, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendChunk.new(url: state.upload_url, offset: state.offset, length: event.bytes_buffered, finalize: true)` | | **`Transmission \| Reading from stream`** | `:chunk_read_eof_empty` | `Event::ChunkRead(bytes_buffered: 0, eof: true)` | `in_flight_length = 0`
`status = :finalizing_sending_finalize` | `Finalizing \| Sending finalize` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :finalizing, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendFinalize.new(url: state.upload_url)` | | **`Transmission \| Sending`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = state.offset + state.in_flight_length`
`in_flight_length = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | | **`Transmission \| Sending`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | | **`Transmission \| Sending`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | -| **`Transmission \| Sending`** | `:request_retries_exhausted` | `Event::RequestFailed(kind: :retries_exhausted)` | `in_flight_length = 0`
`last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Transmission \| Sending`** | `:request_retries_exhausted` / `:request_failed_unknown` | `Event::RequestFailed(kind: :retries_exhausted \| :unknown)` | `in_flight_length = 0`
`last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Transmission \| Sending`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `in_flight_length = 0`
`status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Transmission \| Sending`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **`Finalizing \| Sending with upload`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `offset = state.offset + state.in_flight_length`
`in_flight_length = 0`
`status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | | **`Finalizing \| Sending with upload`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | | **`Finalizing \| Sending with upload`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | -| **`Finalizing \| Sending with upload`** | `:request_retries_exhausted` | `Event::RequestFailed(kind: :retries_exhausted)` | `in_flight_length = 0`
`last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Finalizing \| Sending with upload`** | `:request_retries_exhausted` / `:request_failed_unknown` | `Event::RequestFailed(kind: :retries_exhausted \| :unknown)` | `in_flight_length = 0`
`last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Finalizing \| Sending with upload`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `in_flight_length = 0`
`status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Finalizing \| Sending with upload`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **`Finalizing \| Sending finalize`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | | **`Finalizing \| Sending finalize`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | | **`Finalizing \| Sending finalize`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | -| **`Finalizing \| Sending finalize`** | `:request_retries_exhausted` | `Event::RequestFailed(kind: :retries_exhausted)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Finalizing \| Sending finalize`** | `:request_retries_exhausted` / `:request_failed_unknown` | `Event::RequestFailed(kind: :retries_exhausted \| :unknown)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Finalizing \| Sending finalize`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Finalizing \| Sending finalize`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **`Recovery`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = headers['X-Goog-Upload-Size-Received'].to_i`
`in_flight_length = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | | **`Recovery`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `in_flight_length = 0`
`status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | | **`Recovery`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `status = :recovery` | `Recovery` | `Instruction::SendQuery.new(url: state.upload_url)` | -| **`Recovery`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind:, ...)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Recovery`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:request_failed_unknown` | `Event::RequestFailed(kind:, ...)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Recovery`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Recovery`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **`Transmission \| Reading from stream` / `Transmission \| Sending chunk` / `Finalizing \| Sending with upload` / `Finalizing \| Sending finalize` / `Recovery`** | `:user_cancel` | `Event::Cancel` | `status = :cancelling` | `Cancelling` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :cancelling, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendCancel.new(url: state.upload_url)` | | **`Cancelling`** | `:response_cancelled` | `Event::HttpResponse(200, headers, _)` with `Status: cancelled` | `status = :cancelled` | `Cancelled` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadCancelledError.from(event))` | | **`Cancelling`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | -| **`Cancelling`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:response_fatal_bad_response` | `Event::RequestFailed` or HTTP failure | `last_error = error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Cancelling`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:request_failed_unknown` / `:response_fatal_bad_response` | `Event::RequestFailed` or HTTP failure | `last_error = error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **`Transmission \| Sending chunk` / `Finalizing \| Sending with upload` / `Finalizing \| Sending finalize` / `Recovery`** | `:response_cancelled` | `Event::HttpResponse(200, headers, _)` with `Status: cancelled`, on a session this client never asked to cancel | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::UploadCancelledError.new("Resumable upload session was cancelled (detected while ...)")`
`status = :cancelled` | `Cancelled` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **`Starting` / `Transmission \| Sending chunk` / `Finalizing \| Sending with upload` / `Finalizing \| Sending finalize` / `Recovery` / `Cancelling`** | any HTTP response shape not claimed above | `Event::HttpResponse` whose `X-Goog-Upload-Status` does not match the phase of the request in flight (e.g. `final` during transmission, `active` while finalizing, `cancelled` during initiation) | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event, resume_handle:, prefix: "Resumable upload failed while ")`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **Any Non-Terminal** | `:global_deadline_exceeded` | `Event::GlobalDeadlineExceeded` | `last_error = Gapic::Rest::ResumableUpload::DeadlineExceededError.new`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | @@ -503,41 +503,49 @@ end The implementation distinguishes three categories of network and protocol-level failures: #### 6.1.1 Category 1: Transient Transport Failures -* **Definition**: Standard TCP, network connection timeout, DNS, or server load-shedding errors that do not compromise the protocol session. -* **Examples**: `503 Service Unavailable`, `408 Request Timeout`, `429 Too Many Requests`, `Faraday::ConnectionFailed`, `Faraday::TimeoutError`. -* **Resolution**: The `Driver` intercepts these errors inside the physical execution wrapper and delegates directly to `Gapic::Common::RetryPolicy`. If retries succeed, `Core` receives `Event::HttpResponse`. If retries exhaust attempt/timeout limits, Driver emits `Event::RequestFailed(kind: :retries_exhausted, ...)`. +* **Definition**: Outcomes worth re-sending the same request for: connection and TLS failures, server load-shedding and transient server errors, and gateway noise such as a `200` stripped of its protocol headers. +* **Resolution**: `Driver#make_post_request` runs the retry loop. `ClientStub` is handed a never-retry policy (`Driver::CLIENT_STUB_NO_RETRY`), so every attempt's raw outcome (the response, or the exception it raised) comes back to the Driver, and `Driver::RetryDecider` judges it **before** it is converted to an event. The decider only chooses between re-sending and surfacing; the outcome it declines, or the last one when the budget runs out, is converted unchanged and routed by `Rules` (Sections 6.1.2 and 6.1.3). Under `raise_faraday_errors: false`, a `Gapic::Rest::Error` is judged by its Faraday `cause`, the only object `retry_codes` can read a status from. +* **Decision Table** (first match wins; a `GlobalDeadlineExceeded` short-circuits every row): + + | # | Outcome | `start` / `query` / `cancel` | `upload` / `finalize` | + |---|---|---|---| + | 1 | Non-Faraday error (e.g. credentials refresh) | ask policy | ask policy | + | 2 | `Faraday::TimeoutError`, or a transport error with no response | ask policy | surface | + | 3 | `Faraday::ConnectionFailed`, `Faraday::SSLError` | retry within budget | surface | + | 4 | `200` without `X-Goog-Upload-Status` | retry within budget | surface | + | 5 | Non-200 with `X-Goog-Upload-Status: final` | surface | surface | + | 6 | Any `4xx` | (falls through) | surface | + | 7 | Non-200 in the default `retry_codes` | ask policy (retried by default) | ask policy (retried by default) | + | 8 | Any other non-200 | ask policy (not retried by default) | ask policy (not retried by default) | + | 9 | Any other `2xx`, or `200` with the header | surface | surface | + + * **Retry within budget** is the no-argument `RetryPolicy#call`: it checks the policy deadline and performs the backoff, and never hands the outcome to a caller predicate. + * **Ask policy** is `retry_with_deadline? && call(error)`: the caller's `retry_predicate`, then `retry_codes`. `RetryPolicy#call(error)` does not check the deadline itself, hence the guard. `retry_codes` cannot read a status from a non-Faraday error, so row 1 is retried only when a caller predicate says so. + * Rows 5 and 6 are fixed ahead of the policy: no caller setting can retry a rejection or make the data plane re-send on a `4xx`. The data plane never re-sends after an outcome that leaves the server offset unknown (rows 2, 3, 4, 6); Category 2 `Recovery` owns those. A non-transport error (row 1) fails before anything reaches the wire, so re-sending cannot duplicate bytes. + * Row 2 is inert for timeouts on `start` / `query` / `cancel` until attempts get their own timeouts: each attempt currently receives the whole remaining command budget (Section 6.3), so a timed-out attempt leaves none. #### 6.1.2 Category 2: Recoverable Protocol Failures -* **Definition**: Responses indicating that the client's current offset is misaligned with the server, protocol headers are missing/stripped on completed requests, or unretried transport connection failures during data transmission. -* **Conditions Producing `:response_cat2`**: - 1. **Non-200 Active Responses**: Any response with `X-Goog-Upload-Status: active` where HTTP status is non-200. - 2. **Missing or Empty `X-Goog-Upload-Status` Header**: Any response lacking `X-Goog-Upload-Status` (or empty) whose HTTP status is **not** in `FATAL_STATUS_CODES` (Section 6.1.3). This includes HTTP 200, 5xx server/gateway errors (`500`, `502`, `503`, `504`), and recoverable client errors (`400`, `408`, `409`, `412`, `416`, `429`, `499`). - 3. **Unretried Data Plane Connection Drops or Request Timeouts**: `Event::RequestFailed(kind: :connection_failed)` or `Event::RequestFailed(kind: :timeout)` (`:request_connection_failed`, `:request_timeout`) occurring during `Transmission` or `Finalizing`. -* **Missing Header Handling & Retry Policy Contract**: - * *Why Headers Go Missing*: Intermediate proxies, reverse-proxies, or Google Front End (GFE) edge proxies can strip the protocol response headers or return raw HTML/text error pages on failure. - * *Session Initiation (`start`)*: Missing `X-Goog-Upload-Status` is retriable across any non-fatal status code via two complementary layers: - * **Non-`200` (`4xx`/`5xx`)**: Retried inside `ClientStub` via `start_retry_policy`'s `START_PREDICATE` (which receives the `Faraday::Error` and returns `true` unless the status is in `FATAL_STATUS_CODES`). When `ClientStub` finishes and returns/raises, `Driver#execute_send_start` passes the non-`200` `Event::HttpResponse` immediately to `Core` (`fail_with_bad_response` → `BadResponseError`). - * **`200 OK`**: Faraday does not raise on `200`, so `ClientStub` never runs `start_retry_policy`. Instead, `Driver#execute_send_start` retries headerless `200` responses directly via no-argument `policy.call` (using `start_retry_policy` only for its deadline and backoff schedule, never passing the response to `retry_predicate`). If retries exhaust, `Driver#headerless_start_failure` emits `Event::RequestFailed(kind: :retries_exhausted, source_error: BadResponseError)` (`fail_with_request_error` → `RequestFailedError`). - * *Session Control (`query`, `cancel`)*: `control_plane_retry_policy` does **not** treat missing status headers as retriable, returning the completed `Event::HttpResponse` immediately to `Core` so it can manage protocol recovery or fail fast. - * *Data Plane (`upload`, `upload, finalize`, standalone `finalize`)*: Missing `X-Goog-Upload-Status` is treated as **unretriable** by `data_plane_retry_policy` (retry predicate returns `false`). The Driver immediately returns `Event::HttpResponse` to `Core` so it classifies as `:response_cat2` and initiates Category 2 `Recovery` via `Instruction::SendQuery` rather than blindly re-transmitting data. +* **Definition**: Responses indicating that the client's current offset may be misaligned with the server, protocol headers stripped from an otherwise usable answer, or transport failures during data transmission that leave the server offset unknown. +* **Conditions Producing `:response_cat2`** (`Rules.classify_http_response`): + 1. **Headerless `200`**: A `200` whose `X-Goog-Upload-Status` is missing or empty, regardless of `CAT2_STATUS_CODES`. A proxy that strips the protocol headers from a successful answer leaves the offset unknown, which is exactly what recovery resolves. + 2. **Listed Non-200 Statuses**: A non-200 whose `X-Goog-Upload-Status` is missing, empty or `active`, **and** whose status is in `Rules::CAT2_STATUS_CODES`: `RETRIABLE_4XX_STATUS_CODES` (`409`, `429`, `499`) + `RETRIABLE_5XX_STATUS_CODES` (`500`, `503`, `504`) + `400`, `408`, `412`, `416`, `502`. The list is an allowlist and is load-bearing: any status it does not list is Category 3 (Section 6.1.3). +* **Data Plane Transport Failures**: `:request_connection_failed` and `:request_timeout` during `Transmission` or `Finalizing` also enter `Recovery` (`enter_recovery`), because the Driver does not re-send them on the data plane (Section 6.1.1, rows 2–3). +* **Where Category 2 Arrives, per Plane**: + * *Session Initiation (`start`)*: a headerless `200` and the default-retriable statuses are re-sent by the Driver within the start budget. Whatever surfaces as `:response_cat2` is terminal in `Starting` (`fail_with_bad_response` → `BadResponseError`, status `200` for an exhausted headerless answer): there is no upload URL to query yet. + * *Session Control (`query`, `cancel`)*: same re-sends as `start`. A `:response_cat2` that surfaces in `Recovery` re-queries (`retry_recovery`), bounded by the global deadline; in `Cancelling` it is terminal. + * *Data Plane (`upload`, `upload, finalize`, standalone `finalize`)*: a headerless `200` and every `4xx` surface at once; `500`/`503`/`504` are re-sent optimistically within the data budget (the worst case is a server answer that is itself Category 2). Whatever surfaces as `:response_cat2` enters `Recovery` rather than re-transmitting data blindly. +* *Why Headers Go Missing*: Intermediate proxies, reverse-proxies, or Google Front End (GFE) edge proxies can strip the protocol response headers or return raw HTML/text error pages on failure. * **Resolution**: Core transitions to `Recovery` and emits `Instruction::SendQuery.new(url: state.upload_url)` to obtain `server_offset`. -#### 6.1.3 Category 3: Terminal Failures & Fatal Status Codes -* **Definition**: Irrecoverable errors where either the request is structurally invalid, unauthorized, transport retry limits are exhausted, unseekable rewind is needed, or the server has explicitly aborted/rejected the session. -* **Canonical Fatal Status Codes (`FATAL_STATUS_CODES`)**: - The following status codes indicate structural or authentication failures that cannot be resolved by querying the upload backend: - * `401 Unauthorized`: Authentication token is expired, invalid, or missing. - * `403 Forbidden`: Caller lacks required IAM permissions for the upload destination. - * `404 Not Found`: Session upload URL does not exist or has expired. - * `405 Method Not Allowed`: HTTP method is rejected by the server. - * `410 Gone`: Upload session has been permanently removed. - * `413 Payload Too Large`: Upload chunk or overall size exceeds server limit. - * `415 Unsupported Media Type`: Object content type is rejected. - Responses with these status codes are classified as `:response_fatal_bad_response` even if the `X-Goog-Upload-Status` header is absent. +#### 6.1.3 Category 3: Terminal Failures +* **Definition**: Irrecoverable errors where either the request is structurally invalid or unauthorized, the retry budget is spent on a plane that cannot recover, an unseekable rewind is needed, or the server has explicitly aborted/rejected the session. +* **Fatal Responses (`:response_fatal_bad_response`)**: + * A non-200 whose `X-Goog-Upload-Status` is missing, empty or `active`, and whose status is **not** in `CAT2_STATUS_CODES` (e.g. `401`, `403`, `404`, `405`, `410`, `413`, `415`, and any status never seen before). This is a deliberate inversion of the earlier denylist: an unknown status is fatal rather than silently recovered. + * A non-200 with `X-Goog-Upload-Status: cancelled`, and any response carrying an unrecognised `X-Goog-Upload-Status` value. * **Other Terminal Conditions**: - * **Retry Exhaustion**: Any `Event::RequestFailed(kind: :retries_exhausted)` occurring at any stage. When Category 1 transport retries are exhausted by the `RetryPolicy`, failure is immediate and terminal; it does not enter Category 2 `Recovery`. - * **Session Rejection**: Any response with `X-Goog-Upload-Status: final` and non-2xx status code (`:response_rejected` -> raises `Gapic::Rest::ResumableUpload::UploadRejectedError`). - * **Initiation Failure**: Any 4xx/5xx or `Event::RequestFailed` during `Starting` (`:error` -> raises `Gapic::Rest::ResumableUpload::BadResponseError` or source error). + * **Non-Recoverable Transport Failures**: `:request_failed_unknown` (a non-Faraday error) and `:request_retries_exhausted` (a `Faraday::Error` with no response and no more specific kind) are terminal in every state. `:request_connection_failed` and `:request_timeout` are terminal in `Starting`, `Recovery` and `Cancelling`, where the Driver has already spent the plane's budget on them. + * **Session Rejection**: Any response with `X-Goog-Upload-Status: final` and non-200 status code (`:response_rejected` -> raises `Gapic::Rest::ResumableUpload::UploadRejectedError`). Never re-sent (Section 6.1.1, row 5). + * **Initiation Failure**: Any Category 2 or fatal response, or `Event::RequestFailed`, surfacing during `Starting` (`:error` -> raises `Gapic::Rest::ResumableUpload::BadResponseError` or `RequestFailedError`). * **Session Cancellation**: Cancelled upload sessions raise `Gapic::Rest::ResumableUpload::UploadCancelledError`. * **Global Deadline Expiration**: Monotonic clock exceeding session deadline raises `Gapic::Rest::ResumableUpload::DeadlineExceededError`. * **Unseekable Stream Rewind**: Server offset rolled back behind retained buffer (`server_offset < buffer_start_offset`) on an unseekable stream (raises `Gapic::Rest::ResumableUpload::UnseekableStreamError`). @@ -609,12 +617,15 @@ The total session timeout is resolved in priority order: 3. **Default Base Timeout (`BASE_TIMEOUT`)**: If neither a positive timeout nor `upload_size` is provided (e.g., streaming uploads of unknown length), the timeout defaults to `BASE_TIMEOUT` (`3_600` seconds). #### Bounding Transport Retries by Global Deadline -Transport retries and individual HTTP exchanges must never exceed the remaining global deadline. When `Driver#make_post_request` invokes `ClientStub#make_post_request`, it computes the per-request timeout from the remaining session budget (`max(deadline - monotonic_now, 0)`), additionally capped by `retry_policy.timeout`: +Transport retries and individual HTTP exchanges must never exceed the remaining global deadline or the command's own retry budget. Before each attempt, `Driver#request_timeout` computes the per-attempt timeout from the remaining session budget, clamped to what is left of the command budget (`retry_policy.timeout` minus the time since the command started; `RetryPolicy#deadline` is private, so the Driver measures from its own start time): ```ruby -remaining = [@deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max -timeout = retry_policy&.timeout ? [remaining, retry_policy.timeout].min : remaining +remaining = [@deadline - monotonic_now, 0].max +elapsed = monotonic_now - started_at +timeout = retry_policy&.timeout ? (retry_policy.timeout - elapsed).clamp(0, remaining) : remaining ``` -This timeout is passed in `options[:timeout]`, ensuring that underlying Faraday requests and `Gapic::Common::RetryPolicy` evaluations always respect the remaining upload budget. +This timeout is passed in `options[:timeout]` and shrinks with every attempt. After each backoff delay the loop also checks that the command budget has time left, and it checks the global deadline before every attempt and after every failed one, emitting `Event::GlobalDeadlineExceeded` instead of a transport failure once the deadline has passed. + +Per-attempt timeouts shorter than the command budget are out of scope for now; they arrive in the timeout pass done alongside stall control, and are what will make row 2 of Section 6.1.1 effective. --- @@ -688,7 +699,7 @@ All log entries emitted by `UploadLog` populate structured fields in `Google::Lo * `uploadStatus`: Value of `X-Goog-Upload-Status` response header. * `sizeReceived`: Parsed integer value of `X-Goog-Upload-Size-Received` response header. * `granularity`: Parsed integer value of `X-Goog-Upload-Chunk-Granularity` response header (`wire_receive`). - * `kind`: Transport failure classification symbol (`:timeout`, `:connection_failed`, `:retries_exhausted`). + * `kind`: Transport failure classification symbol (`:timeout`, `:connection_failed`, `:retries_exhausted`, `:unknown`). * `error`: Exception message string (`wire_failure`). * **Buffer Realignment Fields**: * `action`: Realignment strategy string (`"within_buffer"`, `"rewind"`, or `"fast_forward"`). diff --git a/gapic-common/integration/resumable_upload/error_on_start_test.rb b/gapic-common/integration/resumable_upload/error_on_start_test.rb index a3632ae..3e5a268 100644 --- a/gapic-common/integration/resumable_upload/error_on_start_test.rb +++ b/gapic-common/integration/resumable_upload/error_on_start_test.rb @@ -60,19 +60,22 @@ def test_non_fatal_error_on_start_503 assert_equal :completed, phases.last end - # A2. Verifies missing status header / 400 on start is retried and upload completes. - def test_missing_header_retriable_on_start_400 + # A2. Verifies a 400 on start is not retried: it is outside the default retry codes, and a missing status + # header is retried only on a 200. + def test_400_on_start_is_not_retried upload = build_start_error_upload( scenario: "non_fatal_error_on_start", scenario_config: { error_code: 400, failure_count: 1 }, start_retry_policy: FAST_RETRY ) - parsed = JSON.parse upload.start(**start_error_args) + err = assert_raises Gapic::Rest::ResumableUpload::BadResponseError do + upload.start(**start_error_args) + end - assert_equal PAYLOAD_SIZE, parsed["size"] - assert_equal 1, phases.count(:initiating) - assert_equal :completed, phases.last + assert_equal 400, err.status_code + assert_equal 1, @log_output.string.scan('"command":"start"').size + refute_includes phases, :uploading end # A3. Verifies retry exhaustion on start with high failure count times out within ~3s without uploading. diff --git a/gapic-common/integration/resumable_upload/error_recovery_test.rb b/gapic-common/integration/resumable_upload/error_recovery_test.rb index b1ba1f4..bd795d6 100644 --- a/gapic-common/integration/resumable_upload/error_recovery_test.rb +++ b/gapic-common/integration/resumable_upload/error_recovery_test.rb @@ -103,7 +103,27 @@ def test_cat2_failure_on_finalizing_chunk end # B5. Verifies unrecoverable 500 without status header triggers repeated recovery until DeadlineExceededError. + # A 500 is a default retry code on every plane, so the Driver re-sends each request until its plane budget + # runs out; the budgets are kept well inside the upload timeout so the exhausted outcomes reach Rules. def test_no_headers_failure_recovers_until_deadline_exceeded + short_budget = FAST_RETRY.merge timeout: 0.1 + upload = build_upload( + scenario: SCENARIO, + scenario_config: { failure_count: 0, action_after_failures: "terminate" }, + control_plane_retry_policy: short_budget, + data_plane_retry_policy: short_budget + ) + + assert_raises Gapic::Rest::ResumableUpload::DeadlineExceededError do + upload.start(**start_args(upload_timeout: 1)) + end + + assert_operator phases.count(:recovering), :>=, 2 + end + + # B6. Verifies an unrecoverable headerless 500 on the data plane is re-sent in place, without entering + # recovery, while the data plane budget lasts. + def test_no_headers_failure_is_resent_within_the_data_plane_budget upload = build_upload( scenario: SCENARIO, scenario_config: { failure_count: 0, action_after_failures: "terminate" } @@ -113,6 +133,7 @@ def test_no_headers_failure_recovers_until_deadline_exceeded upload.start(**start_args(upload_timeout: 1)) end - assert_operator phases.count(:recovering), :>=, 2 + refute_includes phases, :recovering + assert_operator @log_output.string.scan('"command":"upload"').size, :>=, 2 end end diff --git a/gapic-common/integration/resumable_upload/resume_test.rb b/gapic-common/integration/resumable_upload/resume_test.rb index f751e14..305fa70 100644 --- a/gapic-common/integration/resumable_upload/resume_test.rb +++ b/gapic-common/integration/resumable_upload/resume_test.rb @@ -83,8 +83,9 @@ def test_resume_query_503_absorbed_by_retry refute_includes @log_output.string, "retry_recovery" end - # D3b. Non-fatal 409 error on query during resume triggers protocol retry_recovery without progress notification. - def test_resume_query_409_triggers_retry_recovery + # D3b. Non-fatal 409 error on query during resume is re-sent by the Driver (409 is a default control plane + # retry code), so Rules never sees it and no retry_recovery or progress notification occurs. + def test_resume_query_409_is_retried_by_the_driver upload_url = raw_start( scenario: "non_fatal_error_on_query", scenario_config: { error_code: 409, failure_count: 1 }, @@ -100,7 +101,8 @@ def test_resume_query_409_triggers_retry_recovery assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] assert_equal [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed], phases refute_includes phases, :recovering - assert_includes @log_output.string, "retry_recovery" + refute_includes @log_output.string, "retry_recovery" + assert_includes @log_output.string, '"startAttempt":2,"command":"query"' end # D4. Resume fast-forwards by discarding bytes on an unseekable stream starting at byte 0. diff --git a/gapic-common/lib/gapic/rest/resumable_upload.rb b/gapic-common/lib/gapic/rest/resumable_upload.rb index cb86782..c82ef3d 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload.rb @@ -70,9 +70,8 @@ module ResumableUpload # retry policy. # # Returns a **Hash**, never a policy object: the protocol treats a {Gapic::Common::RetryPolicy} as a - # wholesale replacement and a Hash as a per-key override. Initiation's default policy carries a - # predicate that treats a response missing `X-Goog-Upload-Status` as retriable gateway noise, and - # handing over an object would silently drop it. + # wholesale replacement and a Hash as a per-key override, so a Hash keeps every protocol default the + # caller did not set (notably the initiation `retry_codes`). # # `timeout` is the one setting the caller cannot express here: it is set unconditionally from the # call's own timeout and becomes the local deadline of the initiation request alone — the @@ -84,10 +83,11 @@ module ResumableUpload # # Everything else the caller set is carried across as-is, by asking the policy what it carries # ({Gapic::Common::RetryPolicy#overrides}) rather than inferring it from the readers, so a choice - # that happens to equal a library default still lands. That includes `retry_predicate`: a caller - # who supplies one **replaces** the initiation predicate rather than composing with it, and so - # gives up the missing-`X-Goog-Upload-Status` handling. There is no chaining — a predicate that - # returns `nil` falls through to `retry_codes`, not to the predicate it displaced. + # that happens to equal a library default still lands. That includes `retry_predicate`: the default + # initiation policy has none, so a caller's predicate is consulted as-is, ahead of `retry_codes`. + # The protocol's own retries (connection failures and a `200` without `X-Goog-Upload-Status`) and + # its refusals (a `final` rejection) are decided before the policy is asked, so no predicate can + # disable or override them. # # @param options [Gapic::CallOptions, nil] Per-call options from a generated client method # @return [Hash] Overrides for the initiation retry policy diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb index 41362b6..f2d3311 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb @@ -23,6 +23,7 @@ require "gapic/rest/resumable_upload/events" require "gapic/rest/resumable_upload/instructions" require "gapic/rest/resumable_upload/retry_policies" +require "gapic/rest/resumable_upload/driver/retry_decider" require "gapic/rest/resumable_upload/driver/upload_log" module Gapic @@ -35,8 +36,8 @@ module ResumableUpload # and delegates state transitions to Core. # # The outer tier of the three-tier design. All side effects live here; all protocol decisions live in - # {Rules}, which carries the state graph and the error category taxonomy. Category 1 transient retries - # are absorbed here by `Gapic::Common::RetryPolicy` and never reach {Core}. See + # {Rules}, which carries the state graph and the error category taxonomy. Every retry decision is made + # here by {RetryDecider}; `ClientStub` is handed a never-retry policy. See # `design/resumable_upload/implementation-guide.md` section 2.5 for the buffer and stream position # invariants, and section 6.3 for the deadline model. # @@ -56,6 +57,13 @@ class Driver # @return [Integer] BASE_TIMEOUT = 3_600 + ## + # @private + # Retry policy handed to `ClientStub` for every request, so that each `ClientStub` call is exactly one + # attempt and {RetryDecider} makes every retry decision. `Gapic::CallOptions` accepts a Proc policy. + # @return [Proc] + CLIENT_STUB_NO_RETRY = ->(_error) { false } + # @private # @return [Core] attr_reader :core @@ -364,20 +372,23 @@ def resolve_timeout ## # @private - # Computes the per-request timeout bounded by the global monotonic deadline. + # Computes the per-attempt timeout: whatever is left of the global monotonic deadline, and of the + # command's own retry budget when the command has one. # # @param retry_policy [Gapic::Common::RetryPolicy, nil] Target command retry policy - # @return [Numeric] Effective per-request timeout + # @param started_at [Numeric, nil] Monotonic time the command started; `nil` grants the full budget + # @return [Numeric] Effective per-attempt timeout # - def request_timeout retry_policy + def request_timeout retry_policy, started_at: nil remaining = if @deadline - [@deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max + [@deadline - monotonic_now, 0].max else resolve_timeout end - return [remaining, retry_policy.timeout].min if retry_policy&.timeout + return remaining unless retry_policy&.timeout - remaining + elapsed = started_at ? monotonic_now - started_at : 0 + (retry_policy.timeout - elapsed).clamp 0, remaining end ## @@ -563,50 +574,9 @@ def execute_fill_buffer instruction # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] # def execute_send_start instruction - policy = @start_retry_policy.dup.start! - headers = start_headers instruction - attempt = 1 - - loop do - return Event::GlobalDeadlineExceeded.new if deadline_exceeded? - - event = make_post_request instruction.url, headers: headers, body: instruction.body, - retry_policy: policy, method_name: "#{@method_name_prefix}.start", - start_attempt: attempt - return event unless event.is_a? Event::HttpResponse - - status_hdr = Rules.header_value event.headers, "x-goog-upload-status" - return event unless status_hdr.nil? || status_hdr.empty? - - # Non-200 responses with missing protocol headers (e.g. a 503 from GFE) are retried - # inside ClientStub and should be classified by their status code rather than as a - # missing-header error. - return event unless event.status == 200 - - # Checking for policy here to avoid retrying into timeout - if policy.call - attempt += 1 - next - end - return headerless_start_failure event - end - end - - ## - # @private - # Builds and logs the `:retries_exhausted` failure event for a `200` initiation response - # that lacked `X-Goog-Upload-Status` once the start retry policy is out of budget. - # - # @param event [Event::HttpResponse] Final headerless `200` initiation response - # @return [Event::RequestFailed] - # - def headerless_start_failure event - err = BadResponseError.new "Missing X-Goog-Upload-Status header in start response", - event.status, headers: event.headers - failed_event = Event::RequestFailed.new kind: :retries_exhausted, message: err.message, - source_error: err - @upload_log.wire_failure failed_event - failed_event + make_post_request instruction.url, headers: start_headers(instruction), body: instruction.body, + retry_policy: @start_retry_policy.dup.start!, + method_name: "#{@method_name_prefix}.start" end ## @@ -645,7 +615,7 @@ def execute_send_chunk instruction body = @buffer.byteslice slice_index, instruction.length make_post_request instruction.url, headers: headers, body: body, - retry_policy: @data_plane_retry_policy.dup.start!, + retry_policy: @data_plane_retry_policy.dup.start!, data_plane: true, method_name: "#{@method_name_prefix}.upload" end @@ -663,7 +633,7 @@ def execute_send_finalize instruction "Content-Length" => "0" } make_post_request instruction.url, headers: headers, body: "", - retry_policy: @data_plane_retry_policy.dup.start!, + retry_policy: @data_plane_retry_policy.dup.start!, data_plane: true, method_name: "#{@method_name_prefix}.finalize" end @@ -697,40 +667,78 @@ def execute_send_cancel instruction ## # @private - # Dispatches an HTTP POST request through client stub. + # Sends an HTTP POST request through the client stub, re-sending it for as long as {RetryDecider} says + # so, and converts the final outcome into an event. + # + # Every attempt is logged. The event for the last attempt is returned unchanged: this method only + # chooses whether to send again, never what an outcome means; {Rules} decides that. # # @param url [String] Target URL # @param headers [Hash] Request headers # @param body [String] Request body - # @param retry_policy [Gapic::Common::RetryPolicy] Command retry policy + # @param retry_policy [Gapic::Common::RetryPolicy, nil] Started command retry policy; `nil` sends once + # @param data_plane [Boolean] Whether the request transmits upload bytes (`upload`, `finalize`) # @param method_name [String, nil] RPC method name for logging - # @param start_attempt [Integer] Attempt counter # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] # - def make_post_request url, headers:, body:, retry_policy:, method_name: nil, start_attempt: 1 - return Event::GlobalDeadlineExceeded.new if deadline_exceeded? + def make_post_request url, headers:, body:, retry_policy:, data_plane: false, method_name: nil + decider = retry_policy && RetryDecider.new(retry_policy, data_plane: data_plane) + started_at = monotonic_now + attempt = 1 - options = { - metadata: headers, - retry_policy: retry_policy, - timeout: request_timeout(retry_policy) - } - @upload_log.wire_send method: "POST", url: url, headers: headers, - start_attempt: start_attempt, body_size: body.to_s.bytesize, body: body + loop do + return Event::GlobalDeadlineExceeded.new if deadline_exceeded? - response = @client_stub.make_post_request uri: url, body: body, params: {}, - options: options, method_name: method_name - event = Event::HttpResponse.new status: response.status, headers: response.headers || {}, body: response.body - @upload_log.wire_receive event - event + timeout = request_timeout retry_policy, started_at: started_at + outcome = attempt_post_request url, headers: headers, body: body, timeout: timeout, + method_name: method_name, attempt: attempt + # If the global deadline expired during the HTTP call (e.g. Net::HTTP connection or read timeout + # triggered by request_timeout reaching 0 at @deadline), emit GlobalDeadlineExceeded rather than + # Event::RequestFailed. Otherwise, in states like Recovery where Event::RequestFailed is immediately + # terminal, the state machine would raise the underlying transport error instead of DeadlineExceededError. + return Event::GlobalDeadlineExceeded.new if outcome.is_a?(Exception) && deadline_exceeded? + + event = outcome_event outcome + return event unless decider&.retry?(outcome) && command_budget_left?(retry_policy, started_at) + + attempt += 1 + end + end + + ## + # @private + # Performs one `ClientStub` call, returning the error it raises instead of raising it. + # + # @param url [String] Target URL + # @param headers [Hash] Request headers + # @param body [String] Request body + # @param timeout [Numeric] Per-attempt timeout + # @param method_name [String, nil] RPC method name for logging + # @param attempt [Integer] 1-based attempt number, for logging + # @return [Object, StandardError] The client stub response, or the error it raised + # + def attempt_post_request url, headers:, body:, timeout:, method_name:, attempt: + options = { metadata: headers, retry_policy: CLIENT_STUB_NO_RETRY, timeout: timeout } + @upload_log.wire_send method: "POST", url: url, headers: headers, + start_attempt: attempt, body_size: body.to_s.bytesize, body: body + @client_stub.make_post_request uri: url, body: body, params: {}, options: options, method_name: method_name rescue StandardError => e - # If the global deadline expired during the HTTP call (e.g. Net::HTTP connection or read timeout - # triggered by request_timeout reaching 0 at @deadline), emit GlobalDeadlineExceeded rather than - # Event::RequestFailed. Otherwise, in states like Recovery where Event::RequestFailed is immediately - # terminal, the state machine would raise the underlying transport error instead of DeadlineExceededError. - return Event::GlobalDeadlineExceeded.new if deadline_exceeded? + e + end - event = rescue_request_error e + ## + # @private + # Converts one attempt's outcome into an event and logs it. + # + # @param outcome [Object, StandardError] Client stub response or raised error + # @return [Event::HttpResponse, Event::RequestFailed] + # + def outcome_event outcome + event = if outcome.is_a? Exception + rescue_request_error outcome + else + Event::HttpResponse.new status: outcome.status, headers: outcome.headers || {}, body: outcome.body + end if event.is_a? Event::HttpResponse @upload_log.wire_receive event else @@ -739,28 +747,54 @@ def make_post_request url, headers:, body:, retry_policy:, method_name: nil, sta event end + ## + # @private + # Whether the command's own retry budget has time left, checked after the backoff delay. + # + # `RetryPolicy#deadline` is private, so the budget is measured from when the command started. + # + # @param retry_policy [Gapic::Common::RetryPolicy] Command retry policy + # @param started_at [Numeric] Monotonic time the command started + # @return [Boolean] + # + def command_budget_left? retry_policy, started_at + monotonic_now - started_at < retry_policy.timeout + end + + ## + # @private + # Current monotonic clock reading. + # + # @return [Float] + # + def monotonic_now + Process.clock_gettime Process::CLOCK_MONOTONIC + end + ## # @private # Converts client stub transport exceptions into canonical events. # + # Classified by {RetryDecider.failure_kind}, the same function that decides how the error is retried. + # # @param err [StandardError] Rescued transport error # @return [Event::HttpResponse, Event::RequestFailed] # def rescue_request_error err - case err - when Gapic::Rest::DeadlineExceededError + return rescue_faraday_error err if err.is_a? Faraday::Error + + case RetryDecider.failure_kind err + when :status + Event::HttpResponse.new status: err.status_code, headers: err.headers || {}, body: err.message, + error: err + when :timeout Event::RequestFailed.new kind: :timeout, message: err.message, source_error: err - when Gapic::Rest::Error - if err.status_code - Event::HttpResponse.new status: err.status_code, headers: err.headers || {}, body: err.message, - error: err - else - Event::RequestFailed.new kind: :connection_failed, message: err.message, source_error: err - end - when Faraday::Error - rescue_faraday_error err - else + when :connection_failed Event::RequestFailed.new kind: :connection_failed, message: err.message, source_error: err + else + # Not a transport error at all, e.g. the authorization middleware failing to refresh a token. Nothing + # reached the server, so this is neither a connection failure nor a reason to recover. + Event::RequestFailed.new kind: :unknown, message: err.message, source_error: err end end @@ -772,17 +806,17 @@ def rescue_request_error err # @return [Event::HttpResponse, Event::RequestFailed] # def rescue_faraday_error err - if err.response && err.response[:status] - rest_err = Gapic::Rest::Error.wrap_faraday_error err + case RetryDecider.failure_kind err + when :status Event::HttpResponse.new( status: err.response[:status], headers: err.response[:headers] || {}, body: err.response[:body], - error: rest_err + error: Gapic::Rest::Error.wrap_faraday_error(err) ) - elsif err.is_a? Faraday::TimeoutError + when :timeout Event::RequestFailed.new kind: :timeout, message: err.message, source_error: err - elsif err.is_a? Faraday::ConnectionFailed + when :connection_failed Event::RequestFailed.new kind: :connection_failed, message: err.message, source_error: err else Event::RequestFailed.new kind: :retries_exhausted, message: err.message, source_error: err diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver/retry_decider.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver/retry_decider.rb new file mode 100644 index 0000000..f61538f --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver/retry_decider.rb @@ -0,0 +1,190 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "faraday" +require "gapic/rest/error" +require "gapic/rest/resumable_upload/rules" + +module Gapic + module Rest + module ResumableUpload + class Driver + ## + # @private + # Single owner of every retry decision for one resumable upload command. + # + # `ClientStub` is handed a never-retry policy, so each `ClientStub` call is exactly one attempt and + # every outcome comes back to the Driver. The Driver asks this object whether to send the same request + # again. It only ever chooses between re-sending and surfacing: an outcome it declines, and whatever + # is left when the budget runs out, is converted to an event unchanged and handed to {Rules}, which + # alone decides between recovery and a terminal outcome. + # + # The decision table (first match wins): + # + # | # | Outcome | start / control | data | + # |---|-----------------------------------------------------|------------------------|------------------------| + # | 1 | non-transport error (e.g. auth refresh) | ask policy | ask policy | + # | 2 | `Faraday::TimeoutError`, or no response and no kind | ask policy | surface | + # | 3 | `Faraday::ConnectionFailed`, `Faraday::SSLError` | retry within budget | surface | + # | 4 | `200` without `X-Goog-Upload-Status` | retry within budget | surface | + # | 5 | non-200 with `X-Goog-Upload-Status: final` | surface | surface | + # | 6 | any 4xx | (falls through) | surface | + # | 7 | any other non-200 | ask policy | ask policy | + # | - | any other `2xx`, or `200` with the header | surface | surface | + # + # * **Retry within budget** is the no-argument `RetryPolicy#call`: it checks the policy deadline and + # applies backoff, and hands nothing to a caller predicate. + # * **Ask policy** is `retry_with_deadline? && call(error)`: the caller's `retry_predicate`, then its + # `retry_codes`. `RetryPolicy#call(error)` does not check the deadline itself, hence the guard. + # * The data plane never re-sends after an outcome that leaves the server offset unknown (rows 2, 3, 4, + # 6); recovery owns those. A non-transport error (row 1) fails before anything reaches the wire, so + # re-sending cannot duplicate bytes. + # * Rows 5 and 6 are fixed ahead of the policy, so no caller setting can retry a rejection or make the + # data plane re-send on a 4xx. + # + # Row 2 is inert for timeouts until attempts get their own timeouts: each attempt currently receives the + # whole remaining command budget, so a timed-out attempt leaves none to retry with. + # + # See `design/resumable_upload/transport-error-retry.md`. + # + class RetryDecider + ## + # @private + # Classifies a raised request error by what reached the wire. + # + # Shared with {Driver#rescue_request_error}, so the kind an error is retried as and the kind it is + # surfaced as cannot drift apart. + # + # @param error [StandardError] Error raised by, or through, `ClientStub#make_post_request` + # @return [Symbol] `:status` (an HTTP response arrived), `:timeout`, `:connection_failed`, + # `:no_response` (a transport error of no more specific kind), or `:non_transport` + def self.failure_kind error + case error + when Gapic::Rest::DeadlineExceededError then :timeout + when Gapic::Rest::Error then error.status_code ? :status : :connection_failed + when Faraday::Error then faraday_failure_kind error + else :non_transport + end + end + + ## + # @private + # @param error [Faraday::Error] + # @return [Symbol] See {.failure_kind} + def self.faraday_failure_kind error + if error.response.is_a?(Hash) && error.response[:status] + :status + elsif error.is_a? Faraday::TimeoutError + :timeout + elsif error.is_a?(Faraday::ConnectionFailed) || error.is_a?(Faraday::SSLError) + :connection_failed + else + :no_response + end + end + + ## + # @private + # @param policy [Gapic::Common::RetryPolicy] Started policy for this command + # @param data_plane [Boolean] Whether the command transmits upload bytes (`upload`, `finalize`) + def initialize policy, data_plane: + @policy = policy + @data_plane = data_plane + end + + ## + # @private + # Decides whether to send the same request again, performing the backoff delay if so. + # + # @param outcome [Object, StandardError] The response returned by `ClientStub#make_post_request`, or + # the error it raised + # @return [Boolean] `true` if the request should be sent again (the delay has been performed) + def retry? outcome + return retry_response? outcome unless outcome.is_a? Exception + + error = unwrap outcome + case self.class.failure_kind error + when :status then retry_status? error + when :connection_failed then !@data_plane && @policy.call + when :timeout, :no_response then !@data_plane && ask_policy(error) + else ask_policy error + end + end + + private + + ## + # @private + # Under `raise_faraday_errors: false`, `ClientStub` raises a {Gapic::Rest::Error} from inside its own + # `rescue`, so `cause` is the original Faraday error. `retry_codes` can read a status only from the + # latter. + # + # @param error [StandardError] + # @return [StandardError] + def unwrap error + return error.cause if error.is_a?(Gapic::Rest::Error) && error.cause.is_a?(Faraday::Error) + error + end + + ## + # @private + # Faraday raises on every `4xx`/`5xx`, so a returned response is a `1xx`-`3xx`. Only a headerless + # `200` is worth re-sending. + # + # @param response [Object] Response exposing `status` and `headers` + # @return [Boolean] + def retry_response? response + return false unless response.status == 200 + return false unless upload_status(response.headers).nil? + !@data_plane && @policy.call + end + + ## + # @private + # @param error [Faraday::Error, Gapic::Rest::Error] Error carrying an HTTP status + # @return [Boolean] + def retry_status? error + status, headers = if error.is_a? Gapic::Rest::Error + [error.status_code, error.headers] + else + [error.response[:status], error.response[:headers]] + end + return false if upload_status(headers) == "final" + return false if @data_plane && (400..499).cover?(status) + ask_policy error + end + + ## + # @private + # @param error [StandardError] + # @return [Boolean] + def ask_policy error + @policy.retry_with_deadline? && @policy.call(error) + end + + ## + # @private + # @param headers [Hash, nil] + # @return [String, nil] Downcased `X-Goog-Upload-Status`, or `nil` when missing or empty + def upload_status headers + value = Rules.header_value headers, "x-goog-upload-status" + value.nil? || value.empty? ? nil : value.downcase + end + end + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/events.rb b/gapic-common/lib/gapic/rest/resumable_upload/events.rb index 6a8df2c..f38105c 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/events.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/events.rb @@ -91,7 +91,8 @@ def initialize status:, headers: {}, body: nil, error: nil # Signals an HTTP request failure (e.g. request timeout, transport connection failure, or retries exhausted). # # @!attribute [r] kind - # @return [Symbol] Failure kind: `:timeout`, `:connection_failed`, or `:retries_exhausted` + # @return [Symbol] Failure kind: `:timeout`, `:connection_failed`, `:retries_exhausted` (a transport + # error of no more specific kind), or `:unknown` (a non-transport error, e.g. a credentials failure) # @!attribute [r] message # @return [String, nil] Human-readable failure summary # @!attribute [r] source_error @@ -102,7 +103,7 @@ def initialize status:, headers: {}, body: nil, error: nil # @private # Initializes a RequestFailed event. # - # @param kind [Symbol] Failure kind (`:timeout`, `:connection_failed`, `:retries_exhausted`) + # @param kind [Symbol] Failure kind (`:timeout`, `:connection_failed`, `:retries_exhausted`, `:unknown`) # @param message [String, nil] Human-readable failure summary # @param source_error [StandardError, nil] Original underlying exception # diff --git a/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb index d0cfaa8..feb284f 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +require "gapic/common/error_codes" require "gapic/common/retry_policy" require "gapic/rest/resumable_upload/rules" @@ -24,58 +25,49 @@ module ResumableUpload # @private # Default retry policy generators for control plane and data plane requests. # + # The defaults carry no `retry_predicate`: every protocol-specific retry decision lives in + # {Driver::RetryDecider}, which consults a policy only for its budget and backoff, and for the + # caller-facing `retry_predicate` and `retry_codes`. The `retry_codes` below are derived from the HTTP + # status sets on {Rules}, so HTTP status is the single source of truth. + # + # Keep in sync with the "Retry Policies" section of the {Gapic::ResumableUpload} class doc. + # module RetryPolicies ## # @private - # Retry predicate determining retriability for session initiation requests. - # Retries missing status header on non-fatal codes. - # @return [Proc] - START_PREDICATE = lambda do |error_or_response| - status = extract_status_code error_or_response - return false if Rules::FATAL_STATUS_CODES.include? status - - headers = extract_headers error_or_response - if headers - status_hdr = headers["x-goog-upload-status"] || headers["X-Goog-Upload-Status"] - return true if status_hdr.nil? || status_hdr.empty? - end - nil - end + # Default `retry_codes` for initiation, `query` and `cancel`: 409, 429, 499, 500, 503 and 504. + # @return [Array] + START_AND_CONTROL_PLANE_RETRY_CODES = + (Rules::RETRIABLE_4XX_STATUS_CODES + Rules::RETRIABLE_5XX_STATUS_CODES).map do |status| + Gapic::Common::ErrorCodes.grpc_error_for status + end.freeze ## # @private - # Retry predicate determining retriability for data plane requests. - # Disallows retry when upload status header is missing. - # @return [Proc] - DATA_PLANE_PREDICATE = lambda do |error_or_response| - headers = extract_headers error_or_response - if headers - status_hdr = headers["x-goog-upload-status"] || headers["X-Goog-Upload-Status"] - return false if status_hdr.nil? || status_hdr.empty? - end - nil - end + # Default `retry_codes` for `upload` and `finalize`: 500, 503 and 504. The data plane never + # retries a 4xx, so none is listed. + # @return [Array] + DATA_PLANE_RETRY_CODES = Rules::RETRIABLE_5XX_STATUS_CODES.map do |status| + Gapic::Common::ErrorCodes.grpc_error_for status + end.freeze ## # @private # Default options for start command retry policy. - # Keep in sync with the "Retry Policies" section of the Gapic::ResumableUpload class doc. # @return [Hash] START_DEFAULTS = { - retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, - initial_delay: 1.0, - max_delay: 15.0, - multiplier: 1.3, - retry_predicate: START_PREDICATE + retry_codes: START_AND_CONTROL_PLANE_RETRY_CODES, + initial_delay: 1.0, + max_delay: 15.0, + multiplier: 1.3 }.freeze ## # @private # Default options for query and cancel commands retry policy. - # Keep in sync with the "Retry Policies" section of the Gapic::ResumableUpload class doc. # @return [Hash] CONTROL_PLANE_DEFAULTS = { - retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, + retry_codes: START_AND_CONTROL_PLANE_RETRY_CODES, initial_delay: 1.0, max_delay: 15.0, multiplier: 1.3 @@ -84,22 +76,17 @@ module RetryPolicies ## # @private # Default options for upload and finalize commands retry policy. - # Keep in sync with the "Retry Policies" section of the Gapic::ResumableUpload class doc. # @return [Hash] DATA_PLANE_DEFAULTS = { - retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, - initial_delay: 1.0, - max_delay: 15.0, - multiplier: 1.3, - retry_predicate: DATA_PLANE_PREDICATE + retry_codes: DATA_PLANE_RETRY_CODES, + initial_delay: 1.0, + max_delay: 15.0, + multiplier: 1.3 }.freeze ## # @private # Default retry policy for session initiation requests (start). - # Its predicate treats a missing X-Goog-Upload-Status header as retriable across non-fatal - # 4xx/5xx responses in ClientStub; headerless 200 responses are retried by Driver#execute_send_start - # via the no-argument RetryPolicy#call. # # @return [Gapic::Common::RetryPolicy] def self.default_start @@ -109,7 +96,6 @@ def self.default_start ## # @private # Default retry policy for session control requests (query, cancel). - # Does not retry on missing X-Goog-Upload-Status header. # # @return [Gapic::Common::RetryPolicy] def self.default_control_plane @@ -119,47 +105,11 @@ def self.default_control_plane ## # @private # Default retry policy for data plane requests (upload, finalize). - # Missing X-Goog-Upload-Status header is unretriable (predicate returns false). # # @return [Gapic::Common::RetryPolicy] def self.default_data_plane Gapic::Common::RetryPolicy.new(**DATA_PLANE_DEFAULTS) end - - ## - # @private - # Extracts headers hash from Faraday response or error object. - # - # @param error_or_response [Object] Response, error, or hash object - # @return [Hash, nil] - def self.extract_headers error_or_response - if error_or_response.respond_to? :headers - error_or_response.headers - elsif error_or_response.respond_to? :response_headers - error_or_response.response_headers - elsif error_or_response.respond_to?(:response) && error_or_response.response.is_a?(Hash) - error_or_response.response[:headers] - end - end - - ## - # @private - # Extracts HTTP status code from Faraday response, error, or event object. - # - # @param error_or_response [Object] Response, error, or event object - # @return [Integer, nil] - def self.extract_status_code error_or_response - if error_or_response.respond_to?(:status_code) && error_or_response.status_code.is_a?(Integer) - error_or_response.status_code - elsif error_or_response.respond_to?(:response) && error_or_response.response.is_a?(Hash) && - error_or_response.response[:status].is_a?(Integer) - error_or_response.response[:status] - elsif error_or_response.respond_to?(:response_status) && error_or_response.response_status.is_a?(Integer) - error_or_response.response_status - elsif error_or_response.respond_to?(:status) && error_or_response.status.is_a?(Integer) - error_or_response.status - end - end end end end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb index e6c2ea9..88f4a1c 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb @@ -132,39 +132,51 @@ module Rules # Failures are classified into three categories, which the rest of this module is written in terms of: # - # * **Category 1 (transient transport)** - connection resets, DNS failures, load shedding. Handled - # entirely inside the Driver by `Gapic::Common::RetryPolicy`; Core never sees them. Only their - # exhaustion reaches this module, as `:request_retries_exhausted`. + # * **Category 1 (transient transport)** - connection resets, TLS failures, load shedding. Retried inside + # the Driver by {Driver::RetryDecider} for initiation, `query` and `cancel`, and never retried for + # `upload` and `finalize`, where a lost connection leaves the server offset unknown. Whatever the Driver + # does not retry, or stops retrying, reaches this module as a `:request_*` shape. # * **Category 2 (recoverable protocol)** - the client offset may be misaligned with the server, or a # proxy stripped the protocol headers. Resolved by querying the server for its acknowledged offset # and realigning, never by blindly retransmitting. Shape: `:response_cat2`. # * **Category 3 (terminal)** - structurally invalid, unauthorized, rejected, or out of budget. # Resolved by transitioning to `:error` or `:rejected` and emitting `Instruction::TerminateFailure`. # - # See `design/resumable_upload/implementation-guide.md` section 6.1 for the full classification. + # See `design/resumable_upload/implementation-guide.md` section 6.1 for the full classification, and + # `design/resumable_upload/transport-error-retry.md` for which failures the Driver retries. ## # @private - # HTTP status codes eligible for Category 2 (recovery) handling. + # 4xx status codes the initiation, `query` and `cancel` retry policies retry by default. + # + # The data plane never retries a 4xx; see {Driver::RetryDecider}. # - # Descriptive rather than load-bearing: {Rules.classify_http_response} routes any non-fatal status with a - # missing or empty `X-Goog-Upload-Status` to `:response_cat2`, so this list does not gate the decision. - # It records the codes the upload backend is expected to produce in that situation, and is asserted against - # {Rules.classify_http_response} by the classification tests. + # @return [Array] + RETRIABLE_4XX_STATUS_CODES = [409, 429, 499].freeze + + ## + # @private + # 5xx status codes every retry policy retries by default. # # @return [Array] - CAT2_STATUS_CODES = [400, 408, 409, 412, 416, 429, 499].freeze + RETRIABLE_5XX_STATUS_CODES = [500, 503, 504].freeze ## # @private - # HTTP status codes that are immediately fatal and non-retriable (Category 3). + # HTTP status codes eligible for Category 2 (recovery) handling. + # + # Rules.classify_http_response} classifies a non-200 response whose `X-Goog-Upload-Status` + # is missing, empty or `active` as `:response_cat2` **only** if its status is listed here, and as + # `:response_fatal_bad_response` otherwise. It is an allowlist on purpose: a status nobody anticipated + # fails the upload rather than looping it through recovery. # - # Unlike {CAT2_STATUS_CODES} this list is load-bearing: {Rules.classify_http_response} consults it to decide - # between `:response_fatal_bad_response` and `:response_cat2` when the upload status header is absent, - # and {RetryPolicies::START_PREDICATE} consults it to refuse retries outright. + # Every status either retry policy retries by default is listed, so a response the Driver stopped + # retrying still recovers. 408 and 502 are listed but not retried by default: neither has a gRPC code + # in `Gapic::Common::ErrorCodes`, so neither can be named in `retry_codes`. # # @return [Array] - FATAL_STATUS_CODES = [401, 403, 404, 405, 410, 413, 415].freeze + CAT2_STATUS_CODES = (RETRIABLE_4XX_STATUS_CODES + RETRIABLE_5XX_STATUS_CODES + + [400, 408, 412, 416, 502]).sort.freeze ## # @private @@ -402,8 +414,8 @@ def self.decide state, event, config # {STATUSES} x {SHAPES} product can be enumerated and asserted directly. {Rules.decide} is the only # caller in production code. # - # Arms are written so that no arm matches a pair another arm also matches, which makes their order - # presentational rather than load-bearing — with a single deliberate exception, flagged in place, + # Arms are written so that no arm matches a pair another arm also matches, meaning + # the order does not matter, with a single deliberate exception, flagged in place, # where `enter_recovery` must precede `fail_with_request_error`. # # @param status [Symbol] Current protocol status, one of {STATUSES} @@ -1000,6 +1012,18 @@ def self.resolve_chunk_size user_chunk_size, chunk_granularity # @private # Classifies an HTTP response into a canonical response shape. # + # Shapes are shown without their `response_` prefix. + # + # | `X-Goog-Upload-Status` | `200` | in CAT2_STATUS_CODES | any other non-200 | + # |------------------------|---------------|----------------|-------------------| + # | missing / empty | `cat2` | `cat2` | `fatal_bad_response` | + # | `active` | `active` | `cat2` | `fatal_bad_response` | + # | `final` | `final` | `rejected` | `rejected` | + # | `cancelled` | `cancelled` | `fatal_bad_response` | `fatal_bad_response` | + # | any other value | `fatal_bad_response` | `fatal_bad_response` | `fatal_bad_response` | + # + # A headerless `200` is Category 2 regardless of {CAT2_STATUS_CODES}. + # # @param response [Event::HttpResponse] Response event # @return [Symbol] Canonical response shape def self.classify_http_response response @@ -1007,22 +1031,29 @@ def self.classify_http_response response case status_header when "active" - response.status == 200 ? :response_active : :response_cat2 + response.status == 200 ? :response_active : classify_unsuccessful_status(response.status) when "final" response.status == 200 ? :response_final : :response_rejected when "cancelled" response.status == 200 ? :response_cancelled : :response_fatal_bad_response when nil, "" - if FATAL_STATUS_CODES.include? response.status - :response_fatal_bad_response - else - :response_cat2 - end + response.status == 200 ? :response_cat2 : classify_unsuccessful_status(response.status) else :response_fatal_bad_response end end + ## + # @private + # Classifies a non-200 status whose upload status header is missing, empty or `active`. + # + # @param status [Integer] HTTP status code + # @return [Symbol] `:response_cat2` if the status is in {CAT2_STATUS_CODES}, else + # `:response_fatal_bad_response` + def self.classify_unsuccessful_status status + CAT2_STATUS_CODES.include?(status) ? :response_cat2 : :response_fatal_bad_response + end + ## # @private # Case-insensitive header lookup helper. diff --git a/gapic-common/lib/gapic/resumable_upload.rb b/gapic-common/lib/gapic/resumable_upload.rb index a7f8f27..fb0853f 100644 --- a/gapic-common/lib/gapic/resumable_upload.rb +++ b/gapic-common/lib/gapic/resumable_upload.rb @@ -90,26 +90,27 @@ module Gapic # Retry behavior is partitioned across three policies. Only the initiation policy is caller-supplied; # the other two are the protocol's own and govern the requests no call option describes. # - # All three share the same default retry codes and exponential backoff settings: + # | Policy | Governs | Default `retry_codes` | + # |---|---|---| + # | initiation | session initiation | the 4xx and 5xx sets below | + # | control plane | `query` and `cancel` | the 4xx and 5xx sets below | + # | data plane | `upload` and `finalize` | the 5xx set below only | # - # | Setting | Default | - # |---|---| - # | `retry_codes` | `UNAVAILABLE`, `DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED`, `INTERNAL` | - # | `initial_delay` | `1.0` s | - # | `max_delay` | `15.0` s | - # | `multiplier` | `1.3` | + # * 4xx set: `ALREADY_EXISTS` (HTTP `409`), `RESOURCE_EXHAUSTED` (`429`), `CANCELLED` (`499`). + # * 5xx set: `INTERNAL` (HTTP `500`), `UNAVAILABLE` (`503`), `DEADLINE_EXCEEDED` (`504`). # - # They differ in which requests they govern and how a missing or empty `X-Goog-Upload-Status` response - # header is treated: + # None of the defaults carries a `retry_predicate`, so one supplied by the caller is consulted as-is, + # ahead of `retry_codes`. All three share the same backoff: `initial_delay` `1.0` s, `max_delay` + # `15.0` s, `multiplier` `1.3`. # - # | Policy | Governs | Missing status header | - # |---|---|---| - # | initiation | session initiation | **Retriable** on any status (incl. `200`), unless fatal | - # | control plane | `query` and `cancel` | No predicate; decided on `retry_codes` alone | - # | data plane | `upload` and `finalize` | **Not** retriable | + # Some decisions belong to the protocol and are made before a policy is asked: # - # Initiation treats a response missing `X-Goog-Upload-Status` as gateway noise worth retrying; the data - # plane treats it as a response it cannot interpret and refuses to replay bytes against it. + # * Initiation and control plane requests are re-sent, within the policy's deadline, after a connection + # or TLS failure, and after a `200` response missing `X-Goog-Upload-Status`. + # * Data plane requests are never re-sent after an outcome that leaves the server offset unknown — a + # timeout, a connection failure, a missing status header, or any `4xx`. The upload re-queries the + # session and resumes from the offset the server reports instead. + # * A response carrying `X-Goog-Upload-Status: final` is never retried. # class ResumableUpload ## diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver/retry_decider_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver/retry_decider_test.rb new file mode 100644 index 0000000..97e3ad9 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver/retry_decider_test.rb @@ -0,0 +1,245 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "delegate" +require "faraday" + +## +# Table tests for Driver::RetryDecider: every row of the decision table, on both planes. +# +class DriverRetryDeciderTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + RetryDecider = Driver::RetryDecider + + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + # ============================================================================ + # SUT: failure_kind + # ============================================================================ + + def test_failure_kind + { + Gapic::Rest::DeadlineExceededError.new("deadline", 504) => :timeout, + Gapic::Rest::Error.new("unavailable", 503) => :status, + Gapic::Rest::Error.new("network", nil) => :connection_failed, + http_error(503) => :status, + Faraday::TimeoutError.new("read timeout") => :timeout, + Faraday::ConnectionFailed.new("refused") => :connection_failed, + Faraday::SSLError.new("handshake") => :connection_failed, + Faraday::Error.new("no response") => :no_response, + RuntimeError.new("token refresh failed") => :non_transport + }.each do |error, kind| + assert_equal kind, RetryDecider.failure_kind(error), error.inspect + end + end + + # ============================================================================ + # SUT: retry? — rows of the decision table, start/control vs data + # ============================================================================ + + # Row 1: non-transport error — ask policy on both planes. + def test_row_1_non_transport_error_asks_the_policy + error = RuntimeError.new "token refresh failed" + [false, true].each do |data_plane| + refute decide(error, data_plane: data_plane), "default codes cannot match, data_plane: #{data_plane}" + assert decide(error, data_plane: data_plane, retry_predicate: ->(_e) { true }), "data_plane: #{data_plane}" + end + end + + # Row 2: timeout — ask policy on start/control, surface on data. + def test_row_2_timeout + error = Faraday::TimeoutError.new "read timeout" + refute decide(error) + assert decide(error, retry_predicate: ->(_e) { true }) + refute decide(error, data_plane: true, retry_predicate: ->(_e) { true }) + end + + def test_row_2_a_transport_error_without_a_response_is_treated_as_a_timeout + error = Faraday::Error.new "no response" + assert decide(error, retry_predicate: ->(_e) { true }) + refute decide(error, data_plane: true, retry_predicate: ->(_e) { true }) + end + + # Row 3: connection and TLS failures — retry within budget on start/control, surface on data. + def test_row_3_connection_and_tls_failures + [Faraday::ConnectionFailed.new("refused"), Faraday::SSLError.new("handshake")].each do |error| + assert decide(error, retry_predicate: ->(_e) { false }), "predicate is not consulted: #{error.class}" + refute decide(error, data_plane: true, retry_predicate: ->(_e) { true }), error.class.name + refute decide(error, timeout: 0), "budget exhausted: #{error.class}" + end + end + + # Row 4: headerless 200 — retry within budget on start/control, surface on data. + def test_row_4_headerless_200 + [{}, { "X-Goog-Upload-Status" => "" }].each do |headers| + response = FakeResponse.new status: 200, headers: headers, body: "" + assert decide(response, retry_predicate: ->(_e) { false }), headers.inspect + refute decide(response, data_plane: true, retry_predicate: ->(_e) { true }), headers.inspect + refute decide(response, timeout: 0), "budget exhausted: #{headers.inspect}" + end + end + + def test_a_200_with_the_header_and_other_responses_surface + [ + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: ""), + FakeResponse.new(status: 204, headers: {}, body: ""), + FakeResponse.new(status: 308, headers: {}, body: "") + ].each do |response| + [false, true].each do |data_plane| + refute decide(response, data_plane: data_plane, retry_predicate: ->(_e) { true }), response.inspect + end + end + end + + # Row 5: non-200 with `final` — never retried on either plane, whatever the policy says. + def test_row_5_final_rejection_is_never_retried + [400, 409, 503].each do |status| + error = http_error status, headers: { "X-Goog-Upload-Status" => "Final" } + [false, true].each do |data_plane| + refute decide(error, data_plane: data_plane, retry_predicate: ->(_e) { true }), + "status #{status}, data_plane: #{data_plane}" + end + end + end + + # Row 6: any 4xx on the data plane surfaces, whatever the policy says. + def test_row_6_data_plane_never_retries_a_4xx + [400, 408, 409, 429, 499].each do |status| + refute decide(http_error(status), data_plane: true, retry_predicate: ->(_e) { true }), "status #{status}" + end + end + + # Row 7: a status in the default codes — ask policy, retried by default. + def test_row_7_default_codes_are_retried + [409, 429, 499, 500, 503, 504].each do |status| + assert decide(http_error(status)), "start/control status #{status}" + end + [500, 503, 504].each do |status| + assert decide(http_error(status), data_plane: true), "data status #{status}" + end + end + + def test_row_7_the_predicate_can_decline_a_default_code + refute decide(http_error(503), retry_predicate: ->(_e) { false }) + refute decide(http_error(503), data_plane: true, retry_predicate: ->(_e) { false }) + end + + # Row 8: any other status — ask policy, not retried by default. + def test_row_8_other_statuses_are_not_retried_by_default + [400, 403, 404, 408, 412, 502].each do |status| + refute decide(http_error(status)), "start/control status #{status}" + end + [501, 502, 505].each do |status| + refute decide(http_error(status), data_plane: true), "data status #{status}" + assert decide(http_error(status), data_plane: true, retry_predicate: ->(_e) { true }), "data status #{status}" + end + end + + def test_asking_the_policy_respects_its_deadline + refute decide(http_error(503), timeout: 0) + refute decide(RuntimeError.new("auth"), timeout: 0, retry_predicate: ->(_e) { true }) + end + + # Under `raise_faraday_errors: false`, the Faraday error is the `cause` of a Gapic::Rest::Error, and + # retry_codes can read the status only from the Faraday error. + def test_a_gapic_rest_error_is_judged_by_its_faraday_cause + assert decide(wrapped(http_error(503))) + refute decide(wrapped(http_error(503, headers: { "X-Goog-Upload-Status" => "final" }))) + refute decide(wrapped(http_error(429)), data_plane: true, retry_predicate: ->(_e) { true }) + end + + def test_a_gapic_rest_error_without_a_cause_is_judged_by_its_own_status + error = Gapic::Rest::Error.new "unavailable", 503, headers: { "X-Goog-Upload-Status" => "final" } + refute decide(error, retry_predicate: ->(_e) { true }) + + error = Gapic::Rest::Error.new "conflict", 409, headers: {} + refute decide(error, data_plane: true, retry_predicate: ->(_e) { true }) + end + + def test_a_retry_performs_the_backoff_delay + policy = policy_for({}, data_plane: false) + decider = RetryDecider.new policy, data_plane: false + + assert decider.retry?(http_error(503)) + assert_equal 1, policy.perform_delay_count + end + + # Rows fixed ahead of the policy must not consult it at all: neither its predicate nor its backoff. + def test_surfaced_outcomes_never_consult_the_policy + final = { "X-Goog-Upload-Status" => "final" } + active_200 = FakeResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: "" + headerless_200 = FakeResponse.new status: 200, headers: {}, body: "" + { + true => [Faraday::ConnectionFailed.new("reset"), Faraday::SSLError.new("mac"), + Faraday::TimeoutError.new("read"), Faraday::Error.new("none"), headerless_200, active_200, + http_error(429), http_error(503, headers: final)], + false => [active_200, http_error(503, headers: final)] + }.each do |data_plane, outcomes| + outcomes.each do |outcome| + spy = SpyPolicy.new policy_for({ timeout: 60, retry_predicate: ->(_e) { true } }, data_plane: data_plane) + refute RetryDecider.new(spy, data_plane: data_plane).retry?(outcome), outcome.inspect + assert_equal 0, spy.calls, "policy consulted for #{outcome.inspect}, data_plane: #{data_plane}" + end + end + end + + # Counts RetryPolicy#call invocations. + class SpyPolicy < SimpleDelegator + attr_reader :calls + + def initialize policy + super + @calls = 0 + end + + def call *args + @calls += 1 + __getobj__.call(*args) + end + end + + private + + def decide outcome, data_plane: false, timeout: 60, retry_predicate: nil + overrides = { timeout: timeout } + overrides[:retry_predicate] = retry_predicate if retry_predicate + RetryDecider.new(policy_for(overrides, data_plane: data_plane), data_plane: data_plane).retry?(outcome) + end + + def policy_for overrides, data_plane: + defaults = data_plane ? RetryPolicies::DATA_PLANE_DEFAULTS : RetryPolicies::START_DEFAULTS + Gapic::Common::RetryPolicy.new(**overrides).apply_defaults(defaults).start!(mock_delay: true) + end + + def http_error status, headers: {} + klass = status >= 500 ? Faraday::ServerError : Faraday::ClientError + klass.new "the server responded with status #{status}", { status: status, headers: headers, body: "" } + end + + def wrapped faraday_error + raise faraday_error + rescue Faraday::Error => e + begin + raise Gapic::Rest::Error.wrap_faraday_error(e) + rescue Gapic::Rest::Error => wrapped_error + wrapped_error + end + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb index b7b5e1b..0c8297e 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb @@ -108,21 +108,24 @@ def test_rescue_request_error_rest_error_without_status_code assert_same err, integration_event.source_error end + # A non-transport error (e.g. a credentials refresh failure) reached no server, so it is neither a + # connection failure nor a reason to recover. def test_rescue_request_error_standard_error err = RuntimeError.new "Unexpected low-level runtime error" event = @driver.send :rescue_request_error, err assert_instance_of Event::RequestFailed, event - assert_equal :connection_failed, event.kind + assert_equal :unknown, event.kind assert_equal "Unexpected low-level runtime error", event.message assert_same err, event.source_error + assert_equal :request_failed_unknown, Rules.shape_of(event) # End-to-end via make_post_request @client_stub.error_to_raise = err integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", retry_policy: nil assert_instance_of Event::RequestFailed, integration_event - assert_equal :connection_failed, integration_event.kind + assert_equal :unknown, integration_event.kind assert_same err, integration_event.source_error end @@ -189,6 +192,22 @@ def test_rescue_faraday_error_connection_failed assert_same err, integration_event.source_error end + def test_rescue_faraday_error_ssl_error_is_a_connection_failure + err = Faraday::SSLError.new "SSL_connect returned=1 errno=0 state=error" + event = @driver.send :rescue_faraday_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :connection_failed, event.kind + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :connection_failed, integration_event.kind + end + def test_rescue_faraday_error_generic_without_response err = Faraday::Error.new "Generic transport error without response env" event = @driver.send :rescue_faraday_error, err diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb index 76af4c0..94de0f9 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb @@ -40,7 +40,7 @@ def test_resolve_retry_policy_with_nil_returns_default_policy assert_in_delta 1.0, policy.initial_delay assert_in_delta 15.0, policy.max_delay assert_in_delta 1.3, policy.multiplier - assert_same RetryPolicies::START_PREDICATE, policy.retry_predicate + assert_nil policy.retry_predicate end def test_resolve_retry_policy_with_policy_instance_returns_as_is @@ -52,7 +52,7 @@ def test_resolve_retry_policy_with_policy_instance_returns_as_is assert_empty resolved.retry_codes end - def test_resolve_retry_policy_with_hash_applies_defaults_and_preserves_codes_and_predicate + def test_resolve_retry_policy_with_hash_applies_defaults_and_preserves_codes hash_override = { initial_delay: 0.25, max_delay: 2.0 } resolved = @driver.send :resolve_retry_policy, hash_override, RetryPolicies::START_DEFAULTS @@ -61,17 +61,17 @@ def test_resolve_retry_policy_with_hash_applies_defaults_and_preserves_codes_and assert_in_delta 2.0, resolved.max_delay assert_in_delta 1.3, resolved.multiplier assert_equal RetryPolicies.default_start.retry_codes, resolved.retry_codes - assert_same RetryPolicies::START_PREDICATE, resolved.retry_predicate + assert_nil resolved.retry_predicate end - def test_resolve_retry_policy_with_data_plane_hash_preserves_data_plane_predicate + def test_resolve_retry_policy_with_data_plane_hash_preserves_data_plane_codes hash_override = { timeout: 60.0 } resolved = @driver.send :resolve_retry_policy, hash_override, RetryPolicies::DATA_PLANE_DEFAULTS assert_kind_of Gapic::Common::RetryPolicy, resolved assert_in_delta 60.0, resolved.timeout - assert_equal RetryPolicies.default_data_plane.retry_codes, resolved.retry_codes - assert_same RetryPolicies::DATA_PLANE_PREDICATE, resolved.retry_predicate + assert_equal RetryPolicies::DATA_PLANE_RETRY_CODES, resolved.retry_codes + assert_nil resolved.retry_predicate end def test_resolve_retry_policy_with_empty_retry_codes_array_honors_empty_array @@ -80,7 +80,6 @@ def test_resolve_retry_policy_with_empty_retry_codes_array_honors_empty_array assert_kind_of Gapic::Common::RetryPolicy, resolved assert_empty resolved.retry_codes - assert_same RetryPolicies::START_PREDICATE, resolved.retry_predicate end def test_resolve_retry_policy_with_unknown_hash_key_raises_argument_error @@ -112,32 +111,14 @@ def test_driver_initialize_resolves_hash_overrides_from_config data_policy = driver.instance_variable_get :@data_plane_retry_policy assert_in_delta 0.1, start_policy.initial_delay - assert_same RetryPolicies::START_PREDICATE, start_policy.retry_predicate + assert_equal RetryPolicies::START_AND_CONTROL_PLANE_RETRY_CODES, start_policy.retry_codes assert_in_delta 5.0, control_policy.max_delay - assert_nil control_policy.retry_predicate + assert_equal RetryPolicies::START_AND_CONTROL_PLANE_RETRY_CODES, control_policy.retry_codes assert_in_delta 2.0, data_policy.multiplier - assert_same RetryPolicies::DATA_PLANE_PREDICATE, data_policy.retry_predicate - end - - def test_start_predicate_refutes_fatal_status_codes - [401, 403, 404, 405, 410, 413, 415].each do |code| - response_double = OpenStruct.new status: code, headers: {} - refute RetryPolicies::START_PREDICATE.call(response_double), - "Expected START_PREDICATE to return false for fatal status #{code}" + assert_equal RetryPolicies::DATA_PLANE_RETRY_CODES, data_policy.retry_codes - gapic_err = Gapic::Rest::Error.new "Error", code, status: "FATAL_ERROR", headers: {} - refute RetryPolicies::START_PREDICATE.call(gapic_err), - "Expected START_PREDICATE to return false for Gapic::Rest::Error with fatal status #{code}" - end - end - - def test_start_predicate_retries_missing_header_for_non_fatal_codes - [200, 400, 500, 503].each do |code| - response_double = OpenStruct.new status: code, headers: {} - assert RetryPolicies::START_PREDICATE.call(response_double), - "Expected START_PREDICATE to return true for non-fatal status #{code} with missing status header" - end + [start_policy, control_policy, data_policy].each { |policy| assert_nil policy.retry_predicate } end end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb index f20cf52..9c4ce1c 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb @@ -17,232 +17,369 @@ require "test_helper" require "gapic/rest/resumable_upload" require "stringio" +require "faraday" ## -# Tests for ResumableUpload Driver retry behavior during session start and queries. +# Tests for the ResumableUpload Driver retry loop: which outcomes are re-sent on each plane, and what +# surfaces to Rules when they are not. # +# Non-2xx outcomes are raised as Faraday errors carrying a response env, as Faraday's `raise_error` +# middleware does in production. +# +# rubocop:disable Metrics/ClassLength # rubocop:disable Metrics/MethodLength class DriverRetryTest < Minitest::Test include Gapic::Rest::ResumableUpload FakeResponse = Struct.new :status, :headers, :body, keyword_init: true - # Fake client stub recording calls and yielding scripted responses. + SESSION_URL = "https://example.com/session/1" + + FAST = { initial_delay: 0.001, max_delay: 0.002, timeout: 1.0 }.freeze + + # Fake client stub recording calls and yielding scripted outcomes. An Exception is raised, a Proc is + # called, anything else is returned. class FakeClientStub attr_reader :requests - def initialize responses - @responses = responses + def initialize outcomes + @outcomes = outcomes @requests = [] end def make_post_request uri:, body:, params:, options:, method_name: nil @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } - raise "Unexpected request: no scripted response left" if @responses.empty? + raise "Unexpected request: no scripted response left" if @outcomes.empty? - @responses.shift + outcome = @outcomes.shift + case outcome + when Exception then raise outcome + when Proc then outcome.call + else outcome + end end end - def test_start_retries_when_response_lacks_status_header_even_on_200 - responses = [ - FakeResponse.new(status: 200, headers: {}, body: ""), - FakeResponse.new( - status: 200, - headers: { - "X-Goog-Upload-URL" => "https://example.com/session/1", - "X-Goog-Upload-Status" => "active" - }, - body: "" - ), - FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') - ] - stub = FakeClientStub.new responses - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - upload_size: 4, - chunk_size: 10, - start_retry_policy: { initial_delay: 0.001, max_delay: 0.002, timeout: 1.0 } - ) + # ============================================================================ + # Initiation + # ============================================================================ + + def test_start_retries_a_200_without_status_header + stub = FakeClientStub.new [headerless_200, initiation_response, final_response] + + assert_equal '{"done":true}', run_upload(stub, start_retry_policy: FAST) + assert_commands stub, ["start", "start", "upload, finalize"] + end + + # An exhausted headerless start 200 surfaces as the response itself, which Rules turns into a bad + # response. + def test_start_surfaces_an_exhausted_headerless_200_as_a_bad_response + stub = FakeClientStub.new Array.new(200) { headerless_200 } + + err = assert_raises BadResponseError do + run_upload stub, start_retry_policy: FAST.merge(timeout: 0.01) + end + + assert_equal 200, err.status_code + assert_match(/X-Goog-Upload-Status: missing/, err.message) + assert stub.requests.size > 1 + end + + # The missing-header retry is the protocol's, so it must not depend on anything the policy carries. + def test_start_retries_a_headerless_200_under_a_bare_policy + stub = FakeClientStub.new Array.new(200) { headerless_200 } + policy = Gapic::Common::RetryPolicy.new initial_delay: 0.001, max_delay: 0.002, timeout: 0.05 + assert_empty policy.retry_codes + assert_nil policy.retry_predicate + + assert_raises BadResponseError do + run_upload stub, start_retry_policy: policy + end + + assert stub.requests.size > 1, "Expected the missing-header retry to survive a bare policy" + end - driver = Driver.new client_stub: stub, config: config - result = driver.run + def test_start_retries_the_default_retriable_statuses + [409, 429, 499, 500, 503, 504].each do |status| + stub = FakeClientStub.new [http_error(status), initiation_response, final_response] + + assert_equal '{"done":true}', run_upload(stub, start_retry_policy: FAST), "status #{status}" + assert_commands stub, ["start", "start", "upload, finalize"] + end + end + + def test_start_does_not_retry_other_statuses_by_default + [400, 403, 404, 408, 502].each do |status| + stub = FakeClientStub.new [http_error(status)] + + err = assert_raises BadResponseError do + run_upload stub, start_retry_policy: FAST + end + + assert_equal status, err.status_code + assert_equal 1, stub.requests.size, "status #{status}" + end + end + + def test_start_retries_connection_and_tls_failures + [Faraday::ConnectionFailed.new("refused"), Faraday::SSLError.new("handshake")].each do |error| + stub = FakeClientStub.new [error, initiation_response, final_response] + + assert_equal '{"done":true}', run_upload(stub, start_retry_policy: FAST), error.class.name + assert_commands stub, ["start", "start", "upload, finalize"] + end + end + + # A caller predicate cannot switch off the protocol's own connection retry: it is not consulted. + def test_start_retries_a_connection_failure_without_asking_the_predicate + seen = [] + predicate = lambda { |err| + seen << err + false + } + stub = FakeClientStub.new [Faraday::ConnectionFailed.new("refused"), initiation_response, final_response] + + run_upload stub, start_retry_policy: FAST.merge(retry_predicate: predicate) - assert_equal '{"done":true}', result assert_equal 3, stub.requests.size - assert_start_request stub.requests[0] - assert_start_request stub.requests[1] - assert_chunk_request stub.requests[2], offset: "0", length: "4", body: "0123", finalize: true + assert_empty seen end - def test_start_exhausts_retries_when_200_responses_continually_lack_status_header - responses = Array.new(10) { FakeResponse.new status: 200, headers: {}, body: "" } - stub = FakeClientStub.new responses - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - upload_size: 4, - chunk_size: 10, - start_retry_policy: { initial_delay: 0.001, max_delay: 0.002, timeout: 0.01 } - ) + def test_start_surfaces_an_exhausted_connection_failure_as_request_failed + stub = FakeClientStub.new Array.new(200) { Faraday::ConnectionFailed.new "refused" } - driver = Driver.new client_stub: stub, config: config err = assert_raises RequestFailedError do - driver.run + run_upload stub, start_retry_policy: FAST.merge(timeout: 0.01) end - assert_match(/Missing X-Goog-Upload-Status/, err.message) - assert_equal 200, err.status_code - assert_instance_of BadResponseError, err.cause + assert_instance_of Faraday::ConnectionFailed, err.cause assert stub.requests.size > 1 end - def test_start_does_not_re_retry_non_200_responses_lacking_status_header_outside_client_stub - responses = [FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable")] - stub = FakeClientStub.new responses - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - upload_size: 4, - chunk_size: 10, - start_retry_policy: { initial_delay: 0.001, max_delay: 0.002, timeout: 0.01 } - ) + def test_final_non_200_is_never_retried_even_when_the_predicate_says_yes + stub = FakeClientStub.new [http_error(503, headers: { "X-Goog-Upload-Status" => "final" })] - driver = Driver.new client_stub: stub, config: config - err = assert_raises BadResponseError do - driver.run + assert_raises UploadRejectedError do + run_upload stub, start_retry_policy: FAST.merge(retry_predicate: ->(_err) { true }) end - assert_equal 503, err.status_code - assert_includes err.message, "503" - refute_match(/Missing X-Goog-Upload-Status/, err.message) assert_equal 1, stub.requests.size end - def test_query_does_not_retry_on_missing_status_header_in_driver - responses = [ - FakeResponse.new( - status: 200, - headers: { - "X-Goog-Upload-URL" => "https://example.com/session/1", - "X-Goog-Upload-Status" => "active" - }, - body: "" - ), - FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable"), - FakeResponse.new(status: 200, headers: {}, body: ""), - FakeResponse.new( - status: 200, - headers: { - "X-Goog-Upload-Status" => "active", - "X-Goog-Upload-Size-Received" => "0" - }, - body: "" - ), - FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') - ] - stub = FakeClientStub.new responses - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - upload_size: 4, - chunk_size: 10 - ) + def test_non_transport_error_is_not_retried_by_default + stub = FakeClientStub.new [RuntimeError.new("token refresh failed")] - driver = Driver.new client_stub: stub, config: config - result = driver.run - - assert_equal '{"done":true}', result - assert_equal 5, stub.requests.size - assert_start_request stub.requests[0] - assert_chunk_request stub.requests[1], offset: "0", length: "4", body: "0123", finalize: true - assert_query_request stub.requests[2] - assert_query_request stub.requests[3] - assert_chunk_request stub.requests[4], offset: "0", length: "4", body: "0123", finalize: true - end - - # The initiation missing-header retry is a protocol requirement, so it must not depend on the retry - # policy carrying RetryPolicies::START_PREDICATE. A RetryPolicy instance is passed through verbatim - # by Driver#resolve_retry_policy, defaults and all, so this policy reaches the start loop with no - # predicate at all. Consulting it about the response would answer "no retry" on the first attempt, - # because Event::HttpResponse has no `response_status` for the retry_codes branch to read. - def test_start_retries_missing_status_header_under_a_policy_carrying_no_predicate - responses = Array.new(200) { FakeResponse.new status: 200, headers: {}, body: "" } - stub = FakeClientStub.new responses - policy = Gapic::Common::RetryPolicy.new initial_delay: 0.001, max_delay: 0.002, timeout: 0.05 - assert_nil policy.retry_predicate + err = assert_raises RequestFailedError do + run_upload stub, start_retry_policy: FAST + end - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - upload_size: 4, - chunk_size: 10, - start_retry_policy: policy - ) + assert_match(/token refresh failed/, err.message) + assert_equal 1, stub.requests.size + end - driver = Driver.new client_stub: stub, config: config - err = assert_raises RequestFailedError do - driver.run + def test_non_transport_error_is_retried_when_the_predicate_says_so + predicate = ->(err) { err.is_a?(RuntimeError) || nil } + stub = FakeClientStub.new [RuntimeError.new("token refresh failed"), initiation_response, final_response] + + assert_equal '{"done":true}', run_upload(stub, start_retry_policy: FAST.merge(retry_predicate: predicate)) + assert_commands stub, ["start", "start", "upload, finalize"] + end + + # Under `raise_faraday_errors: false`, ClientStub raises a Gapic::Rest::Error from inside its own rescue. + # Its status is readable by retry_codes only through `cause`. + def test_start_retries_a_gapic_rest_error_through_its_faraday_cause + wrapped = lambda do + raise http_error(503) + rescue Faraday::Error => e + raise Gapic::Rest::Error.wrap_faraday_error(e) end + stub = FakeClientStub.new [wrapped, initiation_response, final_response] - assert_match(/Missing X-Goog-Upload-Status/, err.message) - assert stub.requests.size > 1, "Expected the missing-header retry to survive a predicate-less policy" + assert_equal '{"done":true}', run_upload(stub, start_retry_policy: FAST) + assert_equal 3, stub.requests.size + end + + def test_client_stub_is_handed_a_never_retry_policy + stub = FakeClientStub.new [initiation_response, final_response] + run_upload stub + + stub.requests.each do |req| + policy = req[:options][:retry_policy] + refute policy.call(http_error(503)), "ClientStub must never retry on its own" + refute policy.call(Faraday::ConnectionFailed.new("refused")) + end end - # A caller-supplied predicate displaces START_PREDICATE and is written against the Faraday errors - # ClientStub rescues. The start loop must never hand it an Event, a BadResponseError, or anything - # else from the driver's private vocabulary. - def test_start_never_passes_driver_internals_to_a_caller_supplied_predicate + def test_the_predicate_never_sees_driver_internals seen = [] - responses = Array.new(200) { FakeResponse.new status: 200, headers: {}, body: "" } - stub = FakeClientStub.new responses - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - upload_size: 4, - chunk_size: 10, - start_retry_policy: { - initial_delay: 0.001, - max_delay: 0.002, - timeout: 0.05, - retry_predicate: lambda { |arg| - seen << arg - nil - } - } - ) + predicate = lambda { |arg| + seen << arg + nil + } + outcomes = [headerless_200, http_error(503), RuntimeError.new("auth"), initiation_response, final_response] + stub = FakeClientStub.new outcomes - driver = Driver.new client_stub: stub, config: config assert_raises RequestFailedError do - driver.run + run_upload stub, start_retry_policy: FAST.merge(retry_predicate: predicate) end - assert stub.requests.size > 1, "Expected retries, otherwise the predicate assertion proves nothing" + refute_empty seen refute seen.any? { |arg| arg.is_a?(Event::HttpResponse) || arg.is_a?(BadResponseError) }, "Predicate was handed driver internals: #{seen.map(&:class).uniq.inspect}" end + # ============================================================================ + # Control plane + # ============================================================================ + + def test_query_retries_a_200_without_status_header + outcomes = [ + initiation_response, + Faraday::ConnectionFailed.new("reset"), + headerless_200, + query_response(received: 0), + final_response + ] + stub = FakeClientStub.new outcomes + + assert_equal '{"done":true}', run_upload(stub, control_plane_retry_policy: FAST) + assert_commands stub, ["start", "upload, finalize", "query", "query", "upload, finalize"] + end + + def test_query_retries_connection_and_tls_failures + [Faraday::ConnectionFailed.new("refused"), Faraday::SSLError.new("handshake")].each do |error| + outcomes = [initiation_response, Faraday::ConnectionFailed.new("reset"), error, query_response(received: 0), + final_response] + stub = FakeClientStub.new outcomes + + assert_equal '{"done":true}', run_upload(stub, control_plane_retry_policy: FAST), error.class.name + assert_commands stub, ["start", "upload, finalize", "query", "query", "upload, finalize"] + end + end + + def test_query_retries_a_retriable_4xx + outcomes = [initiation_response, Faraday::ConnectionFailed.new("reset"), http_error(429), + query_response(received: 0), final_response] + stub = FakeClientStub.new outcomes + + assert_equal '{"done":true}', run_upload(stub, control_plane_retry_policy: FAST) + assert_commands stub, ["start", "upload, finalize", "query", "query", "upload, finalize"] + end + + # ============================================================================ + # Data plane + # ============================================================================ + + # Recovery owns every outcome that leaves the server offset unknown: the chunk is not re-sent blindly. + def test_upload_surfaces_connection_and_tls_failures_to_recovery + [Faraday::ConnectionFailed.new("reset"), Faraday::SSLError.new("bad record mac")].each do |error| + stub = FakeClientStub.new [initiation_response, error, query_response(received: 0), final_response] + + assert_equal '{"done":true}', run_upload(stub, data_plane_retry_policy: FAST), error.class.name + assert_commands stub, ["start", "upload, finalize", "query", "upload, finalize"] + end + end + + def test_upload_surfaces_a_headerless_200_to_recovery + stub = FakeClientStub.new [initiation_response, headerless_200, query_response(received: 0), final_response] + + assert_equal '{"done":true}', run_upload(stub, data_plane_retry_policy: FAST) + assert_commands stub, ["start", "upload, finalize", "query", "upload, finalize"] + end + + def test_upload_never_retries_a_4xx_even_when_the_predicate_says_yes + [409, 429].each do |status| + stub = FakeClientStub.new [initiation_response, http_error(status), query_response(received: 0), + final_response] + + result = run_upload stub, data_plane_retry_policy: FAST.merge(retry_predicate: ->(_err) { true }) + + assert_equal '{"done":true}', result, "status #{status}" + assert_commands stub, ["start", "upload, finalize", "query", "upload, finalize"] + end + end + + def test_upload_retries_a_retriable_5xx + stub = FakeClientStub.new [initiation_response, http_error(503), final_response] + + assert_equal '{"done":true}', run_upload(stub, data_plane_retry_policy: FAST) + assert_commands stub, ["start", "upload, finalize", "upload, finalize"] + end + + def test_upload_surfaces_a_timeout_to_recovery + stub = FakeClientStub.new [initiation_response, Faraday::TimeoutError.new("read timeout"), + query_response(received: 0), final_response] + + assert_equal '{"done":true}', run_upload(stub, data_plane_retry_policy: FAST) + assert_commands stub, ["start", "upload, finalize", "query", "upload, finalize"] + end + + # ============================================================================ + # Budgets + # ============================================================================ + + def test_each_attempt_timeout_fits_within_the_shrinking_command_budget + stub = FakeClientStub.new [headerless_200, headerless_200, initiation_response, final_response] + + run_upload stub, start_retry_policy: { initial_delay: 0.02, max_delay: 0.02, timeout: 0.5 } + + timeouts = stub.requests.first(3).map { |req| req[:options][:timeout] } + assert(timeouts.all? { |t| t <= 0.5 }, "Expected every attempt timeout within the budget: #{timeouts}") + assert timeouts[1] < timeouts[0], "Expected the attempt timeout to shrink: #{timeouts}" + assert timeouts[2] < timeouts[1], "Expected the attempt timeout to shrink: #{timeouts}" + end + + def test_the_global_deadline_ends_a_retry_loop + stub = FakeClientStub.new Array.new(200) { headerless_200 } + + assert_raises DeadlineExceededError do + run_upload stub, timeout: 0.05, start_retry_policy: { initial_delay: 0.01, max_delay: 0.01, timeout: 60 } + end + + assert stub.requests.size > 1 + assert stub.requests.size < 200 + end + private - def assert_start_request req - assert_equal "https://example.com/upload", req[:uri] - assert_equal "start", req[:options][:metadata]["X-Goog-Upload-Command"] + def run_upload stub, **config_overrides + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + **config_overrides + ) + Driver.new(client_stub: stub, config: config).run + end + + def headerless_200 + FakeResponse.new status: 200, headers: {}, body: "" + end + + def initiation_response + FakeResponse.new status: 200, headers: { "X-Goog-Upload-URL" => SESSION_URL, "X-Goog-Upload-Status" => "active" }, + body: "" + end + + def query_response received: + FakeResponse.new status: 200, + headers: { "X-Goog-Upload-Status" => "active", "X-Goog-Upload-Size-Received" => received.to_s }, + body: "" + end + + def final_response + FakeResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}' end - def assert_query_request req - assert_equal "https://example.com/session/1", req[:uri] - assert_equal "query", req[:options][:metadata]["X-Goog-Upload-Command"] + def http_error status, headers: {} + klass = status >= 500 ? Faraday::ServerError : Faraday::ClientError + klass.new "the server responded with status #{status}", { status: status, headers: headers, body: "" } end - def assert_chunk_request req, offset:, length:, body:, finalize: - expected_cmd = finalize ? "upload, finalize" : "upload" - metadata = req[:options][:metadata] - assert_equal "https://example.com/session/1", req[:uri] - assert_equal expected_cmd, metadata["X-Goog-Upload-Command"] - assert_equal offset, metadata["X-Goog-Upload-Offset"] - assert_equal length, metadata["Content-Length"] - assert_equal body, req[:body] + def assert_commands stub, expected + assert_equal expected, stub.requests.map { |req| req[:options][:metadata]["X-Goog-Upload-Command"] } end end # rubocop:enable Metrics/MethodLength +# rubocop:enable Metrics/ClassLength diff --git a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb index bd732fe..1f1334c 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb @@ -16,208 +16,78 @@ require "test_helper" require "gapic/rest/resumable_upload" -require "ostruct" require "faraday" ## -# Tests for ResumableUpload RetryPolicies and header extraction. +# Tests for ResumableUpload RetryPolicies defaults and start_retry_policy_for. # class RetryPoliciesTest < Minitest::Test include Gapic::Rest::ResumableUpload # ============================================================================ - # SUT: extract_headers + # SUT: default policies # ============================================================================ - def test_extract_headers_from_headers_method - obj = OpenStruct.new headers: { "X-Test-Header" => "value1" } - assert_equal({ "X-Test-Header" => "value1" }, RetryPolicies.extract_headers(obj)) + def test_start_and_control_plane_retry_codes_cover_the_retriable_4xx_and_5xx_statuses + # 409, 429, 499, 500, 503, 504 in that order. + assert_equal [6, 8, 1, 13, 14, 4], RetryPolicies::START_AND_CONTROL_PLANE_RETRY_CODES end - def test_extract_headers_from_response_headers_method - obj = OpenStruct.new response_headers: { "X-Test-Header" => "value2" } - assert_equal({ "X-Test-Header" => "value2" }, RetryPolicies.extract_headers(obj)) + def test_data_plane_retry_codes_cover_only_the_retriable_5xx_statuses + # 500, 503, 504 in that order. + assert_equal [13, 14, 4], RetryPolicies::DATA_PLANE_RETRY_CODES end - def test_extract_headers_from_faraday_response_hash - err = Faraday::ClientError.new "error message", { headers: { "X-Test-Header" => "value3" } } - assert_equal({ "X-Test-Header" => "value3" }, RetryPolicies.extract_headers(err)) - end - - def test_extract_headers_returns_nil_when_no_headers_present - assert_nil RetryPolicies.extract_headers(StandardError.new("error")) - assert_nil RetryPolicies.extract_headers(nil) - assert_nil RetryPolicies.extract_headers(Object.new) - assert_nil RetryPolicies.extract_headers("string") - assert_nil RetryPolicies.extract_headers({}) - end - - # ============================================================================ - # SUT: RetryPolicies.default_start - # ============================================================================ - - def test_default_start_missing_status_header_retries_unconditionally - policy = RetryPolicies.default_start - - # Retriable code (503) without status header - err_503 = OpenStruct.new response_status: 503, headers: { "Content-Type" => "text/plain" } - assert policy.retry_error?(err_503) - - # Non-retriable code (400) without status header (still retries because missing status header is retriable) - err_400 = OpenStruct.new response_status: 400, headers: { "Content-Type" => "text/plain" } - assert policy.retry_error?(err_400) - - # Status 200 OK without status header - resp_200 = OpenStruct.new response_status: 200, headers: { "Content-Type" => "text/plain" } - assert policy.retry_error?(resp_200) - - # No status code without status header - err_no_code = OpenStruct.new headers: { "Content-Type" => "text/plain" } - assert policy.retry_error?(err_no_code) - - # Empty status header string - err_empty_status = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "" } - assert policy.retry_error?(err_empty_status) - end - - def test_default_start_with_status_header_falls_back_to_codes - policy = RetryPolicies.default_start - - # Retriable code (503) with status header - err_503 = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "active" } - assert policy.retry_error?(err_503) - - # Non-retriable code (400) with status header - err_400 = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "active" } - refute policy.retry_error?(err_400) - - # Without status code with status header - err_no_code = OpenStruct.new headers: { "X-Goog-Upload-Status" => "active" } - refute policy.retry_error?(err_no_code) - end - - def test_default_start_no_headers_falls_back_to_codes - policy = RetryPolicies.default_start - - # Retriable code (503) without headers - err_503 = OpenStruct.new response_status: 503 - assert policy.retry_error?(err_503) - - # Non-retriable code (400) without headers - err_400 = OpenStruct.new response_status: 400 - refute policy.retry_error?(err_400) - - # Without status code and without headers - err_no_code = RuntimeError.new "generic network error" - refute policy.retry_error?(err_no_code) - end - - # ============================================================================ - # SUT: RetryPolicies.default_control_plane - # ============================================================================ - - def test_default_control_plane_missing_status_header_falls_back_to_codes - policy = RetryPolicies.default_control_plane - - # Retriable code (503) without status header - err_503 = OpenStruct.new response_status: 503, headers: { "Content-Type" => "text/plain" } - assert policy.retry_error?(err_503) - - # Non-retriable code (400) without status header - err_400 = OpenStruct.new response_status: 400, headers: { "Content-Type" => "text/plain" } - refute policy.retry_error?(err_400) - - # Without status code without status header - err_no_code = OpenStruct.new headers: { "Content-Type" => "text/plain" } - refute policy.retry_error?(err_no_code) - end - - def test_default_control_plane_with_status_header_falls_back_to_codes - policy = RetryPolicies.default_control_plane - - # Retriable code (503) with status header - err_503 = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "active" } - assert policy.retry_error?(err_503) - - # Non-retriable code (400) with status header - err_400 = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "active" } - refute policy.retry_error?(err_400) - - # Without status code with status header - err_no_code = OpenStruct.new headers: { "X-Goog-Upload-Status" => "active" } - refute policy.retry_error?(err_no_code) - end - - def test_default_control_plane_no_headers_falls_back_to_codes - policy = RetryPolicies.default_control_plane - - # Retriable code (503) without headers - err_503 = OpenStruct.new response_status: 503 - assert policy.retry_error?(err_503) - - # Non-retriable code (400) without headers - err_400 = OpenStruct.new response_status: 400 - refute policy.retry_error?(err_400) - - # Without status code and without headers - err_no_code = RuntimeError.new "generic network error" - refute policy.retry_error?(err_no_code) + def test_retry_codes_are_derived_from_the_rules_status_sets + expected = (Rules::RETRIABLE_4XX_STATUS_CODES + Rules::RETRIABLE_5XX_STATUS_CODES).map do |status| + Gapic::Common::ErrorCodes.grpc_error_for status + end + assert_equal expected, RetryPolicies::START_AND_CONTROL_PLANE_RETRY_CODES + end + + def test_408_and_502_are_not_retried_by_default + unknown = Gapic::Common::ErrorCodes.grpc_error_for 408 + assert_equal Gapic::Common::ErrorCodes.grpc_error_for(502), unknown + refute_includes RetryPolicies::START_AND_CONTROL_PLANE_RETRY_CODES, unknown + refute_includes RetryPolicies::DATA_PLANE_RETRY_CODES, unknown + end + + def test_default_policies_carry_the_per_plane_codes_and_no_predicate + { + RetryPolicies.default_start => RetryPolicies::START_AND_CONTROL_PLANE_RETRY_CODES, + RetryPolicies.default_control_plane => RetryPolicies::START_AND_CONTROL_PLANE_RETRY_CODES, + RetryPolicies.default_data_plane => RetryPolicies::DATA_PLANE_RETRY_CODES + }.each do |policy, codes| + assert_equal codes, policy.retry_codes + assert_nil policy.retry_predicate + assert_in_delta 1.0, policy.initial_delay + assert_in_delta 15.0, policy.max_delay + assert_in_delta 1.3, policy.multiplier + end end - # ============================================================================ - # SUT: RetryPolicies.default_data_plane - # ============================================================================ - - def test_default_data_plane_missing_status_header_unretriable - policy = RetryPolicies.default_data_plane - - # Retriable code (503) without status header (predicate returns false -> unretriable) - err_503 = OpenStruct.new response_status: 503, headers: { "Content-Type" => "text/plain" } - refute policy.retry_error?(err_503) + def test_default_start_retries_a_retriable_4xx_but_the_data_plane_does_not + err429 = Faraday::ClientError.new "Too Many Requests", { status: 429, headers: {} } - # Non-retriable code (400) without status header - err_400 = OpenStruct.new response_status: 400, headers: { "Content-Type" => "text/plain" } - refute policy.retry_error?(err_400) - - # Without status code without status header - err_no_code = OpenStruct.new headers: { "Content-Type" => "text/plain" } - refute policy.retry_error?(err_no_code) - - # Empty status header string - err_empty_status = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "" } - refute policy.retry_error?(err_empty_status) + assert RetryPolicies.default_start.retry_error?(err429) + assert RetryPolicies.default_control_plane.retry_error?(err429) + refute RetryPolicies.default_data_plane.retry_error?(err429) end - def test_default_data_plane_with_status_header_falls_back_to_codes - policy = RetryPolicies.default_data_plane - - # Retriable code (503) with status header - err_503 = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "active" } - assert policy.retry_error?(err_503) + def test_default_policies_retry_a_retriable_5xx + err503 = Faraday::ServerError.new "Service Unavailable", { status: 503, headers: {} } - # Non-retriable code (400) with status header - err_400 = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "active" } - refute policy.retry_error?(err_400) - - # Without status code with status header - err_no_code = OpenStruct.new headers: { "X-Goog-Upload-Status" => "active" } - refute policy.retry_error?(err_no_code) + assert RetryPolicies.default_start.retry_error?(err503) + assert RetryPolicies.default_control_plane.retry_error?(err503) + assert RetryPolicies.default_data_plane.retry_error?(err503) end - def test_default_data_plane_no_headers_falls_back_to_codes - policy = RetryPolicies.default_data_plane - - # Retriable code (503) without headers - err_503 = OpenStruct.new response_status: 503 - assert policy.retry_error?(err_503) - - # Non-retriable code (400) without headers - err_400 = OpenStruct.new response_status: 400 - refute policy.retry_error?(err_400) + def test_default_policies_do_not_retry_an_error_without_a_status + err = RuntimeError.new "generic error" - # Without status code and without headers - err_no_code = RuntimeError.new "generic network error" - refute policy.retry_error?(err_no_code) + refute RetryPolicies.default_start.retry_error?(err) + refute RetryPolicies.default_control_plane.retry_error?(err) + refute RetryPolicies.default_data_plane.retry_error?(err) end # ============================================================================ @@ -262,9 +132,7 @@ def test_start_retry_policy_for_copies_retry_codes_when_given assert_equal [Gapic::Common::ErrorCodes::ERROR_STRING_MAPPING["UNAVAILABLE"]], overrides[:retry_codes] end - # Applying the overrides to the initiation defaults must leave the missing-status-header predicate in - # place: that is the whole reason the conversion returns a Hash rather than a policy object. - def test_start_retry_policy_for_overrides_leave_the_start_predicate_in_place + def test_start_retry_policy_for_overrides_keep_the_initiation_retry_codes options = Gapic::CallOptions.new timeout: 5, retry_policy: { initial_delay: 0.5 } overrides = Gapic::Rest::ResumableUpload.start_retry_policy_for options @@ -272,7 +140,8 @@ def test_start_retry_policy_for_overrides_leave_the_start_predicate_in_place assert_equal 0.5, policy.initial_delay assert_equal 5, policy.timeout - assert_same RetryPolicies::START_PREDICATE, policy.retry_predicate + assert_equal RetryPolicies::START_AND_CONTROL_PLANE_RETRY_CODES, policy.retry_codes + assert_nil policy.retry_predicate end def test_start_retry_policy_for_rejects_a_proc_retry_policy @@ -311,7 +180,7 @@ def test_start_retry_policy_for_carries_a_caller_retry_predicate overrides = Gapic::Rest::ResumableUpload.start_retry_policy_for options policy = Gapic::Common::RetryPolicy.new(**overrides).apply_defaults RetryPolicies::START_DEFAULTS - # The caller's predicate replaces the initiation one wholesale; the two are not composed. + # The initiation defaults carry no predicate, so the caller's is used as-is. assert_same predicate, policy.retry_predicate end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb index 7d9bfa7..5164f5f 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb @@ -97,37 +97,41 @@ def test_classify_http_response_cancelled end end - def test_classify_http_response_missing_header_non_fatal - # HTTP 200 missing header + def test_cat2_status_codes_are_the_retriable_sets_plus_the_recoverable_extras + expected = (Rules::RETRIABLE_4XX_STATUS_CODES + Rules::RETRIABLE_5XX_STATUS_CODES + + [400, 408, 412, 416, 502]).sort + assert_equal expected, Rules::CAT2_STATUS_CODES + assert_equal [400, 408, 409, 412, 416, 429, 499, 500, 502, 503, 504], Rules::CAT2_STATUS_CODES + end + + def test_classify_http_response_headerless_200_is_cat2 resp_200 = Event::HttpResponse.new status: 200, headers: {}, body: "" assert_equal :response_cat2, Rules.classify_http_response(resp_200) - # Recoverable 4xx missing header - Rules::CAT2_STATUS_CODES.each do |code| - resp = Event::HttpResponse.new status: code, headers: {}, body: "" - assert_equal :response_cat2, Rules.classify_http_response(resp), "Expected #{code} to classify as :response_cat2" - end - - # 5xx server/gateway errors missing header - [500, 502, 503, 504].each do |code| - resp = Event::HttpResponse.new status: code, headers: {}, body: "" - assert_equal :response_cat2, Rules.classify_http_response(resp), "Expected #{code} to classify as :response_cat2" - end - - # Empty string header resp_empty = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "" }, body: "" assert_equal :response_cat2, Rules.classify_http_response(resp_empty) end - def test_classify_http_response_missing_header_fatal_status_codes - Rules::FATAL_STATUS_CODES.each do |code| - resp = Event::HttpResponse.new status: code, headers: {}, body: "" - assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp), - "Expected fatal code #{code} to classify as :response_fatal_bad_response" + # A non-200 without a definitive status header is Category 2 only when its status is listed in + # CAT2_STATUS_CODES; missing, empty and `active` headers are treated alike. + def test_classify_http_response_non_200_cat2_statuses + [{}, { "X-Goog-Upload-Status" => "" }, { "X-Goog-Upload-Status" => "active" }].each do |headers| + Rules::CAT2_STATUS_CODES.each do |code| + resp = Event::HttpResponse.new status: code, headers: headers, body: "" + assert_equal :response_cat2, Rules.classify_http_response(resp), + "Expected #{code} with #{headers.inspect} to classify as :response_cat2" + end + end + end - resp_empty = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "" }, body: "" - assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_empty), - "Expected fatal code #{code} with empty header to classify as :response_fatal_bad_response" + # The allowlist is load-bearing: any status it does not list is fatal, including ones never seen before. + def test_classify_http_response_non_200_unlisted_statuses_are_fatal + [{}, { "X-Goog-Upload-Status" => "" }, { "X-Goog-Upload-Status" => "active" }].each do |headers| + [301, 401, 403, 404, 405, 410, 413, 415, 418, 501, 505, 599].each do |code| + resp = Event::HttpResponse.new status: code, headers: headers, body: "" + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp), + "Expected #{code} with #{headers.inspect} to classify as :response_fatal_bad_response" + end end end diff --git a/gapic-common/test/gapic/resumable_upload_test.rb b/gapic-common/test/gapic/resumable_upload_test.rb index ac465de..de3bde8 100644 --- a/gapic-common/test/gapic/resumable_upload_test.rb +++ b/gapic-common/test/gapic/resumable_upload_test.rb @@ -27,6 +27,10 @@ class ResumableUploadTest < Minitest::Test INITIAL_BODY = '{"name":"test.txt"}' SESSION_URL = "https://upload.example.com/session_1" + # A zero budget: the Driver re-sends nothing, so a scripted connection failure on initiation or a query + # surfaces at once instead of being retried against the next scripted response. + NO_RETRY = { timeout: 0 }.freeze + class ScriptedClientStub attr_reader :requests @@ -347,7 +351,7 @@ def test_concurrent_run_raises_session_state_error def test_run_slot_is_released_after_a_failed_run stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), Faraday::ConnectionFailed.new("boom")] - upload = build_upload stub: stub + upload = build_upload stub: stub, control_plane_retry_policy: NO_RETRY assert_raises RequestFailedError do start_upload upload, stream: StringIO.new("01"), upload_size: 2 @@ -383,7 +387,7 @@ def test_run_slot_is_released_after_a_configuration_error def test_configuration_error_leaves_an_earlier_runs_resume_handle_intact stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), Faraday::ConnectionFailed.new("boom")] - upload = build_upload stub: stub + upload = build_upload stub: stub, control_plane_retry_policy: NO_RETRY assert_raises RequestFailedError do start_upload upload, stream: StringIO.new("01"), upload_size: 2 end @@ -401,7 +405,7 @@ def test_a_second_run_replaces_the_previous_runs_resume_handle stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), Faraday::ConnectionFailed.new("boom"), Faraday::ConnectionFailed.new("boom")] - upload = build_upload stub: stub + upload = build_upload stub: stub, start_retry_policy: NO_RETRY, control_plane_retry_policy: NO_RETRY assert_raises RequestFailedError do start_upload upload, stream: StringIO.new("01"), upload_size: 2 end @@ -435,7 +439,7 @@ def test_bare_resume_reuses_the_retained_drivers_handle stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), Faraday::ConnectionFailed.new("boom"), query_response, final_response('{"text":"resumed"}')] - upload = build_upload stub: stub + upload = build_upload stub: stub, control_plane_retry_policy: NO_RETRY assert_raises RequestFailedError do start_upload upload, stream: StringIO.new("01"), upload_size: 2 end @@ -467,7 +471,8 @@ def test_bare_resume_after_a_successful_run_raises_argument_error end def test_bare_resume_after_an_unresumable_failure_raises_argument_error - upload = build_upload stub: ScriptedClientStub.new([Faraday::ConnectionFailed.new("boom")]) + upload = build_upload stub: ScriptedClientStub.new([Faraday::ConnectionFailed.new("boom")]), + start_retry_policy: NO_RETRY assert_raises RequestFailedError do start_upload upload, stream: StringIO.new("01"), upload_size: 2 end @@ -558,7 +563,7 @@ class WrappedError < StandardError; end def failing_upload **kwargs stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), Faraday::ConnectionFailed.new("boom")] - build_upload stub: stub, **kwargs + build_upload stub: stub, control_plane_retry_policy: NO_RETRY, **kwargs end def test_error_handler_replaces_the_raised_error From 02e6a9d5df175ca60618f928739f8f1c97795bd1 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Fri, 25 Sep 2026 22:59:25 +0000 Subject: [PATCH 16/22] fix: mark InternalError as private in the docs, note that `Common::Error` can be raised. --- .../design/resumable_upload/implementation-guide.md | 4 ++-- gapic-common/lib/gapic/rest/resumable_upload.rb | 2 ++ gapic-common/lib/gapic/resumable_upload.rb | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md index 4480a6f..4138ce6 100644 --- a/gapic-common/design/resumable_upload/implementation-guide.md +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -559,8 +559,8 @@ Terminal errors provide actionable context so downstream SDK callers can inspect * `UploadCancelledError < Gapic::Common::Error`: Upload session was cancelled — either by the caller (via the coordinator's cancellation path) or out-of-band by the server/another client, detected when a response carries `X-Goog-Upload-Status: cancelled` while the session is still transmitting, finalizing, or recovering. Does NOT include `HasResumeHandle` (session is terminated permanently). Part of the public error surface. * `DeadlineExceededError < Gapic::Common::Error`: Upload deadline exceeded with optional root cause (`attr_reader :root_cause`), and includes `HasResumeHandle`. * `UnseekableStreamError < Gapic::Common::Error`: Stream rewind required on an unseekable stream; includes `HasResumeHandle`. - * `InternalError < Gapic::Common::Error`: A defect in the upload implementation rather than a condition the caller can act on. Does NOT include `HasResumeHandle`. - * `InvalidTransitionError < InternalError`: Unexpected event dispatched for state. Marked `@private` — it is an implementation detail and is not part of the documented public surface; callers rescue `InternalError` (or `Gapic::Common::Error`). Does NOT include `HasResumeHandle`: an unmatched transition means the state machine's view of the session is untrustworthy, so advertising the session as resumable would be wrong. + * `InternalError < Gapic::Common::Error`: A protocol implementation bug rather than a condition the caller can act on. Marked `@private`: public docs never name it and describe it only as "any other `Gapic::Common::Error` subclass" (the `@raise [Gapic::Common::Error]` tag on `#start`/`#resume` and the closing bullet of the module's `### Error Types` list). Does NOT include `HasResumeHandle`. + * `InvalidTransitionError < InternalError`: Unexpected event dispatched for state. Marked `@private` — it is an implementation detail and is not part of the documented public surface; callers rescue `Gapic::Common::Error`. Does NOT include `HasResumeHandle`: an unmatched transition means the state machine's view of the session is untrustworthy, so advertising the session as resumable would be wrong. * `StreamMismatchError < Gapic::Common::Error`: Stream content or length does not match resumed upload specifications; includes `HasResumeHandle`. * `RequestFailedError < Gapic::Common::Error`: Terminal HTTP request failure (e.g. transport connection failure, request timeout, or retries exhausted). Retains `attr_reader :cause` returning the underlying error, preserves REST error attributes (`status_code`, `status`, `details`, `headers`) when available, and includes `HasResumeHandle`. * `SessionStateError < Gapic::Common::Error`: Raised when an operation violates the upload session lifecycle rules, e.g. starting a second run on a coordinator while one is still in flight. Distinguished from `ArgumentError`, which is raised strictly for invalid argument shapes. diff --git a/gapic-common/lib/gapic/rest/resumable_upload.rb b/gapic-common/lib/gapic/rest/resumable_upload.rb index c82ef3d..6e40f6c 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload.rb @@ -62,6 +62,8 @@ module Rest # upload was in flight. A cancelled session cannot be resumed, so this error carries no {ResumeHandle} (final). # * {SessionStateError} - Upload session lifecycle rule violation, e.g. starting a second run while one # is in flight (final). + # * Any other `Gapic::Common::Error` subclass signals a protocol implementation bug rather than a caller + # or server error (final). # module ResumableUpload ## diff --git a/gapic-common/lib/gapic/resumable_upload.rb b/gapic-common/lib/gapic/resumable_upload.rb index fb0853f..19fa364 100644 --- a/gapic-common/lib/gapic/resumable_upload.rb +++ b/gapic-common/lib/gapic/resumable_upload.rb @@ -212,8 +212,8 @@ def initialize client_stub_proc:, # @raise [Gapic::Rest::ResumableUpload::StreamMismatchError] If stream content or length does not match # protocol expectations # @raise [Gapic::Rest::ResumableUpload::UploadRejectedError] If the server explicitly rejects the upload - # @raise [Gapic::Rest::ResumableUpload::InternalError] If the library detects an internal invariant - # breach; this signals a bug in this library rather than a caller or server error + # @raise [Gapic::Common::Error] Any other subclass signals a protocol implementation bug rather than a + # caller or server error # def start stream:, content_type: nil, @@ -278,8 +278,8 @@ def start stream:, # @raise [Gapic::Rest::ResumableUpload::StreamMismatchError] If stream content or length does not match # the resumed upload # @raise [Gapic::Rest::ResumableUpload::UploadRejectedError] If the server explicitly rejects the upload - # @raise [Gapic::Rest::ResumableUpload::InternalError] If the library detects an internal invariant - # breach; this signals a bug in this library rather than a caller or server error + # @raise [Gapic::Common::Error] Any other subclass signals a protocol implementation bug rather than a + # caller or server error # def resume stream:, resume_handle: nil, From 8a1ff50c2a4806dae95c230d6704136be0e5914e Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Fri, 25 Sep 2026 23:24:26 +0000 Subject: [PATCH 17/22] chore: mark error factories and other API surface elements that are not intended to be public as `@private` --- gapic-common/lib/gapic/rest/resumable_upload.rb | 8 ++++---- .../lib/gapic/rest/resumable_upload/errors.rb | 9 +++++++++ gapic-common/lib/gapic/resumable_upload.rb | 17 +++++++---------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/gapic-common/lib/gapic/rest/resumable_upload.rb b/gapic-common/lib/gapic/rest/resumable_upload.rb index 6e40f6c..fbb685a 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload.rb @@ -47,8 +47,8 @@ module Rest # begin # upload.start stream: File.open("movie.mp4", "rb"), upload_size: File.size("movie.mp4") # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e - # handle = e.resume_handle - # upload.resume stream: File.open("movie.mp4", "rb"), resume_handle: handle + # raise unless e.resume_handle + # upload.resume stream: File.open("movie.mp4", "rb"), resume_handle: e.resume_handle # end # # ### Error Types @@ -58,8 +58,8 @@ module Rest # * {UnseekableStreamError} - Stream rewinding required on an unseekable stream (includes {HasResumeHandle}). # * {StreamMismatchError} - Stream content or length does not match resumed upload (includes {HasResumeHandle}). # * {UploadRejectedError} - Server explicitly rejected the upload session (final). - # * {UploadCancelledError} - Upload session was cancelled, either by this client or on the server while the - # upload was in flight. A cancelled session cannot be resumed, so this error carries no {ResumeHandle} (final). + # * {UploadCancelledError} - Upload session was cancelled on the server while the upload was in flight. A + # cancelled session cannot be resumed, so this error carries no {ResumeHandle} (final). # * {SessionStateError} - Upload session lifecycle rule violation, e.g. starting a second run while one # is in flight (final). # * Any other `Gapic::Common::Error` subclass signals a protocol implementation bug rather than a caller diff --git a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb index 0d5afae..2c8f5b7 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb @@ -177,11 +177,13 @@ module HasResumeHandle attr_reader :resume_handle ## + # @private # Suffix appended to error message when a resume handle is present. # @return [String] RESUMABLE_SUFFIX = " (upload session is resumable: see #resume_handle)" ## + # @private # Appends the uniform resumable suffix if resume_handle is non-nil. # # @param message [String, nil] Error message @@ -278,6 +280,7 @@ def initialize message = nil, resume_handle: nil end ## + # @private # Creates an UnseekableStreamError with optional resume handle. # # @param message [String, nil] Descriptive error message @@ -308,6 +311,7 @@ def initialize message = "Stream content or length does not match resumed upload end ## + # @private # Creates a StreamMismatchError with optional resume handle. # # @param message [String, nil] Error message @@ -352,6 +356,7 @@ def initialize message = nil, status_code = nil, status: nil, details: nil, head end ## + # @private # Creates a BadResponseError from an HTTP response event. # # @param event [Object] HTTP response event @@ -394,6 +399,7 @@ def initialize message = nil, status_code = nil, status: nil, details: nil, head end ## + # @private # Creates an UploadRejectedError from an HTTP response event. # # @param event [Object] HTTP response event @@ -428,6 +434,7 @@ def initialize message = "Upload session was cancelled" end ## + # @private # Creates an UploadCancelledError from a source event or message string. # # @param source [Object, String, nil] Source event or message @@ -468,6 +475,7 @@ def initialize message = "Upload deadline exceeded", root_cause: nil, resume_han end ## + # @private # Creates a DeadlineExceededError with optional resume handle. # # @param message [String, nil] Deadline exceeded message @@ -541,6 +549,7 @@ def cause end ## + # @private # Creates a RequestFailedError from a failure event or error. # # @param event_or_error [Event::RequestFailed, StandardError] Source event or error diff --git a/gapic-common/lib/gapic/resumable_upload.rb b/gapic-common/lib/gapic/resumable_upload.rb index 19fa364..e10e213 100644 --- a/gapic-common/lib/gapic/resumable_upload.rb +++ b/gapic-common/lib/gapic/resumable_upload.rb @@ -65,7 +65,7 @@ module Gapic # | Budget | Set by | Covers | # |---|---|---| # | whole upload | `upload_timeout:` on {#start} and {#resume} | every request, retry and byte of the run | - # | initiation request | per-call `timeout`, or `timeout:` in `start_retry_policy` | creating the session | + # | initiation request | per-call `timeout` | creating the session | # # The per-call `timeout` a client method takes reaches only the initiation request. An upload still # transferring bytes an hour later has long outlived it, and that is expected. To bound the run as a @@ -197,10 +197,9 @@ def initialize client_stub_proc:, # @param on_progress [Proc, nil] Called as `->(progress)` with a {Gapic::Rest::ResumableUpload::Progress} # instance. Runs synchronously on the upload thread and must not block; an exception raised inside it # aborts the run and propagates out of this method. - # @return [Object] The final response decoded into the handle's response type, or the raw response body - # (a String, or `nil` when the final response carried none) when the handle has no response type. - # @raise [ArgumentError] If the initiation headers set a reserved protocol header, if a retry policy is - # neither a {Gapic::Common::RetryPolicy}, a Hash, nor `nil`, or if the client cannot perform REST calls + # @return [Object] The final response decoded into the handle's response type. + # @raise [ArgumentError] If the call metadata sets a reserved `X-Goog-Upload-*` protocol header, or if the + # client cannot perform REST calls # @raise [Gapic::Rest::ResumableUpload::SessionStateError] If a run is already in progress # @raise [Gapic::Rest::ResumableUpload::RequestFailedError] If a transport error, timeout, or retry # exhaustion occurs @@ -262,11 +261,9 @@ def start stream:, # @param upload_timeout [Numeric, nil] Budget in seconds for the **whole run**. See {#start}. # @param on_progress [Proc, nil] Called as `->(progress)` with a {Gapic::Rest::ResumableUpload::Progress} # instance. Runs synchronously on the upload thread and must not block. - # @return [Object] The final response decoded into the handle's response type, or the raw response body - # (a String, or `nil` when the final response carried none) when the handle has no response type. - # @raise [ArgumentError] If there is no upload to resume, if the stream is not positioned at byte 0, if - # a retry policy is neither a {Gapic::Common::RetryPolicy}, a Hash, nor `nil`, or if the client cannot - # perform REST calls + # @return [Object] The final response decoded into the handle's response type. + # @raise [ArgumentError] If there is no upload to resume, if the stream is not positioned at byte 0, or if + # the client cannot perform REST calls # @raise [Gapic::Rest::ResumableUpload::SessionStateError] If a run is already in progress # @raise [Gapic::Rest::ResumableUpload::RequestFailedError] If a transport error, timeout, or retry # exhaustion occurs From a4bcb934f751a9eb3b39de7b587ea5cbcc163e2c Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Sat, 26 Sep 2026 00:03:57 +0000 Subject: [PATCH 18/22] fix: less noise on retry policy tests --- .../resumable_upload/driver_retry_test.rb | 51 +++++++++++++++---- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb index 9c4ce1c..ea82c97 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb @@ -60,6 +60,35 @@ def make_post_request uri:, body:, params:, options:, method_name: nil end end + # Retry policy whose budget is a fixed number of retries instead of a wall-clock deadline, so a test + # that exhausts it does not race the clock on a slow runner. + # + # Only the budget changes: `retry_with_deadline?` is the one check both the no-argument `RetryPolicy#call` + # and `RetryDecider`'s "ask policy" path consult before retrying. `timeout` stays large so the Driver's + # own elapsed-time check (`command_budget_left?`) never ends the loop first. `dup` is overridden because + # `RetryPolicy#dup` builds a plain `RetryPolicy`, and the Driver dups the policy before every command. + class CountedRetryPolicy < Gapic::Common::RetryPolicy + def initialize retries:, **kwargs + @retries = retries + super(timeout: 60, **kwargs) + end + + def dup + self.class.new retries: @retries, **overrides.except(:timeout) + end + + def start! **kwargs + @retries_left = @retries + super + end + + def retry_with_deadline? + return false unless @retries_left.positive? + @retries_left -= 1 + true + end + end + # ============================================================================ # Initiation # ============================================================================ @@ -74,21 +103,21 @@ def test_start_retries_a_200_without_status_header # An exhausted headerless start 200 surfaces as the response itself, which Rules turns into a bad # response. def test_start_surfaces_an_exhausted_headerless_200_as_a_bad_response - stub = FakeClientStub.new Array.new(200) { headerless_200 } + stub = FakeClientStub.new Array.new(10) { headerless_200 } err = assert_raises BadResponseError do - run_upload stub, start_retry_policy: FAST.merge(timeout: 0.01) + run_upload stub, start_retry_policy: CountedRetryPolicy.new(retries: 3, initial_delay: 0.001, max_delay: 0.002) end assert_equal 200, err.status_code assert_match(/X-Goog-Upload-Status: missing/, err.message) - assert stub.requests.size > 1 + assert_equal 4, stub.requests.size end # The missing-header retry is the protocol's, so it must not depend on anything the policy carries. def test_start_retries_a_headerless_200_under_a_bare_policy - stub = FakeClientStub.new Array.new(200) { headerless_200 } - policy = Gapic::Common::RetryPolicy.new initial_delay: 0.001, max_delay: 0.002, timeout: 0.05 + stub = FakeClientStub.new Array.new(10) { headerless_200 } + policy = CountedRetryPolicy.new retries: 3, initial_delay: 0.001, max_delay: 0.002 assert_empty policy.retry_codes assert_nil policy.retry_predicate @@ -96,7 +125,7 @@ def test_start_retries_a_headerless_200_under_a_bare_policy run_upload stub, start_retry_policy: policy end - assert stub.requests.size > 1, "Expected the missing-header retry to survive a bare policy" + assert_equal 4, stub.requests.size, "Expected the missing-header retry to survive a bare policy" end def test_start_retries_the_default_retriable_statuses @@ -146,14 +175,14 @@ def test_start_retries_a_connection_failure_without_asking_the_predicate end def test_start_surfaces_an_exhausted_connection_failure_as_request_failed - stub = FakeClientStub.new Array.new(200) { Faraday::ConnectionFailed.new "refused" } + stub = FakeClientStub.new Array.new(10) { Faraday::ConnectionFailed.new "refused" } err = assert_raises RequestFailedError do - run_upload stub, start_retry_policy: FAST.merge(timeout: 0.01) + run_upload stub, start_retry_policy: CountedRetryPolicy.new(retries: 3, initial_delay: 0.001, max_delay: 0.002) end assert_instance_of Faraday::ConnectionFailed, err.cause - assert stub.requests.size > 1 + assert_equal 4, stub.requests.size end def test_final_non_200_is_never_retried_even_when_the_predicate_says_yes @@ -329,11 +358,13 @@ def test_each_attempt_timeout_fits_within_the_shrinking_command_budget assert timeouts[2] < timeouts[1], "Expected the attempt timeout to shrink: #{timeouts}" end + # Wall-clock by nature: the whole-upload deadline is the Driver's own monotonic clock. The margins leave + # ~450 ms for the second attempt to start, so a slow runner cannot end the loop after one request. def test_the_global_deadline_ends_a_retry_loop stub = FakeClientStub.new Array.new(200) { headerless_200 } assert_raises DeadlineExceededError do - run_upload stub, timeout: 0.05, start_retry_policy: { initial_delay: 0.01, max_delay: 0.01, timeout: 60 } + run_upload stub, timeout: 0.5, start_retry_policy: { initial_delay: 0.05, max_delay: 0.05, timeout: 60 } end assert stub.requests.size > 1 From d1c829515d6cfe98cc78839df0bc21a5d2ccd451 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Mon, 28 Sep 2026 20:31:39 +0000 Subject: [PATCH 19/22] feat: back off recovery queries across a recovery episode --- .../resumable_upload/implementation-guide.md | 51 +++- .../gapic/rest/resumable_upload/data_types.rb | 15 +- .../lib/gapic/rest/resumable_upload/driver.rb | 32 +- .../rest/resumable_upload/driver/abridge.rb | 2 +- .../resumable_upload/driver/retry_decider.rb | 2 +- .../resumable_upload/driver/upload_log.rb | 12 + .../rest/resumable_upload/instructions.rb | 14 +- .../lib/gapic/rest/resumable_upload/rules.rb | 50 ++- gapic-common/lib/gapic/resumable_upload.rb | 2 + .../resumable_upload/driver_logging_test.rb | 29 ++ .../driver_recovery_backoff_test.rb | 284 ++++++++++++++++++ .../rest/resumable_upload/driver_test.rb | 12 +- .../rules_recovery_episode_test.rb | 171 +++++++++++ 13 files changed, 631 insertions(+), 45 deletions(-) create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_recovery_backoff_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/rules_recovery_episode_test.rb diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md index 4138ce6..cd1788a 100644 --- a/gapic-common/design/resumable_upload/implementation-guide.md +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -198,7 +198,8 @@ module Gapic :chunk_size, # [Integer] Resolved effective chunk size :chunk_granularity, # [Integer, nil] Alignment modulus returned by server :in_flight_length, # [Integer] Byte length of in-flight chunk currently being transmitted - :last_error # [StandardError, nil] Terminal exception + :last_error, # [StandardError, nil] Terminal exception + :recovery_offset # [Integer, nil] Offset the open recovery episode began at; nil if none (Section 6.2.1) ) do end @@ -245,7 +246,7 @@ end * `Instruction::SendStart.new(url:, headers:, body:)`: Execute initiation request to establish upload session. * `Instruction::SendChunk.new(url:, offset:, length:, finalize:)`: Transmit buffered chunk of specified `length` starting at `offset`. If `finalize` is true, sends command `upload, finalize`. * `Instruction::SendFinalize.new(url:)`: Send standalone `finalize` command when all data bytes were already acknowledged. -* `Instruction::SendQuery.new(url:)`: Query backend for current acknowledged offset (`query` command). +* `Instruction::SendQuery.new(url:, backoff: false)`: Query backend for current acknowledged offset (`query` command). `backoff: false` opens a new recovery episode and sends at once; `backoff: true` continues the open episode and waits for its next backoff delay first (Section 6.2.1). * `Instruction::SendCancel.new(url:)`: Cancel upload session on server (`cancel` command). * `Instruction::RealignBuffer.new(server_offset:)`: Realign Driver in-memory buffer and stream position to match `server_offset`. * `Instruction::FillBuffer.new(target_bytesize:)`: Read from stream until in-memory buffer reaches `target_bytesize` bytes or stream encounters EOF. @@ -253,7 +254,7 @@ end * `Instruction::TerminateSuccess.new(response:)`: Upload finalized cleanly; Driver returns `response.body`. * `Instruction::TerminateFailure.new(error:)`: Raise terminal exception. -### 2.5 Driver Buffer Invariants & Stream Position Model +### 2.7 Driver Buffer Invariants & Stream Position Model The Driver coordinates stream reading and in-memory buffering using four explicit offset markers: * `server_offset`: Contiguous byte count acknowledged by the server (extracted from `X-Goog-Upload-Size-Received`). @@ -354,7 +355,7 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` | From State | Event Shape | Event & Input Payload | State Mutations | To State | Emitted Instructions & Parameters | | :--- | :--- | :--- | :--- | :--- | :--- | | **`Initializing`** | `:start_upload` | `Event::StartUpload` | `status = :starting` | `Starting` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::SendStart.new(url: config.initial_url, headers: config.initial_headers, body: config.initial_body)` | -| **`Initializing`** | `:resume_upload` | `Event::ResumeUpload` | `upload_url = event.upload_url`
`chunk_size = event.chunk_size`
`offset = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: event.upload_size))`
`Instruction::SendQuery.new(url: event.upload_url)` | +| **`Initializing`** | `:resume_upload` | `Event::ResumeUpload` | `upload_url = event.upload_url`
`chunk_size = event.chunk_size`
`offset = 0`
`recovery_offset = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: event.upload_size))`
`Instruction::SendQuery.new(url: event.upload_url, backoff: false)` | | **`Starting`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `upload_url = headers['X-Goog-Upload-URL']`
`chunk_granularity = headers['...-Granularity']&.to_i`
`chunk_size = resolve(config, chunk_granularity)`
`offset = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | | **`Starting`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Starting`** | `:response_cat2` / `:response_fatal_bad_response` | `Event::HttpResponse` (Non-200, or a headerless `200` after the start budget is spent; see Section 6.1) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | @@ -362,27 +363,27 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` | **`Transmission \| Reading from stream`** | `:chunk_read_full` | `Event::ChunkRead(bytes_buffered, eof: false)` | `in_flight_length = event.bytes_buffered`
`status = :transmission_sending` | `Transmission \| Sending` | `Instruction::SendChunk.new(url: state.upload_url, offset: state.offset, length: event.bytes_buffered, finalize: false)` | | **`Transmission \| Reading from stream`** | `:chunk_read_eof_with_data` | `Event::ChunkRead(bytes_buffered, eof: true)` where `bytes_buffered > 0` | `in_flight_length = event.bytes_buffered`
`status = :finalizing_sending_upload` | `Finalizing \| Sending with upload` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :finalizing, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendChunk.new(url: state.upload_url, offset: state.offset, length: event.bytes_buffered, finalize: true)` | | **`Transmission \| Reading from stream`** | `:chunk_read_eof_empty` | `Event::ChunkRead(bytes_buffered: 0, eof: true)` | `in_flight_length = 0`
`status = :finalizing_sending_finalize` | `Finalizing \| Sending finalize` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :finalizing, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendFinalize.new(url: state.upload_url)` | -| **`Transmission \| Sending`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = state.offset + state.in_flight_length`
`in_flight_length = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | -| **`Transmission \| Sending`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | -| **`Transmission \| Sending`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Transmission \| Sending`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = state.offset + state.in_flight_length`
`in_flight_length = 0`
`recovery_offset = nil`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Transmission \| Sending`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `in_flight_length = 0`
`recovery_offset = state.recovery_offset || state.offset`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url, backoff: !state.recovery_offset.nil?)` | +| **`Transmission \| Sending`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `in_flight_length = 0`
`recovery_offset = state.recovery_offset || state.offset`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url, backoff: !state.recovery_offset.nil?)` | | **`Transmission \| Sending`** | `:request_retries_exhausted` / `:request_failed_unknown` | `Event::RequestFailed(kind: :retries_exhausted \| :unknown)` | `in_flight_length = 0`
`last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Transmission \| Sending`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `in_flight_length = 0`
`status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Transmission \| Sending`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **`Finalizing \| Sending with upload`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `offset = state.offset + state.in_flight_length`
`in_flight_length = 0`
`status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | -| **`Finalizing \| Sending with upload`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | -| **`Finalizing \| Sending with upload`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Finalizing \| Sending with upload`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `in_flight_length = 0`
`recovery_offset = state.recovery_offset || state.offset`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url, backoff: !state.recovery_offset.nil?)` | +| **`Finalizing \| Sending with upload`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `in_flight_length = 0`
`recovery_offset = state.recovery_offset || state.offset`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url, backoff: !state.recovery_offset.nil?)` | | **`Finalizing \| Sending with upload`** | `:request_retries_exhausted` / `:request_failed_unknown` | `Event::RequestFailed(kind: :retries_exhausted \| :unknown)` | `in_flight_length = 0`
`last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Finalizing \| Sending with upload`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `in_flight_length = 0`
`status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Finalizing \| Sending with upload`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **`Finalizing \| Sending finalize`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | -| **`Finalizing \| Sending finalize`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | -| **`Finalizing \| Sending finalize`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Finalizing \| Sending finalize`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `recovery_offset = state.recovery_offset || state.offset`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url, backoff: !state.recovery_offset.nil?)` | +| **`Finalizing \| Sending finalize`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `recovery_offset = state.recovery_offset || state.offset`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url, backoff: !state.recovery_offset.nil?)` | | **`Finalizing \| Sending finalize`** | `:request_retries_exhausted` / `:request_failed_unknown` | `Event::RequestFailed(kind: :retries_exhausted \| :unknown)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Finalizing \| Sending finalize`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Finalizing \| Sending finalize`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | -| **`Recovery`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = headers['X-Goog-Upload-Size-Received'].to_i`
`in_flight_length = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Recovery`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = headers['X-Goog-Upload-Size-Received'].to_i`
`in_flight_length = 0`
`recovery_offset = nil` if the new `offset` exceeds `state.recovery_offset`, else unchanged
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | | **`Recovery`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `in_flight_length = 0`
`status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | -| **`Recovery`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `status = :recovery` | `Recovery` | `Instruction::SendQuery.new(url: state.upload_url)` | +| **`Recovery`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `status = :recovery` | `Recovery` | `Instruction::SendQuery.new(url: state.upload_url, backoff: true)` | | **`Recovery`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:request_failed_unknown` | `Event::RequestFailed(kind:, ...)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Recovery`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Recovery`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | @@ -532,10 +533,10 @@ The implementation distinguishes three categories of network and protocol-level * **Data Plane Transport Failures**: `:request_connection_failed` and `:request_timeout` during `Transmission` or `Finalizing` also enter `Recovery` (`enter_recovery`), because the Driver does not re-send them on the data plane (Section 6.1.1, rows 2–3). * **Where Category 2 Arrives, per Plane**: * *Session Initiation (`start`)*: a headerless `200` and the default-retriable statuses are re-sent by the Driver within the start budget. Whatever surfaces as `:response_cat2` is terminal in `Starting` (`fail_with_bad_response` → `BadResponseError`, status `200` for an exhausted headerless answer): there is no upload URL to query yet. - * *Session Control (`query`, `cancel`)*: same re-sends as `start`. A `:response_cat2` that surfaces in `Recovery` re-queries (`retry_recovery`), bounded by the global deadline; in `Cancelling` it is terminal. + * *Session Control (`query`, `cancel`)*: same re-sends as `start`. A `:response_cat2` that surfaces in `Recovery` re-queries (`retry_recovery`) after the recovery episode's next backoff delay, bounded by the global deadline (Section 6.2.1); in `Cancelling` it is terminal. * *Data Plane (`upload`, `upload, finalize`, standalone `finalize`)*: a headerless `200` and every `4xx` surface at once; `500`/`503`/`504` are re-sent optimistically within the data budget (the worst case is a server answer that is itself Category 2). Whatever surfaces as `:response_cat2` enters `Recovery` rather than re-transmitting data blindly. * *Why Headers Go Missing*: Intermediate proxies, reverse-proxies, or Google Front End (GFE) edge proxies can strip the protocol response headers or return raw HTML/text error pages on failure. -* **Resolution**: Core transitions to `Recovery` and emits `Instruction::SendQuery.new(url: state.upload_url)` to obtain `server_offset`. +* **Resolution**: Core transitions to `Recovery` and emits `Instruction::SendQuery` to obtain `server_offset`, opening or continuing a recovery episode (Section 6.2.1). #### 6.1.3 Category 3: Terminal Failures * **Definition**: Irrecoverable errors where either the request is structurally invalid or unauthorized, the retry budget is spent on a plane that cannot recover, an unseekable rewind is needed, or the server has explicitly aborted/rejected the session. @@ -598,6 +599,19 @@ To realign the upload state, the `Driver` processes `Instruction::RealignBuffer( * If unseekable: reads and discards `server_offset - current_stream_pos` bytes from `stream`. If the stream encounters unexpected EOF before reaching `server_offset`, Driver raises terminal `StreamMismatchError` with `resume_handle`. * The Driver sets `buffer_start_offset = server_offset`. +#### 6.2.1 Recovery Episodes and Backoff +A **recovery episode** is the run of consecutive recovery attempts during which the server-confirmed offset does not advance. Without one, recovery could query back-to-back with no delay: a Category 2 status the control plane `retry_codes` do not cover (`400`, `408`, `412`, `416`, `502`) surfaces after one attempt and `retry_recovery` re-queries at once, and an upload that keeps failing while the query keeps answering `active` loops `upload` → `query` → `upload` re-transmitting a chunk per lap. + +* **Rules** track the episode in `State#recovery_offset` (the confirmed offset it began at, `nil` when none is open): + * `resume_session` opens one at `0`; `enter_recovery` opens one at `state.offset` if none is open, and otherwise continues it. + * `realign_from_recovery` closes it only if the reported offset exceeds `recovery_offset`; `ack_chunk` always closes it. + * The first query of an episode carries `backoff: false`; `retry_recovery` and a continuing `enter_recovery` carry `backoff: true`. +* **Driver** keeps one `Gapic::Common::RetryPolicy` per episode (`@recovery_policy`, a started `dup` of the control plane policy): + * `backoff: false` starts a fresh one; `backoff: true` calls `perform_delay!` on it before sending (after checking the global deadline). + * The same instance is passed to `make_post_request`, so `RetryDecider`'s in-command retries and the re-queries `Rules` decide on advance one delay sequence: `initial_delay`, × `multiplier`, capped at `max_delay`. + * `perform_delay!` rather than `RetryPolicy#call`: `call` stops sleeping once the policy deadline has passed. After that, recovery queries once per `max_delay` until the global deadline. +* **Budget**: the control plane `timeout` budgets the whole episode, not each query. Once it is spent, a transport failure on `query` is no longer re-sent and ends the run through `fail_with_request_error` (resumable). + ### 6.3 Sensible Defaults for Global Deadline Every upload session executed via `Driver#run` must have a finite, guaranteed upper bound on total wall-clock execution time. Without a mandatory global deadline, a session encountering repeated Category 2 protocol recoveries or intermittent network stalls could hang indefinitely. @@ -623,7 +637,7 @@ remaining = [@deadline - monotonic_now, 0].max elapsed = monotonic_now - started_at timeout = retry_policy&.timeout ? (retry_policy.timeout - elapsed).clamp(0, remaining) : remaining ``` -This timeout is passed in `options[:timeout]` and shrinks with every attempt. After each backoff delay the loop also checks that the command budget has time left, and it checks the global deadline before every attempt and after every failed one, emitting `Event::GlobalDeadlineExceeded` instead of a transport failure once the deadline has passed. +This timeout is passed in `options[:timeout]` and shrinks with every attempt. For `query`, `retry_policy` is the recovery episode's policy (Section 6.2.1): its deadline, checked by `RetryDecider`, runs from episode start, while `started_at` here is still the start of the current command. After each backoff delay the loop also checks that the command budget has time left, and it checks the global deadline before every attempt and after every failed one, emitting `Event::GlobalDeadlineExceeded` instead of a transport failure once the deadline has passed. Per-attempt timeouts shorter than the command budget are out of scope for now; they arrive in the timeout pass done alongside stall control, and are what will make row 2 of Section 6.1.1 effective. @@ -657,6 +671,7 @@ The `Driver` emits structured logs across three severity levels (`INFO`, `DEBUG` | `DEBUG` | Wire | Inbound HTTP response (`wire_receive`) | `Received HTTP ` | | `DEBUG` | Wire | Transport exception (`wire_failure`) | `Request failed: ` | | `DEBUG` | Buffer | Stream/buffer realignment (`buffer_realign`) | `Buffer realignment: ` | +| `DEBUG` | Recovery | Backoff before a query that continues a recovery episode (`recovery_backoff`) | `Backing off before recovery query` | | `WARN` | Lifecycle | `:fail_with_deadline_exceeded`, `:fail_with_rejected`, `:fail_with_bad_response`, `:fail_with_request_error` | Resumable upload failed | | `WARN` | Lifecycle | `:fail_with_cancelled` | Resumable upload canceled on the server | | `WARN` | Transition | `InvalidTransitionError` (`unmatched_transition`) | Unmatched transition | @@ -677,6 +692,7 @@ All log entries emitted by `UploadLog` populate structured fields in `Google::Lo * `recipe`: Transition recipe method symbol executed by `Rules`. * `offset`: Current server-confirmed byte offset (`Integer`). * `inFlightLength`: Byte length of the chunk currently in flight (`Integer`). + * `recoveryOffset`: Offset the open recovery episode began at, or `nil` (`Integer`, on decisions). * `instructions`: Array of abridged instruction hashes emitted by the transition. * `uploadSize`: Total expected upload size in bytes from `config.upload_size` (on `:start_session`). * `requestedChunkSize`: Configured chunk size in bytes from `config.chunk_size` (on `:start_session`). @@ -701,6 +717,9 @@ All log entries emitted by `UploadLog` populate structured fields in `Google::Lo * `granularity`: Parsed integer value of `X-Goog-Upload-Chunk-Granularity` response header (`wire_receive`). * `kind`: Transport failure classification symbol (`:timeout`, `:connection_failed`, `:retries_exhausted`, `:unknown`). * `error`: Exception message string (`wire_failure`). +* **Recovery Backoff Fields** (`recovery_backoff`): + * `delay`: Base delay in seconds about to be performed, before jitter and the `max_delay` cap. + * `backoffAttempt`: 1-based index of the delay within the recovery episode, counting in-command retries. * **Buffer Realignment Fields**: * `action`: Realignment strategy string (`"within_buffer"`, `"rewind"`, or `"fast_forward"`). * `serverOffset`: Target byte offset reported by the server (`Integer`). diff --git a/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb b/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb index 5d90520..56c3acc 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb @@ -329,6 +329,11 @@ def initialize upload_url:, chunk_size: # @return [Integer] Byte length of in-flight chunk currently being transmitted # @!attribute [r] last_error # @return [StandardError, nil] Terminal exception if in an error or rejected status + # @!attribute [r] recovery_offset + # @return [Integer, nil] Server-confirmed offset when the open recovery episode began, or `nil` when no + # episode is open. A recovery episode is the run of consecutive recovery attempts during which the + # confirmed offset does not advance; every query in it after the first waits for the next backoff delay. + # See `design/resumable_upload/implementation-guide.md` section 6.2.1. # State = Data.define( :status, @@ -337,7 +342,8 @@ def initialize upload_url:, chunk_size: :chunk_size, :chunk_granularity, :in_flight_length, - :last_error + :last_error, + :recovery_offset ) do ## # @private @@ -350,6 +356,7 @@ def initialize upload_url:, chunk_size: # @param chunk_granularity [Integer, nil] Alignment modulus returned by server # @param in_flight_length [Integer] Byte length of in-flight chunk # @param last_error [StandardError, nil] Terminal exception + # @param recovery_offset [Integer, nil] Offset the open recovery episode began at; `nil` if none is open # def initialize status: :initializing, upload_url: nil, @@ -357,7 +364,8 @@ def initialize status: :initializing, chunk_size: Rules::DEFAULT_CHUNK_SIZE, chunk_granularity: nil, in_flight_length: 0, - last_error: nil + last_error: nil, + recovery_offset: nil super( status: status, upload_url: upload_url, @@ -365,7 +373,8 @@ def initialize status: :initializing, chunk_size: chunk_size, chunk_granularity: chunk_granularity, in_flight_length: in_flight_length, - last_error: last_error + last_error: last_error, + recovery_offset: recovery_offset ) end end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb index f2d3311..fdb5925 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb @@ -38,7 +38,7 @@ module ResumableUpload # The outer tier of the three-tier design. All side effects live here; all protocol decisions live in # {Rules}, which carries the state graph and the error category taxonomy. Every retry decision is made # here by {RetryDecider}; `ClientStub` is handed a never-retry policy. See - # `design/resumable_upload/implementation-guide.md` section 2.5 for the buffer and stream position + # `design/resumable_upload/implementation-guide.md` section 2.7 for the buffer and stream position # invariants, and section 6.3 for the deadline model. # # rubocop:disable Metrics/ClassLength @@ -124,6 +124,8 @@ def initialize client_stub:, config:, core: nil, logger: nil, method_name: nil RetryPolicies::CONTROL_PLANE_DEFAULTS @data_plane_retry_policy = resolve_retry_policy config.data_plane_retry_policy, RetryPolicies::DATA_PLANE_DEFAULTS + # Started copy of the control plane policy for the open recovery episode; see #execute_send_query. + @recovery_policy = nil end ## @@ -641,13 +643,34 @@ def execute_send_finalize instruction # @private # Sends an offset query command over HTTP. # + # Every query runs under the recovery episode's retry policy, a started copy of the control plane + # policy that lives for the whole episode rather than for one command. {RetryDecider} draws the delay + # between attempts of this query from it, and a query that continues the episode + # (`instruction.backoff`) first waits for its next delay via `RetryPolicy#perform_delay!`. So the + # backoff grows across every re-send in the episode, and the control plane `timeout` budgets the + # episode rather than each query. + # + # `perform_delay!` is used rather than `RetryPolicy#call` because `call` stops sleeping once the + # policy's deadline has passed, which would let recovery re-query with no delay at all. Once `delay` + # reaches `max_delay` it stays there, so recovery then queries once per `max_delay` until the global + # deadline. + # # @param instruction [Instruction::SendQuery] SendQuery instruction # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] # def execute_send_query instruction + if instruction.backoff + return Event::GlobalDeadlineExceeded.new if deadline_exceeded? + @recovery_policy ||= @control_plane_retry_policy.dup.start! + @upload_log.recovery_backoff delay: @recovery_policy.delay, + attempt: @recovery_policy.perform_delay_count + 1 + @recovery_policy.perform_delay! + else + @recovery_policy = @control_plane_retry_policy.dup.start! + end headers = { "X-Goog-Upload-Command" => "query", "Content-Length" => "0" } make_post_request instruction.url, headers: headers, body: "", - retry_policy: @control_plane_retry_policy.dup.start!, + retry_policy: @recovery_policy, method_name: "#{@method_name_prefix}.query" end @@ -676,7 +699,10 @@ def execute_send_cancel instruction # @param url [String] Target URL # @param headers [Hash] Request headers # @param body [String] Request body - # @param retry_policy [Gapic::Common::RetryPolicy, nil] Started command retry policy; `nil` sends once + # @param retry_policy [Gapic::Common::RetryPolicy, nil] Started retry policy; `nil` sends once. For + # `query` this is the recovery episode's policy, started before this command, so its deadline and + # delay carry over from earlier commands in the episode. `started_at` still bounds each attempt's + # timeout and the post-delay budget check by this command's own start. # @param data_plane [Boolean] Whether the request transmits upload bytes (`upload`, `finalize`) # @param method_name [String, nil] RPC method name for logging # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb index 36cae29..65e3f9e 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb @@ -138,7 +138,7 @@ def instruction instruction when Instruction::SendFinalize { "type" => "SendFinalize", "url" => url(instruction.url) } when Instruction::SendQuery - { "type" => "SendQuery", "url" => url(instruction.url) } + { "type" => "SendQuery", "url" => url(instruction.url), "backoff" => instruction.backoff } when Instruction::SendCancel { "type" => "SendCancel", "url" => url(instruction.url) } when Instruction::RealignBuffer diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver/retry_decider.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver/retry_decider.rb index f61538f..e01d150 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/driver/retry_decider.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver/retry_decider.rb @@ -58,7 +58,7 @@ class Driver # Row 2 is inert for timeouts until attempts get their own timeouts: each attempt currently receives the # whole remaining command budget, so a timed-out attempt leaves none to retry with. # - # See `design/resumable_upload/transport-error-retry.md`. + # See `design/resumable_upload/implementation-guide.md` section 6.1.1. # class RetryDecider ## diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb index d08fbcc..1345cc1 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb @@ -94,6 +94,7 @@ def decision decision toStatus: decision.next_state.status, offset: decision.next_state.offset, inFlightLength: decision.next_state.in_flight_length, + recoveryOffset: decision.next_state.recovery_offset, instructions: Abridge.instructions(decision.instructions) ) end @@ -183,6 +184,17 @@ def wire_failure event ) end + ## + # @private + # Logs the backoff delay about to be performed before a query that continues a recovery episode. + # + # @param delay [Numeric] Base delay in seconds, before jitter and the `max_delay` cap + # @param attempt [Integer] 1-based index of the delay within the recovery episode + # + def recovery_backoff delay:, attempt: + entry :debug, "Performing backoff delay before recovery query", delay: delay, backoffAttempt: attempt + end + ## # @private # Logs buffer realignment action. diff --git a/gapic-common/lib/gapic/rest/resumable_upload/instructions.rb b/gapic-common/lib/gapic/rest/resumable_upload/instructions.rb index 57d02bd..61ea753 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/instructions.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/instructions.rb @@ -100,18 +100,26 @@ def initialize url: # @private # Query backend for current acknowledged offset. # + # Every query belongs to a recovery episode (see {State#recovery_offset}). The Driver keeps one + # `Gapic::Common::RetryPolicy` per episode and uses it both between attempts of a single query and + # between queries, so the delay grows across the whole episode instead of restarting per command. + # # @!attribute [r] url # @return [String] Session upload URL + # @!attribute [r] backoff + # @return [Boolean] `false` to open a new recovery episode and send at once; `true` to continue the + # open episode, waiting for its next backoff delay before sending # - SendQuery = Data.define :url do + SendQuery = Data.define :url, :backoff do ## # @private # Initializes a SendQuery instruction. # # @param url [String] Session upload URL + # @param backoff [Boolean] Whether to continue the open recovery episode after its next backoff delay # - def initialize url: - super url: url + def initialize url:, backoff: false + super url: url, backoff: backoff end end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb index 88f4a1c..cd6beb4 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb @@ -117,7 +117,9 @@ module ResumableUpload # * `[:starting, :response_cat2]` fails instead of recovering, unlike the same shape during transmission # and finalizing. There is no upload to recover to until initiation yields an upload URL. # * `recovery` re-queries on `:response_cat2` with no attempt cap. Termination is guaranteed only by the - # global deadline the Driver enforces, not by anything in this module. + # global deadline the Driver enforces, not by anything in this module. Re-queries are spaced by the + # recovery episode's backoff, tracked in {State#recovery_offset} and signalled through + # {Instruction::SendQuery#backoff}; see `design/resumable_upload/implementation-guide.md` section 6.2.1. # # See `design/resumable_upload/implementation-guide.md` section 4 for the transition specification and # section 6.1 for the error category taxonomy this module implements. @@ -143,7 +145,7 @@ module Rules # Resolved by transitioning to `:error` or `:rejected` and emitting `Instruction::TerminateFailure`. # # See `design/resumable_upload/implementation-guide.md` section 6.1 for the full classification, and - # `design/resumable_upload/transport-error-retry.md` for which failures the Driver retries. + # section 6.1.1 for which failures the Driver retries. ## # @private @@ -165,7 +167,7 @@ module Rules # @private # HTTP status codes eligible for Category 2 (recovery) handling. # - # Rules.classify_http_response} classifies a non-200 response whose `X-Goog-Upload-Status` + # {Rules.classify_http_response} classifies a non-200 response whose `X-Goog-Upload-Status` # is missing, empty or `active` as `:response_cat2` **only** if its status is listed here, and as # `:response_fatal_bad_response` otherwise. It is an allowlist on purpose: a status nobody anticipated # fails the upload rather than looping it through recovery. @@ -537,16 +539,19 @@ def self.start_session state, _event, config # @private # Resumes an existing upload session by transitioning to recovery and querying backend offset. # + # Opens a recovery episode at offset 0, so the first query is sent at once. + # # @param state [State] Current state # @param _event [Object] Dispatched event # @param config [ResumeUploadConfig] Resume session configuration # @return [Array>] Tuple of [next_state, instructions] def self.resume_session state, _event, config next_state = state.with( - status: :recovery, - upload_url: config.upload_url, - chunk_size: config.chunk_size, - offset: 0 + status: :recovery, + upload_url: config.upload_url, + chunk_size: config.chunk_size, + offset: 0, + recovery_offset: 0 ) progress = Progress.new( phase: :initiating, @@ -664,6 +669,8 @@ def self.send_finalize state, _event, config # @private # Acknowledges transmitted chunk and advances offset. # + # Closes any open recovery episode: the server acknowledged a data command. + # # @param state [State] Current state # @param _event [Object] Dispatched event # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration @@ -673,7 +680,8 @@ def self.ack_chunk state, _event, config next_state = state.with( status: :transmission_reading, offset: new_offset, - in_flight_length: 0 + in_flight_length: 0, + recovery_offset: nil ) progress = Progress.new phase: :uploading, bytes_uploaded: new_offset, total_bytes: config.upload_size instructions = [ @@ -688,26 +696,33 @@ def self.ack_chunk state, _event, config # @private # Transitions to recovery state to query backend byte offset. # + # Opens a recovery episode at the current offset and queries at once, unless an episode is already + # open. An open episode means the server has confirmed no new bytes since the previous recovery (an + # upload keeps failing while the query keeps answering `active`), so the query waits for the episode's + # next backoff delay instead. + # # @param state [State] Current state # @param _event [Object] Dispatched event # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration # @return [Array>] Tuple of [next_state, instructions] def self.enter_recovery state, _event, config + episode_open = !state.recovery_offset.nil? next_state = state.with( status: :recovery, - in_flight_length: 0 + in_flight_length: 0, + recovery_offset: episode_open ? state.recovery_offset : state.offset ) progress = Progress.new phase: :recovering, bytes_uploaded: next_state.offset, total_bytes: config.upload_size instructions = [ Instruction::NotifyProgress.new(progress: progress), - Instruction::SendQuery.new(url: state.upload_url) + Instruction::SendQuery.new(url: state.upload_url, backoff: episode_open) ] [next_state, instructions] end ## # @private - # Retries offset query during recovery. + # Retries offset query during recovery, after the open recovery episode's next backoff delay. # # @param state [State] Current state # @param _event [Object] Dispatched event @@ -718,7 +733,7 @@ def self.retry_recovery state, _event, _config status: :recovery, in_flight_length: 0 ) - [next_state, [Instruction::SendQuery.new(url: state.upload_url)]] + [next_state, [Instruction::SendQuery.new(url: state.upload_url, backoff: true)]] end ## @@ -772,6 +787,11 @@ def self.complete_upload_finalized state, event, _config # @private # Realigns buffer and resumes transmission from recovered offset. # + # Closes the open recovery episode only if the server confirmed bytes beyond the offset the episode + # began at. Otherwise the episode stays open: the command that failed has not yet been re-sent + # successfully, and if it fails again the next query must keep backing off. {Rules.ack_chunk} closes + # the episode once the re-sent command succeeds. + # # @param state [State] Current state # @param event [Event::HttpResponse] Query response containing acknowledged offset # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration @@ -779,10 +799,14 @@ def self.complete_upload_finalized state, event, _config def self.realign_from_recovery state, event, config server_offset_str = header_value event.headers, "x-goog-upload-size-received" server_offset = server_offset_str.to_i + # `>`, not `!=`: a lower offset is a regression, not progress. Closing on it would reset the backoff, + # and a server whose reported offset flips between two values would then query with no delay forever. + progressed = state.recovery_offset.nil? || server_offset > state.recovery_offset next_state = state.with( status: :transmission_reading, offset: server_offset, - in_flight_length: 0 + in_flight_length: 0, + recovery_offset: progressed ? nil : state.recovery_offset ) progress = Progress.new phase: :uploading, bytes_uploaded: server_offset, total_bytes: config.upload_size instructions = [ diff --git a/gapic-common/lib/gapic/resumable_upload.rb b/gapic-common/lib/gapic/resumable_upload.rb index e10e213..57248a2 100644 --- a/gapic-common/lib/gapic/resumable_upload.rb +++ b/gapic-common/lib/gapic/resumable_upload.rb @@ -110,6 +110,8 @@ module Gapic # * Data plane requests are never re-sent after an outcome that leaves the server offset unknown — a # timeout, a connection failure, a missing status header, or any `4xx`. The upload re-queries the # session and resumes from the offset the server reports instead. + # * Consecutive recovery attempts that make no progress back off on a single schedule, with the same + # delays as above; the schedule starts over once the server confirms new bytes. # * A response carrying `X-Goog-Upload-Status: final` is never retried. # class ResumableUpload diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb index 9161f72..cfe0654 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb @@ -174,6 +174,35 @@ def test_recovery_scenario_logs_enter_recovery_and_realign assert_includes info_recipes, "realign_from_recovery" end + def test_recovery_re_query_logs_the_backoff_delay + recording = RecordingLogger.new + responses = [ + FakeResponse.new(200, { "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-URL" => "https://storage.googleapis.com/session?id=123" }, ""), + Faraday::ConnectionFailed.new("reset"), + FakeResponse.new(502, {}, "Bad Gateway"), + FakeResponse.new(200, { "X-Goog-Upload-Status" => "active", "X-Goog-Upload-Size-Received" => "0" }, ""), + FakeResponse.new(200, { "X-Goog-Upload-Status" => "final" }, "done") + ] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("hello world"), + upload_size: 11, + chunk_size: 256, + control_plane_retry_policy: { initial_delay: 0.25, jitter: 0 } + ) + + Kernel.stub :sleep, nil do + Driver.new(client_stub: FakeStub.new(responses), config: config, logger: recording).run + end + + backoff_entries = recording.entries.select { |e| e.message.message == "Backing off before recovery query" } + assert_equal 1, backoff_entries.size + assert_equal Logger::DEBUG, backoff_entries.first.severity + assert_in_delta 0.25, backoff_entries.first.message.fields["delay"] + assert_equal 1, backoff_entries.first.message.fields["backoffAttempt"] + end + def test_resume_upload_logs_resume_session_entry recording = RecordingLogger.new responses = [ diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_recovery_backoff_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_recovery_backoff_test.rb new file mode 100644 index 0000000..ce16ffc --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_recovery_backoff_test.rb @@ -0,0 +1,284 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests that recovery queries are spaced by one `Gapic::Common::RetryPolicy` per recovery episode. +# +# `Kernel.sleep` is stubbed to record delays instead of waiting: `RetryPolicy` performs every backoff through +# it. The control plane policy uses `jitter: 0` so the recorded delays are exact. +# +# See `design/resumable_upload/implementation-guide.md` section 6.2.1. +# +class DriverRecoveryBackoffTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + SESSION_URL = "https://example.com/session/1" + + # Delays 1, 2, 4, then capped at 5. + BACKOFF = { initial_delay: 1, multiplier: 2, max_delay: 5, jitter: 0 }.freeze + + # Fake client stub recording calls and yielding scripted outcomes. An Exception is raised, a Proc is + # called, anything else is returned. + class FakeClientStub + attr_reader :requests + + def initialize outcomes + @outcomes = outcomes + @requests = [] + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } + raise "Unexpected request: no scripted response left" if @outcomes.empty? + + outcome = @outcomes.shift + case outcome + when Exception then raise outcome + when Proc then outcome.call + else outcome + end + end + end + + # Retry policy whose budget is a fixed number of retries per `start!` instead of a wall-clock deadline. + # Because the Driver starts the control plane policy once per recovery episode, the count is shared by + # every query in the episode. `dup` is overridden because `RetryPolicy#dup` builds a plain `RetryPolicy`. + class CountedRetryPolicy < Gapic::Common::RetryPolicy + def initialize retries:, **kwargs + @retries = retries + super(timeout: 60, **kwargs) + end + + def dup + self.class.new retries: @retries, **overrides.except(:timeout) + end + + def start! **kwargs + @retries_left = @retries + super + end + + def retry_with_deadline? + return false unless @retries_left.positive? + @retries_left -= 1 + true + end + end + + # Loop A: a query answered with a Category 2 status that the control plane `retry_codes` do not cover + # used to be re-sent with no delay until the global deadline. + def test_endless_non_retriable_category_2_on_query_backs_off_across_queries + [400, 408, 412, 416, 502].each do |status| + outcomes = [initiation_response, Faraday::ConnectionFailed.new("reset")] + + Array.new(4) { http_error status } + + [query_response(received: 0), final_response] + stub = FakeClientStub.new outcomes + + result, delays = run_recording_delays stub + + assert_equal '{"done":true}', result, "status #{status}" + assert_equal [1.0, 2.0, 4.0, 5.0], delays, "status #{status}" + assert_commands stub, ["start", "upload, finalize"] + Array.new(5, "query") + ["upload, finalize"] + end + end + + # Loop B: the upload keeps failing and the query keeps answering `active` at the same offset. The first + # query of the episode is immediate; every later lap waits for the next delay. + def test_repeated_upload_failures_without_progress_back_off_before_each_later_query + outcomes = [initiation_response] + 3.times { outcomes += [http_error(502), query_response(received: 0)] } + outcomes << final_response + stub = FakeClientStub.new outcomes + + result, delays = run_recording_delays stub + + assert_equal '{"done":true}', result + assert_equal [1.0, 2.0], delays + assert_commands stub, ["start"] + (["upload, finalize", "query"] * 3) + ["upload, finalize"] + end + + # Retries inside one query (RetryDecider) and re-queries decided by Rules draw from one sequence. + def test_in_command_retries_and_re_queries_share_one_backoff_sequence + outcomes = [initiation_response, Faraday::ConnectionFailed.new("reset"), + http_error(429), http_error(429), http_error(502), + query_response(received: 0), final_response] + stub = FakeClientStub.new outcomes + + result, delays = run_recording_delays stub + + assert_equal '{"done":true}', result + assert_equal [1.0, 2.0, 4.0], delays + end + + # A successful upload closes the episode, so the next failure starts a fresh sequence with an immediate + # first query. + def test_an_acknowledged_chunk_resets_the_backoff + outcomes = [ + initiation_response, + http_error(502), http_error(502), query_response(received: 0), # episode 1: one re-query + chunk_ack, # closes episode 1 + http_error(502), http_error(502), query_response(received: 4), # episode 2: one re-query + final_response + ] + stub = FakeClientStub.new outcomes + + result, delays = run_recording_delays stub, stream: "012345", upload_size: 6, chunk_size: 4 + + assert_equal '{"done":true}', result + assert_equal [1.0, 1.0], delays + assert_commands stub, ["start", "upload", "query", "query", "upload", "upload, finalize", "query", "query", + "upload, finalize"] + end + + # A query reporting bytes beyond the episode's starting offset closes the episode, even though the upload + # that failed was never acknowledged. + def test_server_progress_reported_by_a_query_resets_the_backoff + outcomes = [ + initiation_response, + http_error(502), query_response(received: 2), # server stored the chunk despite the 502 + http_error(502), query_response(received: 4), + final_response + ] + stub = FakeClientStub.new outcomes + + result, delays = run_recording_delays stub, chunk_size: 2 + + assert_equal '{"done":true}', result + assert_empty delays + assert_commands stub, ["start", "upload", "query", "upload", "query", "finalize"] + end + + # A server whose reported offset flips between two values without ever exceeding the higher one makes no + # progress, so the backoff keeps growing. Closing the episode on any change of offset would reset it every + # lap and query with no delay. + def test_an_oscillating_offset_does_not_reset_the_backoff + outcomes = [ + initiation_response, + http_error(502), query_response(received: 2), # episode at 0: 2 > 0 closes it + http_error(502), query_response(received: 0), # episode opens at 2; the drop to 0 keeps it open + http_error(502), query_response(received: 2), # back to 2: not beyond 2, still open + http_error(502), query_response(received: 0), + http_error(502), query_response(received: 4), # 4 > 2 closes it + final_response + ] + stub = FakeClientStub.new outcomes + + result, delays = run_recording_delays stub, chunk_size: 2 + + assert_equal '{"done":true}', result + assert_equal [1.0, 2.0, 4.0], delays + assert_commands stub, ["start"] + (["upload", "query"] * 5) + ["finalize"] + end + + # The control plane budget covers the whole episode: once an earlier query has spent it, a connection + # failure on a later query is no longer retried and ends the run. + def test_the_control_plane_budget_is_shared_by_every_query_in_the_episode + outcomes = [initiation_response, Faraday::ConnectionFailed.new("reset"), + Faraday::ConnectionFailed.new("refused"), http_error(502), # query 1: spends the only retry + Faraday::ConnectionFailed.new("refused")] # query 2: nothing left + stub = FakeClientStub.new outcomes + policy = CountedRetryPolicy.new retries: 1, **BACKOFF + + error = assert_raises RequestFailedError do + run_recording_delays stub, control_plane_retry_policy: policy + end + + refute_nil error.resume_handle + assert_commands stub, ["start", "upload, finalize", "query", "query", "query"] + end + + # A backoff never starts once the whole-upload deadline has passed. + def test_no_backoff_is_performed_after_the_global_deadline + outcomes = [initiation_response, http_error(502), query_response(received: 0), http_error(502)] + stub = FakeClientStub.new outcomes + recoveries = 0 + spend_deadline = lambda do |progress| + next unless progress.phase == :recovering + recoveries += 1 + busy_wait 0.6 if recoveries == 2 + end + + delays = [] + Kernel.stub :sleep, ->(seconds) { delays << seconds.to_f } do + assert_raises DeadlineExceededError do + run_upload stub, timeout: 0.5, on_progress: spend_deadline + end + end + + assert_empty delays + assert_commands stub, ["start", "upload, finalize", "query", "upload, finalize"] + end + + private + + def run_recording_delays stub, control_plane_retry_policy: BACKOFF, **overrides + delays = [] + result = Kernel.stub :sleep, ->(seconds) { delays << seconds.to_f } do + run_upload stub, control_plane_retry_policy: control_plane_retry_policy, **overrides + end + [result, delays] + end + + def run_upload stub, stream: "0123", upload_size: 4, chunk_size: 10, **overrides + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new(stream), + upload_size: upload_size, + chunk_size: chunk_size, + **overrides + ) + Driver.new(client_stub: stub, config: config).run + end + + def busy_wait seconds + until_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds + nil while Process.clock_gettime(Process::CLOCK_MONOTONIC) < until_time + end + + def initiation_response + FakeResponse.new status: 200, headers: { "X-Goog-Upload-URL" => SESSION_URL, "X-Goog-Upload-Status" => "active" }, + body: "" + end + + def chunk_ack + FakeResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: "" + end + + def query_response received: + FakeResponse.new status: 200, + headers: { "X-Goog-Upload-Status" => "active", "X-Goog-Upload-Size-Received" => received.to_s }, + body: "" + end + + def final_response + FakeResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}' + end + + def http_error status, headers: {} + klass = status >= 500 ? Faraday::ServerError : Faraday::ClientError + klass.new "the server responded with status #{status}", { status: status, headers: headers, body: "" } + end + + def assert_commands stub, expected + assert_equal expected, stub.requests.map { |req| req[:options][:metadata]["X-Goog-Upload-Command"] } + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb index 1e902c0..65404b9 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb @@ -170,11 +170,13 @@ def test_resume_upload_with_409_recovery_retry ] stub = FakeClientStub.new responses config = ResumeUploadConfig.new( - upload_url: "https://example.com/session/1", - chunk_size: 4, - stream: StringIO.new("0123456789"), - upload_size: 10, - on_progress: ->(p) { progress_records << p } + upload_url: "https://example.com/session/1", + chunk_size: 4, + stream: StringIO.new("0123456789"), + upload_size: 10, + on_progress: ->(p) { progress_records << p }, + # The re-query backs off on the control plane policy; keep the delay negligible. + control_plane_retry_policy: { initial_delay: 0.001, max_delay: 0.002, jitter: 0 } ) driver = Driver.new client_stub: stub, config: config diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_recovery_episode_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_recovery_episode_test.rb new file mode 100644 index 0000000..f1696c8 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_recovery_episode_test.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for recovery episode tracking in Rules: which recipes open, continue and close an episode +# (`State#recovery_offset`), and which queries wait for the episode's backoff (`SendQuery#backoff`). +# +# See `design/resumable_upload/implementation-guide.md` section 6.2.1. +# +class RulesRecoveryEpisodeTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + SESSION_URL = "https://example.com/session" + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4 + ) + end + + def test_send_query_defaults_to_opening_an_episode + refute Instruction::SendQuery.new(url: SESSION_URL).backoff + end + + def test_state_starts_with_no_open_episode + assert_nil State.new.recovery_offset + end + + def test_resume_session_opens_an_episode_at_offset_zero_and_queries_at_once + config = ResumeUploadConfig.new upload_url: SESSION_URL, chunk_size: 4, stream: StringIO.new("0123"), + upload_size: 4 + next_state, instructions = Rules.step State.new, Event::ResumeUpload.new, config + + assert_equal 0, next_state.recovery_offset + refute query_in(instructions).backoff + end + + def test_enter_recovery_with_no_open_episode_opens_one_at_the_current_offset + state = sending_state offset: 4 + next_state, instructions = Rules.step state, cat2_response, @config + + assert_equal :recovery, next_state.status + assert_equal 4, next_state.recovery_offset + refute query_in(instructions).backoff + end + + def test_enter_recovery_with_an_open_episode_backs_off_and_keeps_the_episode_start + state = sending_state(offset: 4).with recovery_offset: 4 + next_state, instructions = Rules.step state, cat2_response, @config + + assert_equal 4, next_state.recovery_offset + assert query_in(instructions).backoff + end + + def test_enter_recovery_on_a_connection_failure_follows_the_same_rule + failure = Event::RequestFailed.new kind: :connection_failed, message: "reset" + + _, opening = Rules.step sending_state(offset: 0), failure, @config + _, continuing = Rules.step sending_state(offset: 0).with(recovery_offset: 0), failure, @config + + refute query_in(opening).backoff + assert query_in(continuing).backoff + end + + def test_retry_recovery_backs_off_and_keeps_the_episode_start + state = recovery_state offset: 4, recovery_offset: 4 + next_state, instructions = Rules.step state, cat2_response, @config + + assert_equal :recovery, next_state.status + assert_equal 4, next_state.recovery_offset + assert query_in(instructions).backoff + end + + def test_realign_without_progress_keeps_the_episode_open + state = recovery_state offset: 4, recovery_offset: 4 + next_state, = Rules.step state, query_response(received: 4), @config + + assert_equal :transmission_reading, next_state.status + assert_equal 4, next_state.recovery_offset + end + + # A lower offset is a regression, not progress: the episode stays open at its starting offset while the + # client rewinds. + def test_realign_to_a_lower_offset_keeps_the_episode_open + state = recovery_state offset: 4, recovery_offset: 4 + next_state, = Rules.step state, query_response(received: 2), @config + + assert_equal 2, next_state.offset + assert_equal 4, next_state.recovery_offset + end + + def test_realign_with_progress_closes_the_episode + state = recovery_state offset: 4, recovery_offset: 4 + next_state, = Rules.step state, query_response(received: 8), @config + + assert_equal 8, next_state.offset + assert_nil next_state.recovery_offset + end + + def test_ack_chunk_closes_the_episode + state = sending_state(offset: 4).with recovery_offset: 4 + next_state, = Rules.step state, Event::HttpResponse.new(status: 200, headers: { "x-goog-upload-status" => "active" }), + @config + + assert_equal 8, next_state.offset + assert_nil next_state.recovery_offset + end + + # Loop B from the design doc: an upload that keeps failing while the query keeps answering `active` + # at the same offset. Only the first query of the episode is sent at once. + def test_repeated_failures_without_progress_back_off_every_query_after_the_first + state = sending_state offset: 0 + backoffs = [] + + 3.times do + state, instructions = Rules.step state, cat2_response, @config + backoffs << query_in(instructions).backoff + state, = Rules.step state, query_response(received: 0), @config + state = state.with status: :transmission_sending, in_flight_length: 4 + end + + assert_equal [false, true, true], backoffs + end + + private + + def sending_state offset: + State.new status: :transmission_sending, upload_url: SESSION_URL, offset: offset, chunk_size: 4, + in_flight_length: 4 + end + + def recovery_state offset:, recovery_offset: + State.new status: :recovery, upload_url: SESSION_URL, offset: offset, chunk_size: 4, + recovery_offset: recovery_offset + end + + def cat2_response + Event::HttpResponse.new status: 502, headers: {} + end + + def query_response received: + Event::HttpResponse.new status: 200, + headers: { "x-goog-upload-status" => "active", "x-goog-upload-size-received" => received.to_s } + end + + def query_in instructions + queries = instructions.grep Instruction::SendQuery + assert_equal 1, queries.size, "Expected exactly one SendQuery in #{instructions.map(&:class)}" + queries.first + end +end From 04947ffa302913fb219f693b4140df7afdfe47d7 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Mon, 28 Sep 2026 20:50:17 +0000 Subject: [PATCH 20/22] fix: handle missing/malformed granularity and offset --- .../resumable_upload/implementation-guide.md | 16 +- .../resumable_upload/driver/upload_log.rb | 12 +- .../lib/gapic/rest/resumable_upload/rules.rb | 27 ++- .../resumable_upload/driver_logging_test.rb | 2 +- .../resumable_upload/header_parsing_test.rb | 195 ++++++++++++++++++ 5 files changed, 237 insertions(+), 15 deletions(-) create mode 100644 gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md index cd1788a..47be785 100644 --- a/gapic-common/design/resumable_upload/implementation-guide.md +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -338,7 +338,7 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` * `X-Goog-Upload-Header-Content-Type: config.content_type` * `X-Goog-Upload-Header-Content-Length: config.upload_size` (if known upfront). * Callers cannot supply either of these two headers via `initial_headers`; see the reserved-headers rule in Section 2 (doing so raises an `ArgumentError`). Other `X-Goog-Upload-Header-*` pass-through headers are permitted. -2. **Offset Extraction**: On `query` responses, the acknowledged byte count is extracted from `X-Goog-Upload-Size-Received` as an integer (`server_offset`). +2. **Offset Extraction**: On `query` responses, the acknowledged byte count is extracted from `X-Goog-Upload-Size-Received` as an integer (`server_offset`). Parsing is strict (`Rules.parse_header_positive_integer`): only a positive decimal integer, optionally surrounded by whitespace, is accepted. A missing, empty, negative or otherwise malformed value (`"-5"`, `"12abc"`, `"1.5"`) yields `server_offset = 0`, so the client realigns to the start of the upload. Validating the reported offset beyond that is out of scope: a server that answers `active` while stripping only this header is not a supported scenario. 3. **Request Modification on 4xx**: Retrying Category 2 errors requires querying the backend for `server_offset` first. 4. **Standard Retry Configuration & Distinct Policies**: The Driver resolves one `Gapic::Common::RetryPolicy` per plane. A policy supplies the retry budget (`timeout`), the backoff schedule, and the caller-facing `retry_predicate` / `retry_codes`; which outcomes are re-sent at all is decided by `Driver::RetryDecider` (Section 6.1.1). None of the defaults carries a `retry_predicate`, so a caller-supplied one is consulted as-is, ahead of `retry_codes`. `retry_codes` are derived from the HTTP status sets on `Rules` via `Gapic::Common::ErrorCodes.grpc_error_for`. * **Start Policy (`start_retry_policy`)**: Applies to session initiation (`start`). Default `retry_codes`: `RETRIABLE_4XX_STATUS_CODES` + `RETRIABLE_5XX_STATUS_CODES` (HTTP `409`, `429`, `499`, `500`, `503`, `504` → `ALREADY_EXISTS`, `RESOURCE_EXHAUSTED`, `CANCELLED`, `INTERNAL`, `UNAVAILABLE`, `DEADLINE_EXCEEDED`). HTTP `408` and `502` map to `UNKNOWN` and are deliberately not retried. @@ -356,7 +356,7 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` | :--- | :--- | :--- | :--- | :--- | :--- | | **`Initializing`** | `:start_upload` | `Event::StartUpload` | `status = :starting` | `Starting` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::SendStart.new(url: config.initial_url, headers: config.initial_headers, body: config.initial_body)` | | **`Initializing`** | `:resume_upload` | `Event::ResumeUpload` | `upload_url = event.upload_url`
`chunk_size = event.chunk_size`
`offset = 0`
`recovery_offset = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: event.upload_size))`
`Instruction::SendQuery.new(url: event.upload_url, backoff: false)` | -| **`Starting`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `upload_url = headers['X-Goog-Upload-URL']`
`chunk_granularity = headers['...-Granularity']&.to_i`
`chunk_size = resolve(config, chunk_granularity)`
`offset = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Starting`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `upload_url = headers['X-Goog-Upload-URL']`
`chunk_granularity = parse_header_positive_integer(headers['...-Granularity'])`
`chunk_size = resolve(config, chunk_granularity)`
`offset = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | | **`Starting`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Starting`** | `:response_cat2` / `:response_fatal_bad_response` | `Event::HttpResponse` (Non-200, or a headerless `200` after the start budget is spent; see Section 6.1) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **`Starting`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:request_failed_unknown` | `Event::RequestFailed(kind:, message:, source_error:)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | @@ -381,7 +381,7 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` | **`Finalizing \| Sending finalize`** | `:request_retries_exhausted` / `:request_failed_unknown` | `Event::RequestFailed(kind: :retries_exhausted \| :unknown)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Finalizing \| Sending finalize`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Finalizing \| Sending finalize`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | -| **`Recovery`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = headers['X-Goog-Upload-Size-Received'].to_i`
`in_flight_length = 0`
`recovery_offset = nil` if the new `offset` exceeds `state.recovery_offset`, else unchanged
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Recovery`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = parse_header_positive_integer(headers['X-Goog-Upload-Size-Received']) \|\| 0`
`in_flight_length = 0`
`recovery_offset = nil` if the new `offset` exceeds `state.recovery_offset`, else unchanged
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | | **`Recovery`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `in_flight_length = 0`
`status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | | **`Recovery`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `status = :recovery` | `Recovery` | `Instruction::SendQuery.new(url: state.upload_url, backoff: true)` | | **`Recovery`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:request_failed_unknown` | `Event::RequestFailed(kind:, ...)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | @@ -460,12 +460,12 @@ Upon receiving `200 OK` from the `start` request, `Core` inspects the response h ### 5.1 Variable Definitions * `DEFAULT_CHUNK_SIZE`: Default chunk size of `8_388_608` bytes (8 MB). * `user_chunk_size`: Explicit chunk size specified in `StartUploadConfig.chunk_size` (or `nil` if unspecified). -* `chunk_granularity`: Required byte alignment modulus parsed from header `X-Goog-Upload-Chunk-Granularity` as an Integer (or `nil` if header is absent). +* `chunk_granularity`: Required byte alignment modulus parsed from header `X-Goog-Upload-Chunk-Granularity` as a positive Integer, or `nil`. Parsing is strict (`Rules.parse_header_positive_integer`): a missing, empty, zero, negative or otherwise malformed value is `nil`, exactly as if the server had not sent the header. * `effective_chunk_size`: Final calculated byte size used by Driver for in-memory buffering and chunk transmission. ### 5.2 Resolution Rules -#### Rule 1: No Server Granularity Specified (`chunk_granularity` is nil or 0) +#### Rule 1: No Server Granularity Specified (`chunk_granularity` is nil) When the server does not specify a granularity requirement: * If `user_chunk_size` is provided: `effective_chunk_size = user_chunk_size`. * If `user_chunk_size` is omitted: `effective_chunk_size = DEFAULT_CHUNK_SIZE`. @@ -671,7 +671,7 @@ The `Driver` emits structured logs across three severity levels (`INFO`, `DEBUG` | `DEBUG` | Wire | Inbound HTTP response (`wire_receive`) | `Received HTTP ` | | `DEBUG` | Wire | Transport exception (`wire_failure`) | `Request failed: ` | | `DEBUG` | Buffer | Stream/buffer realignment (`buffer_realign`) | `Buffer realignment: ` | -| `DEBUG` | Recovery | Backoff before a query that continues a recovery episode (`recovery_backoff`) | `Backing off before recovery query` | +| `DEBUG` | Recovery | Backoff before a query that continues a recovery episode (`recovery_backoff`) | `Performing backoff delay before recovery query` | | `WARN` | Lifecycle | `:fail_with_deadline_exceeded`, `:fail_with_rejected`, `:fail_with_bad_response`, `:fail_with_request_error` | Resumable upload failed | | `WARN` | Lifecycle | `:fail_with_cancelled` | Resumable upload canceled on the server | | `WARN` | Transition | `InvalidTransitionError` (`unmatched_transition`) | Unmatched transition | @@ -713,8 +713,8 @@ All log entries emitted by `UploadLog` populate structured fields in `Google::Lo * `body`: Abridged payload or error body snippet. * `status`: HTTP response status code (`Integer`, on `wire_receive`). * `uploadStatus`: Value of `X-Goog-Upload-Status` response header. - * `sizeReceived`: Parsed integer value of `X-Goog-Upload-Size-Received` response header. - * `granularity`: Parsed integer value of `X-Goog-Upload-Chunk-Granularity` response header (`wire_receive`). + * `sizeReceived`: Parsed integer value of `X-Goog-Upload-Size-Received` response header. Present only when the header parses as a positive integer; a malformed value is still visible in `headers`. + * `granularity`: Parsed integer value of `X-Goog-Upload-Chunk-Granularity` response header (`wire_receive`). Present only when the header parses as a positive integer. * `kind`: Transport failure classification symbol (`:timeout`, `:connection_failed`, `:retries_exhausted`, `:unknown`). * `error`: Exception message string (`wire_failure`). * **Recovery Backoff Fields** (`recovery_backoff`): diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb index 1345cc1..e369c79 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb @@ -153,8 +153,12 @@ def wire_send method:, url:, headers:, start_attempt:, body_size: nil, body: nil # def wire_receive event upload_status = Rules.header_value event.headers, "x-goog-upload-status" - size_recv = Rules.header_value event.headers, "x-goog-upload-size-received" - gran = Rules.header_value event.headers, "x-goog-upload-chunk-granularity" + size_recv = Rules.parse_header_positive_integer( + Rules.header_value(event.headers, "x-goog-upload-size-received") + ) + gran = Rules.parse_header_positive_integer( + Rules.header_value(event.headers, "x-goog-upload-chunk-granularity") + ) err = event.error if event.respond_to? :error fields = { status: event.status, @@ -163,8 +167,8 @@ def wire_receive event } fields[:uploadStatus] = upload_status if upload_status fields[:errorStatus] = err.status if err&.status - fields[:sizeReceived] = size_recv.to_i if size_recv - fields[:granularity] = gran.to_i if gran + fields[:sizeReceived] = size_recv if size_recv + fields[:granularity] = gran if gran entry :debug, "Received HTTP #{event.status}", **fields end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb index cd6beb4..1b1e6f5 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb @@ -575,7 +575,7 @@ def self.resume_session state, _event, config # @return [Array>] Tuple of [next_state, instructions] def self.begin_transmission state, event, config granularity_str = header_value event.headers, "x-goog-upload-chunk-granularity" - granularity = granularity_str&.to_i + granularity = parse_header_positive_integer granularity_str chunk_size = resolve_chunk_size config.chunk_size, granularity upload_url = header_value event.headers, "x-goog-upload-url" next_state = state.with( @@ -798,7 +798,9 @@ def self.complete_upload_finalized state, event, _config # @return [Array>] Tuple of [next_state, instructions] def self.realign_from_recovery state, event, config server_offset_str = header_value event.headers, "x-goog-upload-size-received" - server_offset = server_offset_str.to_i + # A missing or malformed header realigns to 0. The server does not strip this header alone while + # still reporting `active`, so this only has to be safe, not smart. + server_offset = parse_header_positive_integer(server_offset_str) || 0 # `>`, not `!=`: a lower offset is a regression, not progress. Closing on it would reset the backoff, # and a server whose reported offset flips between two values would then query with no delay forever. progressed = state.recovery_offset.nil? || server_offset > state.recovery_offset @@ -1017,6 +1019,27 @@ def self.describe_event event, shape end end + ## + # @private + # Parses a numeric protocol header value strictly. + # + # Accepts only a positive decimal integer, optionally surrounded by whitespace. Anything else — + # `nil`, empty, `0`, signed, fractional, or trailing garbage such as `"12abc"` — returns `nil`, so + # callers treat a malformed header the same as a missing one. `String#to_i` is deliberately not + # used: it turns `"12abc"` into `12` and `"-5"` into `-5`. + # + # @param value [String, nil] Raw header value + # @return [Integer, nil] Parsed value, or `nil` if the value is not a positive integer + def self.parse_header_positive_integer value + return nil unless value.is_a? String + + stripped = value.strip + return nil unless stripped.match?(/\A\d+\z/) + + parsed = Integer stripped, 10 + parsed.positive? ? parsed : nil + end + ## # @private # Resolves effective chunk size given user specification and backend granularity. diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb index cfe0654..45c2b88 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb @@ -196,7 +196,7 @@ def test_recovery_re_query_logs_the_backoff_delay Driver.new(client_stub: FakeStub.new(responses), config: config, logger: recording).run end - backoff_entries = recording.entries.select { |e| e.message.message == "Backing off before recovery query" } + backoff_entries = recording.entries.select { |e| e.message.message == "Performing backoff delay before recovery query" } assert_equal 1, backoff_entries.size assert_equal Logger::DEBUG, backoff_entries.first.severity assert_in_delta 0.25, backoff_entries.first.message.fields["delay"] diff --git a/gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb b/gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb new file mode 100644 index 0000000..c442574 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb @@ -0,0 +1,195 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for strict parsing of the numeric protocol headers `X-Goog-Upload-Size-Received` and +# `X-Goog-Upload-Chunk-Granularity`. A malformed value is treated exactly like a missing one: the offset +# falls back to 0 and the granularity to `nil`. +# +# See `design/resumable_upload/implementation-guide.md` sections 3 and 5. +# +class HeaderParsingTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + SESSION_URL = "https://example.com/session" + + # Values that must parse as if the header were absent. + UNPARSEABLE = [nil, "", " ", "0", "00", "-5", "+5", "1.5", "12abc", "abc", "0x10", "1e3", "1_000"].freeze + + class FakeClientStub + attr_reader :commands + + def initialize outcomes + @outcomes = outcomes + @commands = [] + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + _ = [uri, body, params, method_name] + @commands << options[:metadata]["X-Goog-Upload-Command"] + outcome = @outcomes.shift + raise "Unexpected request: no scripted response left" if outcome.nil? + raise outcome if outcome.is_a? Exception + outcome + end + end + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4 + ) + end + + # --- Rules.parse_header_positive_integer --- + + def test_parses_positive_decimal_integers + assert_equal 123, Rules.parse_header_positive_integer("123") + assert_equal 123, Rules.parse_header_positive_integer(" 123 ") + assert_equal 8, Rules.parse_header_positive_integer("008") + assert_equal 2**40, Rules.parse_header_positive_integer((2**40).to_s) + end + + def test_rejects_everything_else + UNPARSEABLE.each do |value| + assert_nil Rules.parse_header_positive_integer(value), "value #{value.inspect}" + end + end + + # --- X-Goog-Upload-Chunk-Granularity on the start response --- + + def test_unparseable_granularity_is_treated_as_absent + absent_state, absent_instructions = Rules.step State.new(status: :starting), start_response(nil), @config + + UNPARSEABLE.each do |value| + next_state, instructions = Rules.step State.new(status: :starting), start_response(value), @config + + assert_nil next_state.chunk_granularity, "value #{value.inspect}" + assert_equal absent_state, next_state, "value #{value.inspect}" + assert_equal absent_instructions, instructions, "value #{value.inspect}" + end + end + + def test_valid_granularity_aligns_the_chunk_size + next_state, = Rules.step State.new(status: :starting), start_response(" 3 "), @config + + assert_equal 3, next_state.chunk_granularity + assert_equal 3, next_state.chunk_size + end + + # --- X-Goog-Upload-Size-Received on the query response --- + + def test_unparseable_offset_realigns_to_zero + UNPARSEABLE.each do |value| + state = State.new status: :recovery, upload_url: SESSION_URL, offset: 4, chunk_size: 4, recovery_offset: 4 + next_state, instructions = Rules.step state, query_response(value), @config + + assert_equal :transmission_reading, next_state.status, "value #{value.inspect}" + assert_equal 0, next_state.offset, "value #{value.inspect}" + assert_equal 4, next_state.recovery_offset, "value #{value.inspect}" + realign = instructions.grep(Instruction::RealignBuffer).first + assert_equal 0, realign.server_offset, "value #{value.inspect}" + progress = instructions.grep(Instruction::NotifyProgress).first.progress + assert_equal 0, progress.bytes_uploaded, "value #{value.inspect}" + end + end + + def test_trailing_garbage_does_not_yield_a_partial_offset + state = State.new status: :recovery, upload_url: SESSION_URL, offset: 0, chunk_size: 4, recovery_offset: 0 + next_state, = Rules.step state, query_response("8abc"), @config + + assert_equal 0, next_state.offset + end + + def test_valid_offset_is_used + state = State.new status: :recovery, upload_url: SESSION_URL, offset: 4, chunk_size: 4, recovery_offset: 4 + next_state, = Rules.step state, query_response(" 8 "), @config + + assert_equal 8, next_state.offset + assert_nil next_state.recovery_offset + end + + # --- Driver --- + + # A negative offset used to reach `stream.seek(-5)` and escape as `Errno::EINVAL`. It now realigns to 0 + # and re-sends the data from the start. + def test_negative_offset_restarts_the_upload_from_zero + outcomes = [ + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-URL" => SESSION_URL, "X-Goog-Upload-Status" => "active" }, + body: ""), + FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable"), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active", "X-Goog-Upload-Size-Received" => "-5" }, + body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + stub = FakeClientStub.new outcomes + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + data_plane_retry_policy: { retry_codes: [] }, + control_plane_retry_policy: { initial_delay: 0.01, jitter: 0 } + ) + + result = Driver.new(client_stub: stub, config: config).run + + assert_equal '{"done":true}', result + assert_equal ["start", "upload, finalize", "query", "upload, finalize"], stub.commands + end + + # --- UploadLog --- + + def test_wire_receive_logs_only_parseable_numeric_headers + fields = wire_receive_fields "X-Goog-Upload-Size-Received" => "12abc", "X-Goog-Upload-Chunk-Granularity" => "-1" + refute fields.key?("sizeReceived") + refute fields.key?("granularity") + + fields = wire_receive_fields "X-Goog-Upload-Size-Received" => "12", "X-Goog-Upload-Chunk-Granularity" => "256" + assert_equal 12, fields["sizeReceived"] + assert_equal 256, fields["granularity"] + end + + private + + def start_response granularity + headers = { "x-goog-upload-status" => "active", "x-goog-upload-url" => SESSION_URL } + headers["x-goog-upload-chunk-granularity"] = granularity unless granularity.nil? + Event::HttpResponse.new status: 200, headers: headers + end + + def query_response received + headers = { "x-goog-upload-status" => "active" } + headers["x-goog-upload-size-received"] = received unless received.nil? + Event::HttpResponse.new status: 200, headers: headers + end + + def wire_receive_fields headers + recording = RecordingLogger.new + stub_logger = Gapic::LoggingConcerns::StubLogger.new logger: recording, service: "ResumableUpload" + upload_log = Driver::UploadLog.new stub_logger, upload_id: "test-upload" + upload_log.wire_receive Event::HttpResponse.new(status: 200, headers: headers, body: "") + recording.entries.last.message.fields + end +end From b9f5122f111dd8f7d9a637f95821bb6b7342f65b Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Mon, 28 Sep 2026 21:05:56 +0000 Subject: [PATCH 21/22] fix: re-query when a query answer has no parseable offset 04947ff realigned to offset 0 when `X-Goog-Upload-Size-Received` was missing or malformed, which silently restarts the upload. Such an answer is now Category 2 and is re-queried with the recovery episode's backoff. * The classifier takes the state status (`shape_of(event, state_status)`, `classify_http_response(response, state_status)`). An `active` 200 that lacks a header the awaited command requires is `:response_cat2` (`Rules.required_headers_present?`); for now that is the offset on a query answer in `:recovery`. The router is unchanged. * The helper becomes `parse_header_non_negative_integer`, so a literal "0" is a valid offset again; granularity keeps treating 0 as absent. * `wire_receive` logs `sizeReceived: 0` again. --- .../resumable_upload/implementation-guide.md | 16 +- .../resumable_upload/driver/upload_log.rb | 6 +- .../lib/gapic/rest/resumable_upload/rules.rb | 74 ++++-- .../driver_error_mapping_test.rb | 2 +- .../resumable_upload/header_parsing_test.rb | 218 ++++++++++++------ .../rules_classification_test.rb | 72 +++--- 6 files changed, 250 insertions(+), 138 deletions(-) diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md index 47be785..79404c5 100644 --- a/gapic-common/design/resumable_upload/implementation-guide.md +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -299,7 +299,7 @@ The authoritative implementation is the source itself, under `lib/gapic/rest/res ### 3.1 Rules Module (`Gapic::Rest::ResumableUpload::Rules`) The `Rules` module is a pure functional transition engine with zero state awareness and zero side effects. It provides two primary entry points: -* `Rules.shape_of(event)`: Classifies raw input events (`Event::StartUpload`, `Event::ChunkRead`, `Event::HttpResponse`, `Event::RequestFailed`, `Event::Cancel`, `Event::GlobalDeadlineExceeded`) into canonical symbols. +* `Rules.shape_of(event, state_status)`: Classifies raw input events (`Event::StartUpload`, `Event::ChunkRead`, `Event::HttpResponse`, `Event::RequestFailed`, `Event::Cancel`, `Event::GlobalDeadlineExceeded`) into canonical symbols. Only `Event::HttpResponse` consults `state_status`, to check the headers the awaited command requires (Section 6.1.2, condition 3). * `Rules.decide(state, event, config)`: Evaluates `case [state.status, shape]` pattern matching to select a transition recipe symbol, dispatches via `public_send(recipe, state, event, config)`, and returns a `Decision` snapshot (`from_status`, `shape`, `recipe`, `next_state`, `instructions`). * `Rules.step(state, event, config)`: Convenience tuple wrapper around `Rules.decide` returning `[decision.next_state, decision.instructions]`. @@ -338,7 +338,7 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` * `X-Goog-Upload-Header-Content-Type: config.content_type` * `X-Goog-Upload-Header-Content-Length: config.upload_size` (if known upfront). * Callers cannot supply either of these two headers via `initial_headers`; see the reserved-headers rule in Section 2 (doing so raises an `ArgumentError`). Other `X-Goog-Upload-Header-*` pass-through headers are permitted. -2. **Offset Extraction**: On `query` responses, the acknowledged byte count is extracted from `X-Goog-Upload-Size-Received` as an integer (`server_offset`). Parsing is strict (`Rules.parse_header_positive_integer`): only a positive decimal integer, optionally surrounded by whitespace, is accepted. A missing, empty, negative or otherwise malformed value (`"-5"`, `"12abc"`, `"1.5"`) yields `server_offset = 0`, so the client realigns to the start of the upload. Validating the reported offset beyond that is out of scope: a server that answers `active` while stripping only this header is not a supported scenario. +2. **Offset Extraction**: On `query` responses, the acknowledged byte count is extracted from `X-Goog-Upload-Size-Received` as an integer (`server_offset`). Parsing is strict (`Rules.parse_header_non_negative_integer`): only a non-negative decimal integer, optionally surrounded by whitespace, is accepted; `String#to_i` is not used, because it reads `"12abc"` as `12` and `"-5"` as `-5`. An `active` answer whose offset is missing, empty, negative or otherwise malformed (`"-5"`, `"12abc"`, `"1.5"`) is Category 2 (Section 6.1.2, condition 3): the client re-queries with backoff rather than assuming an offset. 3. **Request Modification on 4xx**: Retrying Category 2 errors requires querying the backend for `server_offset` first. 4. **Standard Retry Configuration & Distinct Policies**: The Driver resolves one `Gapic::Common::RetryPolicy` per plane. A policy supplies the retry budget (`timeout`), the backoff schedule, and the caller-facing `retry_predicate` / `retry_codes`; which outcomes are re-sent at all is decided by `Driver::RetryDecider` (Section 6.1.1). None of the defaults carries a `retry_predicate`, so a caller-supplied one is consulted as-is, ahead of `retry_codes`. `retry_codes` are derived from the HTTP status sets on `Rules` via `Gapic::Common::ErrorCodes.grpc_error_for`. * **Start Policy (`start_retry_policy`)**: Applies to session initiation (`start`). Default `retry_codes`: `RETRIABLE_4XX_STATUS_CODES` + `RETRIABLE_5XX_STATUS_CODES` (HTTP `409`, `429`, `499`, `500`, `503`, `504` → `ALREADY_EXISTS`, `RESOURCE_EXHAUSTED`, `CANCELLED`, `INTERNAL`, `UNAVAILABLE`, `DEADLINE_EXCEEDED`). HTTP `408` and `502` map to `UNKNOWN` and are deliberately not retried. @@ -356,7 +356,7 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` | :--- | :--- | :--- | :--- | :--- | :--- | | **`Initializing`** | `:start_upload` | `Event::StartUpload` | `status = :starting` | `Starting` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::SendStart.new(url: config.initial_url, headers: config.initial_headers, body: config.initial_body)` | | **`Initializing`** | `:resume_upload` | `Event::ResumeUpload` | `upload_url = event.upload_url`
`chunk_size = event.chunk_size`
`offset = 0`
`recovery_offset = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: event.upload_size))`
`Instruction::SendQuery.new(url: event.upload_url, backoff: false)` | -| **`Starting`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `upload_url = headers['X-Goog-Upload-URL']`
`chunk_granularity = parse_header_positive_integer(headers['...-Granularity'])`
`chunk_size = resolve(config, chunk_granularity)`
`offset = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Starting`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `upload_url = headers['X-Goog-Upload-URL']`
`chunk_granularity = parse_header_non_negative_integer(headers['...-Granularity'])`, `nil` unless positive
`chunk_size = resolve(config, chunk_granularity)`
`offset = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | | **`Starting`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Starting`** | `:response_cat2` / `:response_fatal_bad_response` | `Event::HttpResponse` (Non-200, or a headerless `200` after the start budget is spent; see Section 6.1) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **`Starting`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:request_failed_unknown` | `Event::RequestFailed(kind:, message:, source_error:)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | @@ -381,7 +381,7 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` | **`Finalizing \| Sending finalize`** | `:request_retries_exhausted` / `:request_failed_unknown` | `Event::RequestFailed(kind: :retries_exhausted \| :unknown)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Finalizing \| Sending finalize`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | | **`Finalizing \| Sending finalize`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | -| **`Recovery`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = parse_header_positive_integer(headers['X-Goog-Upload-Size-Received']) \|\| 0`
`in_flight_length = 0`
`recovery_offset = nil` if the new `offset` exceeds `state.recovery_offset`, else unchanged
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Recovery`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = parse_header_non_negative_integer(headers['X-Goog-Upload-Size-Received'])` (always an Integer: an answer without a parseable offset is `:response_cat2`)
`in_flight_length = 0`
`recovery_offset = nil` if the new `offset` exceeds `state.recovery_offset`, else unchanged
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | | **`Recovery`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `in_flight_length = 0`
`status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | | **`Recovery`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `status = :recovery` | `Recovery` | `Instruction::SendQuery.new(url: state.upload_url, backoff: true)` | | **`Recovery`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:request_failed_unknown` | `Event::RequestFailed(kind:, ...)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | @@ -460,7 +460,7 @@ Upon receiving `200 OK` from the `start` request, `Core` inspects the response h ### 5.1 Variable Definitions * `DEFAULT_CHUNK_SIZE`: Default chunk size of `8_388_608` bytes (8 MB). * `user_chunk_size`: Explicit chunk size specified in `StartUploadConfig.chunk_size` (or `nil` if unspecified). -* `chunk_granularity`: Required byte alignment modulus parsed from header `X-Goog-Upload-Chunk-Granularity` as a positive Integer, or `nil`. Parsing is strict (`Rules.parse_header_positive_integer`): a missing, empty, zero, negative or otherwise malformed value is `nil`, exactly as if the server had not sent the header. +* `chunk_granularity`: Required byte alignment modulus parsed from header `X-Goog-Upload-Chunk-Granularity` as a positive Integer, or `nil`. Parsing is strict (`Rules.parse_header_non_negative_integer`, then `nil` unless positive): a missing, empty, zero, negative or otherwise malformed value is `nil`, exactly as if the server had not sent the header. * `effective_chunk_size`: Final calculated byte size used by Driver for in-memory buffering and chunk transmission. ### 5.2 Resolution Rules @@ -530,6 +530,8 @@ The implementation distinguishes three categories of network and protocol-level * **Conditions Producing `:response_cat2`** (`Rules.classify_http_response`): 1. **Headerless `200`**: A `200` whose `X-Goog-Upload-Status` is missing or empty, regardless of `CAT2_STATUS_CODES`. A proxy that strips the protocol headers from a successful answer leaves the offset unknown, which is exactly what recovery resolves. 2. **Listed Non-200 Statuses**: A non-200 whose `X-Goog-Upload-Status` is missing, empty or `active`, **and** whose status is in `Rules::CAT2_STATUS_CODES`: `RETRIABLE_4XX_STATUS_CODES` (`409`, `429`, `499`) + `RETRIABLE_5XX_STATUS_CODES` (`500`, `503`, `504`) + `400`, `408`, `412`, `416`, `502`. The list is an allowlist and is load-bearing: any status it does not list is Category 3 (Section 6.1.3). + 3. **`active` `200` Missing a Required Header**: A `200` with `X-Goog-Upload-Status: active` that lacks a header the awaited command requires (`Rules.required_headers_present?`). This is the only classification that depends on the state: `Rules.shape_of(event, state.status)` passes the status through, and each status that awaits a response awaits exactly one command. The required headers are: + * `query` (in `Recovery`): `X-Goog-Upload-Size-Received` must parse as a non-negative integer (Section 4.1, item 2). The client does not guess an offset; it re-queries with backoff. * **Data Plane Transport Failures**: `:request_connection_failed` and `:request_timeout` during `Transmission` or `Finalizing` also enter `Recovery` (`enter_recovery`), because the Driver does not re-send them on the data plane (Section 6.1.1, rows 2–3). * **Where Category 2 Arrives, per Plane**: * *Session Initiation (`start`)*: a headerless `200` and the default-retriable statuses are re-sent by the Driver within the start budget. Whatever surfaces as `:response_cat2` is terminal in `Starting` (`fail_with_bad_response` → `BadResponseError`, status `200` for an exhausted headerless answer): there is no upload URL to query yet. @@ -713,8 +715,8 @@ All log entries emitted by `UploadLog` populate structured fields in `Google::Lo * `body`: Abridged payload or error body snippet. * `status`: HTTP response status code (`Integer`, on `wire_receive`). * `uploadStatus`: Value of `X-Goog-Upload-Status` response header. - * `sizeReceived`: Parsed integer value of `X-Goog-Upload-Size-Received` response header. Present only when the header parses as a positive integer; a malformed value is still visible in `headers`. - * `granularity`: Parsed integer value of `X-Goog-Upload-Chunk-Granularity` response header (`wire_receive`). Present only when the header parses as a positive integer. + * `sizeReceived`: Parsed integer value of `X-Goog-Upload-Size-Received` response header. Present only when the header parses as a non-negative integer; a malformed value is still visible in `headers`. + * `granularity`: Parsed integer value of `X-Goog-Upload-Chunk-Granularity` response header (`wire_receive`). Present only when the header parses as a non-negative integer. * `kind`: Transport failure classification symbol (`:timeout`, `:connection_failed`, `:retries_exhausted`, `:unknown`). * `error`: Exception message string (`wire_failure`). * **Recovery Backoff Fields** (`recovery_backoff`): diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb index e369c79..425b987 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb @@ -153,10 +153,10 @@ def wire_send method:, url:, headers:, start_attempt:, body_size: nil, body: nil # def wire_receive event upload_status = Rules.header_value event.headers, "x-goog-upload-status" - size_recv = Rules.parse_header_positive_integer( + size_recv = Rules.parse_header_non_negative_integer( Rules.header_value(event.headers, "x-goog-upload-size-received") ) - gran = Rules.parse_header_positive_integer( + gran = Rules.parse_header_non_negative_integer( Rules.header_value(event.headers, "x-goog-upload-chunk-granularity") ) err = event.error if event.respond_to? :error @@ -241,7 +241,7 @@ def unmatched_transition state, event, error :warn, "Unmatched transition", status: state.status, - shape: Rules.shape_of(event), + shape: Rules.shape_of(event, state.status), error: error.message ) end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb index 1b1e6f5..5c949f6 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb @@ -30,8 +30,8 @@ module ResumableUpload # # ### Model # - # {Rules.decide} is the protocol. It is a total function of `[state.status, shape_of(event)]` returning a - # {Decision} that carries the next {State} and the instructions for the Driver to execute. Three + # {Rules.decide} is the protocol. It is a total function of `[state.status, shape_of(event, state.status)]` + # returning a {Decision} that carries the next {State} and the instructions for the Driver to execute. Three # vocabularies define it, each published as a frozen constant: # # * {STATUSES} - protocol lifecycle statuses a {State} may hold. @@ -354,8 +354,10 @@ module Rules # Classifies incoming event into a canonical shape symbol. # # @param event [Object] Input event + # @param state_status [Symbol] Status of the state the event arrives in. Only HTTP responses use it; see + # {Rules.classify_http_response}. # @return [Symbol] Canonical event shape - def self.shape_of event + def self.shape_of event, state_status case event when Event::StartUpload, Event::StartUpload.singleton_class :start_upload @@ -370,7 +372,7 @@ def self.shape_of event when Event::RequestFailed classify_request_failed event when Event::HttpResponse - classify_http_response event + classify_http_response event, state_status when Class classify_event_class event else @@ -380,7 +382,8 @@ def self.shape_of event ## # @private - # Top-level transition decision engine. Routes `[state.status, shape_of(event)]` to a recipe and runs it. + # Top-level transition decision engine. Routes `[state.status, shape_of(event, state.status)]` to a recipe + # and runs it. # # @param state [State] Current state # @param event [Object] Input event @@ -388,7 +391,7 @@ def self.shape_of event # @return [Decision] Decision snapshot # @raise [InternalError] If {Rules.shape_of} or {Rules.route} produces an unlisted value def self.decide state, event, config - shape = shape_of event + shape = shape_of event, state.status unless SHAPES.include? shape raise InternalError, "Resumable upload internal error: shape_of returned unknown shape #{shape.inspect}" end @@ -575,7 +578,8 @@ def self.resume_session state, _event, config # @return [Array>] Tuple of [next_state, instructions] def self.begin_transmission state, event, config granularity_str = header_value event.headers, "x-goog-upload-chunk-granularity" - granularity = parse_header_positive_integer granularity_str + granularity = parse_header_non_negative_integer granularity_str + granularity = nil unless granularity&.positive? chunk_size = resolve_chunk_size config.chunk_size, granularity upload_url = header_value event.headers, "x-goog-upload-url" next_state = state.with( @@ -798,9 +802,9 @@ def self.complete_upload_finalized state, event, _config # @return [Array>] Tuple of [next_state, instructions] def self.realign_from_recovery state, event, config server_offset_str = header_value event.headers, "x-goog-upload-size-received" - # A missing or malformed header realigns to 0. The server does not strip this header alone while - # still reporting `active`, so this only has to be safe, not smart. - server_offset = parse_header_positive_integer(server_offset_str) || 0 + # Never nil here: {Rules.classify_http_response} classifies a query answer without a parseable offset + # as Category 2, so it is routed to {Rules.retry_recovery} instead. + server_offset = parse_header_non_negative_integer server_offset_str # `>`, not `!=`: a lower offset is a regression, not progress. Closing on it would reset the backoff, # and a server whose reported offset flips between two values would then query with no delay forever. progressed = state.recovery_offset.nil? || server_offset > state.recovery_offset @@ -984,7 +988,7 @@ def self.fail_with_request_error state, event, _config # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration # @raise [InvalidTransitionError] def self.fail_with_unmatched_transition state, event, _config - shape = shape_of event + shape = shape_of event, state.status action = STATE_DESCRIPTIONS[state.status] || "processing #{state.status}" happened = describe_event event, shape message = "Resumable upload failed while #{action}: #{happened}." @@ -1023,21 +1027,20 @@ def self.describe_event event, shape # @private # Parses a numeric protocol header value strictly. # - # Accepts only a positive decimal integer, optionally surrounded by whitespace. Anything else — - # `nil`, empty, `0`, signed, fractional, or trailing garbage such as `"12abc"` — returns `nil`, so - # callers treat a malformed header the same as a missing one. `String#to_i` is deliberately not - # used: it turns `"12abc"` into `12` and `"-5"` into `-5`. + # Accepts only a non-negative decimal integer, optionally surrounded by whitespace. Anything else — + # `nil`, empty, signed, fractional, or trailing garbage such as `"12abc"` — returns `nil`, so callers + # treat a malformed header the same as a missing one. `String#to_i` is deliberately not used: it turns + # `"12abc"` into `12` and `"-5"` into `-5`. # # @param value [String, nil] Raw header value - # @return [Integer, nil] Parsed value, or `nil` if the value is not a positive integer - def self.parse_header_positive_integer value + # @return [Integer, nil] Parsed value, or `nil` if the value is not a non-negative integer + def self.parse_header_non_negative_integer value return nil unless value.is_a? String stripped = value.strip return nil unless stripped.match?(/\A\d+\z/) - parsed = Integer stripped, 10 - parsed.positive? ? parsed : nil + Integer stripped, 10 end ## @@ -1064,21 +1067,27 @@ def self.resolve_chunk_size user_chunk_size, chunk_granularity # | `X-Goog-Upload-Status` | `200` | in CAT2_STATUS_CODES | any other non-200 | # |------------------------|---------------|----------------|-------------------| # | missing / empty | `cat2` | `cat2` | `fatal_bad_response` | - # | `active` | `active` | `cat2` | `fatal_bad_response` | + # | `active` | `active` (1) | `cat2` | `fatal_bad_response` | # | `final` | `final` | `rejected` | `rejected` | # | `cancelled` | `cancelled` | `fatal_bad_response` | `fatal_bad_response` | # | any other value | `fatal_bad_response` | `fatal_bad_response` | `fatal_bad_response` | # # A headerless `200` is Category 2 regardless of {CAT2_STATUS_CODES}. # + # (1) `cat2` instead if the response lacks a header that the command it answers requires (see + # {Rules.required_headers_present?}). This is the only cell that depends on `state_status`: every status + # that awaits a response awaits exactly one command, so the status says which headers are required. + # # @param response [Event::HttpResponse] Response event + # @param state_status [Symbol] Status of the state the response arrives in # @return [Symbol] Canonical response shape - def self.classify_http_response response + def self.classify_http_response response, state_status status_header = header_value(response.headers, "x-goog-upload-status")&.downcase case status_header when "active" - response.status == 200 ? :response_active : classify_unsuccessful_status(response.status) + return classify_unsuccessful_status response.status unless response.status == 200 + required_headers_present?(response, state_status) ? :response_active : :response_cat2 when "final" response.status == 200 ? :response_final : :response_rejected when "cancelled" @@ -1090,6 +1099,27 @@ def self.classify_http_response response end end + ## + # @private + # Whether a `200` `active` response carries the headers the command it answers needs to be acted on. + # + # * `query` (awaited in `:recovery`): `X-Goog-Upload-Size-Received` must parse as a non-negative integer. + # Treating a missing or malformed offset as `0` would silently restart the upload. + # + # A response that fails this check is Category 2, like a response with no upload status header at all. + # + # @param response [Event::HttpResponse] Response event + # @param state_status [Symbol] Status of the state the response arrives in + # @return [Boolean] + def self.required_headers_present? response, state_status + case state_status + when :recovery + !parse_header_non_negative_integer(header_value(response.headers, "x-goog-upload-size-received")).nil? + else + true + end + end + ## # @private # Classifies a non-200 status whose upload status header is missing, empty or `active`. diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb index 0c8297e..988fdad 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb @@ -118,7 +118,7 @@ def test_rescue_request_error_standard_error assert_equal :unknown, event.kind assert_equal "Unexpected low-level runtime error", event.message assert_same err, event.source_error - assert_equal :request_failed_unknown, Rules.shape_of(event) + assert_equal :request_failed_unknown, Rules.shape_of(event, :transmission_sending) # End-to-end via make_post_request @client_stub.error_to_raise = err diff --git a/gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb b/gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb index c442574..cbb8e60 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb @@ -20,10 +20,12 @@ ## # Tests for strict parsing of the numeric protocol headers `X-Goog-Upload-Size-Received` and -# `X-Goog-Upload-Chunk-Granularity`. A malformed value is treated exactly like a missing one: the offset -# falls back to 0 and the granularity to `nil`. +# `X-Goog-Upload-Chunk-Granularity`. # -# See `design/resumable_upload/implementation-guide.md` sections 3 and 5. +# * A query answer whose offset is missing or malformed is Category 2 and is re-queried with backoff. +# * A granularity that is missing, malformed or zero is treated as absent. +# +# See `design/resumable_upload/implementation-guide.md` sections 4.1, 4.2 and 5. # class HeaderParsingTest < Minitest::Test include Gapic::Rest::ResumableUpload @@ -32,8 +34,11 @@ class HeaderParsingTest < Minitest::Test SESSION_URL = "https://example.com/session" - # Values that must parse as if the header were absent. - UNPARSEABLE = [nil, "", " ", "0", "00", "-5", "+5", "1.5", "12abc", "abc", "0x10", "1e3", "1_000"].freeze + # Delays 1, 2, 4, then capped at 5. + BACKOFF = { initial_delay: 1, multiplier: 2, max_delay: 5, jitter: 0 }.freeze + + # Values that are not a non-negative decimal integer. + MALFORMED = ["", " ", "-5", "+5", "1.5", "12abc", "abc", "0x10", "1e3", "1_000"].freeze class FakeClientStub attr_reader :commands @@ -62,101 +67,131 @@ def setup ) end - # --- Rules.parse_header_positive_integer --- + # --- Rules.parse_header_non_negative_integer --- - def test_parses_positive_decimal_integers - assert_equal 123, Rules.parse_header_positive_integer("123") - assert_equal 123, Rules.parse_header_positive_integer(" 123 ") - assert_equal 8, Rules.parse_header_positive_integer("008") - assert_equal 2**40, Rules.parse_header_positive_integer((2**40).to_s) + def test_parses_non_negative_decimal_integers + assert_equal 0, Rules.parse_header_non_negative_integer("0") + assert_equal 123, Rules.parse_header_non_negative_integer("123") + assert_equal 123, Rules.parse_header_non_negative_integer(" 123 ") + assert_equal 8, Rules.parse_header_non_negative_integer("008") + assert_equal 2**40, Rules.parse_header_non_negative_integer((2**40).to_s) end def test_rejects_everything_else - UNPARSEABLE.each do |value| - assert_nil Rules.parse_header_positive_integer(value), "value #{value.inspect}" + ([nil] + MALFORMED).each do |value| + assert_nil Rules.parse_header_non_negative_integer(value), "value #{value.inspect}" end end - # --- X-Goog-Upload-Chunk-Granularity on the start response --- - - def test_unparseable_granularity_is_treated_as_absent - absent_state, absent_instructions = Rules.step State.new(status: :starting), start_response(nil), @config - - UNPARSEABLE.each do |value| - next_state, instructions = Rules.step State.new(status: :starting), start_response(value), @config + # --- X-Goog-Upload-Size-Received on the query answer --- - assert_nil next_state.chunk_granularity, "value #{value.inspect}" - assert_equal absent_state, next_state, "value #{value.inspect}" - assert_equal absent_instructions, instructions, "value #{value.inspect}" + def test_query_answer_without_a_parseable_offset_is_category_2 + ([nil] + MALFORMED).each do |value| + assert_equal :response_cat2, Rules.classify_http_response(query_response(value), :recovery), + "value #{value.inspect}" end end - def test_valid_granularity_aligns_the_chunk_size - next_state, = Rules.step State.new(status: :starting), start_response(" 3 "), @config - - assert_equal 3, next_state.chunk_granularity - assert_equal 3, next_state.chunk_size + def test_query_answer_with_a_parseable_offset_is_active + ["0", "8", " 8 "].each do |value| + assert_equal :response_active, Rules.classify_http_response(query_response(value), :recovery), + "value #{value.inspect}" + end end - # --- X-Goog-Upload-Size-Received on the query response --- - - def test_unparseable_offset_realigns_to_zero - UNPARSEABLE.each do |value| - state = State.new status: :recovery, upload_url: SESSION_URL, offset: 4, chunk_size: 4, recovery_offset: 4 - next_state, instructions = Rules.step state, query_response(value), @config - - assert_equal :transmission_reading, next_state.status, "value #{value.inspect}" - assert_equal 0, next_state.offset, "value #{value.inspect}" - assert_equal 4, next_state.recovery_offset, "value #{value.inspect}" - realign = instructions.grep(Instruction::RealignBuffer).first - assert_equal 0, realign.server_offset, "value #{value.inspect}" - progress = instructions.grep(Instruction::NotifyProgress).first.progress - assert_equal 0, progress.bytes_uploaded, "value #{value.inspect}" + # The offset is required only of the query answer. Chunk acknowledgements do not carry it. + def test_the_offset_is_not_required_outside_recovery + Rules::STATUSES.reject { |status| status == :recovery }.each do |status| + assert_equal :response_active, Rules.classify_http_response(query_response(nil), status), + "status #{status.inspect}" end end - def test_trailing_garbage_does_not_yield_a_partial_offset - state = State.new status: :recovery, upload_url: SESSION_URL, offset: 0, chunk_size: 4, recovery_offset: 0 - next_state, = Rules.step state, query_response("8abc"), @config + def test_missing_offset_re_queries_with_backoff_instead_of_realigning + state = recovery_state offset: 4, recovery_offset: 4 + decision = Rules.decide state, query_response(nil), @config + + assert_equal :retry_recovery, decision.recipe + assert_equal :response_cat2, decision.shape + assert_equal :recovery, decision.next_state.status + assert_equal 4, decision.next_state.offset + assert_equal 4, decision.next_state.recovery_offset + assert_empty decision.instructions.grep(Instruction::RealignBuffer) + assert decision.instructions.grep(Instruction::SendQuery).first.backoff + end + + def test_zero_offset_realigns_to_zero + state = recovery_state offset: 0, recovery_offset: 0 + next_state, instructions = Rules.step state, query_response("0"), @config + assert_equal :transmission_reading, next_state.status assert_equal 0, next_state.offset + assert_equal 0, instructions.grep(Instruction::RealignBuffer).first.server_offset end def test_valid_offset_is_used - state = State.new status: :recovery, upload_url: SESSION_URL, offset: 4, chunk_size: 4, recovery_offset: 4 + state = recovery_state offset: 4, recovery_offset: 4 next_state, = Rules.step state, query_response(" 8 "), @config assert_equal 8, next_state.offset assert_nil next_state.recovery_offset end + # --- X-Goog-Upload-Chunk-Granularity on the start answer --- + + def test_unparseable_or_zero_granularity_is_treated_as_absent + absent_state, absent_instructions = Rules.step State.new(status: :starting), start_response(nil), @config + + (MALFORMED + ["0", "00"]).each do |value| + next_state, instructions = Rules.step State.new(status: :starting), start_response(value), @config + + assert_nil next_state.chunk_granularity, "value #{value.inspect}" + assert_equal absent_state, next_state, "value #{value.inspect}" + assert_equal absent_instructions, instructions, "value #{value.inspect}" + end + end + + def test_valid_granularity_aligns_the_chunk_size + next_state, = Rules.step State.new(status: :starting), start_response(" 3 "), @config + + assert_equal 3, next_state.chunk_granularity + assert_equal 3, next_state.chunk_size + end + # --- Driver --- - # A negative offset used to reach `stream.seek(-5)` and escape as `Errno::EINVAL`. It now realigns to 0 - # and re-sends the data from the start. - def test_negative_offset_restarts_the_upload_from_zero - outcomes = [ - FakeResponse.new(status: 200, headers: { "X-Goog-Upload-URL" => SESSION_URL, "X-Goog-Upload-Status" => "active" }, - body: ""), - FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable"), - FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active", "X-Goog-Upload-Size-Received" => "-5" }, - body: ""), - FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') - ] + # A negative offset used to reach `stream.seek(-5)` and escape as `Errno::EINVAL`. It is now re-queried. + def test_malformed_offset_is_re_queried_after_a_backoff + outcomes = [initiation_response, http_error(503), query_answer("-5"), query_answer("0"), final_response] stub = FakeClientStub.new outcomes - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("0123"), - upload_size: 4, - chunk_size: 10, - data_plane_retry_policy: { retry_codes: [] }, - control_plane_retry_policy: { initial_delay: 0.01, jitter: 0 } - ) - result = Driver.new(client_stub: stub, config: config).run + result, delays = run_recording_delays stub assert_equal '{"done":true}', result - assert_equal ["start", "upload, finalize", "query", "upload, finalize"], stub.commands + assert_equal [1.0], delays + assert_equal ["start", "upload, finalize", "query", "query", "upload, finalize"], stub.commands + end + + def test_endless_missing_offset_backs_off_until_the_deadline + outcomes = [initiation_response, http_error(503)] + Array.new(50) { query_answer(nil) } + stub = FakeClientStub.new outcomes + + delays = [] + clock = 0.0 + sleep_stub = lambda do |seconds| + delays << seconds.to_f + clock += seconds + end + Process.stub :clock_gettime, ->(*) { clock } do + Kernel.stub :sleep, sleep_stub do + assert_raises DeadlineExceededError do + run_upload stub, timeout: 20 + end + end + end + + assert_equal [1.0, 2.0, 4.0, 5.0, 5.0], delays.first(5) + refute_includes stub.commands.drop(2), "upload, finalize" end # --- UploadLog --- @@ -166,13 +201,18 @@ def test_wire_receive_logs_only_parseable_numeric_headers refute fields.key?("sizeReceived") refute fields.key?("granularity") - fields = wire_receive_fields "X-Goog-Upload-Size-Received" => "12", "X-Goog-Upload-Chunk-Granularity" => "256" - assert_equal 12, fields["sizeReceived"] + fields = wire_receive_fields "X-Goog-Upload-Size-Received" => "0", "X-Goog-Upload-Chunk-Granularity" => "256" + assert_equal 0, fields["sizeReceived"] assert_equal 256, fields["granularity"] end private + def recovery_state offset:, recovery_offset: + State.new status: :recovery, upload_url: SESSION_URL, offset: offset, chunk_size: 4, + recovery_offset: recovery_offset + end + def start_response granularity headers = { "x-goog-upload-status" => "active", "x-goog-upload-url" => SESSION_URL } headers["x-goog-upload-chunk-granularity"] = granularity unless granularity.nil? @@ -185,6 +225,46 @@ def query_response received Event::HttpResponse.new status: 200, headers: headers end + def run_recording_delays stub + delays = [] + result = Kernel.stub :sleep, ->(seconds) { delays << seconds.to_f } do + run_upload stub + end + [result, delays] + end + + def run_upload stub, **overrides + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + data_plane_retry_policy: { retry_codes: [] }, + control_plane_retry_policy: BACKOFF, + **overrides + ) + Driver.new(client_stub: stub, config: config).run + end + + def initiation_response + FakeResponse.new status: 200, headers: { "X-Goog-Upload-URL" => SESSION_URL, "X-Goog-Upload-Status" => "active" }, + body: "" + end + + def query_answer received + headers = { "X-Goog-Upload-Status" => "active" } + headers["X-Goog-Upload-Size-Received"] = received unless received.nil? + FakeResponse.new status: 200, headers: headers, body: "" + end + + def final_response + FakeResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}' + end + + def http_error status + Faraday::ServerError.new "the server responded with status #{status}", { status: status, headers: {}, body: "" } + end + def wire_receive_fields headers recording = RecordingLogger.new stub_logger = Gapic::LoggingConcerns::StubLogger.new logger: recording, service: "ResumableUpload" diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb index 5164f5f..97fda62 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb @@ -48,52 +48,52 @@ def test_header_value_missing_or_non_hash def test_classify_http_response_active resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: "" - assert_equal :response_active, Rules.classify_http_response(resp_200) + assert_equal :response_active, Rules.classify_http_response(resp_200, :transmission_sending) # Value case variations ["Active", "ACTIVE", "aCtIvE"].each do |val| resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => val }, body: "" - assert_equal :response_active, Rules.classify_http_response(resp) + assert_equal :response_active, Rules.classify_http_response(resp, :transmission_sending) end # Non-200 with active maps to Category 2 [503, 500, 400, 408].each do |code| resp_non_200 = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "active" }, body: "" - assert_equal :response_cat2, Rules.classify_http_response(resp_non_200) + assert_equal :response_cat2, Rules.classify_http_response(resp_non_200, :transmission_sending) end end def test_classify_http_response_final resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: "" - assert_equal :response_final, Rules.classify_http_response(resp_200) + assert_equal :response_final, Rules.classify_http_response(resp_200, :transmission_sending) # Value case variations ["Final", "FINAL"].each do |val| resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => val }, body: "" - assert_equal :response_final, Rules.classify_http_response(resp) + assert_equal :response_final, Rules.classify_http_response(resp, :transmission_sending) end # Non-200 with final maps to response_rejected [400, 404, 500].each do |code| resp_non_200 = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "final" }, body: "" - assert_equal :response_rejected, Rules.classify_http_response(resp_non_200) + assert_equal :response_rejected, Rules.classify_http_response(resp_non_200, :transmission_sending) end end def test_classify_http_response_cancelled resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "cancelled" }, body: "" - assert_equal :response_cancelled, Rules.classify_http_response(resp_200) + assert_equal :response_cancelled, Rules.classify_http_response(resp_200, :transmission_sending) # Value case variations ["Cancelled", "CANCELLED"].each do |val| resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => val }, body: "" - assert_equal :response_cancelled, Rules.classify_http_response(resp) + assert_equal :response_cancelled, Rules.classify_http_response(resp, :transmission_sending) end # Non-200 with cancelled maps to fatal bad response [400, 500].each do |code| resp_non_200 = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "cancelled" }, body: "" - assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_non_200) + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_non_200, :transmission_sending) end end @@ -106,10 +106,10 @@ def test_cat2_status_codes_are_the_retriable_sets_plus_the_recoverable_extras def test_classify_http_response_headerless_200_is_cat2 resp_200 = Event::HttpResponse.new status: 200, headers: {}, body: "" - assert_equal :response_cat2, Rules.classify_http_response(resp_200) + assert_equal :response_cat2, Rules.classify_http_response(resp_200, :transmission_sending) resp_empty = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "" }, body: "" - assert_equal :response_cat2, Rules.classify_http_response(resp_empty) + assert_equal :response_cat2, Rules.classify_http_response(resp_empty, :transmission_sending) end # A non-200 without a definitive status header is Category 2 only when its status is listed in @@ -118,7 +118,7 @@ def test_classify_http_response_non_200_cat2_statuses [{}, { "X-Goog-Upload-Status" => "" }, { "X-Goog-Upload-Status" => "active" }].each do |headers| Rules::CAT2_STATUS_CODES.each do |code| resp = Event::HttpResponse.new status: code, headers: headers, body: "" - assert_equal :response_cat2, Rules.classify_http_response(resp), + assert_equal :response_cat2, Rules.classify_http_response(resp, :transmission_sending), "Expected #{code} with #{headers.inspect} to classify as :response_cat2" end end @@ -129,7 +129,7 @@ def test_classify_http_response_non_200_unlisted_statuses_are_fatal [{}, { "X-Goog-Upload-Status" => "" }, { "X-Goog-Upload-Status" => "active" }].each do |headers| [301, 401, 403, 404, 405, 410, 413, 415, 418, 501, 505, 599].each do |code| resp = Event::HttpResponse.new status: code, headers: headers, body: "" - assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp), + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp, :transmission_sending), "Expected #{code} with #{headers.inspect} to classify as :response_fatal_bad_response" end end @@ -138,10 +138,10 @@ def test_classify_http_response_non_200_unlisted_statuses_are_fatal def test_classify_http_response_unknown_header_values ["absconded", "pending", "in_progress", "error", "unknown"].each do |unknown_val| resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => unknown_val }, body: "" - assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_200) + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_200, :transmission_sending) resp_400 = Event::HttpResponse.new status: 400, headers: { "X-Goog-Upload-Status" => unknown_val }, body: "" - assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_400) + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_400, :transmission_sending) end end @@ -149,55 +149,55 @@ def test_classify_http_response_header_key_casing keys = ["x-goog-upload-status", "X-GOOG-UPLOAD-STATUS", "X-Goog-Upload-Status", :"x-goog-upload-status"] keys.each do |key| resp = Event::HttpResponse.new status: 200, headers: { key => "active" }, body: "" - assert_equal :response_active, Rules.classify_http_response(resp) + assert_equal :response_active, Rules.classify_http_response(resp, :transmission_sending) end end def test_shape_of_control_events - assert_equal :start_upload, Rules.shape_of(Event::StartUpload.new) - assert_equal :start_upload, Rules.shape_of(Event::StartUpload) - assert_equal :resume_upload, Rules.shape_of(Event::ResumeUpload.new) - assert_equal :resume_upload, Rules.shape_of(Event::ResumeUpload) - assert_equal :user_cancel, Rules.shape_of(Event::Cancel.new) - assert_equal :user_cancel, Rules.shape_of(Event::Cancel) - assert_equal :global_deadline_exceeded, Rules.shape_of(Event::GlobalDeadlineExceeded.new) - assert_equal :global_deadline_exceeded, Rules.shape_of(Event::GlobalDeadlineExceeded) + assert_equal :start_upload, Rules.shape_of(Event::StartUpload.new, :transmission_sending) + assert_equal :start_upload, Rules.shape_of(Event::StartUpload, :transmission_sending) + assert_equal :resume_upload, Rules.shape_of(Event::ResumeUpload.new, :transmission_sending) + assert_equal :resume_upload, Rules.shape_of(Event::ResumeUpload, :transmission_sending) + assert_equal :user_cancel, Rules.shape_of(Event::Cancel.new, :transmission_sending) + assert_equal :user_cancel, Rules.shape_of(Event::Cancel, :transmission_sending) + assert_equal :global_deadline_exceeded, Rules.shape_of(Event::GlobalDeadlineExceeded.new, :transmission_sending) + assert_equal :global_deadline_exceeded, Rules.shape_of(Event::GlobalDeadlineExceeded, :transmission_sending) end def test_shape_of_chunk_read full_chunk = Event::ChunkRead.new bytes_buffered: 4096, eof: false - assert_equal :chunk_read_full, Rules.shape_of(full_chunk) + assert_equal :chunk_read_full, Rules.shape_of(full_chunk, :transmission_sending) eof_data = Event::ChunkRead.new bytes_buffered: 1024, eof: true - assert_equal :chunk_read_eof_with_data, Rules.shape_of(eof_data) + assert_equal :chunk_read_eof_with_data, Rules.shape_of(eof_data, :transmission_sending) eof_empty = Event::ChunkRead.new bytes_buffered: 0, eof: true - assert_equal :chunk_read_eof_empty, Rules.shape_of(eof_empty) + assert_equal :chunk_read_eof_empty, Rules.shape_of(eof_empty, :transmission_sending) end def test_shape_of_request_failed timeout = Event::RequestFailed.new kind: :timeout, message: "read timeout" - assert_equal :request_timeout, Rules.shape_of(timeout) + assert_equal :request_timeout, Rules.shape_of(timeout, :transmission_sending) exhausted = Event::RequestFailed.new kind: :retries_exhausted, message: "exhausted" - assert_equal :request_retries_exhausted, Rules.shape_of(exhausted) + assert_equal :request_retries_exhausted, Rules.shape_of(exhausted, :transmission_sending) conn_failed = Event::RequestFailed.new kind: :connection_failed, message: "dropped" - assert_equal :request_connection_failed, Rules.shape_of(conn_failed) + assert_equal :request_connection_failed, Rules.shape_of(conn_failed, :transmission_sending) other = Event::RequestFailed.new kind: :other, message: "unknown error" - assert_equal :request_failed_unknown, Rules.shape_of(other) + assert_equal :request_failed_unknown, Rules.shape_of(other, :transmission_sending) end def test_shape_of_http_response_delegates_to_classify resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: "" - assert_equal :response_active, Rules.shape_of(resp) + assert_equal :response_active, Rules.shape_of(resp, :transmission_sending) end def test_shape_of_unknown_event - assert_equal :unknown, Rules.shape_of(Object.new) - assert_equal :unknown, Rules.shape_of(nil) - assert_equal :unknown, Rules.shape_of("unrecognized_event") + assert_equal :unknown, Rules.shape_of(Object.new, :transmission_sending) + assert_equal :unknown, Rules.shape_of(nil, :transmission_sending) + assert_equal :unknown, Rules.shape_of("unrecognized_event", :transmission_sending) end ## @@ -234,7 +234,7 @@ def test_shapes_constant_is_exhaustive_and_minimal corpus = shape_corpus corpus.each do |expected_shape, event| - assert_equal expected_shape, Rules.shape_of(event), + assert_equal expected_shape, Rules.shape_of(event, :transmission_sending), "Corpus event for #{expected_shape} no longer classifies as that shape" end From b0854f254678bd5a153d873a1738652aa954baa4 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Mon, 28 Sep 2026 21:09:13 +0000 Subject: [PATCH 22/22] fix: fail a start response that has no upload URL `begin_transmission` read `X-Goog-Upload-URL` without checking it, so an `active` 200 without the header put `upload_url: nil` into state and the first chunk was sent to nil. `Rules.required_headers_present?` now requires a non-blank upload URL on the answer to `start`. Such an answer is Category 2, which in `:starting` routes to `fail_with_bad_response`: a `BadResponseError` with no resume handle, since there is no session to resume. It is not re-sent within the start budget, because `RetryDecider` only re-sends a 200 that lacks `X-Goog-Upload-Status`. --- .../resumable_upload/implementation-guide.md | 3 +- .../lib/gapic/rest/resumable_upload/rules.rb | 6 ++ .../resumable_upload/header_parsing_test.rb | 57 +++++++++++++++---- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md index 79404c5..ec11361 100644 --- a/gapic-common/design/resumable_upload/implementation-guide.md +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -358,7 +358,7 @@ Source: `lib/gapic/rest/resumable_upload/driver.rb` | **`Initializing`** | `:resume_upload` | `Event::ResumeUpload` | `upload_url = event.upload_url`
`chunk_size = event.chunk_size`
`offset = 0`
`recovery_offset = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: event.upload_size))`
`Instruction::SendQuery.new(url: event.upload_url, backoff: false)` | | **`Starting`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `upload_url = headers['X-Goog-Upload-URL']`
`chunk_granularity = parse_header_non_negative_integer(headers['...-Granularity'])`, `nil` unless positive
`chunk_size = resolve(config, chunk_granularity)`
`offset = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | | **`Starting`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | -| **`Starting`** | `:response_cat2` / `:response_fatal_bad_response` | `Event::HttpResponse` (Non-200, or a headerless `200` after the start budget is spent; see Section 6.1) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Starting`** | `:response_cat2` / `:response_fatal_bad_response` | `Event::HttpResponse` (Non-200, a headerless `200` after the start budget is spent, or an `active` `200` without `X-Goog-Upload-URL`; see Section 6.1) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | | **`Starting`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:request_failed_unknown` | `Event::RequestFailed(kind:, message:, source_error:)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | | **`Transmission \| Reading from stream`** | `:chunk_read_full` | `Event::ChunkRead(bytes_buffered, eof: false)` | `in_flight_length = event.bytes_buffered`
`status = :transmission_sending` | `Transmission \| Sending` | `Instruction::SendChunk.new(url: state.upload_url, offset: state.offset, length: event.bytes_buffered, finalize: false)` | | **`Transmission \| Reading from stream`** | `:chunk_read_eof_with_data` | `Event::ChunkRead(bytes_buffered, eof: true)` where `bytes_buffered > 0` | `in_flight_length = event.bytes_buffered`
`status = :finalizing_sending_upload` | `Finalizing \| Sending with upload` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :finalizing, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendChunk.new(url: state.upload_url, offset: state.offset, length: event.bytes_buffered, finalize: true)` | @@ -531,6 +531,7 @@ The implementation distinguishes three categories of network and protocol-level 1. **Headerless `200`**: A `200` whose `X-Goog-Upload-Status` is missing or empty, regardless of `CAT2_STATUS_CODES`. A proxy that strips the protocol headers from a successful answer leaves the offset unknown, which is exactly what recovery resolves. 2. **Listed Non-200 Statuses**: A non-200 whose `X-Goog-Upload-Status` is missing, empty or `active`, **and** whose status is in `Rules::CAT2_STATUS_CODES`: `RETRIABLE_4XX_STATUS_CODES` (`409`, `429`, `499`) + `RETRIABLE_5XX_STATUS_CODES` (`500`, `503`, `504`) + `400`, `408`, `412`, `416`, `502`. The list is an allowlist and is load-bearing: any status it does not list is Category 3 (Section 6.1.3). 3. **`active` `200` Missing a Required Header**: A `200` with `X-Goog-Upload-Status: active` that lacks a header the awaited command requires (`Rules.required_headers_present?`). This is the only classification that depends on the state: `Rules.shape_of(event, state.status)` passes the status through, and each status that awaits a response awaits exactly one command. The required headers are: + * `start` (in `Starting`): `X-Goog-Upload-URL` must be present and non-blank. There is no session to query, so this is terminal (`fail_with_bad_response` → `BadResponseError`, no resume handle). Unlike a headerless `200`, it is not re-sent within the start budget: `RetryDecider` sees an upload status header and surfaces it. * `query` (in `Recovery`): `X-Goog-Upload-Size-Received` must parse as a non-negative integer (Section 4.1, item 2). The client does not guess an offset; it re-queries with backoff. * **Data Plane Transport Failures**: `:request_connection_failed` and `:request_timeout` during `Transmission` or `Finalizing` also enter `Recovery` (`enter_recovery`), because the Driver does not re-send them on the data plane (Section 6.1.1, rows 2–3). * **Where Category 2 Arrives, per Plane**: diff --git a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb index 5c949f6..f22f661 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb @@ -1103,16 +1103,22 @@ def self.classify_http_response response, state_status # @private # Whether a `200` `active` response carries the headers the command it answers needs to be acted on. # + # * `start` (awaited in `:starting`): `X-Goog-Upload-URL` must be present and non-blank. Without it there + # is no session to upload to. # * `query` (awaited in `:recovery`): `X-Goog-Upload-Size-Received` must parse as a non-negative integer. # Treating a missing or malformed offset as `0` would silently restart the upload. # # A response that fails this check is Category 2, like a response with no upload status header at all. + # {Rules.route} then decides per status: `:starting` fails with {BadResponseError}, `:recovery` + # re-queries. # # @param response [Event::HttpResponse] Response event # @param state_status [Symbol] Status of the state the response arrives in # @return [Boolean] def self.required_headers_present? response, state_status case state_status + when :starting + !header_value(response.headers, "x-goog-upload-url").to_s.strip.empty? when :recovery !parse_header_non_negative_integer(header_value(response.headers, "x-goog-upload-size-received")).nil? else diff --git a/gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb b/gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb index cbb8e60..cdccf24 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/header_parsing_test.rb @@ -19,13 +19,14 @@ require "stringio" ## -# Tests for strict parsing of the numeric protocol headers `X-Goog-Upload-Size-Received` and -# `X-Goog-Upload-Chunk-Granularity`. +# Tests for the protocol headers an `active` answer must carry, and for strict parsing of the numeric ones. # -# * A query answer whose offset is missing or malformed is Category 2 and is re-queried with backoff. -# * A granularity that is missing, malformed or zero is treated as absent. +# * A query answer whose `X-Goog-Upload-Size-Received` is missing or malformed is Category 2 and is re-queried +# with backoff. +# * A start answer without `X-Goog-Upload-URL` is Category 2, which is terminal in `:starting`. +# * An `X-Goog-Upload-Chunk-Granularity` that is missing, malformed or zero is treated as absent. # -# See `design/resumable_upload/implementation-guide.md` sections 4.1, 4.2 and 5. +# See `design/resumable_upload/implementation-guide.md` sections 4.1, 4.2, 5 and 6.1.2. # class HeaderParsingTest < Minitest::Test include Gapic::Rest::ResumableUpload @@ -99,14 +100,45 @@ def test_query_answer_with_a_parseable_offset_is_active end end - # The offset is required only of the query answer. Chunk acknowledgements do not carry it. - def test_the_offset_is_not_required_outside_recovery - Rules::STATUSES.reject { |status| status == :recovery }.each do |status| - assert_equal :response_active, Rules.classify_http_response(query_response(nil), status), - "status #{status.inspect}" + # Each status requires only the headers of the command it awaits. Chunk acknowledgements carry neither. + def test_no_header_is_required_outside_starting_and_recovery + bare_active = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "active" } + (Rules::STATUSES - [:starting, :recovery]).each do |status| + assert_equal :response_active, Rules.classify_http_response(bare_active, status), "status #{status.inspect}" end end + # --- X-Goog-Upload-URL on the start answer --- + + def test_start_answer_without_an_upload_url_is_category_2 + [nil, "", " "].each do |url| + assert_equal :response_cat2, Rules.classify_http_response(start_response(nil, url: url), :starting), + "url #{url.inspect}" + end + assert_equal :response_active, Rules.classify_http_response(start_response(nil), :starting) + end + + def test_start_answer_without_an_upload_url_fails_with_a_bad_response + decision = Rules.decide State.new(status: :starting), start_response(nil, url: nil), @config + + assert_equal :fail_with_bad_response, decision.recipe + assert_equal :error, decision.next_state.status + assert_instance_of BadResponseError, decision.next_state.last_error + assert_nil Rules.resume_handle_from(decision.next_state) + end + + def test_driver_fails_without_a_resume_handle_when_the_upload_url_is_missing + initiation = FakeResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: "" + stub = FakeClientStub.new [initiation] + + error = assert_raises BadResponseError do + run_upload stub + end + + assert_nil error.resume_handle + assert_equal ["start"], stub.commands + end + def test_missing_offset_re_queries_with_backoff_instead_of_realigning state = recovery_state offset: 4, recovery_offset: 4 decision = Rules.decide state, query_response(nil), @config @@ -213,8 +245,9 @@ def recovery_state offset:, recovery_offset: recovery_offset: recovery_offset end - def start_response granularity - headers = { "x-goog-upload-status" => "active", "x-goog-upload-url" => SESSION_URL } + def start_response granularity, url: SESSION_URL + headers = { "x-goog-upload-status" => "active" } + headers["x-goog-upload-url"] = url unless url.nil? headers["x-goog-upload-chunk-granularity"] = granularity unless granularity.nil? Event::HttpResponse.new status: 200, headers: headers end