From 49cdbcddfc140b2f15198721175cab2b32e1f36b Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Fri, 31 Jul 2026 18:07:29 -0700 Subject: [PATCH 01/10] Verify Swift against the shared conformance fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Swift was the only SDK not fixture-verified — the #508/#509 retry-lifecycle bugs lived exactly where no runner watched. This runner executes every mock fixture in conformance/tests through the SDK's public Transport seam (no @testable): a scripted per-test response queue with network-error injection, monotonic request timing, and full request capture, feeding the shared assertion vocabulary (headerAbsent, index-aware headerPresent, errorMessage substrings, responseMeta, responseBody with 64-bit integer fidelity). Ported from the Kotlin runner at current main, including its policies: a mock body that flunks required-field decoding is a FAIL (a fixture bug to fix), not a silent skip; link-header requestCount fixtures are skipped because the SDK auto-paginates by design. Swift-specific mechanics: - The package depends on the ROOT distribution manifest: SwiftPM derives package identity from the directory name, so a path dependency on ".../swift" collides with conformance/runner/swift's own identity and is treated as a self-reference. Both manifests build the same swift/Sources/Basecamp sources. - HTTPS enforcement is a preconditionFailure (a trap, not a thrown error), so the runner re-executes itself as a --https-probe subprocess and expects the child to die; a surviving child is a FAIL, never a silent pass. Result today: 121 passed, 0 failed, 3 skipped of 124 — identical to Kotlin. The two download hop-1 retry skips are TEMPORARY until B4 threads retry through performDownloadRequest (proven to fail today via SWIFT_CONFORMANCE_NO_SKIPS=1: 1 request where 4 and 3 are expected). --- .gitignore | 3 + conformance/runner/swift/Package.swift | 33 ++ .../ConformanceRunner/Assertions.swift | 344 ++++++++++++++++ .../Sources/ConformanceRunner/Dispatch.swift | 384 ++++++++++++++++++ .../Sources/ConformanceRunner/Fixtures.swift | 177 ++++++++ .../Sources/ConformanceRunner/Runner.swift | 257 ++++++++++++ .../ConformanceRunner/ScriptedTransport.swift | 143 +++++++ 7 files changed, 1341 insertions(+) create mode 100644 conformance/runner/swift/Package.swift create mode 100644 conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift create mode 100644 conformance/runner/swift/Sources/ConformanceRunner/Dispatch.swift create mode 100644 conformance/runner/swift/Sources/ConformanceRunner/Fixtures.swift create mode 100644 conformance/runner/swift/Sources/ConformanceRunner/Runner.swift create mode 100644 conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift diff --git a/.gitignore b/.gitignore index 971672f490..77a6066e03 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ conformance/runner/typescript/node_modules # test run cannot dirty the working tree and trip the release clean-tree guard. node_modules/ .vite/ +conformance/runner/swift/.build/ +conformance/runner/swift/.swiftpm/ +conformance/runner/swift/Package.resolved # Claude Code session files (except skills, which are committed) .claude/* diff --git a/conformance/runner/swift/Package.swift b/conformance/runner/swift/Package.swift new file mode 100644 index 0000000000..77a492339d --- /dev/null +++ b/conformance/runner/swift/Package.swift @@ -0,0 +1,33 @@ +// swift-tools-version: 6.0 +import PackageDescription + +// Development-only conformance runner. Depends on the Swift SDK via the ROOT +// distribution manifest (../../../Package.swift), whose Basecamp target builds +// the same swift/Sources/Basecamp sources as the development manifest. +// +// Why not swift/Package.swift directly: SwiftPM derives package identity from +// the directory name, so a path dependency on ".../swift" collides with this +// package's own "conformance/runner/swift" identity and is silently treated +// as a self-reference ("product 'Basecamp' ... not found"). The root manifest +// avoids the collision without renaming either directory. +let package = Package( + name: "ConformanceRunner", + platforms: [ + .macOS(.v12) + ], + dependencies: [ + .package(name: "Basecamp", path: "../../..") + ], + targets: [ + .executableTarget( + name: "ConformanceRunner", + dependencies: [ + "Basecamp" + ], + path: "Sources/ConformanceRunner", + swiftSettings: [ + .swiftLanguageMode(.v6) + ] + ) + ] +) diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift new file mode 100644 index 0000000000..b28fc53b3b --- /dev/null +++ b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift @@ -0,0 +1,344 @@ +import Basecamp +import Foundation + +/// Outcome of a single conformance test. +struct TestResult { + let passed: Bool + let message: String + var skipped: Bool = false + + static func fail(_ message: String) -> TestResult { + TestResult(passed: false, message: message) + } +} + +/// SDK-observed values captured from a dispatched operation. +struct DispatchResult { + /// X-Total-Count as parsed by the SDK into ListResult.meta.totalCount. + var totalCount: Int? = nil + /// True when the SDK truncated results (maxPages/maxItems cap hit). + var truncated: Bool? = nil + /// The deserialized SDK response re-serialized to JSON (responseBody assertions). + var resultJSON: JSON? = nil +} + +/// Maps a BasecampError onto the conformance error-code vocabulary shared by +/// every runner (auth_required, not_found, ...). +func conformanceCode(_ error: BasecampError) -> String { + switch error { + case .auth: "auth_required" + case .forbidden: "forbidden" + case .notFound: "not_found" + case .rateLimit: "rate_limit" + case .validation: "validation" + case .api: "api_error" + case .network: "network" + case .usage: "usage" + case .ambiguous: "ambiguous" + } +} + +private let knownErrorTypes: Set = [ + "not_found", "auth_required", "forbidden", "rate_limit", + "validation", "api_error", "usage", "network", +] + +/// Resolves a per-request assertion index (0-based; negative = from the end) +/// against the number of recorded requests. Nil when out of range. +private func resolveRequestIndex(_ index: Int, _ count: Int) -> Int? { + let resolved = index < 0 ? count + index : index + return (0.. String? { + guard let expected else { return "\(label): expected value is null in assertion" } + if let expInt = expected.intValue, let actInt = actual.intValue { + return expInt == actInt ? nil : "Expected \(label) = \(expInt), got \(actInt)" + } + return expected == actual ? nil : "Expected \(label) = \(expected.display), got \(actual.display)" +} + +/// Compares an expected fixture value against a loosely-typed actual value +/// (error fields, response meta). Nil means equal. +private func compareValue(_ label: String, _ expected: JSON?, _ actual: Any?) -> String? { + guard let expected else { return "\(label): expected value is null in assertion" } + switch expected { + case .bool(let b): + if (actual as? Bool) != b { return "Expected \(label) = \(b), got \(String(describing: actual))" } + case .int(let i): + let actualInt: Int64? = switch actual { + case let n as Int: Int64(n) + case let n as Int64: n + default: nil + } + if actualInt != i { return "Expected \(label) = \(i), got \(String(describing: actual))" } + case .string(let s): + let actualString = actual.map { "\($0)" } + if actualString != s { return "Expected \(label) = \"\(s)\", got \(String(describing: actual))" } + default: + let actualString = actual.map { "\($0)" } + if actualString != expected.display { return "Expected \(label) = \(expected.display), got \(String(describing: actual))" } + } + return nil +} + +/// Evaluates every assertion in the fixture against the recorded transport +/// traffic and dispatch outcome. Direct port of the Kotlin evaluator. +func evaluateAssertions( + _ tc: TestCase, + transport: ScriptedTransport, + caughtError: BasecampError?, + httpStatus: Int?, + dispatch: DispatchResult, + autoPaginates: Bool +) -> TestResult { + let captured = transport.captured + let requestCount = captured.count + + // Implicit method invariant: the scripted transport answers any verb, so a + // wrong-verb request (e.g. a PUT regressing to POST) would consume a queued + // response silently. When the fixture declares a method and carries no + // explicit requestMethod assertions, the first request must use it. + let fixtureMethod = tc.fixtureMethod.uppercased() + if !fixtureMethod.isEmpty, + !tc.allAssertions.contains(where: { $0.type == "requestMethod" }), + let first = captured.first, first.method != fixtureMethod { + return .fail("Expected first request method \(fixtureMethod), got \(first.method)") + } + + for assertion in tc.allAssertions { + switch assertion.type { + case "requestCount": + guard let expected = assertion.expected?.intValue.map(Int.init) else { + return .fail("requestCount assertion missing expected value") + } + if autoPaginates { + if requestCount < expected { + return .fail("Expected >= \(expected) requests (SDK auto-paginates), got \(requestCount)") + } + } else if requestCount != expected { + return .fail("Expected \(expected) requests, got \(requestCount)") + } + + case "statusCode", "responseStatus": + guard let expected = assertion.expected?.intValue.map(Int.init) else { + return .fail("\(assertion.type) assertion missing expected value") + } + guard let actual = httpStatus else { + return .fail("Expected status code \(expected), but got no response") + } + if actual != expected { + return .fail("Expected status code \(expected), got \(actual)") + } + + case "responseBody": + let fieldPath = assertion.fieldPath + guard let result = dispatch.resultJSON else { + return .fail("responseBody.\(fieldPath): no result captured from operation") + } + guard let actual = result.navigate(fieldPath) else { + return .fail("responseBody.\(fieldPath): field not found in result") + } + if let failure = compareJSON("responseBody.\(fieldPath)", assertion.expected, actual) { + return .fail(failure) + } + + case "noError": + if let caughtError { + return .fail("Expected no error, got: \(caughtError.message)") + } + + case "requestPath": + guard let expected = assertion.expected?.stringValue else { + return .fail("requestPath assertion missing expected value") + } + guard let idx = resolveRequestIndex(assertion.requestIndex, requestCount) else { + return .fail("requestPath[\(assertion.requestIndex)]: no request recorded at that index (\(requestCount) requests)") + } + if captured[idx].path != expected { + return .fail("Expected request path \"\(expected)\" at index \(assertion.requestIndex), got \"\(captured[idx].path)\"") + } + + case "requestMethod": + guard let expected = assertion.expected?.stringValue?.uppercased() else { + return .fail("requestMethod assertion missing expected value") + } + guard let idx = resolveRequestIndex(assertion.requestIndex, requestCount) else { + return .fail("requestMethod[\(assertion.requestIndex)]: no request recorded at that index (\(requestCount) requests)") + } + if captured[idx].method != expected { + return .fail("Expected request method \(expected) at index \(assertion.requestIndex), got \(captured[idx].method)") + } + + case "requestBody": + let key = assertion.fieldPath + guard let idx = resolveRequestIndex(assertion.requestIndex, requestCount) else { + return .fail("requestBody.\(key)[\(assertion.requestIndex)]: no request recorded at that index (\(requestCount) requests)") + } + guard let body = captured[idx].bodyJSON else { + return .fail("requestBody.\(key)[\(assertion.requestIndex)]: request has no JSON body") + } + guard let actual = body.navigate(key) else { + return .fail("requestBody.\(key)[\(assertion.requestIndex)]: key not present in request body") + } + if let failure = compareJSON("requestBody.\(key)[\(assertion.requestIndex)]", assertion.expected, actual) { + return .fail(failure) + } + + case "requestBodyAbsent": + let key = assertion.fieldPath + guard let idx = resolveRequestIndex(assertion.requestIndex, requestCount) else { + return .fail("requestBodyAbsent.\(key)[\(assertion.requestIndex)]: no request recorded at that index (\(requestCount) requests)") + } + if let body = captured[idx].bodyJSON, body.navigate(key) != nil { + return .fail("requestBodyAbsent.\(key)[\(assertion.requestIndex)]: key unexpectedly present in request body") + } + + case "delayBetweenRequests": + if captured.count >= 2 { + let delay = captured[1].monotonicMs - captured[0].monotonicMs + let minDelay = UInt64(assertion.minValue) + if delay < minDelay { + return .fail("Expected delay >= \(minDelay)ms, got \(delay)ms") + } + } + + case "headerValue": + let headerName = assertion.fieldPath + guard let expected = assertion.expected?.stringValue else { + return .fail("headerValue assertion missing expected value") + } + if headerName.lowercased() == "x-total-count" { + let actual = dispatch.totalCount.map(String.init) + if actual != expected { + return .fail("SDK meta.totalCount: expected \(expected), got \(actual ?? "nil")") + } + } else { + guard let first = tc.responses.first else { + return .fail("Expected response header \(headerName)=\(expected), but no mock responses defined") + } + let actual = first.allHeaders[headerName] + if actual != expected { + return .fail("Expected response header \(headerName)=\(expected), got \(actual ?? "nil")") + } + } + + case "errorType", "errorCode": + guard let expected = assertion.expected?.stringValue else { + return .fail("\(assertion.type) assertion missing expected value") + } + guard let caughtError else { + return .fail("Expected error \(assertion.type == "errorType" ? "type" : "code") \"\(expected)\", but got no error") + } + if assertion.type == "errorType", !knownErrorTypes.contains(expected) { + return .fail("Unknown conformance error type \"\(expected)\"") + } + let actual = conformanceCode(caughtError) + if actual != expected { + return .fail("Expected error code \"\(expected)\", got \"\(actual)\"") + } + + case "errorMessage": + guard let expected = assertion.expected?.stringValue else { + return .fail("errorMessage assertion missing expected value") + } + guard let caughtError else { + return .fail("Expected error message containing \"\(expected)\", but got no error") + } + if !caughtError.message.contains(expected) { + return .fail("Expected error message containing \"\(expected)\", got \"\(caughtError.message)\"") + } + + case "errorField": + let fieldPath = assertion.fieldPath + guard let caughtError else { + return .fail("Expected error field \(fieldPath), but got no error") + } + let actual: Any? = switch fieldPath { + case "httpStatus": caughtError.httpStatusCode + case "retryable": caughtError.isRetryable + case "code": conformanceCode(caughtError) + case "message": caughtError.message + case "requestId": caughtError.requestId + default: nil + } + if actual == nil, !["httpStatus", "retryable", "code", "message", "requestId"].contains(fieldPath) { + return .fail("Unknown error field: \(fieldPath)") + } + if let failure = compareValue("error.\(fieldPath)", assertion.expected, actual) { + return .fail(failure) + } + + case "headerInjected": + let headerName = assertion.fieldPath + guard let expected = assertion.expected?.stringValue else { + return .fail("headerInjected assertion missing expected value") + } + guard let first = captured.first else { + return .fail("Expected header \(headerName)=\"\(expected)\", but no requests were recorded") + } + let actual = first.header(headerName) + // Content-Type may include charset (e.g., "application/json; charset=utf-8") + let matches: Bool = if headerName.lowercased() == "content-type" { + actual?.lowercased().hasPrefix(expected.lowercased()) ?? false + } else { + actual == expected + } + if !matches { + return .fail("Expected header \(headerName)=\"\(expected)\", got \"\(actual ?? "nil")\"") + } + + case "headerPresent": + let headerName = assertion.fieldPath + guard let idx = resolveRequestIndex(assertion.requestIndex, requestCount) else { + return .fail("headerPresent \(headerName)[\(assertion.requestIndex)]: no request recorded at that index (\(requestCount) requests)") + } + let actual = captured[idx].header(headerName) + if actual == nil || actual?.isEmpty == true { + return .fail("Expected header \(headerName) present on request index \(idx), but it was empty or missing") + } + + case "headerAbsent": + let headerName = assertion.fieldPath + guard let idx = resolveRequestIndex(assertion.requestIndex, requestCount) else { + return .fail("headerAbsent \(headerName)[\(assertion.requestIndex)]: no request recorded at that index (\(requestCount) requests)") + } + // A present-but-empty header must fail an absence assertion, same + // as the Go runner's Values check. + if let actual = captured[idx].header(headerName) { + return .fail("Expected header \(headerName) absent on request index \(idx), got \"\(actual)\"") + } + + case "requestScheme": + if assertion.expected?.stringValue == "https", caughtError == nil { + return .fail("Expected HTTPS enforcement error, but request succeeded over HTTP") + } + + case "urlOrigin": + if assertion.expected?.stringValue == "rejected", requestCount > 1 { + return .fail("Expected cross-origin URL rejection (1 request), but \(requestCount) requests were made") + } + + case "responseMeta": + let fieldPath = assertion.fieldPath + let actual: Any? = switch fieldPath { + case "totalCount": dispatch.totalCount + case "truncated": dispatch.truncated + default: nil + } + if !["totalCount", "truncated"].contains(fieldPath) { + return .fail("Unknown response meta field: \(fieldPath)") + } + if let failure = compareValue("meta.\(fieldPath)", assertion.expected, actual) { + return .fail(failure) + } + + default: + return .fail("Unknown assertion type: \(assertion.type)") + } + } + + return TestResult(passed: true, message: "All assertions passed") +} diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Dispatch.swift b/conformance/runner/swift/Sources/ConformanceRunner/Dispatch.swift new file mode 100644 index 0000000000..06f64a9b50 --- /dev/null +++ b/conformance/runner/swift/Sources/ConformanceRunner/Dispatch.swift @@ -0,0 +1,384 @@ +import Basecamp +import Foundation + +enum RunnerError: Error, CustomStringConvertible { + case unknownOperation(String) + + var description: String { + switch self { + case .unknownOperation(let op): "Unknown operation: \(op)" + } + } +} + +// MARK: - Fixture parameter helpers + +extension Optional where Wrapped == [String: JSON] { + func longParam(_ key: String) -> Int { + (self?[key]?.intValue).map(Int.init) ?? 0 + } + + func stringParam(_ key: String) -> String { + self?[key]?.stringValue ?? "" + } + + func optString(_ key: String) -> String? { + self?[key]?.stringValue + } + + func optBool(_ key: String) -> Bool? { + self?[key]?.boolValue + } + + func intArray(_ key: String) -> [Int]? { + self?[key]?.arrayValue?.compactMap { $0.intValue.map(Int.init) } + } +} + +/// Re-serializes a decoded SDK model through the SDK's own encoder (snake_case +/// keys) so responseBody assertions see wire-shaped field names. +private func resultJSON(_ value: T) throws -> JSON? { + JSON.parse(try BaseService.encoder.encode(value)) +} + +/// Dispatches the test operation against the SDK and returns observed metadata. +/// Direct port of the Kotlin dispatch table. +func dispatchOperation(_ tc: TestCase, _ account: AccountClient) async throws -> DispatchResult { + let pathParams = tc.pathParams + let rb = tc.requestBody + + switch tc.operation { + case "ListProjects": + let maxItems = tc.configOverrides?.maxItems + let options = (maxItems ?? 0) > 0 ? ListProjectOptions(maxItems: maxItems) : nil + let result = try await account.projects.list(options: options) + return DispatchResult(totalCount: result.meta.totalCount, truncated: result.meta.truncated) + + case "GetProject": + let project = try await account.projects.get(projectId: pathParams.longParam("projectId")) + return DispatchResult(resultJSON: try resultJSON(project)) + + case "CreateProject": + _ = try await account.projects.create(req: CreateProjectRequest(name: rb.stringParam("name"))) + return DispatchResult() + + case "UpdateProject": + _ = try await account.projects.update( + projectId: pathParams.longParam("projectId"), + req: UpdateProjectRequest(name: rb.stringParam("name"))) + return DispatchResult() + + case "TrashProject": + try await account.projects.trash(projectId: pathParams.longParam("projectId")) + return DispatchResult() + + case "ListTodos": + let result = try await account.todos.list(todolistId: pathParams.longParam("todolistId")) + return DispatchResult(totalCount: result.meta.totalCount, truncated: result.meta.truncated) + + case "UpdateTodo": + _ = try await account.todos.update( + todoId: pathParams.longParam("todoId"), + req: UpdateTodoRequest( + assigneeIds: rb.intArray("assignee_ids"), + completionSubscriberIds: rb.intArray("completion_subscriber_ids"), + content: rb.optString("content"), + description: rb.optString("description"), + dueOn: rb.optString("due_on"), + notify: rb.optBool("notify"), + startsOn: rb.optString("starts_on"))) + return DispatchResult() + + // Participants are presence-bearing: an absent key must not become an + // empty list on the wire, or BC3 clears the participants. + case "UpdateScheduleEntry": + _ = try await account.schedules.updateEntry( + entryId: pathParams.longParam("entryId"), + req: UpdateScheduleEntryRequest( + endsAt: rb.optString("ends_at"), + participantIds: rb.intArray("participant_ids"), + startsAt: rb.optString("starts_at"), + summary: rb.optString("summary"))) + return DispatchResult() + + // Merge-safe composite: GET then PUT, resending the fetched due_on. + // An explicit empty due_on means clear (single PUT, no GET); an absent + // key means preserve (GET first). + case "UpdateCard": + let dueOn: CardsService.DueDate = if let raw = rb.optString("due_on") { + raw.isEmpty ? .clear : .on(raw) + } else { + .preserve + } + _ = try await account.cards.update( + cardId: pathParams.longParam("cardId"), + title: rb.optString("title"), + content: rb.optString("content"), + dueOn: dueOn, + assigneeIds: rb.intArray("assignee_ids")) + return DispatchResult() + + // Raw single PUT, no read-before-write. + case "UpdateCardVerbatim": + _ = try await account.cards.updateVerbatim( + cardId: pathParams.longParam("cardId"), + req: UpdateCardRequest( + assigneeIds: rb.intArray("assignee_ids"), + content: rb.optString("content"), + dueOn: rb.optString("due_on"), + title: rb.optString("title"))) + return DispatchResult() + + // Synthetic scenario key (not a wire operation): exercises the + // read-modify-write edit closure by assigning each fixture key + // onto the corresponding TodoFields member. + case "EditTodo": + _ = try await account.todos.edit(todoId: pathParams.longParam("todoId")) { fields in + if let content = rb.optString("content") { fields.content = content } + if let description = rb.optString("description") { fields.description = description } + if let assigneeIds = rb.intArray("assignee_ids") { fields.assigneeIds = assigneeIds } + if let subscriberIds = rb.intArray("completion_subscriber_ids") { fields.completionSubscriberIds = subscriberIds } + if let dueOn = rb.optString("due_on") { fields.dueOn = dueOn } + if let startsOn = rb.optString("starts_on") { fields.startsOn = startsOn } + if let notify = rb.optBool("notify") { fields.notify = notify } + } + return DispatchResult() + + case "ReplaceTodo": + _ = try await account.todos.replace( + todoId: pathParams.longParam("todoId"), + req: ReplaceTodoRequest( + assigneeIds: rb.intArray("assignee_ids"), + completionSubscriberIds: rb.intArray("completion_subscriber_ids"), + content: rb.stringParam("content"), + description: rb.optString("description"), + dueOn: rb.optString("due_on"), + notify: rb.optBool("notify"), + startsOn: rb.optString("starts_on"))) + return DispatchResult() + + case "CreateTodo": + _ = try await account.todos.create( + todolistId: pathParams.longParam("todolistId"), + req: CreateTodoRequest(content: rb.stringParam("content"))) + return DispatchResult() + + case "CreateTodosetTodo": + _ = try await account.todos.createTodosetTodo( + bucketId: pathParams.longParam("bucketId"), + todosetId: pathParams.longParam("todosetId"), + req: CreateTodosetTodoRequest(content: rb.stringParam("content"))) + return DispatchResult() + + case "CompleteTodo": + try await account.todos.complete(todoId: pathParams.longParam("todoId")) + return DispatchResult() + + case "Subscribe": + _ = try await account.subscriptions.subscribe(recordingId: pathParams.longParam("recordingId")) + return DispatchResult() + + case "ListMyBookmarks": + _ = try await account.bookmarks.listMyBookmarks() + return DispatchResult() + + case "ListMyDrafts": + _ = try await account.drafts.listMyDrafts() + return DispatchResult() + + case "GetMyNote": + _ = try await account.myNotes.getMyNote() + return DispatchResult() + + case "PrioritizeAssignment": + try await account.myAssignments.prioritizeAssignment( + req: PrioritizeAssignmentRequest(id: rb.longParam("id"))) + return DispatchResult() + + case "DeprioritizeAssignment": + try await account.myAssignments.deprioritizeAssignment(recordingId: pathParams.longParam("recordingId")) + return DispatchResult() + + case "ReorderUpNext": + try await account.myAssignments.reorderUpNext( + req: ReorderUpNextRequest( + position: Int32(rb.longParam("position")), + sourceId: rb.longParam("source_id"))) + return DispatchResult() + + case "GetCalendar": + _ = try await account.calendars.getCalendar(calendarId: pathParams.longParam("calendarId")) + return DispatchResult() + + case "UpdateCalendar": + let calendar = rb?["calendar"]?.objectValue + _ = try await account.calendars.updateCalendar( + calendarId: pathParams.longParam("calendarId"), + req: UpdateCalendarRequest(calendar: CalendarAttributes(color: calendar.stringParam("color")))) + return DispatchResult() + + case "UpdateMyNote": + let note = rb?["note"]?.objectValue + _ = try await account.myNotes.updateMyNote( + req: UpdateMyNoteRequest(note: MyNoteAttributes(content: note.stringParam("content")))) + return DispatchResult() + + case "GetBookmark": + _ = try await account.bookmarks.getBookmark(recordingId: pathParams.longParam("recordingId")) + return DispatchResult() + + case "CreateBookmark": + _ = try await account.bookmarks.createBookmark(recordingId: pathParams.longParam("recordingId")) + return DispatchResult() + + case "DeleteBookmark": + try await account.bookmarks.deleteBookmark(recordingId: pathParams.longParam("recordingId")) + return DispatchResult() + + case "GetTimesheetEntry": + var entryId = pathParams.longParam("timesheetEntryId") + if entryId == 0 { entryId = pathParams.longParam("entryId") } + _ = try await account.timesheets.get(entryId: entryId) + return DispatchResult() + + case "CreateTimesheetEntry": + _ = try await account.timesheets.create( + recordingId: pathParams.longParam("recordingId"), + req: CreateTimesheetEntryRequest( + date: rb.stringParam("date"), + description: rb.optString("description"), + hours: rb.stringParam("hours"))) + return DispatchResult() + + case "UpdateTimesheetEntry": + var entryId = pathParams.longParam("entryId") + if entryId == 0 { entryId = pathParams.longParam("timesheetEntryId") } + _ = try await account.timesheets.update( + entryId: entryId, + req: UpdateTimesheetEntryRequest( + date: rb.optString("date"), + description: rb.optString("description"), + hours: rb.optString("hours"))) + return DispatchResult() + + case "GetProjectTimeline": + _ = try await account.timeline.projectTimeline(projectId: pathParams.longParam("projectId")) + return DispatchResult() + + case "GetProgressReport": + _ = try await account.reports.progress() + return DispatchResult() + + case "GetPersonProgress": + _ = try await account.reports.personProgress(personId: pathParams.longParam("personId")) + return DispatchResult() + + case "GetProjectTimesheet": + _ = try await account.timesheets.forProject(projectId: pathParams.longParam("projectId")) + return DispatchResult() + + case "ListWebhooks": + _ = try await account.webhooks.list(bucketId: pathParams.longParam("bucketId")) + return DispatchResult() + + case "CreateWebhook": + let types = rb?["types"]?.arrayValue?.compactMap(\.stringValue) ?? [] + _ = try await account.webhooks.create( + bucketId: pathParams.longParam("bucketId"), + req: CreateWebhookRequest(payloadUrl: rb.stringParam("payload_url"), types: types)) + return DispatchResult() + + case "GetTool": + _ = try await account.tools.get(toolId: pathParams.longParam("toolId")) + return DispatchResult() + + case "CreateTool": + _ = try await account.tools.create( + bucketId: pathParams.longParam("bucketId"), + req: CreateToolRequest(title: rb.optString("title"), toolType: rb.stringParam("tool_type"))) + return DispatchResult() + + case "EnableTool": + try await account.tools.enable(toolId: pathParams.longParam("toolId")) + return DispatchResult() + + case "GetEverythingMessages": + _ = try await account.everything.everythingMessages() + return DispatchResult() + + case "GetEverythingComments": + _ = try await account.everything.everythingComments() + return DispatchResult() + + case "GetEverythingCheckins": + _ = try await account.everything.everythingCheckins() + return DispatchResult() + + case "GetEverythingForwards": + _ = try await account.everything.everythingForwards() + return DispatchResult() + + case "GetEverythingFiles": + _ = try await account.everything.everythingFiles() + return DispatchResult() + + case "GetEverythingOverdueTodos": + _ = try await account.everything.everythingOverdueTodos() + return DispatchResult() + + case "GetEverythingOverdueCards": + _ = try await account.everything.everythingOverdueCards() + return DispatchResult() + + case "GetEverythingOpenTodos": + _ = try await account.everything.everythingOpenTodos() + return DispatchResult() + + case "GetEverythingCompletedTodos": + _ = try await account.everything.everythingCompletedTodos() + return DispatchResult() + + case "GetEverythingUnassignedTodos": + _ = try await account.everything.everythingUnassignedTodos() + return DispatchResult() + + case "GetEverythingNoDueDateTodos": + _ = try await account.everything.everythingNoDueDateTodos() + return DispatchResult() + + case "GetEverythingOpenCards": + _ = try await account.everything.everythingOpenCards() + return DispatchResult() + + case "GetEverythingCompletedCards": + _ = try await account.everything.everythingCompletedCards() + return DispatchResult() + + case "GetEverythingUnassignedCards": + _ = try await account.everything.everythingUnassignedCards() + return DispatchResult() + + case "GetEverythingNoDueDateCards": + _ = try await account.everything.everythingNoDueDateCards() + return DispatchResult() + + case "GetEverythingNotNowCards": + _ = try await account.everything.everythingNotNowCards() + return DispatchResult() + + case "DownloadURL": + // Construct an absolute URL the SDK will accept. downloadURL rewrites + // the scheme+host to the configured baseURL, so the synthetic host here + // is never actually hit — only tc.path matters for mock routing. Same + // shape as the Go and Kotlin runners. + _ = try await account.downloadURL("https://storage.3.basecamp.com" + tc.fixturePath) + return DispatchResult() + + case "UploadsDownload": + _ = try await account.uploads.download(uploadId: pathParams.longParam("uploadId")) + return DispatchResult() + + default: + throw RunnerError.unknownOperation(tc.operation) + } +} diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Fixtures.swift b/conformance/runner/swift/Sources/ConformanceRunner/Fixtures.swift new file mode 100644 index 0000000000..f8beceeb10 --- /dev/null +++ b/conformance/runner/swift/Sources/ConformanceRunner/Fixtures.swift @@ -0,0 +1,177 @@ +import Foundation + +/// Arbitrary JSON value that preserves 64-bit integer precision. +/// +/// Fixture bodies carry IDs beyond 2^53 (integer-precision.json), so numbers +/// are decoded as `Int64` first and only fall back to `Double` when the value +/// is not an integer. Re-encoding an `.int` therefore round-trips the exact +/// digits to the wire. +indirect enum JSON: Codable, Equatable, Sendable { + case null + case bool(Bool) + case int(Int64) + case double(Double) + case string(String) + case array([JSON]) + case object([String: JSON]) + + init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if container.decodeNil() { + self = .null + } else if let b = try? container.decode(Bool.self) { + self = .bool(b) + } else if let i = try? container.decode(Int64.self) { + self = .int(i) + } else if let d = try? container.decode(Double.self) { + self = .double(d) + } else if let s = try? container.decode(String.self) { + self = .string(s) + } else if let a = try? container.decode([JSON].self) { + self = .array(a) + } else if let o = try? container.decode([String: JSON].self) { + self = .object(o) + } else { + throw DecodingError.dataCorruptedError( + in: container, debugDescription: "Unsupported JSON value") + } + } + + func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self { + case .null: try container.encodeNil() + case .bool(let b): try container.encode(b) + case .int(let i): try container.encode(i) + case .double(let d): try container.encode(d) + case .string(let s): try container.encode(s) + case .array(let a): try container.encode(a) + case .object(let o): try container.encode(o) + } + } + + // MARK: - Accessors + + var intValue: Int64? { + switch self { + case .int(let i): i + case .double(let d): d == d.rounded() ? Int64(exactly: d.rounded()) : nil + default: nil + } + } + + var stringValue: String? { + if case .string(let s) = self { return s } + return nil + } + + var boolValue: Bool? { + if case .bool(let b) = self { return b } + return nil + } + + var arrayValue: [JSON]? { + if case .array(let a) = self { return a } + return nil + } + + var objectValue: [String: JSON]? { + if case .object(let o) = self { return o } + return nil + } + + /// Display form used in failure messages. + var display: String { + switch self { + case .null: "null" + case .bool(let b): String(b) + case .int(let i): String(i) + case .double(let d): String(d) + case .string(let s): "\"\(s)\"" + case .array, .object: + (try? String(data: JSONEncoder().encode(self), encoding: .utf8) ?? "?") ?? "?" + } + } + + /// Serializes this value to JSON `Data`. + func serialized() throws -> Data { + // JSONEncoder refuses top-level fragments on older platforms; wrap and + // slice is unnecessary on modern macOS — encode directly. + try JSONEncoder().encode(self) + } + + /// Parses raw data into a JSON value, or nil when not valid JSON. + static func parse(_ data: Data) -> JSON? { + try? JSONDecoder().decode(JSON.self, from: data) + } + + /// Navigates a dot-separated key path through nested objects. + func navigate(_ path: String) -> JSON? { + var current = self + for key in path.split(separator: ".") { + guard let next = current.objectValue?[String(key)] else { return nil } + current = next + } + return current + } +} + +// MARK: - Fixture models + +/// One conformance test case, matching conformance/tests.schema.json. +struct TestCase: Decodable { + let name: String + let operation: String + private let method: String? + private let path: String? + let pathParams: [String: JSON]? + let queryParams: [String: JSON]? + let requestBody: [String: JSON]? + private let mockResponses: [MockResponse]? + private let assertions: [Assertion]? + private let tags: [String]? + let configOverrides: ConfigOverrides? + private let mode: String? + + var fixtureMethod: String { method ?? "" } + var fixturePath: String { path ?? "" } + var responses: [MockResponse] { mockResponses ?? [] } + var allAssertions: [Assertion] { assertions ?? [] } + var allTags: [String] { tags ?? [] } + /// Live tests are TS-only (canonical wire-capturer); other runners filter + /// them out at load time. + var isMock: Bool { (mode ?? "mock") == "mock" } +} + +struct ConfigOverrides: Decodable { + let baseUrl: String? + let maxPages: Int? + let maxItems: Int? +} + +struct MockResponse: Decodable, Sendable { + let status: Int? + private let networkError: Bool? + private let headers: [String: String]? + let body: JSON? + private let delay: Int? + + var isNetworkError: Bool { networkError ?? false } + var allHeaders: [String: String] { headers ?? [:] } + var delayMs: Int { delay ?? 0 } +} + +struct Assertion: Decodable { + let type: String + let expected: JSON? + private let min: Double? + private let max: Double? + private let path: String? + /// Request index for per-request assertions (0-based; negative = from end). + private let index: Int? + + var minValue: Double { min ?? 0 } + var maxValue: Double { max ?? 0 } + var fieldPath: String { path ?? "" } + var requestIndex: Int { index ?? 0 } +} diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift b/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift new file mode 100644 index 0000000000..efc4f33bb4 --- /dev/null +++ b/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift @@ -0,0 +1,257 @@ +import Basecamp +import Foundation + +/// Default account ID for conformance tests. +private let testAccountID = "999" + +/// Tests the Swift runner must skip, with reasons. Keep near-empty: Swift is +/// three-gate (status, network, and idempotent-POST retry), so capability +/// skips are rare. +/// +/// SWIFT_CONFORMANCE_NO_SKIPS=1 empties the roster so a skip can be proven to +/// still fail (or shown ready to flip live once the capability lands). +private let swiftSkips: [String: String] = ProcessInfo.processInfo.environment["SWIFT_CONFORMANCE_NO_SKIPS"] != nil ? [:] : [ + // TEMPORARY until B4 (Wave 2) threads retry through the download first + // hop: performDownloadRequest is one-shot today, so hop-1 retry fixtures + // fail against a correct runner. Flip these live when B4 merges. + "DownloadURL retries on 503 at the auth'd first hop": + "Swift download hop 1 does not retry yet (B4)", + "DownloadURL honors Retry-After on 429 at the auth'd first hop": + "Swift download hop 1 does not retry yet (B4)", +] + +@main +struct Runner { + static func main() async { + // Child-process mode for the HTTPS-enforcement probe: constructing a + // client with a non-HTTPS, non-localhost base URL must trap + // (preconditionFailure), which only a subprocess can observe. Exit 0 + // means construction survived — the parent treats that as a failure. + if CommandLine.arguments.count >= 3, CommandLine.arguments[1] == "--https-probe" { + _ = BasecampClient( + tokenProvider: StaticTokenProvider("conformance-test-token"), + userAgent: "basecamp-conformance-runner/1.0", + config: BasecampConfig(baseURL: CommandLine.arguments[2]), + transport: ScriptedTransport(responses: [], autoPaginates: false) + ) + exit(0) + } + + let testsDir = URL(fileURLWithPath: "../../tests", isDirectory: true) + let files: [URL] + do { + files = try FileManager.default + .contentsOfDirectory(at: testsDir, includingPropertiesForKeys: nil) + .filter { $0.pathExtension == "json" } + .sorted { $0.lastPathComponent < $1.lastPathComponent } + } catch { + print("No test files found in \(testsDir.path): \(error)") + exit(1) + } + if files.isEmpty { + print("No test files found in \(testsDir.path)") + exit(1) + } + + var passed = 0 + var failed = 0 + var skipped = 0 + + for file in files { + // Live tests are TS-only (canonical wire-capturer); filter them out + // so the offline Swift runner never sees unresolved ${...} fixtures. + let testCases: [TestCase] + do { + let data = try Data(contentsOf: file) + testCases = try JSONDecoder().decode([TestCase].self, from: data).filter(\.isMock) + } catch { + print("\n=== \(file.lastPathComponent) ===") + print(" FAIL: could not decode fixture file: \(error)") + failed += 1 + continue + } + if testCases.isEmpty { continue } + print("\n=== \(file.lastPathComponent) ===") + + for tc in testCases { + // The Swift SDK auto-paginates list operations (like TS and + // Kotlin), so tests that assert requestCount=1 with Link + // headers are not applicable. + if tc.allTags.contains("link-header") { + skipped += 1 + print(" SKIP: \(tc.name)") + print(" Swift SDK auto-paginates (follows Link headers by design)") + continue + } + if let reason = swiftSkips[tc.name] { + skipped += 1 + print(" SKIP: \(tc.name)") + print(" \(reason)") + continue + } + + let result = await runTest(tc) + if result.skipped { + skipped += 1 + print(" SKIP: \(tc.name)") + print(" \(result.message)") + } else if result.passed { + passed += 1 + print(" PASS: \(tc.name)") + } else { + failed += 1 + print(" FAIL: \(tc.name)") + print(" \(result.message)") + } + } + } + + print("\n=== Summary ===") + print("Passed: \(passed), Failed: \(failed), Skipped: \(skipped), Total: \(passed + failed + skipped)") + + exit(failed > 0 ? 1 : 0) + } + + static func runTest(_ tc: TestCase) async -> TestResult { + // Defense-in-depth backstop for the operationally-harmful mockResponses + // shapes: neither mode set (would be served as `status ?? 200`, a false + // positive) or both active. The AUTHORITATIVE oneOf enforcement is + // `make conformance-fixtures-check`, which runs before the runners. + for (i, mock) in tc.responses.enumerated() { + if (mock.status != nil) == mock.isNetworkError { + return .fail("mockResponses[\(i)] must set exactly one of status or networkError (got status=\(mock.status.map(String.init) ?? "nil"), networkError=\(mock.isNetworkError))") + } + } + + // Detect if the fixture uses Link next headers (SDK will auto-paginate). + let autoPaginates = tc.responses.contains { mock in + mock.allHeaders.contains { key, value in + key.lowercased() == "link" && value.contains("rel=\"next\"") + } + } + + let transport = ScriptedTransport(responses: tc.responses, autoPaginates: autoPaginates) + let baseURL = tc.configOverrides?.baseUrl ?? "http://localhost:3000" + + var caughtError: BasecampError? + var httpStatus: Int? + var dispatch = DispatchResult() + + if requiresHTTPSCrashProbe(baseURL) { + // The SDK enforces HTTPS with preconditionFailure — a trap, not a + // thrown error — so it can only be observed from outside the + // process. The probe re-runs this binary in --https-probe mode and + // expects the child to die; a surviving child means enforcement + // did not fire. + switch runHTTPSProbe(baseURL) { + case .enforced: + caughtError = .usage(message: "Base URL must use HTTPS: \(baseURL)", hint: nil) + case .constructionSucceeded: + return .fail("client construction with non-HTTPS base URL unexpectedly succeeded") + case .probeFailure(let message): + return .fail("HTTPS probe failed to run: \(message)") + } + } else { + let client = BasecampClient( + tokenProvider: StaticTokenProvider("conformance-test-token"), + userAgent: "basecamp-conformance-runner/1.0", + config: BasecampConfig( + baseURL: baseURL, + maxPages: tc.configOverrides?.maxPages ?? 10_000 + ), + transport: transport + ) + let account = client.forAccount(testAccountID) + + do { + dispatch = try await dispatchOperation(tc, account) + httpStatus = transport.lastConsumedIndex.flatMap { tc.responses[$0].status } + } catch let error as BasecampError { + caughtError = error + httpStatus = error.httpStatusCode + } catch let error as DecodingError { + // A mock body that fails the model's required-field validation + // is a fixture bug, not a runner limitation: fail loudly so it + // gets fixed (canonical bodies live in spec/fixtures/) instead + // of silently degrading coverage. Kotlin's #555 policy, adopted + // from day one. + return .fail("Mock body lacks required Swift model fields: \(describeDecodingError(error))") + } catch { + return .fail("Unexpected exception: \(type(of: error)): \(error)") + } + } + + return evaluateAssertions( + tc, + transport: transport, + caughtError: caughtError, + httpStatus: httpStatus, + dispatch: dispatch, + autoPaginates: autoPaginates + ) + } +} + +// MARK: - HTTPS enforcement probe + +private enum HTTPSProbeOutcome { + case enforced + case constructionSucceeded + case probeFailure(String) +} + +/// Mirrors the SDK's localhost carve-out (loopback, *.localhost per RFC 6761, +/// HTTP(S)-only) just to ROUTE the test: carved-out URLs are safe to construct +/// in-process; everything else must go through the crash probe. The probe +/// itself exercises the SDK's real enforcement, so a routing mistake here +/// surfaces as a loud failure, never a silent pass. +private func requiresHTTPSCrashProbe(_ baseURL: String) -> Bool { + guard let url = URL(string: baseURL), url.scheme?.lowercased() == "http" else { + return false + } + guard var host = url.host?.lowercased() else { return true } + if host.hasPrefix("["), host.hasSuffix("]") { + host = String(host.dropFirst().dropLast()) + } + let isLocal = host == "localhost" || host == "127.0.0.1" || host == "::1" + || host.hasSuffix(".localhost") + return !isLocal +} + +private func runHTTPSProbe(_ baseURL: String) -> HTTPSProbeOutcome { + let probe = Process() + probe.executableURL = URL(fileURLWithPath: CommandLine.arguments[0]) + probe.arguments = ["--https-probe", baseURL] + probe.standardError = Pipe() // suppress the expected crash banner + probe.standardOutput = Pipe() + do { + try probe.run() + probe.waitUntilExit() + } catch { + return .probeFailure("\(error)") + } + let crashed = probe.terminationReason == .uncaughtSignal || probe.terminationStatus != 0 + return crashed ? .enforced : .constructionSucceeded +} + +// MARK: - Decoding-error rendering + +/// Renders a DecodingError with the missing key and coding path, which is the +/// actionable part when a fixture body under-specifies a model. +private func describeDecodingError(_ error: DecodingError) -> String { + func renderPath(_ path: [any CodingKey]) -> String { + path.map(\.stringValue).joined(separator: ".") + } + switch error { + case .keyNotFound(let key, let context): + return "missing key \"\(key.stringValue)\" at \(renderPath(context.codingPath))" + case .typeMismatch(let type, let context): + return "type mismatch (expected \(type)) at \(renderPath(context.codingPath)): \(context.debugDescription)" + case .valueNotFound(let type, let context): + return "null for non-optional \(type) at \(renderPath(context.codingPath))" + case .dataCorrupted(let context): + return "corrupted data at \(renderPath(context.codingPath)): \(context.debugDescription)" + @unknown default: + return "\(error)" + } +} diff --git a/conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift b/conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift new file mode 100644 index 0000000000..36579624d8 --- /dev/null +++ b/conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift @@ -0,0 +1,143 @@ +import Basecamp +import Foundation + +/// One outbound request captured by the scripted transport. +struct CapturedRequest: @unchecked Sendable { + /// The full URLRequest as handed to the transport (headers, body, method). + let request: URLRequest + /// Monotonic capture time in milliseconds (DispatchTime, immune to + /// wall-clock adjustments), for delayBetweenRequests assertions. + let monotonicMs: UInt64 + + var method: String { request.httpMethod?.uppercased() ?? "" } + var path: String { request.url?.path ?? "" } + var bodyJSON: JSON? { request.httpBody.flatMap { JSON.parse($0) } } + + /// Case-insensitive request-header lookup. + func header(_ name: String) -> String? { + request.value(forHTTPHeaderField: name) + } +} + +/// Transport that answers each request from the fixture's scripted response +/// queue, in order, recording every request it sees. +/// +/// This is the public `Transport` seam — no `@testable` anywhere. Both +/// `data(for:)` and `dataNoRedirect(for:)` draw from the same queue, so +/// multi-hop flows (downloads) consume entries in wire order exactly like the +/// Kotlin MockEngine port model. +final class ScriptedTransport: Transport, @unchecked Sendable { + private let lock = NSLock() + private let responses: [MockResponse] + /// When the fixture advertises Link rel="next" headers the SDK will + /// auto-paginate past the scripted queue; answer the overflow with an + /// empty terminal page instead of an error, mirroring the Kotlin runner. + private let autoPaginates: Bool + private var _captured: [CapturedRequest] = [] + private var served = 0 + + init(responses: [MockResponse], autoPaginates: Bool) { + self.responses = responses + self.autoPaginates = autoPaginates + } + + var captured: [CapturedRequest] { + lock.withLock { _captured } + } + + var requestCount: Int { + lock.withLock { _captured.count } + } + + /// The fixture index of the last consumed queue entry, or nil when no + /// entry (or only synthetic overflow pages) served the final request. + var lastConsumedIndex: Int? { + lock.withLock { + let last = served - 1 + return (last >= 0 && last < responses.count) ? last : nil + } + } + + func data(for request: URLRequest) async throws -> (Data, URLResponse) { + try await serve(request) + } + + func dataNoRedirect(for request: URLRequest) async throws -> (Data, URLResponse) { + try await serve(request) + } + + private func serve(_ request: URLRequest) async throws -> (Data, URLResponse) { + let index: Int? = lock.withLock { + _captured.append(CapturedRequest( + request: request, + monotonicMs: DispatchTime.now().uptimeNanoseconds / 1_000_000 + )) + let i = served + served += 1 + return i < responses.count ? i : nil + } + + let url = request.url ?? URL(string: "http://localhost:3000/")! + + guard let index else { + if autoPaginates { + // Terminal empty page: no Link header ends pagination cleanly. + return ( + Data("[]".utf8), + HTTPURLResponse( + url: url, statusCode: 200, httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"])! + ) + } + return ( + Data(#"{"error": "No more mock responses"}"#.utf8), + HTTPURLResponse( + url: url, statusCode: 500, httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"])! + ) + } + + let mock = responses[index] + + if mock.delayMs > 0 { + try await Task.sleep(nanoseconds: UInt64(mock.delayMs) * 1_000_000) + } + + // Genuine transport failure for this queued entry: throw a plain + // URLError (NOT a BasecampError — the SDK rethrows those untouched, + // which would bypass the network-retry path under test). + if mock.isNetworkError { + throw URLError(.networkConnectionLost) + } + + var headerFields = ["Content-Type": "application/json"] + for (key, value) in mock.allHeaders { + headerFields[key] = value + } + + let body: Data + if let fixtureBody = mock.body { + body = try Self.normalize(fixtureBody).serialized() + } else { + body = Data() + } + + // The fixture schema guarantees a status on every non-networkError + // entry; the runner backstop re-checks before dispatch. + let response = HTTPURLResponse( + url: url, statusCode: mock.status ?? 200, + httpVersion: "HTTP/1.1", headerFields: headerFields)! + return (body, response) + } + + /// Unwraps `{"key": [...]}` single-key array wrappers: some fixtures wrap + /// list bodies in an object, but the SDK's list operations decode a raw + /// JSON array (same normalization as the Kotlin runner). + private static func normalize(_ body: JSON) -> JSON { + if let object = body.objectValue, object.count == 1, + let sole = object.values.first, sole.arrayValue != nil { + return sole + } + return body + } +} From 5d138148930b27d5b61d87b0ad4937db8867c954 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Fri, 31 Jul 2026 18:07:35 -0700 Subject: [PATCH 02/10] Gate CI on Swift conformance, not just Swift tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run conformance-swift inside the test-swift job (via the IS_MACOS-gated make target, same single-source-of-truth pattern as the drift gate) and add test-swift to the required "Conformance Tests" fan-in's needs — without the fan-in edge, Swift conformance could fail while the required check stayed green. The conformance-swift target lives in the Makefile's Swift section because its IS_MACOS ifdef must parse after the variable is defined; the conformance aggregate picks it up by name. --- .github/workflows/test.yml | 9 ++++++++- Makefile | 20 ++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 15ff80bd91..dfdcb6cb83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -381,6 +381,13 @@ jobs: # target must run from repo root. working-directory: . run: make swift-check-drift + + - name: Run Swift conformance tests + # Same single-source-of-truth pattern as the drift gate: the make + # target carries the platform gate (IS_MACOS), and this macos runner + # satisfies it. The target must run from repo root. + working-directory: . + run: make conformance-swift test-kotlin: name: Kotlin Tests runs-on: ubuntu-latest @@ -446,7 +453,7 @@ jobs: # which includes its conformance step — did not succeed. name: Conformance Tests runs-on: ubuntu-latest - needs: [test-go, test-typescript, test-ruby, test-kotlin, test-python] + needs: [test-go, test-typescript, test-ruby, test-kotlin, test-python, test-swift] if: always() permissions: contents: read diff --git a/Makefile b/Makefile index 8e5247128d..4f1d140366 100644 --- a/Makefile +++ b/Makefile @@ -408,7 +408,11 @@ py-clean: # Conformance Test targets #------------------------------------------------------------------------------ -.PHONY: conformance conformance-runner-tests conformance-go conformance-go-replay conformance-kotlin conformance-kotlin-replay conformance-typescript conformance-typescript-live conformance-ruby conformance-ruby-replay conformance-python conformance-python-replay conformance-build conformance-live conformance-canary oauth-fixtures-check oauth-token-fixtures-check conformance-fixtures-check +.PHONY: conformance conformance-runner-tests conformance-go conformance-go-replay conformance-kotlin conformance-kotlin-replay conformance-typescript conformance-typescript-live conformance-ruby conformance-ruby-replay conformance-python conformance-python-replay conformance-swift conformance-swift-runner-tests conformance-build conformance-live conformance-canary oauth-fixtures-check oauth-token-fixtures-check conformance-fixtures-check + +# NOTE: conformance-swift and conformance-swift-runner-tests are defined in the +# Swift SDK targets section below — their IS_MACOS conditional must parse after +# that variable is defined. # Pinned validator for the data-only OAuth discovery fixtures. Run via uvx so the # version is reproducible without a global install; the schema is separate from @@ -530,7 +534,7 @@ conformance-python-replay: cd conformance/runner/python && uv sync && uv run python replay_runner.py # Run all conformance tests -conformance: oauth-fixtures-check oauth-token-fixtures-check conformance-fixtures-check conformance-runner-tests conformance-go conformance-kotlin conformance-typescript conformance-ruby conformance-python +conformance: oauth-fixtures-check oauth-token-fixtures-check conformance-fixtures-check conformance-runner-tests conformance-go conformance-kotlin conformance-typescript conformance-ruby conformance-python conformance-swift @echo "==> Conformance tests passed" # Orchestrate one canary pass against a single backend: @@ -730,6 +734,17 @@ else @echo "SKIP: swift-check (macOS only)" endif +# Run Swift conformance tests (macOS only — the SDK requires Apple platforms). +# Defined here rather than in the conformance section so the IS_MACOS ifdef +# parses after the variable is defined above. +conformance-swift: +ifdef IS_MACOS + @echo "==> Running Swift conformance tests..." + cd conformance/runner/swift && swift run ConformanceRunner +else + @echo "SKIP: conformance-swift (macOS only)" +endif + # Regenerate Swift SDK services from OpenAPI spec (needs swift on any platform) swift-generate: ifdef HAS_SWIFT @@ -999,6 +1014,7 @@ help: @echo " conformance-ruby-replay Decode TS-captured wire snapshots through Ruby SDK" @echo " conformance-python Run Python conformance tests" @echo " conformance-python-replay Decode TS-captured wire snapshots through Python SDK" + @echo " conformance-swift Run Swift conformance tests (macOS only)" @echo " conformance-build Build Go conformance test runner" @echo " oauth-fixtures-check Validate OAuth discovery fixtures against their schema" @echo " oauth-token-fixtures-check Validate OAuth token wire-behavior fixtures against their schema" From 97e700ee3456ece43135e1a9ebe78cf15e76049d Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 1 Aug 2026 02:27:54 -0700 Subject: [PATCH 03/10] Close the Swift runner's own false greens before it gates anything MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A verification sweep found the new runner reporting green on things it was not checking. A conformance runner that does that is worse than no runner: it converts an untested SDK into a tested-looking one. The delayBetweenRequests evaluator measured gap 0 and ignored the assertion's index, and skipped the check entirely below two requests — the identical defect #568 had just fixed in the other five. It mattered here and not hypothetically: the two download-retry fixtures this branch un-skips carry index 0 AND index 1 assertions, so the second backoff was never measured. checkDelayGaps moves into an SDK-free ConformanceSupport target with #568's semantics and its committed test roster ported; 12 of the 13 cases fail against the evaluator as shipped. An executable target carrying @main cannot host XCTest, which is why the split exists. The two download skips are gone. #563 landed the authenticated hop-1 retry, and both cases now pass live rather than being deleted on faith. Nothing capability-shaped is skipped any more: the roster is empty and the only standing exclusion is the architectural link-header branch Kotlin and TypeScript share. The rest are review findings on this branch's own code, each a way a malformed fixture could have passed: - A case with no assertions ran an operation and verified nothing. An empty assertions array is schema-legal, so the fixture gate does not catch it. - An absent mockResponses key decoded as an empty queue, which is a deliberate declaration for the HTTPS case and a malformed fixture everywhere else. The two no longer collapse. - networkError: false alongside a status slipped past the exactly-one-of backstop and was served as a plain success. The schema pins the literal true; so does the runner now. - Path parameters coerced to 0 when missing or non-integral, so the request went to a different resource and the scripted queue answered it anyway. They throw. The two timesheet arms used a `== 0` sentinel to pick between two spellings of one key, which could not tell an absent key from an id of zero; they ask for the first key present instead. - configOverrides.maxItems reaches the SDK through one dispatch arm. Any other operation would have paginated unbounded while the fixture believed it had capped the walk, so that now fails loudly instead. - headerInjected ignored its index and validated the first request, which the schema documents as index-aware and the other runners implement. - errorType: "ambiguous" could never pass despite Swift mapping it. - An explicit expected: null compared nil against the literal "null" and always failed. - The HTTPS probe routed only http:// to the crash child, but the SDK traps on every non-HTTPS scheme outside the localhost carve-out, so an ftp:// or ws:// fixture would have taken the whole run down mid-suite. - SWIFT_CONFORMANCE_NO_SKIPS was enabled by the variable's presence, so an inherited empty value silently changed coverage. It compares to "1". One of these was not a runner bug at all. The single-key array unwrap that lets list fixtures decode also rewrote {"payload_url": ["is not a valid URL"]} into a bare array, so the SDK found no field errors and reported "bad request" — a false FAIL, and the same heuristic could as easily manufacture a pass. Kotlin took the success-only status guard in #549; this port predated it and now carries it too. SPEC retires the "where a runner does not exist yet, e.g. Swift" carve-out, adds swift to the §21 gate roster, and rosters the one architectural skip. --- .github/workflows/test.yml | 7 ++ Makefile | 15 +++ SPEC.md | 13 +- conformance/runner/swift/Package.swift | 26 +++- .../ConformanceRunner/Assertions.swift | 50 +++++--- .../Sources/ConformanceRunner/Dispatch.swift | 110 ++++++++++++---- .../Sources/ConformanceRunner/Fixtures.swift | 42 ++++++- .../Sources/ConformanceRunner/Runner.swift | 102 +++++++++++---- .../ConformanceRunner/ScriptedTransport.swift | 11 +- .../ConformanceSupport/DelayGaps.swift | 75 +++++++++++ .../ConformanceSupport/RequestIndex.swift | 25 ++++ .../DelayGapsTests.swift | 119 ++++++++++++++++++ .../RequestIndexTests.swift | 46 +++++++ conformance/tests/network-retry.json | 2 +- 14 files changed, 569 insertions(+), 74 deletions(-) create mode 100644 conformance/runner/swift/Sources/ConformanceSupport/DelayGaps.swift create mode 100644 conformance/runner/swift/Sources/ConformanceSupport/RequestIndex.swift create mode 100644 conformance/runner/swift/Tests/ConformanceSupportTests/DelayGapsTests.swift create mode 100644 conformance/runner/swift/Tests/ConformanceSupportTests/RequestIndexTests.swift diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dfdcb6cb83..cb9f4771b7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -382,6 +382,13 @@ jobs: working-directory: . run: make swift-check-drift + - name: Run Swift conformance runner unit tests + # Unit-tests the runner's own assertion helpers. Their bounds + # branches never execute against a fixture that passes, so a + # vacuous assertion (#563) survives a fully green conformance run. + working-directory: . + run: make conformance-swift-runner-tests + - name: Run Swift conformance tests # Same single-source-of-truth pattern as the drift gate: the make # target carries the platform gate (IS_MACOS), and this macos runner diff --git a/Makefile b/Makefile index 4f1d140366..53f7c088ba 100644 --- a/Makefile +++ b/Makefile @@ -459,6 +459,7 @@ conformance-runner-tests: cd conformance/runner/python && uv run python -m pytest -q test_delay_gaps.py cd conformance/runner/ruby && bundle install --quiet && bundle exec ruby delay_gaps_test.rb cd kotlin && ./gradlew --quiet :conformance:test + @$(MAKE) --no-print-directory conformance-swift-runner-tests # Build conformance test runner conformance-build: @@ -745,6 +746,19 @@ else @echo "SKIP: conformance-swift (macOS only)" endif +# Unit-test the Swift runner's own assertion helpers (macOS only). Same reason +# as the other five: the bounds branches never execute against a fixture that +# passes, so a vacuous assertion survives a fully green conformance run. +# Reached from conformance-runner-tests, which is platform-agnostic and defers +# the gate to this target. +conformance-swift-runner-tests: +ifdef IS_MACOS + @echo "==> Running Swift conformance runner unit tests..." + cd conformance/runner/swift && swift test +else + @echo "SKIP: conformance-swift-runner-tests (macOS only)" +endif + # Regenerate Swift SDK services from OpenAPI spec (needs swift on any platform) swift-generate: ifdef HAS_SWIFT @@ -1015,6 +1029,7 @@ help: @echo " conformance-python Run Python conformance tests" @echo " conformance-python-replay Decode TS-captured wire snapshots through Python SDK" @echo " conformance-swift Run Swift conformance tests (macOS only)" + @echo " conformance-swift-runner-tests Unit-test the Swift runner's assertion helpers (macOS only)" @echo " conformance-build Build Go conformance test runner" @echo " oauth-fixtures-check Validate OAuth discovery fixtures against their schema" @echo " oauth-token-fixtures-check Validate OAuth token wire-behavior fixtures against their schema" diff --git a/SPEC.md b/SPEC.md index a8d4404c1a..23b8e13b9d 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1635,7 +1635,7 @@ All wire operations are generated (rubric 1A.6). One narrow exception is sanctio 1. **No hand-written wire I/O.** Every request flows through public generated wire methods (Go: through the shared generated-client transport). No manual path construction or verb selection. Bodies use the generated request types, with one Go-specific carve-out: where zero-value + `omitempty` request structs cannot express always-send-empty semantics, the composite's private transport MAY marshal an explicit body map and call the operation's generated `*WithBody` variant — the generated wrapper still owns path, verb, content type, and response decoding, and the operation identity still reaches hooks and retry. This is the only sanctioned use of hand-marshaled bodies; sparse public methods keep using the generated request types. 2. **Composition, not substitution.** It composes existing generated operations (e.g. GET → overlay → full PUT); it never introduces a wire operation the spec lacks — fix the spec and regenerate instead. 3. **Native hook identities.** Hooks observe the constituent wire operations under their normal per-language identities; composites never mint synthetic operation names. -4. **Conformance-covered.** The composite's behavior is encoded in `conformance/tests/` fixtures run by every runner (with native test mirrors where a runner does not exist yet, e.g. Swift). +4. **Conformance-covered.** The composite's behavior is encoded in `conformance/tests/` fixtures run by every runner. All six SDKs now have one, so a native test mirror is no longer a substitute for fixture coverage. 5. **Declared placement.** The composite lives in the language's designated hand-written extension point (Kotlin generator `EXTENSIBLE_SERVICES`/`HAND_WRITTEN_SERVICES`, TS `src/services/*-extensions.ts` wired in `client.ts`, Ruby zeitwerk `prepend` module, Python service subclass re-exported by the client, Swift same-module extension) so regeneration can never silently drop or fork it. 6. **The raw operation stays reachable.** When a composite takes over the plain method name, the generated single-request method is renamed (via `METHOD_NAME_OVERRIDES`) rather than hidden, and gets its own conformance case asserting it makes exactly one request with no read-before-write. Without that second case, later generator drift could silently turn both public methods into composite behavior and nothing would notice. @@ -1753,6 +1753,15 @@ logic is covered by `TestIsSameOrigin` unit tests: is empty): - "List operation returns first page with Link header" — skipped via the `link-header` tag branch, not `KOTLIN_SKIPS`: Kotlin auto-paginates by design, so a first-page-only requestCount assertion is inapplicable (architectural). +**Swift** (`conformance/runner/swift/.../Runner.swift` — one tag-based branch; +`temporarySkips` is empty): +- "List operation returns first page with Link header" — skipped via the `link-header` tag branch, not `temporarySkips`: Swift auto-paginates by design, so a first-page-only requestCount assertion is inapplicable (architectural, identical to Kotlin and TypeScript). + +Swift carries no capability skips. It is three-gate on retry (status, network, +idempotent POST) and, since #563, retries the authenticated download hop, so +`SWIFT_CONFORMANCE_NO_SKIPS=1` is a no-op today — the mechanism is kept live so +a future temporary skip must be proven genuine before it is added. + The TypeScript live canary additionally reports one placeholder skip when `BASECAMP_LIVE` is unset (`live-runner.test.ts`) — that is the opt-in gate for `live-my-surface.json` documented in the category table above, not a @@ -1798,7 +1807,7 @@ The following are must-pass criteria from the rubric. Each maps to a spec sectio | `rb-check` | Ruby: test + rubocop | | `kt-check` | Kotlin: build + test | | `swift-check` | Swift: build + test | -| `conformance` | All conformance test categories pass with documented waivers (go, kotlin, python, ruby, typescript runners) | +| `conformance` | All conformance test categories pass with documented waivers (go, kotlin, python, ruby, swift, typescript runners) | Representative dependency chain (see the Makefile `check:` line for the authoritative, complete list): `check: … sync-api-version-check url-routes-check go-check-drift … kt-check-drift … go-check ts-check rb-check kt-check swift-check py-check conformance …` diff --git a/conformance/runner/swift/Package.swift b/conformance/runner/swift/Package.swift index 77a492339d..d5a6eb5d4e 100644 --- a/conformance/runner/swift/Package.swift +++ b/conformance/runner/swift/Package.swift @@ -19,15 +19,39 @@ let package = Package( .package(name: "Basecamp", path: "../../..") ], targets: [ + // Assertion contracts with no SDK dependency, split out of the + // executable so their bounds branches can be unit-tested: a target + // carrying @main cannot host XCTest cleanly, and these branches never + // execute against a fixture that passes. #563 shipped a + // delayBetweenRequests check that vacuously passed when the gap it + // named did not exist, in four runners at once. + .target( + name: "ConformanceSupport", + path: "Sources/ConformanceSupport", + swiftSettings: [ + .swiftLanguageMode(.v6) + ] + ), .executableTarget( name: "ConformanceRunner", dependencies: [ - "Basecamp" + "Basecamp", + "ConformanceSupport" ], path: "Sources/ConformanceRunner", swiftSettings: [ .swiftLanguageMode(.v6) ] + ), + .testTarget( + name: "ConformanceSupportTests", + dependencies: [ + "ConformanceSupport" + ], + path: "Tests/ConformanceSupportTests", + swiftSettings: [ + .swiftLanguageMode(.v6) + ] ) ] ) diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift index b28fc53b3b..4097b33b67 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift @@ -1,4 +1,5 @@ import Basecamp +import ConformanceSupport import Foundation /// Outcome of a single conformance test. @@ -38,22 +39,18 @@ func conformanceCode(_ error: BasecampError) -> String { } } +/// The vocabulary `conformanceCode` can produce. It must stay in sync with that +/// mapping: a member missing here can never be asserted, so the guard meant to +/// catch a typo'd error type silently forbids a real one instead. private let knownErrorTypes: Set = [ "not_found", "auth_required", "forbidden", "rate_limit", - "validation", "api_error", "usage", "network", + "validation", "api_error", "usage", "network", "ambiguous", ] -/// Resolves a per-request assertion index (0-based; negative = from the end) -/// against the number of recorded requests. Nil when out of range. -private func resolveRequestIndex(_ index: Int, _ count: Int) -> Int? { - let resolved = index < 0 ? count + index : index - return (0.. String? { - guard let expected else { return "\(label): expected value is null in assertion" } + guard let expected else { return "\(label): assertion has no expected value" } if let expInt = expected.intValue, let actInt = actual.intValue { return expInt == actInt ? nil : "Expected \(label) = \(expInt), got \(actInt)" } @@ -63,8 +60,13 @@ private func compareJSON(_ label: String, _ expected: JSON?, _ actual: JSON) -> /// Compares an expected fixture value against a loosely-typed actual value /// (error fields, response meta). Nil means equal. private func compareValue(_ label: String, _ expected: JSON?, _ actual: Any?) -> String? { - guard let expected else { return "\(label): expected value is null in assertion" } + guard let expected else { return "\(label): assertion has no expected value" } switch expected { + case .null: + // An explicit `expected: null` asserts the observed field is absent. + // Falling through to the string fallback compared nil against the + // literal "null" and always failed. + if let actual { return "Expected \(label) = null, got \(actual)" } case .bool(let b): if (actual as? Bool) != b { return "Expected \(label) = \(b), got \(String(describing: actual))" } case .int(let i): @@ -197,12 +199,17 @@ func evaluateAssertions( } case "delayBetweenRequests": - if captured.count >= 2 { - let delay = captured[1].monotonicMs - captured[0].monotonicMs - let minDelay = UInt64(assertion.minValue) - if delay < minDelay { - return .fail("Expected delay >= \(minDelay)ms, got \(delay)ms") - } + // Delegated to ConformanceSupport so the bounds branches are + // unit-testable. This evaluator used to measure gap 0 and ignore + // the assertion's index, and skipped the check entirely on a + // single-request run — the #563/#568 false-green, which downloads + // fixtures (index 0 AND index 1) walk straight into. + if let failure = checkDelayGaps( + captured.map(\.monotonicMs), + minDelayMs: assertion.minDelayMs, + index: assertion.gapIndex + ) { + return .fail(failure) } case "headerValue": @@ -276,10 +283,13 @@ func evaluateAssertions( guard let expected = assertion.expected?.stringValue else { return .fail("headerInjected assertion missing expected value") } - guard let first = captured.first else { - return .fail("Expected header \(headerName)=\"\(expected)\", but no requests were recorded") + // Index-aware like headerPresent/headerAbsent and the other + // runners: reading captured.first regardless would validate the + // initial attempt when the fixture named a retry or a second hop. + guard let idx = resolveRequestIndex(assertion.requestIndex, requestCount) else { + return .fail("headerInjected \(headerName)[\(assertion.requestIndex)]: no request recorded at that index (\(requestCount) requests)") } - let actual = first.header(headerName) + let actual = captured[idx].header(headerName) // Content-Type may include charset (e.g., "application/json; charset=utf-8") let matches: Bool = if headerName.lowercased() == "content-type" { actual?.lowercased().hasPrefix(expected.lowercased()) ?? false @@ -287,7 +297,7 @@ func evaluateAssertions( actual == expected } if !matches { - return .fail("Expected header \(headerName)=\"\(expected)\", got \"\(actual ?? "nil")\"") + return .fail("Expected header \(headerName)=\"\(expected)\" on request index \(idx), got \"\(actual ?? "nil")\"") } case "headerPresent": diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Dispatch.swift b/conformance/runner/swift/Sources/ConformanceRunner/Dispatch.swift index 06f64a9b50..88394858ea 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Dispatch.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Dispatch.swift @@ -3,35 +3,90 @@ import Foundation enum RunnerError: Error, CustomStringConvertible { case unknownOperation(String) + /// A fixture parameter the dispatch table cannot use as written. + case badParameter(String) var description: String { switch self { case .unknownOperation(let op): "Unknown operation: \(op)" + case .badParameter(let detail): "Fixture parameter: \(detail)" } } } // MARK: - Fixture parameter helpers +/// These throw rather than substituting a default. A missing or non-integral +/// `projectId` that quietly became `0` still produced a request the scripted +/// transport answered from the queue — a green test for a call to the wrong +/// resource, which is the exact false-green class this runner exists to catch. +/// A wrong-typed optional is the same fault one step quieter: it drops the +/// field and the requestBody assertion never sees what it was meant to pin. extension Optional where Wrapped == [String: JSON] { - func longParam(_ key: String) -> Int { - (self?[key]?.intValue).map(Int.init) ?? 0 + func longParam(_ key: String) throws -> Int { + guard let value = self?[key] else { + throw RunnerError.badParameter("missing integer parameter \"\(key)\"") + } + guard let int = value.intValue, let narrowed = Int(exactly: int) else { + throw RunnerError.badParameter( + "parameter \"\(key)\" must be an integer, got \(value.display)") + } + return narrowed } - func stringParam(_ key: String) -> String { - self?[key]?.stringValue ?? "" + /// Reads the first key that is present, for the operations whose fixtures + /// spell one path parameter two ways. Replaces a `== 0` sentinel that could + /// not tell an absent key from an id that legitimately read zero. + func longParam(anyOf keys: [String]) throws -> Int { + for key in keys where self?[key] != nil { + return try longParam(key) + } + let tried = keys.map { "\"\($0)\"" }.joined(separator: " or ") + throw RunnerError.badParameter("missing integer parameter \(tried)") } - func optString(_ key: String) -> String? { - self?[key]?.stringValue + func stringParam(_ key: String) throws -> String { + guard let value = self?[key] else { + throw RunnerError.badParameter("missing string parameter \"\(key)\"") + } + guard let string = value.stringValue else { + throw RunnerError.badParameter( + "parameter \"\(key)\" must be a string, got \(value.display)") + } + return string } - func optBool(_ key: String) -> Bool? { - self?[key]?.boolValue + func optString(_ key: String) throws -> String? { + guard let value = self?[key] else { return nil } + guard let string = value.stringValue else { + throw RunnerError.badParameter( + "parameter \"\(key)\" must be a string, got \(value.display)") + } + return string } - func intArray(_ key: String) -> [Int]? { - self?[key]?.arrayValue?.compactMap { $0.intValue.map(Int.init) } + func optBool(_ key: String) throws -> Bool? { + guard let value = self?[key] else { return nil } + guard let bool = value.boolValue else { + throw RunnerError.badParameter( + "parameter \"\(key)\" must be a boolean, got \(value.display)") + } + return bool + } + + func intArray(_ key: String) throws -> [Int]? { + guard let value = self?[key] else { return nil } + guard let array = value.arrayValue else { + throw RunnerError.badParameter( + "parameter \"\(key)\" must be an array, got \(value.display)") + } + return try array.map { element in + guard let int = element.intValue, let narrowed = Int(exactly: int) else { + throw RunnerError.badParameter( + "parameter \"\(key)\" must contain only integers, got \(element.display)") + } + return narrowed + } } } @@ -105,7 +160,7 @@ func dispatchOperation(_ tc: TestCase, _ account: AccountClient) async throws -> // An explicit empty due_on means clear (single PUT, no GET); an absent // key means preserve (GET first). case "UpdateCard": - let dueOn: CardsService.DueDate = if let raw = rb.optString("due_on") { + let dueOn: CardsService.DueDate = if let raw = try rb.optString("due_on") { raw.isEmpty ? .clear : .on(raw) } else { .preserve @@ -133,14 +188,24 @@ func dispatchOperation(_ tc: TestCase, _ account: AccountClient) async throws -> // read-modify-write edit closure by assigning each fixture key // onto the corresponding TodoFields member. case "EditTodo": + // Read every fixture key before the call: the edit closure is + // non-throwing, and validating up front means a malformed parameter + // fails the test instead of reaching the wire half-applied. + let editContent = try rb.optString("content") + let editDescription = try rb.optString("description") + let editAssigneeIds = try rb.intArray("assignee_ids") + let editSubscriberIds = try rb.intArray("completion_subscriber_ids") + let editDueOn = try rb.optString("due_on") + let editStartsOn = try rb.optString("starts_on") + let editNotify = try rb.optBool("notify") _ = try await account.todos.edit(todoId: pathParams.longParam("todoId")) { fields in - if let content = rb.optString("content") { fields.content = content } - if let description = rb.optString("description") { fields.description = description } - if let assigneeIds = rb.intArray("assignee_ids") { fields.assigneeIds = assigneeIds } - if let subscriberIds = rb.intArray("completion_subscriber_ids") { fields.completionSubscriberIds = subscriberIds } - if let dueOn = rb.optString("due_on") { fields.dueOn = dueOn } - if let startsOn = rb.optString("starts_on") { fields.startsOn = startsOn } - if let notify = rb.optBool("notify") { fields.notify = notify } + if let editContent { fields.content = editContent } + if let editDescription { fields.description = editDescription } + if let editAssigneeIds { fields.assigneeIds = editAssigneeIds } + if let editSubscriberIds { fields.completionSubscriberIds = editSubscriberIds } + if let editDueOn { fields.dueOn = editDueOn } + if let editStartsOn { fields.startsOn = editStartsOn } + if let editNotify { fields.notify = editNotify } } return DispatchResult() @@ -236,9 +301,8 @@ func dispatchOperation(_ tc: TestCase, _ account: AccountClient) async throws -> return DispatchResult() case "GetTimesheetEntry": - var entryId = pathParams.longParam("timesheetEntryId") - if entryId == 0 { entryId = pathParams.longParam("entryId") } - _ = try await account.timesheets.get(entryId: entryId) + _ = try await account.timesheets.get( + entryId: pathParams.longParam(anyOf: ["timesheetEntryId", "entryId"])) return DispatchResult() case "CreateTimesheetEntry": @@ -251,10 +315,8 @@ func dispatchOperation(_ tc: TestCase, _ account: AccountClient) async throws -> return DispatchResult() case "UpdateTimesheetEntry": - var entryId = pathParams.longParam("entryId") - if entryId == 0 { entryId = pathParams.longParam("timesheetEntryId") } _ = try await account.timesheets.update( - entryId: entryId, + entryId: pathParams.longParam(anyOf: ["entryId", "timesheetEntryId"]), req: UpdateTimesheetEntryRequest( date: rb.optString("date"), description: rb.optString("description"), diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Fixtures.swift b/conformance/runner/swift/Sources/ConformanceRunner/Fixtures.swift index f8beceeb10..3e5eb00af9 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Fixtures.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Fixtures.swift @@ -138,6 +138,11 @@ struct TestCase: Decodable { var responses: [MockResponse] { mockResponses ?? [] } var allAssertions: [Assertion] { assertions ?? [] } var allTags: [String] { tags ?? [] } + /// Whether the fixture stated a queue at all. An EMPTY queue is a + /// deliberate declaration (the HTTPS-enforcement case makes no request); + /// an absent key is a malformed fixture, and collapsing the two lets one + /// through as a test that exercises nothing. + var declaresMockResponses: Bool { mockResponses != nil } /// Live tests are TS-only (canonical wire-capturer); other runners filter /// them out at load time. var isMock: Bool { (mode ?? "mock") == "mock" } @@ -151,18 +156,26 @@ struct ConfigOverrides: Decodable { struct MockResponse: Decodable, Sendable { let status: Int? - private let networkError: Bool? + /// Raw flag, kept rather than folded into `isNetworkError`: the schema pins + /// the literal `true`, so `networkError: false` is not legal. Collapsing it + /// to "not a network error" lets a `status` + `networkError: false` entry + /// slip past the exactly-one-of backstop and be served as a plain success. + let networkError: Bool? private let headers: [String: String]? let body: JSON? private let delay: Int? - var isNetworkError: Bool { networkError ?? false } + var isNetworkError: Bool { networkError == true } var allHeaders: [String: String] { headers ?? [:] } var delayMs: Int { delay ?? 0 } } struct Assertion: Decodable { let type: String + /// An explicit `expected: null` is a real expectation (the field must be + /// absent), distinct from an omitted key (the assertion is malformed). The + /// synthesized `decodeIfPresent` collapses both to nil, which turns the + /// former into "assertion missing expected value" — a false FAIL. let expected: JSON? private let min: Double? private let max: Double? @@ -170,8 +183,31 @@ struct Assertion: Decodable { /// Request index for per-request assertions (0-based; negative = from end). private let index: Int? - var minValue: Double { min ?? 0 } + private enum CodingKeys: String, CodingKey { + case type, expected, min, max, path, index + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + type = try container.decode(String.self, forKey: .type) + expected = container.contains(.expected) + ? try container.decode(JSON.self, forKey: .expected) + : nil + min = try container.decodeIfPresent(Double.self, forKey: .min) + max = try container.decodeIfPresent(Double.self, forKey: .max) + path = try container.decodeIfPresent(String.self, forKey: .path) + index = try container.decodeIfPresent(Int.self, forKey: .index) + } + var maxValue: Double { max ?? 0 } var fieldPath: String { path ?? "" } var requestIndex: Int { index ?? 0 } + /// Raw minimum for `delayBetweenRequests`. `checkDelayGaps` applies the + /// default itself so no call site can gate the assertion away — a `min` of + /// zero silently disabled the whole check in two other runners. + var minDelayMs: Double? { min } + /// Raw index for `delayBetweenRequests`, which must tell "gap 0" from + /// "every gap". Distinct from `requestIndex`, whose 0 default is correct + /// for the per-request assertions. + var gapIndex: Int? { index } } diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift b/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift index efc4f33bb4..f4fe72eb87 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift @@ -4,21 +4,30 @@ import Foundation /// Default account ID for conformance tests. private let testAccountID = "999" -/// Tests the Swift runner must skip, with reasons. Keep near-empty: Swift is -/// three-gate (status, network, and idempotent-POST retry), so capability -/// skips are rare. +/// Operations whose dispatch arm passes `configOverrides.maxItems` into the +/// SDK. Kept beside the backstop that enforces it so the two cannot drift: +/// widening one without the other is caught the first time a fixture asks. +private let operationsHonoringMaxItems: Set = ["ListProjects"] + +/// Temporary capability skips, keyed by exact test name. +/// +/// EMPTY, and meant to stay that way. Swift is three-gate (status, network and +/// idempotent-POST retry) and since #563 retries the authenticated download hop +/// too, so no fixture asks for a capability the SDK lacks. The one standing +/// exclusion is architectural rather than a gap — the `link-header` tag branch +/// in the run loop, which no name-keyed entry can express. +private let temporarySkips: [String: String] = [:] + +/// The roster the run loop consults. `SWIFT_CONFORMANCE_NO_SKIPS=1` empties it, +/// so a temporary skip can be proven genuine before it is added and proven +/// ready to flip once the capability lands. With `temporarySkips` empty the +/// switch is a no-op — it is the mechanism kept live, not a claim that anything +/// is being skipped. /// -/// SWIFT_CONFORMANCE_NO_SKIPS=1 empties the roster so a skip can be proven to -/// still fail (or shown ready to flip live once the capability lands). -private let swiftSkips: [String: String] = ProcessInfo.processInfo.environment["SWIFT_CONFORMANCE_NO_SKIPS"] != nil ? [:] : [ - // TEMPORARY until B4 (Wave 2) threads retry through the download first - // hop: performDownloadRequest is one-shot today, so hop-1 retry fixtures - // fail against a correct runner. Flip these live when B4 merges. - "DownloadURL retries on 503 at the auth'd first hop": - "Swift download hop 1 does not retry yet (B4)", - "DownloadURL honors Retry-After on 429 at the auth'd first hop": - "Swift download hop 1 does not retry yet (B4)", -] +/// The value is compared exactly: an inherited empty or `=0` variable must not +/// quietly change what the suite covers. +private let swiftSkips: [String: String] = + ProcessInfo.processInfo.environment["SWIFT_CONFORMANCE_NO_SKIPS"] == "1" ? [:] : temporarySkips @main struct Runner { @@ -113,16 +122,51 @@ struct Runner { } static func runTest(_ tc: TestCase) async -> TestResult { - // Defense-in-depth backstop for the operationally-harmful mockResponses - // shapes: neither mode set (would be served as `status ?? 200`, a false - // positive) or both active. The AUTHORITATIVE oneOf enforcement is - // `make conformance-fixtures-check`, which runs before the runners. + // Defense-in-depth backstops for fixture shapes that would otherwise + // produce a PASS while testing nothing. The AUTHORITATIVE enforcement + // is `make conformance-fixtures-check` against conformance/schema.json, + // which runs before the runners — but a runner that reports green on a + // fixture it cannot honor is the failure mode this whole rock exists to + // remove, so it fails loudly here too. + + // A case with no assertions runs an operation and verifies nothing. + // An empty `assertions: []` is schema-legal, so the schema gate does + // not catch it. + if tc.allAssertions.isEmpty { + return .fail("test case declares no assertions — it would pass without verifying anything") + } + + // An EMPTY response queue is a deliberate declaration (the HTTPS + // enforcement case makes no request at all); an ABSENT key is a + // malformed fixture, and the two must not collapse. + if !tc.declaresMockResponses { + return .fail("mock test case is missing mockResponses (an empty queue must be stated explicitly)") + } + for (i, mock) in tc.responses.enumerated() { + // The schema pins the literal `true`. `networkError: false` + // alongside a status otherwise reads as a plain success and slips + // past the exactly-one-of check below. + if let flag = mock.networkError, !flag { + return .fail("mockResponses[\(i)]: networkError must be the literal true when present, got false") + } + // Neither mode set would be served as `status ?? 200`, a false + // positive; both active is ambiguous. if (mock.status != nil) == mock.isNetworkError { return .fail("mockResponses[\(i)] must set exactly one of status or networkError (got status=\(mock.status.map(String.init) ?? "nil"), networkError=\(mock.isNetworkError))") } } + // maxItems reaches the SDK only through the per-operation options of + // the arms that thread it. Any other operation would run UNBOUNDED + // pagination while the fixture believed it had capped the walk, so the + // request-count assertion would be measuring something else entirely. + // Fail rather than silently ignore; add the operation to the dispatch + // arm and to this roster together. + if tc.configOverrides?.maxItems != nil, !operationsHonoringMaxItems.contains(tc.operation) { + return .fail("configOverrides.maxItems is set but \(tc.operation)'s dispatch does not thread it through — it would paginate unbounded") + } + // Detect if the fixture uses Link next headers (SDK will auto-paginate). let autoPaginates = tc.responses.contains { mock in mock.allHeaders.contains { key, value in @@ -169,6 +213,12 @@ struct Runner { } catch let error as BasecampError { caughtError = error httpStatus = error.httpStatusCode + } catch let error as RunnerError { + // A fixture the dispatch table cannot honor as written: an + // unknown operation, or a parameter that would have been + // coerced into a call against the wrong resource. Both are + // fixture bugs to fix, not runner limitations to skip. + return .fail(error.description) } catch let error as DecodingError { // A mock body that fails the model's required-field validation // is a fixture bug, not a runner limitation: fail loudly so it @@ -205,10 +255,20 @@ private enum HTTPSProbeOutcome { /// in-process; everything else must go through the crash probe. The probe /// itself exercises the SDK's real enforcement, so a routing mistake here /// surfaces as a loud failure, never a silent pass. +/// +/// The SDK traps on EVERY parsed non-HTTPS scheme outside the carve-out, not +/// just `http` — `BasecampClient` tests `scheme != "https"`. Routing only +/// `http` through the probe meant a fixture with, say, `ftp://` or +/// `ws://localhost` would trap in-process and take the entire conformance run +/// down with it, losing every result after it. private func requiresHTTPSCrashProbe(_ baseURL: String) -> Bool { - guard let url = URL(string: baseURL), url.scheme?.lowercased() == "http" else { - return false - } + // An unparseable URL never reaches the scheme check in the SDK either: the + // guard is `if let url = URL(string:)`, so construction survives. + guard let url = URL(string: baseURL) else { return false } + let scheme = url.scheme?.lowercased() + if scheme == "https" { return false } + // The carve-out is HTTP(S)-only, matching the SDK's isLocalhost. + guard scheme == "http" else { return true } guard var host = url.host?.lowercased() else { return true } if host.hasPrefix("["), host.hasSuffix("]") { host = String(host.dropFirst().dropLast()) diff --git a/conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift b/conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift index 36579624d8..fede7fc51c 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift @@ -117,7 +117,7 @@ final class ScriptedTransport: Transport, @unchecked Sendable { let body: Data if let fixtureBody = mock.body { - body = try Self.normalize(fixtureBody).serialized() + body = try Self.normalize(fixtureBody, status: mock.status).serialized() } else { body = Data() } @@ -133,7 +133,14 @@ final class ScriptedTransport: Transport, @unchecked Sendable { /// Unwraps `{"key": [...]}` single-key array wrappers: some fixtures wrap /// list bodies in an object, but the SDK's list operations decode a raw /// JSON array (same normalization as the Kotlin runner). - private static func normalize(_ body: JSON) -> JSON { + /// + /// SUCCESS bodies only. An error body with one array-valued key is the + /// unwrapped Rails field map (`{"payload_url": ["is not a valid URL"]}`), + /// and unwrapping it rewrites the fixture on the wire — the SDK then sees a + /// bare array, finds no field errors, and reports the generic status text. + /// Kotlin took the status guard in #549; this port predated it. + private static func normalize(_ body: JSON, status: Int?) -> JSON { + guard (status ?? 200) < 400 else { return body } if let object = body.objectValue, object.count == 1, let sole = object.values.first, sole.arrayValue != nil { return sole diff --git a/conformance/runner/swift/Sources/ConformanceSupport/DelayGaps.swift b/conformance/runner/swift/Sources/ConformanceSupport/DelayGaps.swift new file mode 100644 index 0000000000..3a5dd7e6ed --- /dev/null +++ b/conformance/runner/swift/Sources/ConformanceSupport/DelayGaps.swift @@ -0,0 +1,75 @@ +/// The `delayBetweenRequests` assertion contract, kept in its own SDK-free +/// target so its bounds branches are unit-testable (`ConformanceSupportTests`). +/// An executable target carrying `@main` cannot host XCTest cleanly, and these +/// branches never execute against a fixture that passes. + +/// Validates one assertion against the recorded request times, returning `nil` +/// when it holds and a failure message otherwise. +/// +/// Gap i is the interval between request i and request i+1, so N requests yield +/// N-1 gaps. The contract in conformance/schema.json: +/// +/// - A NAMED index selects exactly that gap, bounds-checked unconditionally. A +/// gap the run never produced is a failure, not a silent pass — the whole +/// point of a timing pin is to catch a dropped backoff, and a dropped backoff +/// is precisely what removes the gap. +/// - An OMITTED index requires the minimum on EVERY gap. Zero gaps means +/// nothing was measured, so that fails too: an "every gap" rule with no gaps +/// left would otherwise wave through a run that dropped every retry. +/// - Negative indexes are rejected rather than wrapping to the end the way the +/// per-request assertions do. There is no sensible "last gap" when the point +/// of naming one is to pin a specific backoff. +/// +/// The bounds test compares against the gap COUNT and never adds one to the +/// index: `index + 1 >= requestTimes.count` overflows for `Int.max`, which in +/// Swift traps and takes the whole runner down instead of failing the +/// assertion — the same shape that read out of bounds in Go and Kotlin. +public func checkDelayGaps( + _ requestTimes: [UInt64], + minDelayMs: Double?, + index: Int? +) -> String? { + // An absent or zero minimum still asserts that the gap EXISTS. The default + // lands HERE rather than at the call site so a truthiness gate cannot + // quietly reduce the assertion to nothing. + let minimum = minDelayMs ?? 0 + let gaps = requestTimes.count - 1 + + func shortfall(_ gap: Int) -> String? { + // Saturating rather than wrapping: the capture clock is monotonic, so a + // decreasing pair cannot happen, and if it ever did a 0ms gap fails a + // positive minimum — the fail-closed direction. Unsigned subtraction + // traps on underflow, which would crash the runner instead. + let later = requestTimes[gap + 1] + let earlier = requestTimes[gap] + let delay = later >= earlier ? later - earlier : 0 + return Double(delay) < minimum + ? "Expected delay >= \(formatMs(minimum))ms at gap \(gap), got \(delay)ms" + : nil + } + + if let index { + if index < 0 { + return "delayBetweenRequests gap index must be non-negative, got \(index)" + } + if index >= gaps { + return "Expected a delay at gap \(index), but only \(requestTimes.count) request(s) were made" + } + return shortfall(index) + } + + if gaps < 1 { + return "Expected a delay between requests, but only \(requestTimes.count) request(s) were made" + } + for gap in 0.. String { + value == value.rounded() && value.magnitude < 1e15 ? String(Int64(value)) : String(value) +} diff --git a/conformance/runner/swift/Sources/ConformanceSupport/RequestIndex.swift b/conformance/runner/swift/Sources/ConformanceSupport/RequestIndex.swift new file mode 100644 index 0000000000..1f2f5159d5 --- /dev/null +++ b/conformance/runner/swift/Sources/ConformanceSupport/RequestIndex.swift @@ -0,0 +1,25 @@ +/// Resolves a per-request assertion index against the number of recorded +/// requests, returning `nil` when it names no request that was made. +/// +/// 0-based, and NEGATIVE INDEXES WRAP: -1 is the last request, -2 the one +/// before it. That is the opposite of `checkDelayGaps`, which rejects a +/// negative index outright — the two contracts share a fixture key (`index`) +/// and are easy to conflate, so both are pinned by tests. +/// +/// The difference is deliberate and stated in conformance/schema.json: "the +/// last request" is a meaningful thing to assert a header on, whereas naming a +/// gap exists to pin one specific backoff, and "the last backoff" would let a +/// dropped retry re-target the assertion at a gap that did survive. +/// +/// Out of range is nil rather than a clamp, so the caller fails the assertion +/// instead of validating a different request than the fixture asked for. +/// +/// Unlike the delay-gap bounds test, `count + index` needs no overflow guard: +/// `count` is a non-negative array length and the branch only runs for a +/// negative `index`, so the sum always moves toward zero. These tests are +/// characterization, not a regression proof — they pin a contract that was +/// already correct, next to one that was not. +public func resolveRequestIndex(_ index: Int, _ count: Int) -> Int? { + let resolved = index < 0 ? count + index : index + return (0.. [UInt64] { + var out: [UInt64] = [0] + for ms in gapsMs { out.append(out[out.count - 1] + ms) } + return out + } + + func testOmittedIndexCatchesALaterFailingGap() throws { + // Swift measured gap 0 and stopped, so a second backoff that never + // happened passed unnoticed. + let failure = try XCTUnwrap(checkDelayGaps(times(1000, 5), minDelayMs: 500, index: nil)) + XCTAssertTrue(failure.contains("at gap 1"), failure) + } + + func testOmittedIndexPassesWhenEveryGapClearsTheMinimum() { + XCTAssertNil(checkDelayGaps(times(1000, 2000, 800), minDelayMs: 500, index: nil)) + } + + func testOmittedIndexFailsWhenThereAreNoGapsAtAll() { + // An "every gap" rule with no gaps left must not wave the run through: + // a fully dropped retry lands exactly here. The old evaluator's + // `if captured.count >= 2` guard made this the silent pass. + XCTAssertEqual( + "Expected a delay between requests, but only 1 request(s) were made", + checkDelayGaps(times(), minDelayMs: 500, index: nil) + ) + } + + func testNamedGapFailsWhenTheRunNeverProducedIt() { + XCTAssertEqual( + "Expected a delay at gap 1, but only 2 request(s) were made", + checkDelayGaps(times(1000), minDelayMs: 500, index: 1) + ) + } + + func testNamedGapFailsOnASingleRequestRun() { + XCTAssertEqual( + "Expected a delay at gap 0, but only 1 request(s) were made", + checkDelayGaps(times(), minDelayMs: 500, index: 0) + ) + } + + func testNegativeGapIndexIsRejected() { + // Rejected categorically, not wrapped to the end the way + // headerPresent's index is. + XCTAssertEqual( + "delayBetweenRequests gap index must be non-negative, got -1", + checkDelayGaps(times(1000, 2000), minDelayMs: 500, index: -1) + ) + } + + func testIntMaxGapIndexFailsWithoutOverflowing() throws { + // `gap + 1 >= count` computes the addition first; in Swift that traps + // on Int.max and takes the runner down rather than failing the + // assertion. + let failure = try XCTUnwrap( + checkDelayGaps(times(1000, 2000), minDelayMs: 500, index: Int.max)) + XCTAssertTrue(failure.contains("Expected a delay at gap"), failure) + } + + func testZeroMinimumStillAssertsThatTheGapExists() { + // A zero minimum is trivially met; the EXISTENCE requirement is not. + XCTAssertEqual( + "Expected a delay between requests, but only 1 request(s) were made", + checkDelayGaps(times(), minDelayMs: 0, index: nil) + ) + XCTAssertEqual( + "Expected a delay at gap 0, but only 1 request(s) were made", + checkDelayGaps(times(), minDelayMs: 0, index: 0) + ) + XCTAssertNil(checkDelayGaps(times(5), minDelayMs: 0, index: nil)) + } + + func testAbsentMinimumStillAssertsThatTheGapExists() { + // The default lands inside the function, so an omitted `min` cannot + // gate the whole assertion away at the call site. + XCTAssertEqual( + "Expected a delay between requests, but only 1 request(s) were made", + checkDelayGaps(times(), minDelayMs: nil, index: nil) + ) + XCTAssertNil(checkDelayGaps(times(5), minDelayMs: nil, index: nil)) + } + + func testNamedGapPassesWhenItClearsTheMinimum() { + XCTAssertNil(checkDelayGaps(times(5, 2000), minDelayMs: 500, index: 1)) + } + + func testNamedGapFailsWhenItIsBelowTheMinimum() throws { + let failure = try XCTUnwrap(checkDelayGaps(times(2000, 5), minDelayMs: 500, index: 1)) + XCTAssertTrue(failure.contains("at gap 1"), failure) + } + + func testIntegralMinimumRendersWithoutATrailingDecimal() throws { + let failure = try XCTUnwrap(checkDelayGaps(times(5), minDelayMs: 1000, index: 0)) + XCTAssertEqual("Expected delay >= 1000ms at gap 0, got 5ms", failure) + } + + func testEmptyRequestListFailsRatherThanReadingOutOfBounds() { + XCTAssertEqual( + "Expected a delay between requests, but only 0 request(s) were made", + checkDelayGaps([], minDelayMs: 500, index: nil) + ) + XCTAssertEqual( + "Expected a delay at gap 0, but only 0 request(s) were made", + checkDelayGaps([], minDelayMs: 500, index: 0) + ) + } +} diff --git a/conformance/runner/swift/Tests/ConformanceSupportTests/RequestIndexTests.swift b/conformance/runner/swift/Tests/ConformanceSupportTests/RequestIndexTests.swift new file mode 100644 index 0000000000..368ccb7905 --- /dev/null +++ b/conformance/runner/swift/Tests/ConformanceSupportTests/RequestIndexTests.swift @@ -0,0 +1,46 @@ +import XCTest + +@testable import ConformanceSupport + +/// The per-request `index` contract, pinned alongside the delay-gap one +/// because they share a fixture key and disagree about negatives on purpose. +/// +/// Characterization rather than regression: this resolver was already correct, +/// and every case here passes against the pre-fix code. It earns its place by +/// making the disagreement with `checkDelayGaps` explicit, so a later "tidy-up" +/// cannot quietly unify them. +final class RequestIndexTests: XCTestCase { + func testZeroSelectsTheFirstRequest() { + XCTAssertEqual(0, resolveRequestIndex(0, 3)) + } + + func testPositiveIndexSelectsThatRequest() { + XCTAssertEqual(2, resolveRequestIndex(2, 3)) + } + + func testNegativeIndexWrapsFromTheEnd() { + // Unlike checkDelayGaps, which rejects negatives outright. + XCTAssertEqual(2, resolveRequestIndex(-1, 3)) + XCTAssertEqual(0, resolveRequestIndex(-3, 3)) + } + + func testOutOfRangeIsNilRatherThanClamped() { + // A clamp would validate a different request than the fixture named, + // which is the shape that lets a dropped retry pass unnoticed. + XCTAssertNil(resolveRequestIndex(3, 3)) + XCTAssertNil(resolveRequestIndex(-4, 3)) + } + + func testNoRequestsRecordedResolvesToNil() { + XCTAssertNil(resolveRequestIndex(0, 0)) + XCTAssertNil(resolveRequestIndex(-1, 0)) + } + + func testExtremeIndexesResolveToNil() { + // No overflow to guard against here — `count` is a non-negative length + // and the negative branch only ever moves the sum toward zero, which + // is why this pair passes against the pre-fix resolver too. + XCTAssertNil(resolveRequestIndex(Int.min, 3)) + XCTAssertNil(resolveRequestIndex(Int.max, 3)) + } +} diff --git a/conformance/tests/network-retry.json b/conformance/tests/network-retry.json index 1779dd78bb..d2f215b148 100644 --- a/conformance/tests/network-retry.json +++ b/conformance/tests/network-retry.json @@ -1,7 +1,7 @@ [ { "name": "Network error on a non-idempotent POST is not retried", - "description": "A transport-level failure on CreateTodo (a non-idempotent POST) must surface as a network error with no re-send — no SDK re-sends a non-idempotent POST on a network blip. Exercises the transport-exception path that the 503 safety case cannot reach. Runs and passes in all five runners.", + "description": "A transport-level failure on CreateTodo (a non-idempotent POST) must surface as a network error with no re-send — no SDK re-sends a non-idempotent POST on a network blip. Exercises the transport-exception path that the 503 safety case cannot reach. Runs and passes in all six runners.", "operation": "CreateTodo", "method": "POST", "path": "/todolists/{todolistId}/todos.json", From 1c1387898e6325c9715303a87acdce502b31884b Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 1 Aug 2026 02:43:12 -0700 Subject: [PATCH 04/10] Count requests exactly, including where the SDK auto-paginates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit requestCount only checked a lower bound on fixtures carrying a Link rel="next" header, so an SDK that fetched too many pages passed. That is backwards for the fixtures it covered: "Pagination stops at maxPages safety cap" and "maxItems caps results across pages" both queue THREE pages and expect TWO requests, and the whole point is that the walk stops early. A lower bound cannot fail in the direction those cases exist to test. The one fixture where a count genuinely does not apply to an auto-paginating SDK — first page only, Link header present — is excluded by its own link-header tag before it reaches the evaluator, so the relaxation was protecting nothing that was still running. Proven both ways against a simulated regression that drops the maxPages cap: with the exact comparison the count assertion itself fails ("Expected 2 requests, got 3"); with the lower bound it stays silent and only the unrelated meta.truncated assertion fires. The multi-page fixture has no such companion assertion at all — requestCount and noError are the only two, so an over-fetch there had nothing to catch it. The transport keeps its auto-pagination tolerance. Answering an over-walk with a terminal empty page rather than a 500 is what lets the tightened assertion report a clean count mismatch instead of a decode error. The suite is unchanged at 136 passed, 0 failed, 1 skipped: Swift already made exactly the expected number of requests everywhere, so the lower bound was pure false-green surface with no coverage behind it. Kotlin's evaluator has the identical shape and the same fixtures; filed separately rather than widened here. --- .../Sources/ConformanceRunner/Assertions.swift | 17 ++++++++++------- .../Sources/ConformanceRunner/Runner.swift | 7 +++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift index 4097b33b67..3ed98a17b2 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift @@ -93,8 +93,7 @@ func evaluateAssertions( transport: ScriptedTransport, caughtError: BasecampError?, httpStatus: Int?, - dispatch: DispatchResult, - autoPaginates: Bool + dispatch: DispatchResult ) -> TestResult { let captured = transport.captured let requestCount = captured.count @@ -116,11 +115,15 @@ func evaluateAssertions( guard let expected = assertion.expected?.intValue.map(Int.init) else { return .fail("requestCount assertion missing expected value") } - if autoPaginates { - if requestCount < expected { - return .fail("Expected >= \(expected) requests (SDK auto-paginates), got \(requestCount)") - } - } else if requestCount != expected { + // Exact, including the auto-paginating fixtures. A lower bound + // makes the cap assertions vacuous in the direction that matters: + // "Pagination stops at maxPages safety cap" and "maxItems caps + // results across pages" both queue THREE pages and expect TWO + // requests, so `>=` passes an SDK that ignored the cap and walked + // every page. The first-page-only fixture, the one case where the + // count genuinely does not apply to an auto-paginating SDK, is + // excluded by its own `link-header` tag before it reaches here. + if requestCount != expected { return .fail("Expected \(expected) requests, got \(requestCount)") } diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift b/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift index f4fe72eb87..25009e38a1 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift @@ -168,6 +168,10 @@ struct Runner { } // Detect if the fixture uses Link next headers (SDK will auto-paginate). + // This only relaxes the TRANSPORT, which answers an over-walk with a + // terminal empty page instead of a 500. The count assertion stays exact, + // so an SDK that fetches too many pages reports a clean + // "Expected N requests, got N+1" rather than a decode error. let autoPaginates = tc.responses.contains { mock in mock.allHeaders.contains { key, value in key.lowercased() == "link" && value.contains("rel=\"next\"") @@ -236,8 +240,7 @@ struct Runner { transport: transport, caughtError: caughtError, httpStatus: httpStatus, - dispatch: dispatch, - autoPaginates: autoPaginates + dispatch: dispatch ) } } From 3aa08fdf7c4cc369b9eda219998d71f886990bc9 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 1 Aug 2026 02:57:20 -0700 Subject: [PATCH 05/10] Hold each operation to the endpoint its fixture names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner checked the first request's METHOD against the fixture but never its PATH, and the scripted transport answers any URL. So an operation aimed at the wrong endpoint consumed the queued responses and passed anyway: its retry, status, error-mapping, auth-header and pagination assertions all hold just as well against a resource the fixture never named. Both invariants exist for the same reason and only one of them was there. The first request must now match the fixture's declared path with pathParams substituted, unless the fixture carries an explicit requestPath assertion — the same carve-out the method invariant already used. The comparison is EXACT, against both the account-scoped form the SDK builds and the absolute form the download fixtures state. Deliberately not a suffix test: /999/my/ projects.json ends with /projects.json and both endpoints exist, so a suffix match would wave through exactly the confusion this is meant to catch. A placeholder the params do not supply is reported by name rather than left to fail as a mismatch, which is the difference between a fixable fixture and a puzzling diff. Proven by pointing GetProject at a hardcoded id: 20 fixtures that passed before now fail, each naming the endpoint it expected and the one it got. Every one of those had been verifying error mapping or retry behavior against the wrong project. The renderer and the match rule live in ConformanceSupport with nine tests of their own, including the suffix collision and the wrong-account case. Unchanged at 136 passed, 0 failed, 1 skipped: all 135 fixtures that reach the check already went where they said they would. --- .../ConformanceRunner/Assertions.swift | 22 ++++++ .../Sources/ConformanceRunner/Runner.swift | 6 +- .../ConformanceSupport/FixturePath.swift | 52 ++++++++++++++ .../FixturePathTests.swift | 69 +++++++++++++++++++ 4 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 conformance/runner/swift/Sources/ConformanceSupport/FixturePath.swift create mode 100644 conformance/runner/swift/Tests/ConformanceSupportTests/FixturePathTests.swift diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift index 3ed98a17b2..cb30044c77 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift @@ -109,6 +109,28 @@ func evaluateAssertions( return .fail("Expected first request method \(fixtureMethod), got \(first.method)") } + // Implicit PATH invariant, for the same reason: the transport answers any + // URL, so an operation aimed at the wrong endpoint consumes the queued + // responses and passes its retry, status, auth and pagination assertions + // against a resource the fixture never named. Checking the verb alone left + // that open. When the fixture declares a path and carries no explicit + // requestPath assertion, the first request must go there. + if !tc.fixturePath.isEmpty, + !tc.allAssertions.contains(where: { $0.type == "requestPath" }), + let first = captured.first { + let params = (tc.pathParams ?? [:]).compactMapValues { + $0.stringValue ?? $0.intValue.map(String.init) + } + switch renderFixturePath(tc.fixturePath, params) { + case .unsubstituted(let name): + return .fail("fixture path \"\(tc.fixturePath)\" has no pathParams entry for \"\(name)\"") + case .rendered(let expected): + if !requestPathMatches(first.path, fixturePath: expected, accountID: testAccountID) { + return .fail("Expected first request path /\(testAccountID)\(expected), got \(first.path)") + } + } + } + for assertion in tc.allAssertions { switch assertion.type { case "requestCount": diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift b/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift index 25009e38a1..7627af0afe 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Runner.swift @@ -1,8 +1,10 @@ import Basecamp import Foundation -/// Default account ID for conformance tests. -private let testAccountID = "999" +/// Default account ID for conformance tests. Not private: the path invariant +/// in the evaluator needs it to reconstruct the account-scoped URL the SDK +/// builds from an account-relative fixture path. +let testAccountID = "999" /// Operations whose dispatch arm passes `configOverrides.maxItems` into the /// SDK. Kept beside the backstop that enforces it so the two cannot drift: diff --git a/conformance/runner/swift/Sources/ConformanceSupport/FixturePath.swift b/conformance/runner/swift/Sources/ConformanceSupport/FixturePath.swift new file mode 100644 index 0000000000..3b854eb715 --- /dev/null +++ b/conformance/runner/swift/Sources/ConformanceSupport/FixturePath.swift @@ -0,0 +1,52 @@ +import Foundation + +/// Rendering a fixture's declared `path` template against its `pathParams`, +/// so the runner can hold an operation to the endpoint the fixture names. + +/// The outcome of rendering a path template. +public enum RenderedPath: Equatable, Sendable { + case rendered(String) + /// A `{placeholder}` the params did not supply. Fail-closed: an + /// unsubstituted template can never equal a real request path, but saying + /// WHICH parameter is missing is the difference between a fixable fixture + /// and a puzzling mismatch. + case unsubstituted(String) +} + +/// Substitutes `{name}` placeholders in a fixture path template. +/// +/// Values arrive already stringified by the caller, because a path parameter +/// that is neither a string nor an integer is a fixture bug the dispatch +/// accessors reject first — this function only has to render what survived. +public func renderFixturePath(_ template: String, _ params: [String: String]) -> RenderedPath { + var out = template + for (name, value) in params { + out = out.replacingOccurrences(of: "{\(name)}", with: value) + } + if let leftover = firstPlaceholder(in: out) { + return .unsubstituted(leftover) + } + return .rendered(out) +} + +/// Names the first `{...}` still present, or nil when the template is fully +/// rendered. Scans rather than using a regex so the target stays dependency-free. +private func firstPlaceholder(in path: String) -> String? { + guard let open = path.firstIndex(of: "{") else { return nil } + let afterOpen = path.index(after: open) + guard let close = path[afterOpen...].firstIndex(of: "}") else { return nil } + return String(path[afterOpen.. Bool { + actual == fixturePath || actual == "/\(accountID)\(fixturePath)" +} diff --git a/conformance/runner/swift/Tests/ConformanceSupportTests/FixturePathTests.swift b/conformance/runner/swift/Tests/ConformanceSupportTests/FixturePathTests.swift new file mode 100644 index 0000000000..1447ca9020 --- /dev/null +++ b/conformance/runner/swift/Tests/ConformanceSupportTests/FixturePathTests.swift @@ -0,0 +1,69 @@ +import XCTest + +@testable import ConformanceSupport + +/// The implicit path invariant. The scripted transport answers any URL, so +/// without this an operation pointed at the wrong endpoint still consumes the +/// queued responses and passes its retry, status and auth assertions. +final class FixturePathTests: XCTestCase { + func testSubstitutesEveryPlaceholder() { + XCTAssertEqual( + .rendered("/buckets/456/todosets/700/todos.json"), + renderFixturePath( + "/buckets/{bucketId}/todosets/{todosetId}/todos.json", + ["bucketId": "456", "todosetId": "700"]) + ) + } + + func testTemplateWithoutPlaceholdersIsUnchanged() { + XCTAssertEqual(.rendered("/projects.json"), renderFixturePath("/projects.json", [:])) + } + + func testMissingParameterIsNamedRatherThanLeftToMismatch() { + XCTAssertEqual( + .unsubstituted("bucketId"), + renderFixturePath("/buckets/{bucketId}/webhooks.json", [:]) + ) + } + + func testExtraParametersAreIgnored() { + XCTAssertEqual( + .rendered("/todos/456"), + renderFixturePath("/todos/{todoId}", ["todoId": "456", "unused": "1"]) + ) + } + + func testAccountRelativeAndAbsoluteFormsBothMatch() { + // Most fixtures state the account-relative path and the SDK prefixes + // the account id; the download fixtures state it already absolute. + XCTAssertTrue(requestPathMatches( + "/999/projects.json", fixturePath: "/projects.json", accountID: "999")) + XCTAssertTrue(requestPathMatches( + "/999999999/blobs/abcd/download/doc.pdf", + fixturePath: "/999999999/blobs/abcd/download/doc.pdf", accountID: "999")) + } + + func testASiblingEndpointDoesNotMatch() { + XCTAssertFalse(requestPathMatches( + "/999/todos.json", fixturePath: "/projects.json", accountID: "999")) + } + + func testSuffixCollisionDoesNotMatch() { + // The reason this is an equality test and not `hasSuffix`: an operation + // that hit /999/my/projects.json instead of /999/projects.json would + // otherwise pass, and both endpoints exist. + XCTAssertFalse(requestPathMatches( + "/999/my/projects.json", fixturePath: "/projects.json", accountID: "999")) + } + + func testWrongAccountDoesNotMatch() { + XCTAssertFalse(requestPathMatches( + "/111/projects.json", fixturePath: "/projects.json", accountID: "999")) + } + + func testUnsubstitutedTemplateCannotMatchARealPath() { + XCTAssertFalse(requestPathMatches( + "/999/buckets/456/webhooks.json", + fixturePath: "/buckets/{bucketId}/webhooks.json", accountID: "999")) + } +} From a9b7dcf280b864fecb0bc4f8739de49285dca638 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 1 Aug 2026 03:11:45 -0700 Subject: [PATCH 06/10] Exempt the implicit invariants only where an assertion covers request zero MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The implicit method and path checks stood down whenever the fixture carried ANY assertion of that type, regardless of which request it named. One fixture has exactly that shape: edit-clear pins requestPath at index 1, the composite's PUT, and nothing at index 0. So the leading GET of a read-modify-write composite was unchecked. If it regressed to a different todo while the PUT stayed correct, the method, body, count and index-1 path assertions all still held and the fixture reported PASS. The exemption now requires an explicit assertion that actually resolves to request 0. Both invariants take it, since both had the same coarse test. Isolated by diverging the composite's GET alone — the SDK's edit reads todo 777777 and writes todo 456: coarse exemption: PASS: edit-clear ... narrowed exemption: FAIL: edit-clear ... Expected first request path /999/todos/456, got /999/todos/777777 An earlier attempt changed the id the runner passed in, which moved both hops and so proved nothing about the carve-out; only patching the composite isolates the variable. Unchanged at 136 passed, 0 failed, 1 skipped. --- .../ConformanceRunner/Assertions.swift | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift index cb30044c77..78a787bfae 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift @@ -98,13 +98,25 @@ func evaluateAssertions( let captured = transport.captured let requestCount = captured.count + // The implicit invariants below defer to an explicit assertion, but only + // when that assertion actually covers the FIRST request. "Any assertion of + // this type exists" is too coarse: the EditTodo edit-clear fixture pins + // requestPath at index 1 only, so a coarse exemption left the composite's + // leading GET unchecked — a regression there would keep the method, body, + // count and index-1 path assertions all green. + func explicitAssertionCoversFirstRequest(_ type: String) -> Bool { + tc.allAssertions.contains { + $0.type == type && resolveRequestIndex($0.requestIndex, requestCount) == 0 + } + } + // Implicit method invariant: the scripted transport answers any verb, so a // wrong-verb request (e.g. a PUT regressing to POST) would consume a queued - // response silently. When the fixture declares a method and carries no - // explicit requestMethod assertions, the first request must use it. + // response silently. When the fixture declares a method and nothing + // explicitly pins the first request's method, it must use that verb. let fixtureMethod = tc.fixtureMethod.uppercased() if !fixtureMethod.isEmpty, - !tc.allAssertions.contains(where: { $0.type == "requestMethod" }), + !explicitAssertionCoversFirstRequest("requestMethod"), let first = captured.first, first.method != fixtureMethod { return .fail("Expected first request method \(fixtureMethod), got \(first.method)") } @@ -113,10 +125,9 @@ func evaluateAssertions( // URL, so an operation aimed at the wrong endpoint consumes the queued // responses and passes its retry, status, auth and pagination assertions // against a resource the fixture never named. Checking the verb alone left - // that open. When the fixture declares a path and carries no explicit - // requestPath assertion, the first request must go there. + // that open. if !tc.fixturePath.isEmpty, - !tc.allAssertions.contains(where: { $0.type == "requestPath" }), + !explicitAssertionCoversFirstRequest("requestPath"), let first = captured.first { let params = (tc.pathParams ?? [:]).compactMapValues { $0.stringValue ?? $0.intValue.map(String.init) From f15f0cfb9d3ffd4d049d31f63cc476ef4475eeaf Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 1 Aug 2026 03:32:37 -0700 Subject: [PATCH 07/10] Hold every hop to the fixture path, not just the first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The path invariant checked captured.first and stopped, so the second hop of a read-modify-write composite was unpinned. update-preserves-due-on GETs a card and PUTs it back; if the PUT regressed to a different endpoint while the GET stayed correct, the method, body, count and noError assertions all still held and the fixture reported PASS. Same shape as the leading-GET gap fixed one commit ago, from the other end. Every captured request must now match the fixture's declared path unless an explicit requestPath assertion names that exact index. Extending it that far is only sound because the hops that legitimately go elsewhere are a closed set. Measuring every fixture, exactly five diverge and all five are the download redirect and delegation flows — a 302 sends the SDK to a signed URL, and UploadsDownload fetches the upload then follows its download_url. Those now carry their own indexed requestPath assertions, so the general rule needs no exception inside the runner. Retries, pagination and every composite stay on the fixture's path already. The fixture assertions are the better half of this change: they are shared, so all six runners now pin the redirect target rather than only counting the hops. Verified by running all six. Proven by diverging the card composite's PUT alone — the SDK writes card 888888 and reads 1069479350: FAIL: update-preserves-due-on: composite refetches and resends the due date Expected request 1 at path /999/card_tables/cards/1069479350, got /999/card_tables/cards/888888 Six runners green: Go 135/0/2, Kotlin 136/0/1, TypeScript 168 passed, Ruby 126/0/11, Python 137/0/0, Swift 136/0/1. --- .../ConformanceRunner/Assertions.swift | 35 ++++++++++++------- conformance/tests/downloads.json | 12 ++++--- conformance/tests/uploads_download.json | 4 ++- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift index 78a787bfae..bfb1c6c9f8 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift @@ -99,14 +99,14 @@ func evaluateAssertions( let requestCount = captured.count // The implicit invariants below defer to an explicit assertion, but only - // when that assertion actually covers the FIRST request. "Any assertion of - // this type exists" is too coarse: the EditTodo edit-clear fixture pins - // requestPath at index 1 only, so a coarse exemption left the composite's - // leading GET unchecked — a regression there would keep the method, body, - // count and index-1 path assertions all green. - func explicitAssertionCoversFirstRequest(_ type: String) -> Bool { + // for the exact request that assertion names. "Any assertion of this type + // exists" is too coarse: the EditTodo edit-clear fixture pins requestPath + // at index 1 only, so a coarse exemption left the composite's leading GET + // unchecked — a regression there would keep the method, body, count and + // index-1 path assertions all green. + func explicitAssertionCovers(_ type: String, request index: Int) -> Bool { tc.allAssertions.contains { - $0.type == type && resolveRequestIndex($0.requestIndex, requestCount) == 0 + $0.type == type && resolveRequestIndex($0.requestIndex, requestCount) == index } } @@ -116,7 +116,7 @@ func evaluateAssertions( // explicitly pins the first request's method, it must use that verb. let fixtureMethod = tc.fixtureMethod.uppercased() if !fixtureMethod.isEmpty, - !explicitAssertionCoversFirstRequest("requestMethod"), + !explicitAssertionCovers("requestMethod", request: 0), let first = captured.first, first.method != fixtureMethod { return .fail("Expected first request method \(fixtureMethod), got \(first.method)") } @@ -126,9 +126,15 @@ func evaluateAssertions( // responses and passes its retry, status, auth and pagination assertions // against a resource the fixture never named. Checking the verb alone left // that open. - if !tc.fixturePath.isEmpty, - !explicitAssertionCoversFirstRequest("requestPath"), - let first = captured.first { + // + // EVERY hop, not just the first. Retries and pagination stay on the + // fixture's path, and so do the read-modify-write composites — a card or + // todo edit GETs and PUTs the same resource, so a regression in either + // hop alone is exactly what this catches. The hops that legitimately go + // elsewhere are the download redirect and delegation flows, and those say + // so with their own indexed requestPath assertions rather than being + // waved through by a rule in here. + if !tc.fixturePath.isEmpty, !captured.isEmpty { let params = (tc.pathParams ?? [:]).compactMapValues { $0.stringValue ?? $0.intValue.map(String.init) } @@ -136,8 +142,11 @@ func evaluateAssertions( case .unsubstituted(let name): return .fail("fixture path \"\(tc.fixturePath)\" has no pathParams entry for \"\(name)\"") case .rendered(let expected): - if !requestPathMatches(first.path, fixturePath: expected, accountID: testAccountID) { - return .fail("Expected first request path /\(testAccountID)\(expected), got \(first.path)") + for (i, request) in captured.enumerated() { + if explicitAssertionCovers("requestPath", request: i) { continue } + if !requestPathMatches(request.path, fixturePath: expected, accountID: testAccountID) { + return .fail("Expected request \(i) at path /\(testAccountID)\(expected), got \(request.path)") + } } } } diff --git a/conformance/tests/downloads.json b/conformance/tests/downloads.json index 418f362b2a..c5bb622ce6 100644 --- a/conformance/tests/downloads.json +++ b/conformance/tests/downloads.json @@ -13,7 +13,8 @@ {"type": "requestCount", "expected": 2}, {"type": "noError"}, {"type": "headerPresent", "path": "Authorization", "index": 0}, - {"type": "headerAbsent", "path": "Authorization", "index": -1} + {"type": "headerAbsent", "path": "Authorization", "index": -1}, + {"type": "requestPath", "expected": "/signed/logo.png", "index": -1} ], "tags": ["download", "redirect"] }, @@ -53,7 +54,8 @@ {"type": "headerPresent", "path": "Authorization", "index": 0}, {"type": "headerPresent", "path": "Authorization", "index": 1}, {"type": "headerPresent", "path": "Authorization", "index": 2}, - {"type": "headerAbsent", "path": "Authorization", "index": -1} + {"type": "headerAbsent", "path": "Authorization", "index": -1}, + {"type": "requestPath", "expected": "/signed/logo.png", "index": -1} ], "tags": ["download", "retry", "503"] }, @@ -74,7 +76,8 @@ {"type": "noError"}, {"type": "headerPresent", "path": "Authorization", "index": 0}, {"type": "headerPresent", "path": "Authorization", "index": 1}, - {"type": "headerAbsent", "path": "Authorization", "index": -1} + {"type": "headerAbsent", "path": "Authorization", "index": -1}, + {"type": "requestPath", "expected": "/signed/logo.png", "index": -1} ], "tags": ["download", "retry", "network"] }, @@ -110,7 +113,8 @@ {"type": "delayBetweenRequests", "min": 1000, "index": 0}, {"type": "noError"}, {"type": "headerPresent", "path": "Authorization", "index": 0}, - {"type": "headerAbsent", "path": "Authorization", "index": -1} + {"type": "headerAbsent", "path": "Authorization", "index": -1}, + {"type": "requestPath", "expected": "/signed/logo.png", "index": -1} ], "tags": ["download", "retry", "429", "retry-after"] }, diff --git a/conformance/tests/uploads_download.json b/conformance/tests/uploads_download.json index fa2ad98bcc..d966dc99bc 100644 --- a/conformance/tests/uploads_download.json +++ b/conformance/tests/uploads_download.json @@ -16,7 +16,9 @@ {"type": "noError"}, {"type": "headerPresent", "path": "Authorization", "index": 0}, {"type": "headerPresent", "path": "Authorization", "index": 1}, - {"type": "headerAbsent", "path": "Authorization", "index": -1} + {"type": "headerAbsent", "path": "Authorization", "index": -1}, + {"type": "requestPath", "expected": "/999999999/blobs/abcd1234/download/logo.png", "index": -2}, + {"type": "requestPath", "expected": "/signed/logo.png", "index": -1} ], "tags": ["upload", "download", "redirect"] }, From 7c24d42b3bdd15474fc11b0da7f1dc3d4265d358 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 1 Aug 2026 03:55:27 -0700 Subject: [PATCH 08/10] Require the account prefix, and follow the link the response actually gave MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more ways a request could go somewhere other than where the fixture said and still be answered from the queue. The path match accepted either the account-scoped form or the bare fixture path, so an SDK that dropped the account prefix and asked for /projects.json satisfied a fixture meaning /999/projects.json. Which form is required is now decided by the FIXTURE, not by whichever the SDK happened to send: a fixture path that already carries an account segment — only the download flows, which the SDK dials literally — must arrive verbatim, and every other path must arrive scoped. The captured path also dropped the query string, so pagination that refetched page 1 was indistinguishable from following the link to page 2 — and the transport handed it page 2's body anyway. Three requests, three pages, green, while production would have looped. Requests are captured with their query now, and a response advertising rel="next" pins the request that follows it to exactly that URL, resolved the way the SDK had to resolve it. Tightening the first check exposed why the second is the right shape: with account scoping enforced, five pagination fixtures failed, because their Link headers are root-relative and the SDK correctly follows them unscoped. The hop was being judged by two rules that disagreed. A hop that follows a link is now governed by the link rule alone — the most specific rule that applies, and the stronger one, since it pins the query too. Proven by dropping the query in the SDK's parseNextLink: with the link invariant: FAIL x5 Response 0 advertised rel="next" /projects.json?page=2, so request 1 must fetch /projects.json?page=2, got /projects.json without it: PASS x5, suite 136/0/1 Eleven more unit tests on the two rules, including an unscoped request against an account-relative fixture and a Link header with no next. Six runners green: Go 135/0/2, Kotlin 136/0/1, TypeScript 168 passed, Ruby 126/0/11, Python 137/0/0, Swift 136/0/1. --- .../ConformanceRunner/Assertions.swift | 43 ++++++++++- .../ConformanceRunner/ScriptedTransport.swift | 9 +++ .../ConformanceSupport/FixturePath.swift | 61 +++++++++++++-- .../FixturePathTests.swift | 74 +++++++++++++++++++ 4 files changed, 181 insertions(+), 6 deletions(-) diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift index bfb1c6c9f8..2a0a9de12e 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift @@ -121,6 +121,18 @@ func evaluateAssertions( return .fail("Expected first request method \(fixtureMethod), got \(first.method)") } + // Requests that follow a rel="next" link are governed by the LINK + // invariant below instead: they go where the previous response SAID to go, + // which the fixtures state root-relative, so they legitimately arrive + // unscoped. Each hop is pinned by exactly one rule — the most specific one + // that applies — rather than by two rules that disagree. + let linkFollowers: Set = Set( + tc.responses.enumerated().compactMap { i, mock in + mock.allHeaders.contains { $0.key.lowercased() == "link" && nextLinkTarget($0.value) != nil } + ? i + 1 : nil + } + ) + // Implicit PATH invariant, for the same reason: the transport answers any // URL, so an operation aimed at the wrong endpoint consumes the queued // responses and passes its retry, status, auth and pagination assertions @@ -143,14 +155,43 @@ func evaluateAssertions( return .fail("fixture path \"\(tc.fixturePath)\" has no pathParams entry for \"\(name)\"") case .rendered(let expected): for (i, request) in captured.enumerated() { + if linkFollowers.contains(i) { continue } if explicitAssertionCovers("requestPath", request: i) { continue } if !requestPathMatches(request.path, fixturePath: expected, accountID: testAccountID) { - return .fail("Expected request \(i) at path /\(testAccountID)\(expected), got \(request.path)") + let want = expectedRequestPath(expected, accountID: testAccountID) + return .fail("Expected request \(i) at path \(want), got \(request.path)") } } } } + // Implicit LINK invariant: a response that advertises rel="next" says + // exactly which URL to fetch, so the following request must be that URL — + // query string included. The path check above cannot see the query, and + // the transport answers any URL from the same queue, so pagination that + // refetched page 1 was handed page 2's body and reported three requests, + // three pages, all green. + // + // Only constrains a hop that actually happened: a walk stopped by a cap, + // or a link the SDK is meant to refuse (cross-origin, protocol downgrade), + // simply has no following request to check. + for (i, mock) in tc.responses.enumerated() where i + 1 < captured.count { + guard let link = mock.allHeaders.first(where: { $0.key.lowercased() == "link" })?.value, + let target = nextLinkTarget(link) + else { continue } + let follower = captured[i + 1] + // Resolve against the request that carried the link, so a relative + // target is compared the way the SDK had to resolve it. + let resolved = URL(string: target, relativeTo: follower.request.url) + let wanted = resolved.map { url -> String in + guard let query = url.query, !query.isEmpty else { return url.path } + return "\(url.path)?\(query)" + } ?? target + if follower.pathAndQuery != wanted { + return .fail("Response \(i) advertised rel=\"next\" \(target), so request \(i + 1) must fetch \(wanted), got \(follower.pathAndQuery)") + } + } + for assertion in tc.allAssertions { switch assertion.type { case "requestCount": diff --git a/conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift b/conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift index fede7fc51c..ca691cc126 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/ScriptedTransport.swift @@ -11,6 +11,15 @@ struct CapturedRequest: @unchecked Sendable { var method: String { request.httpMethod?.uppercased() ?? "" } var path: String { request.url?.path ?? "" } + /// Path WITH the query string. `path` alone cannot tell `/projects.json` + /// from `/projects.json?page=2`, so pagination that refetched page 1 was + /// answered from the queue with page 2's body and passed — three requests, + /// three pages, all green, while production would have looped on page 1. + var pathAndQuery: String { + guard let url = request.url else { return "" } + guard let query = url.query, !query.isEmpty else { return url.path } + return "\(url.path)?\(query)" + } var bodyJSON: JSON? { request.httpBody.flatMap { JSON.parse($0) } } /// Case-insensitive request-header lookup. diff --git a/conformance/runner/swift/Sources/ConformanceSupport/FixturePath.swift b/conformance/runner/swift/Sources/ConformanceSupport/FixturePath.swift index 3b854eb715..5cbab81270 100644 --- a/conformance/runner/swift/Sources/ConformanceSupport/FixturePath.swift +++ b/conformance/runner/swift/Sources/ConformanceSupport/FixturePath.swift @@ -38,15 +38,66 @@ private func firstPlaceholder(in path: String) -> String? { return String(path[afterOpen.. Bool { + guard fixturePath.hasPrefix("/") else { return false } + let rest = fixturePath.dropFirst() + let digits = rest.prefix(while: \.isNumber) + return !digits.isEmpty && rest.dropFirst(digits.count).hasPrefix("/") +} + /// Whether an observed request path matches a rendered fixture path. /// -/// The SDK prefixes the account id, and fixtures state the account-relative -/// path — except the download fixtures, whose path is already absolute because -/// the SDK dials a literal URL there. Both forms are accepted. +/// The form is decided by the FIXTURE, not by accepting whichever the SDK +/// happened to send. Accepting either let an unscoped request through: a +/// regression that dropped the account prefix and asked for `/projects.json` +/// satisfied a fixture meaning `/999/projects.json`, and the transport serves +/// any URL, so nothing else noticed. /// -/// EXACT on both, never a suffix test: `/999/my/projects.json` ends with +/// EXACT, never a suffix test: `/999/my/projects.json` ends with /// `/projects.json`, so a suffix match would wave through an operation that /// hit a neighbouring endpoint — the very thing this invariant exists to catch. public func requestPathMatches(_ actual: String, fixturePath: String, accountID: String) -> Bool { - actual == fixturePath || actual == "/\(accountID)\(fixturePath)" + fixtureIsAccountScoped(fixturePath) + ? actual == fixturePath + : actual == "/\(accountID)\(fixturePath)" +} + +/// The expected request path for a fixture path, for failure messages. +public func expectedRequestPath(_ fixturePath: String, accountID: String) -> String { + fixtureIsAccountScoped(fixturePath) ? fixturePath : "/\(accountID)\(fixturePath)" +} + +/// Extracts the `rel="next"` target from a `Link` header value, or nil when the +/// header names no next page. +/// +/// Deliberately tolerant of the surrounding syntax (multiple comma-separated +/// links, arbitrary parameter order and spacing) and strict about the target +/// itself, which is returned verbatim between the angle brackets. +public func nextLinkTarget(_ headerValue: String) -> String? { + for link in headerValue.split(separator: ",") { + let parts = link.split(separator: ";") + guard let target = parts.first?.trimmingCharacters(in: .whitespaces), + target.hasPrefix("<"), target.hasSuffix(">"), + parts.dropFirst().contains(where: { isRelNext($0) }) + else { continue } + return String(target.dropFirst().dropLast()) + } + return nil +} + +private func isRelNext(_ parameter: Substring) -> Bool { + let cleaned = parameter + .trimmingCharacters(in: .whitespaces) + .replacingOccurrences(of: " ", with: "") + .replacingOccurrences(of: "\"", with: "") + .lowercased() + return cleaned == "rel=next" } diff --git a/conformance/runner/swift/Tests/ConformanceSupportTests/FixturePathTests.swift b/conformance/runner/swift/Tests/ConformanceSupportTests/FixturePathTests.swift index 1447ca9020..973b944c9a 100644 --- a/conformance/runner/swift/Tests/ConformanceSupportTests/FixturePathTests.swift +++ b/conformance/runner/swift/Tests/ConformanceSupportTests/FixturePathTests.swift @@ -67,3 +67,77 @@ final class FixturePathTests: XCTestCase { fixturePath: "/buckets/{bucketId}/webhooks.json", accountID: "999")) } } + +/// Account scoping and Link parsing, the two rules that decide WHERE a hop was +/// allowed to go. +final class AccountScopingTests: XCTestCase { + func testAccountRelativeFixturePathsAreNotTreatedAsScoped() { + XCTAssertFalse(fixtureIsAccountScoped("/projects.json")) + XCTAssertFalse(fixtureIsAccountScoped("/buckets/456/webhooks.json")) + XCTAssertFalse(fixtureIsAccountScoped("/my/notes.json")) + } + + func testDownloadFixturePathsAreTreatedAsScoped() { + // The only shape that carries its own account segment; the SDK dials + // these literally. + XCTAssertTrue(fixtureIsAccountScoped("/999999999/blobs/abcd1234/download/logo.png")) + } + + func testANumericSegmentMustBeWholeAndLeading() { + XCTAssertFalse(fixtureIsAccountScoped("/999projects.json")) + XCTAssertFalse(fixtureIsAccountScoped("/todos/456")) + XCTAssertFalse(fixtureIsAccountScoped("")) + XCTAssertFalse(fixtureIsAccountScoped("999/projects.json")) + } + + func testAnUnscopedRequestFailsAnAccountRelativeFixture() { + // Accepting either form let a dropped account prefix pass: the + // transport serves any URL, so nothing else would have noticed. + XCTAssertFalse(requestPathMatches( + "/projects.json", fixturePath: "/projects.json", accountID: "999")) + XCTAssertTrue(requestPathMatches( + "/999/projects.json", fixturePath: "/projects.json", accountID: "999")) + } + + func testAScopedFixtureRequiresTheLiteralPath() { + let blob = "/999999999/blobs/abcd1234/download/logo.png" + XCTAssertTrue(requestPathMatches(blob, fixturePath: blob, accountID: "999")) + XCTAssertFalse(requestPathMatches("/999" + blob, fixturePath: blob, accountID: "999")) + } + + func testExpectedRequestPathReportsTheFormThatWasRequired() { + XCTAssertEqual("/999/projects.json", expectedRequestPath("/projects.json", accountID: "999")) + XCTAssertEqual("/9/blobs/x", expectedRequestPath("/9/blobs/x", accountID: "999")) + } +} + +final class NextLinkTests: XCTestCase { + func testExtractsARelativeNextTarget() { + XCTAssertEqual("/projects.json?page=2", nextLinkTarget("; rel=\"next\"")) + } + + func testExtractsAnAbsoluteNextTarget() { + XCTAssertEqual( + "https://evil.example.com/projects.json?page=2", + nextLinkTarget("; rel=\"next\"") + ) + } + + func testPicksNextOutOfSeveralLinks() { + XCTAssertEqual( + "/projects.json?page=3", + nextLinkTarget("; rel=\"prev\", ; rel=\"next\"") + ) + } + + func testToleratesUnquotedRelAndExtraSpacing() { + XCTAssertEqual("/p?page=2", nextLinkTarget(";rel=next")) + XCTAssertEqual("/p?page=2", nextLinkTarget(" ; rel = \"NEXT\"")) + } + + func testHeaderWithoutANextRelYieldsNil() { + XCTAssertNil(nextLinkTarget("; rel=\"prev\"")) + XCTAssertNil(nextLinkTarget("")) + XCTAssertNil(nextLinkTarget("garbage")) + } +} From ca900e4f6bddcdf34e9e4c40265f6bd05fed667e Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 1 Aug 2026 04:09:13 -0700 Subject: [PATCH 09/10] Fail a fixture whose operation never reached the transport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every invariant in the evaluator is guarded on having captured a request, so an operation that short-circuits before the transport skips all of them at once. The "HTTP allowed for localhost" fixture asserts only noError, so a ListProjects that returned locally without dialing would have reported PASS while exercising nothing — the runner green on a call it never watched. A fixture that queues mock responses is testing a wire operation, so one has to have happened. An EMPTY queue stays the deliberate no-request case: the HTTPS-enforcement fixture makes no call at all and says so with requestCount 0. It is the only fixture in the suite with an empty queue, and no fixture queues responses while expecting none to be used. Proven by returning early from the ListProjects dispatch arm: with the guard: FAIL: HTTP allowed for localhost fixture queues 1 mock response(s) but the operation made no request — it never reached the transport without it: PASS, suite 136/0/1 Six runners green: Go 135/0/2, Kotlin 136/0/1, TypeScript 168 passed, Ruby 126/0/11, Python 137/0/0, Swift 136/0/1. --- .../swift/Sources/ConformanceRunner/Assertions.swift | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift index 2a0a9de12e..8c1162d013 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift @@ -98,6 +98,18 @@ func evaluateAssertions( let captured = transport.captured let requestCount = captured.count + // A fixture that queues responses is testing a wire operation, so one must + // have happened. Every invariant below is guarded on having captured a + // request, so an operation short-circuited before the transport would slip + // through all of them and pass on a bare noError assertion — the runner + // reporting green on a call it never watched. + // + // An EMPTY queue is the deliberate no-request case: the HTTPS-enforcement + // fixture makes no call at all, and says so with requestCount 0. + if !tc.responses.isEmpty, captured.isEmpty { + return .fail("fixture queues \(tc.responses.count) mock response(s) but the operation made no request — it never reached the transport") + } + // The implicit invariants below defer to an explicit assertion, but only // for the exact request that assertion names. "Any assertion of this type // exists" is too coarse: the EditTodo edit-clear fixture pins requestPath From 0b06dccefad0a87c9e27da4a984ba6246873195f Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Sat, 1 Aug 2026 04:25:06 -0700 Subject: [PATCH 10/10] Check the method on every hop, matching the path invariant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The path invariant covers every request; the method invariant still stopped at the first. So a download could POST its signed final hop and satisfy everything else — the transport answers any verb, and the path, authorization, count and noError assertions were all still true. Both now cover every request not named by an indexed assertion. Measuring first: no hop in the suite currently diverges. Retries repeat the verb, a redirect followed after a GET stays a GET, and the read-modify-write composites — the one place a later hop legitimately differs — already pin their hops with indexed requestMethod assertions. Proven by making the redirect-following hop a POST in HTTPClient: every hop: FAIL x5, e.g. Expected request 1 to use method GET, got POST Expected request 3 to use method GET, got POST first only: PASS x5, suite 136/0/1 Six runners green: Go 135/0/2, Kotlin 136/0/1, TypeScript 168 passed, Ruby 126/0/11, Python 137/0/0, Swift 136/0/1. --- .../ConformanceRunner/Assertions.swift | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift index 8c1162d013..5dad284c1b 100644 --- a/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift +++ b/conformance/runner/swift/Sources/ConformanceRunner/Assertions.swift @@ -124,13 +124,22 @@ func evaluateAssertions( // Implicit method invariant: the scripted transport answers any verb, so a // wrong-verb request (e.g. a PUT regressing to POST) would consume a queued - // response silently. When the fixture declares a method and nothing - // explicitly pins the first request's method, it must use that verb. + // response silently. + // + // EVERY hop, like the path invariant below. Retries repeat the verb, a + // redirect followed after a GET stays a GET, and the read-modify-write + // composites — the one place a later hop legitimately differs — pin their + // hops with indexed requestMethod assertions already. Checking only the + // first left the download flows able to POST their signed final hop and + // still satisfy path, authorization, count and noError. let fixtureMethod = tc.fixtureMethod.uppercased() - if !fixtureMethod.isEmpty, - !explicitAssertionCovers("requestMethod", request: 0), - let first = captured.first, first.method != fixtureMethod { - return .fail("Expected first request method \(fixtureMethod), got \(first.method)") + if !fixtureMethod.isEmpty { + for (i, request) in captured.enumerated() { + if explicitAssertionCovers("requestMethod", request: i) { continue } + if request.method != fixtureMethod { + return .fail("Expected request \(i) to use method \(fixtureMethod), got \(request.method)") + } + } } // Requests that follow a rel="next" link are governed by the LINK