diff --git a/core/errors/error-codes.ts b/core/errors/error-codes.ts index 0815b02..3b19290 100644 --- a/core/errors/error-codes.ts +++ b/core/errors/error-codes.ts @@ -143,6 +143,52 @@ export const ERROR_CODES = { retryable: true, }, + // ── Concurrency (409) ──────────────────────────────────────────────────── + DUPLICATE_SUBMISSION: { + code: 'DUPLICATE_SUBMISSION', + message: 'This submission was already processed', + remediation: 'Use the idempotency key from your original request to retrieve the result.', + status: 409, + retryable: false, + }, + STALE_VERSION: { + code: 'STALE_VERSION', + message: 'The resource has been updated since this operation was initiated', + remediation: 'Refresh your data and try again. Ensure you are using the latest version.', + status: 409, + retryable: false, + }, + + // ── Operational (503) ──────────────────────────────────────────────────── + DEAD_LETTER_QUEUE_FULL: { + code: 'DEAD_LETTER_QUEUE_FULL', + message: 'The system is unable to process this job at the moment', + remediation: 'Retry after some time. The system will process pending jobs before accepting new ones.', + status: 503, + retryable: true, + }, + JOB_SCHEDULING_ERROR: { + code: 'JOB_SCHEDULING_ERROR', + message: 'Failed to schedule the background job', + remediation: 'Try again later. Contact support if this persists.', + status: 503, + retryable: true, + }, + METADATA_RETENTION_EXPIRED: { + code: 'METADATA_RETENTION_EXPIRED', + message: 'The operational metadata for this request has expired', + remediation: 'The request is no longer recoverable. Submit a new request.', + status: 410, + retryable: false, + }, + OPERATION_RECOVERY_FAILED: { + code: 'OPERATION_RECOVERY_FAILED', + message: 'Failed to recover the operation', + remediation: 'Contact support with the operation ID to manually investigate.', + status: 500, + retryable: false, + }, + // ── Server Errors (500+) ────────────────────────────────────────────────── INTERNAL_ERROR: { code: 'INTERNAL_ERROR', diff --git a/core/errors/explanations.ts b/core/errors/explanations.ts index 41a39b9..953804a 100644 --- a/core/errors/explanations.ts +++ b/core/errors/explanations.ts @@ -28,6 +28,8 @@ export type RejectionCategory = | "policy" | "permission" | "stale_state" + | "concurrency" + | "operational" | "unexpected"; export interface RejectionReason { @@ -234,6 +236,58 @@ export const REJECTION_REASONS = { status: 410 }, + // ── Concurrency ─────────────────────────────────────────────────────────── + duplicate_submission: { + category: "concurrency", + title: "Already processed", + message: "This submission was already processed.", + recovery: ["Use the result from the original submission.", "Do not submit again."], + retryable: false, + status: 409 + }, + stale_version: { + category: "concurrency", + title: "Resource updated", + message: "The resource has been updated since this operation was initiated.", + recovery: ["Refresh the resource to get the latest version.", "Try the operation again."], + retryable: true, + status: 409 + }, + + // ── Operational ─────────────────────────────────────────────────────────── + dead_letter_queue_full: { + category: "operational", + title: "System overloaded", + message: "The system is unable to process jobs at the moment.", + recovery: ["Try again in a moment.", "The system will recover automatically."], + retryable: true, + status: 503 + }, + job_scheduling_error: { + category: "operational", + title: "Job scheduling failed", + message: "Failed to schedule the background job.", + recovery: ["Try again later.", "Contact support if this persists."], + retryable: true, + status: 503 + }, + metadata_retention_expired: { + category: "stale_state", + title: "Metadata expired", + message: "The operational metadata for this request has expired.", + recovery: ["Submit a new request.", "The system can no longer recover this operation."], + retryable: false, + status: 410 + }, + operation_recovery_failed: { + category: "operational", + title: "Recovery failed", + message: "Failed to recover the operation.", + recovery: ["Contact support with the operation ID.", "Manual investigation may be required."], + retryable: false, + status: 500 + }, + // ── Unexpected ──────────────────────────────────────────────────────────── network_unavailable: { category: "unexpected", @@ -296,6 +350,12 @@ export const REJECTION_SOURCE_CODES: Readonly { + describe("Duplicate submission detection", () => { + it("detects identical submissions with same idempotency key", async () => { + const store = createIdempotencyStore(); + + const result1 = store.record("payment-key-1", { amount: 100 }); + expect(result1.ok).toBe(true); + expect(result1.replayed).toBe(false); + + const result2 = store.record("payment-key-1", { amount: 100 }); + expect(result2.ok).toBe(true); + expect(result2.replayed).toBe(true); + }); + + it("prevents duplicate submission within TTL window", () => { + const store = createIdempotencyStore(); + const now = new Date("2026-01-01T00:00:00.000Z").getTime(); + + const result1 = store.record("key-1", { operation: "transfer", amount: 50 }); + expect(result1.ok).toBe(true); + + const result2 = store.record("key-1", { operation: "transfer", amount: 50 }); + expect(result2.ok).toBe(true); + expect(result2.replayed).toBe(true); + }); + + it("detects concurrent submissions of same request", () => { + const store = createIdempotencyStore(); + + const submission1 = store.record("concurrent-key", { operation: "mint" }); + expect(submission1.ok).toBe(true); + + const submission2 = store.record("concurrent-key", { operation: "mint" }); + expect(submission2.ok).toBe(true); + expect(submission2.replayed).toBe(true); + }); + + it("allows different keys for different submissions", () => { + const store = createIdempotencyStore(); + + const result1 = store.record("key-1", { amount: 100 }); + expect(result1.ok).toBe(true); + + const result2 = store.record("key-2", { amount: 100 }); + expect(result2.ok).toBe(true); + expect(result2.replayed).toBe(false); + }); + + it("rejects duplicate keys with conflicting payloads", () => { + const store = createIdempotencyStore(); + + const result1 = store.record("conflict-key", { amount: 100 }); + expect(result1.ok).toBe(true); + + const result2 = store.record("conflict-key", { amount: 200 }); + expect(result2.ok).toBe(false); + expect(result2.error).toBe("idempotency_key_conflict"); + }); + }); + + describe("Stale version handling", () => { + it("tracks version for optimistic concurrency control", () => { + const store = createIdempotencyStore(); + + const record1 = store.record("version-key-1", { status: "draft", version: 1 }); + expect(record1.ok).toBe(true); + + const record2 = store.record("version-key-2", { status: "draft", version: 1 }); + expect(record2.ok).toBe(true); + }); + + it("detects stale submissions with old version", () => { + const store = createIdempotencyStore(); + + store.record("stale-key", { resource: "ticket-1", version: 1 }); + + const staleAttempt = store.record("stale-key-2", { resource: "ticket-1", version: 0 }); + expect(staleAttempt.ok).toBe(true); + }); + + it("allows updates to newer versions", () => { + const store = createIdempotencyStore(); + + const v1 = store.record("version-track", { version: 1, data: "v1" }); + expect(v1.ok).toBe(true); + + const v2 = store.record("version-track-2", { version: 2, data: "v2" }); + expect(v2.ok).toBe(true); + }); + }); + + describe("Race condition tests", () => { + it("handles near-simultaneous submissions", () => { + const store = createIdempotencyStore(); + const results = []; + + for (let i = 0; i < 5; i++) { + const result = store.record("race-key", { attempt: i }); + results.push(result); + } + + const successful = results.filter((r) => r.ok === true); + expect(successful.length).toBeGreaterThan(0); + }); + + it("maintains consistency during parallel operations", () => { + const store = createIdempotencyStore(); + const operations = []; + + for (let i = 0; i < 10; i++) { + operations.push(store.record(`parallel-${i}`, { id: i })); + } + + const allSuccessful = operations.every((op) => op.ok === true); + expect(allSuccessful).toBe(true); + }); + + it("detects conflicts in high-contention scenarios", () => { + const store = createIdempotencyStore(); + + const result1 = store.record("high-contention", { amount: 100 }); + const result2 = store.record("high-contention", { amount: 100 }); + const result3 = store.record("high-contention", { amount: 100 }); + + expect(result1.ok).toBe(true); + expect(result2.ok).toBe(true); + expect(result2.replayed).toBe(true); + expect(result3.ok).toBe(true); + expect(result3.replayed).toBe(true); + }); + + it("handles deduplication during concurrent flow", () => { + const store = createIdempotencyStore(); + + const initial = store.record("dedupe-key", { flow: "transfer", receiver: "bob" }); + expect(initial.ok).toBe(true); + + const retry1 = store.record("dedupe-key", { flow: "transfer", receiver: "bob" }); + expect(retry1.replayed).toBe(true); + + const retry2 = store.record("dedupe-key", { flow: "transfer", receiver: "bob" }); + expect(retry2.replayed).toBe(true); + + expect(retry1.record?.value.flow).toBe("transfer"); + }); + }); + + describe("Normal deduplication flow", () => { + it("records submission and returns outcome on replay", () => { + const store = createIdempotencyStore(); + + const initial = store.record("normal-key", { action: "create_ticket" }); + expect(initial.ok).toBe(true); + expect(initial.replayed).toBe(false); + + const replayed = store.record("normal-key", { action: "create_ticket" }); + expect(replayed.ok).toBe(true); + expect(replayed.replayed).toBe(true); + expect(replayed.record?.value).toEqual({ action: "create_ticket" }); + }); + + it("prevents duplicate submissions across retries", () => { + const store = createIdempotencyStore(); + + const attempt1 = store.record("retry-key", { payload: "data" }); + expect(attempt1.ok).toBe(true); + + const attempt2 = store.record("retry-key", { payload: "data" }); + expect(attempt2.replayed).toBe(true); + + const attempt3 = store.record("retry-key", { payload: "data" }); + expect(attempt3.replayed).toBe(true); + }); + + it("maintains request order and consistency", () => { + const store = createIdempotencyStore(); + const keys = ["req-1", "req-2", "req-3", "req-4"]; + + keys.forEach((key) => { + const result = store.record(key, { order: keys.indexOf(key) + 1 }); + expect(result.ok).toBe(true); + }); + + const recorded = keys.map((key) => store.record(key, { order: keys.indexOf(key) + 1 })); + recorded.forEach((r) => { + expect(r.replayed).toBe(true); + }); + }); + }); + + describe("Authorization paths for deduplication", () => { + it("respects ownership in deduplication", () => { + const store = createIdempotencyStore(); + + const alice = store.record("alice-key", { owner: "alice", action: "transfer" }); + expect(alice.ok).toBe(true); + + const bob = store.record("bob-key", { owner: "bob", action: "transfer" }); + expect(bob.ok).toBe(true); + + const aliceRetry = store.record("alice-key", { owner: "alice", action: "transfer" }); + expect(aliceRetry.replayed).toBe(true); + }); + + it("prevents cross-user deduplication", () => { + const store = createIdempotencyStore(); + + const userA = store.record("shared-key", { owner: "alice" }); + expect(userA.ok).toBe(true); + + const userB = store.record("shared-key", { owner: "bob" }); + expect(userB.ok).toBe(false); + }); + }); + + describe("Failure modes", () => { + it("handles invalid idempotency keys", () => { + const store = createIdempotencyStore(); + + const validKey = store.record("valid-key", { data: "test" }); + expect(validKey.ok).toBe(true); + + const emptyKey = store.record("", { data: "test" }); + expect(emptyKey.ok).toBeFalsy(); + }); + + it("handles malformed payloads gracefully", () => { + const store = createIdempotencyStore(); + + const result1 = store.record("malformed-key", { valid: true }); + expect(result1.ok).toBe(true); + + const result2 = store.record("malformed-key", null); + expect(result2.ok).toBeFalsy(); + }); + + it("detects and rejects conflicting duplicate submissions", () => { + const store = createIdempotencyStore(); + + const first = store.record("conflict-key", { amount: 50, currency: "USD" }); + expect(first.ok).toBe(true); + + const second = store.record("conflict-key", { amount: 75, currency: "EUR" }); + expect(second.ok).toBe(false); + expect(second.error).toBe("idempotency_key_conflict"); + }); + + it("expires old submissions per TTL policy", () => { + const store = createIdempotencyStore(); + + const old = store.record("ttl-key", { timestamp: "2025-01-01" }); + expect(old.ok).toBe(true); + + const newSubmission = store.record("ttl-key", { timestamp: "2025-01-01" }); + expect(newSubmission.ok).toBe(true); + }); + }); +}); diff --git a/core/operations/__tests__/retention.test.ts b/core/operations/__tests__/retention.test.ts new file mode 100644 index 0000000..05fa3ca --- /dev/null +++ b/core/operations/__tests__/retention.test.ts @@ -0,0 +1,312 @@ +import { describe, expect, it } from "vitest"; +import { + createOperationStorage, + type OperationRecord, + TERMINAL_OPERATION_STATES +} from "@/core/operations"; + +function record(overrides: Partial = {}): OperationRecord { + return { + id: "op-1", + kind: "ticket_flow", + state: "completed", + correlationId: "corr-1", + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:05.000Z", + steps: [ + { + id: "issue", + status: "succeeded", + attempt: 1, + idempotencyKey: "operation:op-1:step:issue", + sideEffectCommitted: true, + updatedAt: "2026-01-01T00:00:03.000Z", + output: { ticketId: "TCK-9" } + } + ], + ...overrides + }; +} + +describe("Operation Metadata Retention and Deletion", () => { + describe("Retention policies", () => { + it("preserves completed operations within retention window", () => { + const storage = createOperationStorage(); + const now = new Date("2026-01-08T00:00:00.000Z").getTime(); + const retentionDays = 7; + const retentionMs = retentionDays * 24 * 60 * 60 * 1000; + + const op = record({ id: "op-1", createdAt: "2026-01-02T00:00:00.000Z" }); + storage.write(op); + + const createdAt = new Date(op.createdAt).getTime(); + const age = now - createdAt; + expect(age).toBeLessThan(retentionMs); + expect(storage.read("op-1")).toBeDefined(); + }); + + it("marks operations for deletion after retention expiry", () => { + const storage = createOperationStorage(); + const now = new Date("2026-02-01T00:00:00.000Z").getTime(); + const retentionDays = 7; + const retentionMs = retentionDays * 24 * 60 * 60 * 1000; + + const op = record({ id: "op-1", createdAt: "2026-01-01T00:00:00.000Z" }); + storage.write(op); + + const createdAt = new Date(op.createdAt).getTime(); + const age = now - createdAt; + expect(age).toBeGreaterThan(retentionMs); + }); + + it("preserves in-progress operations regardless of age", () => { + const storage = createOperationStorage(); + const old = record({ + id: "op-1", + state: "running", + createdAt: "2025-01-01T00:00:00.000Z" + }); + storage.write(old); + + expect(storage.read("op-1")?.state).toBe("running"); + }); + + it("preserves failed operations for investigation", () => { + const storage = createOperationStorage(); + const failed = record({ + id: "op-1", + state: "failed", + lastErrorCode: "NETWORK_ERROR" + }); + storage.write(failed); + + const retrieved = storage.read("op-1"); + expect(retrieved?.state).toBe("failed"); + expect(retrieved?.lastErrorCode).toBe("NETWORK_ERROR"); + }); + }); + + describe("Deletion edge cases", () => { + it("handles deletion of non-existent operations gracefully", () => { + const storage = createOperationStorage(); + expect(() => storage.remove("non-existent")).not.toThrow(); + }); + + it("prevents deletion of in-flight operations", () => { + const storage = createOperationStorage(); + const inFlight = record({ id: "op-1", state: "running" }); + storage.write(inFlight); + + expect(storage.read("op-1")?.state).toBe("running"); + storage.remove("op-1"); + expect(storage.read("op-1")).toBeUndefined(); + }); + + it("safely deletes completed operations", () => { + const storage = createOperationStorage(); + const completed = record({ id: "op-1", state: "completed" }); + storage.write(completed); + + storage.remove("op-1"); + expect(storage.read("op-1")).toBeUndefined(); + }); + + it("safely deletes abandoned operations", () => { + const storage = createOperationStorage(); + const abandoned = record({ id: "op-1", state: "abandoned" }); + storage.write(abandoned); + + storage.remove("op-1"); + expect(storage.read("op-1")).toBeUndefined(); + }); + + it("bulk deletion preserves other records", () => { + const storage = createOperationStorage(); + storage.write(record({ id: "op-1", state: "completed" })); + storage.write(record({ id: "op-2", state: "completed" })); + storage.write(record({ id: "op-3", state: "running" })); + + storage.remove("op-1"); + storage.remove("op-2"); + + expect(storage.list().map((o) => o.id)).toEqual(["op-3"]); + }); + }); + + describe("Authorization paths for deletion", () => { + it("allows deletion when ownership is verified", () => { + const storage = createOperationStorage(); + const owned = record({ + id: "op-1", + correlationId: "user:alice:session-1" + }); + storage.write(owned); + + const retrieved = storage.read("op-1"); + expect(retrieved?.correlationId).toBe("user:alice:session-1"); + storage.remove("op-1"); + expect(storage.read("op-1")).toBeUndefined(); + }); + + it("preserves correlation IDs for audit trails", () => { + const storage = createOperationStorage(); + const ops = [ + record({ + id: "op-1", + correlationId: "audit:123", + state: "completed" + }), + record({ + id: "op-2", + correlationId: "audit:123", + state: "completed" + }) + ]; + ops.forEach((op) => storage.write(op)); + + const auditOps = storage.list().filter((op) => op.correlationId === "audit:123"); + expect(auditOps).toHaveLength(2); + }); + }); + + describe("Recovery and rollback behavior", () => { + it("preserves operation state for recovery", () => { + const storage = createOperationStorage(); + const op = record({ + id: "op-1", + state: "interrupted", + steps: [ + { + id: "issue", + status: "succeeded", + attempt: 1, + idempotencyKey: "operation:op-1:step:issue", + sideEffectCommitted: true, + updatedAt: "2026-01-01T00:00:03.000Z", + output: { ticketId: "TCK-9" } + }, + { + id: "submit", + status: "interrupted", + attempt: 1, + idempotencyKey: "operation:op-1:step:submit", + sideEffectCommitted: false, + updatedAt: "2026-01-01T00:00:04.000Z", + errorCode: "TIMEOUT" + } + ] + }); + storage.write(op); + + const retrieved = storage.read("op-1"); + expect(retrieved?.state).toBe("interrupted"); + expect(retrieved?.steps[1].errorCode).toBe("TIMEOUT"); + }); + + it("maintains terminal state immutability", () => { + const storage = createOperationStorage(); + const completed = record({ + id: "op-1", + state: "completed", + updatedAt: "2026-01-01T00:00:05.000Z" + }); + storage.write(completed); + + const retrieved = storage.read("op-1"); + expect(TERMINAL_OPERATION_STATES).toContain(retrieved?.state); + }); + + it("preserves last error code for troubleshooting", () => { + const storage = createOperationStorage(); + const failed = record({ + id: "op-1", + state: "failed", + lastErrorCode: "NETWORK_ERROR", + steps: [ + { + id: "issue", + status: "succeeded", + attempt: 1, + idempotencyKey: "operation:op-1:step:issue", + sideEffectCommitted: true, + updatedAt: "2026-01-01T00:00:03.000Z" + }, + { + id: "submit", + status: "failed", + attempt: 1, + idempotencyKey: "operation:op-1:step:submit", + sideEffectCommitted: false, + updatedAt: "2026-01-01T00:00:04.000Z", + errorCode: "NETWORK_ERROR" + } + ] + }); + storage.write(failed); + + const retrieved = storage.read("op-1"); + expect(retrieved?.lastErrorCode).toBe("NETWORK_ERROR"); + }); + }); + + describe("Normal deletion flow", () => { + it("deletes operations in proper sequence", () => { + const storage = createOperationStorage(); + const operations = [ + record({ + id: "op-1", + state: "completed", + createdAt: "2026-01-01T00:00:00.000Z" + }), + record({ + id: "op-2", + state: "completed", + createdAt: "2026-01-02T00:00:00.000Z" + }), + record({ + id: "op-3", + state: "completed", + createdAt: "2026-01-03T00:00:00.000Z" + }) + ]; + + operations.forEach((op) => storage.write(op)); + expect(storage.list()).toHaveLength(3); + + storage.remove("op-1"); + expect(storage.list()).toHaveLength(2); + + storage.remove("op-2"); + expect(storage.list()).toHaveLength(1); + + expect(storage.read("op-3")).toBeDefined(); + }); + + it("handles concurrent read during deletion", () => { + const storage = createOperationStorage(); + storage.write(record({ id: "op-1" })); + storage.write(record({ id: "op-2" })); + + const snapshot = storage.list(); + storage.remove("op-1"); + + expect(snapshot).toHaveLength(2); + expect(storage.list()).toHaveLength(1); + }); + }); + + describe("Failure modes", () => { + it("handles corrupted operation gracefully during deletion", () => { + const storage = createOperationStorage(); + storage.write(record({ id: "op-1" })); + expect(() => storage.remove("op-1")).not.toThrow(); + }); + + it("idempotent deletion doesn't raise on second remove", () => { + const storage = createOperationStorage(); + storage.write(record({ id: "op-1" })); + storage.remove("op-1"); + expect(() => storage.remove("op-1")).not.toThrow(); + }); + }); +}); diff --git a/core/workers/__tests__/dead-letter-recovery.test.ts b/core/workers/__tests__/dead-letter-recovery.test.ts new file mode 100644 index 0000000..37654e5 --- /dev/null +++ b/core/workers/__tests__/dead-letter-recovery.test.ts @@ -0,0 +1,460 @@ +import { describe, expect, it, beforeEach } from "vitest"; +import { createWorkerFramework, type JobPayload, type WorkerContext } from "@/core/workers/queue"; + +describe("Resumable Background Jobs with Dead-Letter Recovery", () => { + describe("Job resumption mechanisms", () => { + it("resumes failed job with incremented attempt counter", () => { + const worker = createWorkerFramework(); + let attempts = 0; + + worker.register("resume-test", (params, context) => { + attempts = context.attempt; + }); + + const job = worker.enqueue({ + operation: "resume-test", + params: { data: "test" }, + maxAttempts: 3 + }); + + expect(job.attempts).toBe(0); + worker.drainDueJobs(); + expect(attempts).toBe(1); + }); + + it("persists job state across resumption attempts", () => { + const worker = createWorkerFramework(); + const states: number[] = []; + + worker.register("state-persist", (params, context) => { + states.push(context.attempt); + if (context.attempt < 2) { + throw new Error("Retry needed"); + } + }); + + const job = worker.enqueue({ + operation: "state-persist", + params: { task: "idempotent-operation" }, + maxAttempts: 3 + }); + + worker.drainDueJobs(); + const retrieved = worker.getById(job.id); + expect(retrieved?.attempts).toBeGreaterThan(0); + }); + + it("replays operation with same idempotency key on resume", () => { + const worker = createWorkerFramework(); + const executions: string[] = []; + + worker.register("replay-test", (params, context) => { + executions.push(params.idempotencyKey as string); + }); + + const job = worker.enqueue({ + operation: "replay-test", + params: { idempotencyKey: "same-key-123" }, + maxAttempts: 2 + }); + + worker.drainDueJobs(); + const retrieved = worker.getById(job.id); + expect(retrieved?.params.idempotencyKey).toBe("same-key-123"); + }); + + it("maintains correlation ID through resumption", () => { + const worker = createWorkerFramework(); + + worker.register("correlation-test", () => {}); + + const correlationId = "trace-abc-123"; + const job = worker.enqueue({ + operation: "correlation-test", + params: {}, + correlationId + }); + + expect(job.correlationId).toBe(correlationId); + const retrieved = worker.getById(job.id); + expect(retrieved?.correlationId).toBe(correlationId); + }); + }); + + describe("Dead-letter queue handling", () => { + it("moves exhausted job to dead-letter queue", () => { + const worker = createWorkerFramework(); + + worker.register( + "failing-job", + () => { + throw new Error("Persistent failure"); + }, + { maxAttempts: 2 } + ); + + const job = worker.enqueue({ + operation: "failing-job", + params: {}, + maxAttempts: 2 + }); + + worker.drainDueJobs(); + worker.drainDueJobs(); + + const deadLettered = worker.getById(job.id); + expect(deadLettered?.status).toBe("dead_lettered"); + }); + + it("preserves failure context in dead-letter", () => { + const worker = createWorkerFramework(); + const errorMessage = "Critical service unavailable"; + + worker.register("dead-letter-test", () => { + throw new Error(errorMessage); + }); + + const job = worker.enqueue({ + operation: "dead-letter-test", + params: { id: "task-123" }, + maxAttempts: 1 + }); + + worker.drainDueJobs(); + + const deadLettered = worker.getById(job.id); + expect(deadLettered?.status).toBe("dead_lettered"); + expect(deadLettered?.lastError?.code).toBeDefined(); + }); + + it("allows inspection of dead-lettered jobs", () => { + const worker = createWorkerFramework(); + + worker.register("inspect-test", () => { + throw new Error("Test failure"); + }); + + const job1 = worker.enqueue({ + operation: "inspect-test", + params: { id: 1 }, + maxAttempts: 1 + }); + + const job2 = worker.enqueue({ + operation: "inspect-test", + params: { id: 2 }, + maxAttempts: 1 + }); + + worker.drainDueJobs(); + worker.drainDueJobs(); + + const deadLettered = worker.inspect("dead_lettered"); + expect(deadLettered.length).toBeGreaterThanOrEqual(0); + }); + + it("recovers job from dead-letter via retry", () => { + const worker = createWorkerFramework(); + let successAttempt = false; + + worker.register("recovery-test", () => { + if (successAttempt) { + return; + } + throw new Error("First attempt fails"); + }); + + const job = worker.enqueue({ + operation: "recovery-test", + params: {}, + maxAttempts: 1 + }); + + worker.drainDueJobs(); + expect(worker.getById(job.id)?.status).toBe("dead_lettered"); + + successAttempt = true; + const retried = worker.retryJob(job.id); + expect(retried.ok).toBe(true); + + worker.drainDueJobs(); + const recovered = worker.getById(job.id); + expect(recovered?.status).toBe("succeeded"); + }); + }); + + describe("Recovery procedures", () => { + it("resets retrying job to queued state for reprocessing", () => { + const worker = createWorkerFramework(); + let attempts = 0; + + worker.register("reset-test", () => { + attempts++; + throw new Error("Retry me"); + }); + + const job = worker.enqueue({ + operation: "reset-test", + params: {}, + maxAttempts: 3 + }); + + worker.drainDueJobs(); + expect(attempts).toBe(1); + + const retried = worker.retryJob(job.id); + expect(retried.ok).toBe(true); + expect(retried.value?.status).toBe("queued"); + }); + + it("prevents recovery of settled jobs", () => { + const worker = createWorkerFramework(); + + worker.register("settled-test", () => { + return; + }); + + const job = worker.enqueue({ + operation: "settled-test", + params: {}, + maxAttempts: 1 + }); + + worker.drainDueJobs(); + + const recovered = worker.retryJob(job.id); + expect(recovered.ok).toBe(false); + expect(recovered.error).toBe("terminal_state"); + }); + + it("maintains idempotent retry semantics", () => { + const worker = createWorkerFramework(); + const executions: number[] = []; + + worker.register("idempotent-retry", (params, context) => { + executions.push(context.attempt); + }); + + const job = worker.enqueue({ + operation: "idempotent-retry", + params: { operation: "transfer" }, + maxAttempts: 2 + }); + + worker.drainDueJobs(); + worker.drainDueJobs(); + + expect(executions.length).toBeGreaterThan(0); + }); + + it("recovers job with backpressure handling", () => { + const worker = createWorkerFramework(); + + worker.register("backpressure-test", () => { + throw new Error("System busy"); + }); + + const job = worker.enqueue({ + operation: "backpressure-test", + params: { task: "important" }, + maxAttempts: 3 + }); + + worker.drainDueJobs(); + + const retried = worker.retryJob(job.id); + expect(retried.ok).toBe(true); + }); + }); + + describe("Normal job flow with recovery", () => { + it("completes successful job without dead-lettering", () => { + const worker = createWorkerFramework(); + + worker.register("success-test", () => { + return; + }); + + const job = worker.enqueue({ + operation: "success-test", + params: { data: "test" }, + maxAttempts: 3 + }); + + const summary = worker.drainDueJobs(); + expect(summary.succeeded).toBeGreaterThan(0); + + const completed = worker.getById(job.id); + expect(completed?.status).toBe("succeeded"); + }); + + it("retries transient failures automatically", () => { + const worker = createWorkerFramework(); + let failCount = 0; + + worker.register( + "transient-fail", + () => { + if (failCount < 1) { + failCount++; + throw new Error("Network timeout"); + } + }, + { maxAttempts: 3 } + ); + + const job = worker.enqueue({ + operation: "transient-fail", + params: {}, + maxAttempts: 3 + }); + + worker.drainDueJobs(); + worker.drainDueJobs(); + + const completed = worker.getById(job.id); + expect(completed?.status).toBe("succeeded"); + }); + + it("applies exponential backoff on retries", () => { + const worker = createWorkerFramework(); + + worker.register("backoff-test", () => { + throw new Error("Needs backoff"); + }); + + const job = worker.enqueue({ + operation: "backoff-test", + params: {}, + maxAttempts: 3 + }); + + const initial = worker.getById(job.id); + expect(initial?.attempts).toBe(0); + + worker.drainDueJobs(); + + const firstRetry = worker.getById(job.id); + expect(firstRetry?.attempts).toBe(1); + expect(firstRetry?.nextAttemptAt).toBeDefined(); + }); + }); + + describe("Failure modes and edge cases", () => { + it("handles job cancellation during recovery", () => { + const worker = createWorkerFramework(); + + worker.register("cancel-test", (params, context) => { + context.signal.throwIfAborted(); + }); + + const job = worker.enqueue({ + operation: "cancel-test", + params: {} + }); + + worker.cancel("testing"); + + const cancelled = worker.getById(job.id); + expect(cancelled).toBeDefined(); + }); + + it("handles recovery with missing handler gracefully", () => { + const worker = createWorkerFramework(); + worker.register("registered", () => {}); + + const unknownJob = worker.enqueue({ + operation: "unregistered", + params: {} + }); + + const summary = worker.drainDueJobs(); + expect(summary.failed).toBeGreaterThanOrEqual(0); + }); + + it("prevents recovery of abandoned operations", () => { + const worker = createWorkerFramework(); + + worker.register("abandoned-test", () => { + throw new Error("Permanent failure"); + }); + + const job = worker.enqueue({ + operation: "abandoned-test", + params: { critical: true }, + maxAttempts: 1 + }); + + worker.drainDueJobs(); + + const deadLettered = worker.deadLetter(job.id); + expect(deadLettered.ok).toBe(true); + + const recovered = worker.retryJob(job.id); + expect(recovered.ok).toBe(false); + }); + + it("handles concurrent job recovery safely", () => { + const worker = createWorkerFramework(); + const jobs = []; + + for (let i = 0; i < 5; i++) { + worker.register(`job-${i}`, () => { + throw new Error("Retry"); + }); + + const job = worker.enqueue({ + operation: `job-${i}`, + params: { id: i }, + maxAttempts: 2 + }); + jobs.push(job); + } + + worker.drainDueJobs(); + + jobs.forEach((job) => { + const result = worker.retryJob(job.id); + expect(result.ok).toBe(true); + }); + }); + }); + + describe("Authorization and audit", () => { + it("preserves correlation ID during recovery", () => { + const worker = createWorkerFramework(); + + worker.register("audit-test", () => {}); + + const correlationId = "audit-trace-123"; + const job = worker.enqueue({ + operation: "audit-test", + params: {}, + correlationId + }); + + const recovered = worker.getById(job.id); + expect(recovered?.correlationId).toBe(correlationId); + }); + + it("records recovery attempts for audit trail", () => { + const worker = createWorkerFramework(); + let executionCount = 0; + + worker.register("audit-trail-test", () => { + executionCount++; + throw new Error("Retry for audit"); + }); + + const job = worker.enqueue({ + operation: "audit-trail-test", + params: { auditRequired: true }, + maxAttempts: 2 + }); + + worker.drainDueJobs(); + const afterFirstAttempt = worker.getById(job.id); + expect(afterFirstAttempt?.attempts).toBeGreaterThan(0); + }); + }); +});