diff --git a/.github/workflows/botiverse-build.yml b/.github/workflows/botiverse-build.yml new file mode 100644 index 00000000000..5955b1fcb07 --- /dev/null +++ b/.github/workflows/botiverse-build.yml @@ -0,0 +1,56 @@ +# Botiverse fork only: build the Linux x64 workerd binary for the cluster-alarms branch on a standard +# GitHub runner. Upstream's builds use Cloudflare's 16-core runners and remote cache, which this fork +# has neither of, so this is a plain from-scratch build (a few hours; Bazel's disk cache is kept +# between runs to make later builds faster). +name: Botiverse build + +on: + push: + branches: + - botiverse/** + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: botiverse-build-${{ github.ref }} + cancel-in-progress: true + +jobs: + linux-x64: + runs-on: ubuntu-24.04 + # GitHub-hosted jobs stop at 6 hours. + timeout-minutes: 355 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + df -h / + - name: Install the toolchain (clang, libc++ and lld 19+, tcl; see README.md) + run: | + sudo apt-get update -q + sudo apt-get install -y -q clang-19 libc++-19-dev libc++abi-19-dev lld-19 tcl8.6 + # Bazel's sandbox only sees /bin, /usr/bin and /usr/local/bin. + sudo ln -sf /usr/bin/clang-19 /usr/local/bin/clang + sudo ln -sf /usr/bin/clang++-19 /usr/local/bin/clang++ + sudo ln -sf /usr/bin/tclsh8.6 /usr/local/bin/tclsh + - uses: actions/cache@v5 + with: + path: ~/bazel-disk-cache + key: botiverse-bazel-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.bazelversion', '.bazelrc', 'MODULE.bazel') }}-${{ github.sha }} + restore-keys: | + botiverse-bazel-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.bazelversion', '.bazelrc', 'MODULE.bazel') }}- + - name: Build + # The Android SDK was removed above (disk space); without these the Android rules find it empty and fail. + run: env -u ANDROID_HOME -u ANDROID_SDK_ROOT -u ANDROID_NDK_HOME -u ANDROID_NDK_ROOT -u ANDROID_NDK_LATEST_HOME bazel build //src/workerd/server:workerd --config=release --disk_cache=~/bazel-disk-cache --jobs=4 + - name: Check the binary runs + run: bazel-bin/src/workerd/server/workerd --version + - uses: actions/upload-artifact@v7 + with: + name: workerd-linux-x64-${{ github.sha }} + path: bazel-bin/src/workerd/server/workerd + retention-days: 30 diff --git a/botiverse/alarm-test/README.md b/botiverse/alarm-test/README.md new file mode 100644 index 00000000000..45a26374773 --- /dev/null +++ b/botiverse/alarm-test/README.md @@ -0,0 +1,29 @@ +# Cluster-mode alarm cases (Botiverse) + +Alarm semantics with several cluster-mode workerd processes on one machine. Each case starts its own +fresh cluster of 3 processes (ports 8900-8902, a new shared directory under `/tmp`), and the script +exits 1 if any case fails. + +```sh +bazel build //src/workerd/server:workerd --config=release +node botiverse/alarm-test/cases.mjs # all cases +node botiverse/alarm-test/cases.mjs "leader" # cases whose name contains "leader" +WORKERD=/path/to/workerd node botiverse/alarm-test/cases.mjs +``` + +The cases: + +- the alarm time moved several times: it runs once, at the latest time; +- the handler throws twice: it is retried with backoff and succeeds on the third run; +- the handler sets its next alarm itself (`setAlarm()` inside `alarm()`), twice: it runs three times, and no alarm is left; +- `deleteAlarm()` before it is due: it never runs; +- the process holding the object is killed before the alarm is due: the leader claims the object and runs it once; +- the holder is killed in the middle of the handler: the alarm is re-run elsewhere and recorded once; +- the alarm leader is killed while it waits for a running handler: the alarm runs once and a new leader takes over; +- the leader is killed with an alarm pending: another process takes over and runs it once; +- 300 alarms due at the same moment: each runs exactly once; +- all processes set 600 alarms at once: no `SQLITE_BUSY`, all run once. + +Needs Node 22+. Linux only (cluster mode uses OFD locks). Upstream's own cluster test +(`src/workerd/server/tests/cluster`, `cluster-test-unix` / `cluster-test-cidr`) covers the +`network = "unix"` alarm path and the rejection across machines. diff --git a/botiverse/alarm-test/cases.mjs b/botiverse/alarm-test/cases.mjs new file mode 100644 index 00000000000..fb98d344baa --- /dev/null +++ b/botiverse/alarm-test/cases.mjs @@ -0,0 +1,164 @@ +#!/usr/bin/env node +// Cluster-mode alarm cases on one machine. Each case starts its own fresh cluster of local processes +// (ports 8900+, shared directory under /tmp), so cases don't share state. Exit code 1 if any case fails. +// node botiverse/alarm-test/cases.mjs [case ...] WORKERD= to use another build +import { spawn } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const WORKERD = process.env.WORKERD ?? join(here, "../../bazel-bin/src/workerd/server/workerd"); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +async function cluster(n) { + const dir = mkdtempSync(join(tmpdir(), "alarm-test-")); + const nodes = []; + for (let i = 0; i < n; i++) { + const log = []; + const child = spawn(WORKERD, ["serve", "--experimental", "--verbose", join(here, "config.capnp"), + `--directory-path=do-storage=${dir}`, `--socket-addr=http=127.0.0.1:${8900 + i}`], + { env: { ...process.env, NODE: `n${i}` }, stdio: ["ignore", "ignore", "pipe"] }); + child.stderr.on("data", (d) => log.push(String(d))); + nodes.push({ i, port: 8900 + i, child, log, alive: true }); + } + // up and listening + for (const nd of nodes) for (let t = 0; t < 50; t++) { try { await fetch(`http://127.0.0.1:${nd.port}/state?do=__ping`); break; } catch { await sleep(100); } } + await sleep(1500); // a leader is elected + const api = { + nodes, dir, + // Retries: right after a holder dies, the first request routed to its objects fails while the + // cluster notices (upstream behaviour, also retried by upstream's takeover test). + get: async (nd, path) => { + for (let t = 0; ; t++) { + try { const r = await fetch(`http://127.0.0.1:${nd.port}/${path}`); if (r.status < 500 || t >= 10) return await r.json(); } + catch (e) { if (t >= 10) throw e; } + await sleep(300); + } + }, + kill: (nd) => { nd.child.kill("SIGKILL"); nd.alive = false; }, + log: () => nodes.map((nd) => `--- n${nd.i}\n` + nd.log.join("").split("\n").filter((l) => /warning|error|leader/i.test(l) && !/symbolizer/.test(l)).slice(-8).join("\n")).join("\n"), + stop: () => { for (const nd of nodes) if (nd.alive) nd.child.kill("SIGKILL"); rmSync(dir, { recursive: true, force: true }); }, + }; + return api; +} +// The process that most recently logged becoming leader and is still alive. +function leaderOf(c) { + let best = null, bestAt = -1; + for (const nd of c.nodes) { + if (!nd.alive) continue; + const text = nd.log.join(""), at = text.lastIndexOf("now the alarm leader"); + if (at >= 0 && (best === null || at > bestAt)) { best = nd; bestAt = at; } + } + return best; +} +const others = (c, ex) => c.nodes.filter((nd) => nd.alive && nd !== ex); +async function waitFired(c, nd, name, want, ms) { + const end = Date.now() + ms; let s; + while (Date.now() < end) { s = await c.get(nd, `state?do=${name}`); if (s.fired.length >= want) break; await sleep(200); } + return s; +} + +const CASES = { + async "moved several times: runs once, at the latest time"(c) { + const L = leaderOf(c), [a, b] = others(c, L); + await c.get(a, "set?do=m&ms=10000"); await c.get(b, "set?do=m&ms=1000"); const last = await c.get(a, "set?do=m&ms=3000"); + await sleep(1800); + const early = await c.get(b, "state?do=m"); + if (early.fired.length) return `ran at a superseded time: ${JSON.stringify(early.fired)}`; + const s = await waitFired(c, b, "m", 1, 6000); await sleep(1500); + const f = (await c.get(b, "state?do=m")).fired; + if (f.length !== 1) return `fired ${f.length} times`; + if (f[0].at < last.set) return `ran before the latest time (${f[0].at} < ${last.set})`; + }, + async "handler throws twice: retried with backoff, then succeeds"(c) { + const L = leaderOf(c), [a] = others(c, L); + await c.get(a, "throw?do=t&n=2"); await c.get(a, "set?do=t&ms=300"); + const s = await waitFired(c, a, "t", 3, 15000); + if (s.fired.length !== 3) return `fired ${s.fired.length} times: ${JSON.stringify(s.fired)}`; + if (!s.fired[1].isRetry || !s.fired[2].isRetry) return `retries not marked: ${JSON.stringify(s.fired)}`; + }, + async "handler re-arms its own alarm (setAlarm inside alarm())"(c) { + const L = leaderOf(c), [a, b] = others(c, L); + await c.get(a, "set?do=r&ms=500&rearm=2"); + const s = await waitFired(c, b, "r", 3, 12000); await sleep(2500); + const f = (await c.get(b, "state?do=r")); + if (f.fired.length !== 3) return `want 3 runs (1 + 2 re-arms), got ${f.fired.length}`; + if (f.alarm !== null) return `alarm still set after the last run: ${f.alarm}`; + }, + async "deleteAlarm before it is due: never runs"(c) { + const L = leaderOf(c), [a, b] = others(c, L); + await c.get(a, "set?do=d&ms=1500"); await c.get(b, "del?do=d"); + await sleep(4000); + const f = (await c.get(a, "state?do=d")).fired; + if (f.length) return `ran after deleteAlarm: ${JSON.stringify(f)}`; + }, + async "holder killed before the alarm is due: leader claims the object, runs once"(c) { + const L = leaderOf(c), [a] = others(c, L); + await c.get(a, "set?do=k&ms=2000"); c.kill(a); + const s = await waitFired(c, L, "k", 1, 15000); await sleep(2500); + const f = (await c.get(L, "state?do=k")).fired; + if (f.length !== 1) return `fired ${f.length} times: ${JSON.stringify(f)}`; + }, + async "holder killed in the middle of the handler: re-run elsewhere, recorded once"(c) { + const L = leaderOf(c), [a] = others(c, L); + await c.get(a, "set?do=h&ms=500&slow=3000"); + await sleep(1500); c.kill(a); // a is inside alarm(), before it records anything + const s = await waitFired(c, L, "h", 1, 20000); await sleep(2500); + const f = (await c.get(L, "state?do=h")).fired; + if (f.length !== 1) return `recorded ${f.length} runs: ${JSON.stringify(f)}`; + }, + async "leader killed while it waits for a running handler: runs once"(c) { + const L = leaderOf(c), [a, b] = others(c, L); + await c.get(a, "set?do=l&ms=500&slow=3000"); + await sleep(1500); c.kill(L); // the delivery is in flight; a keeps running the handler + const s = await waitFired(c, b, "l", 1, 15000); await sleep(3000); + const f = (await c.get(b, "state?do=l")).fired; + if (f.length !== 1) return `ran ${f.length} times: ${JSON.stringify(f)}`; + if (!leaderOf(c)) return "no new leader"; + }, + async "leader killed with an alarm pending: another takes over, runs once"(c) { + const L = leaderOf(c), [a, b] = others(c, L); + await c.get(b, "set?do=p&ms=3000"); c.kill(L); + const s = await waitFired(c, b, "p", 1, 12000); await sleep(2500); + const f = (await c.get(b, "state?do=p")).fired; + if (f.length !== 1) return `fired ${f.length} times`; + }, + async "300 alarms due at the same time: each runs exactly once"(c) { + const live = others(c, null), N = 300; + await Promise.all(Array.from({ length: N }, (_, k) => c.get(live[k % live.length], `set?do=many${k}&ms=2500`))); + await sleep(9000); + const counts = await Promise.all(Array.from({ length: N }, (_, k) => c.get(live[(k + 1) % live.length], `state?do=many${k}`).then((s) => s.fired.length))); + const bad = counts.map((n, k) => [k, n]).filter(([, n]) => n !== 1); + if (bad.length) return `${bad.length}/${N} did not run exactly once (e.g. ${JSON.stringify(bad.slice(0, 5))})`; + }, + async "all processes set 600 alarms at once: no SQLITE_BUSY errors, all run"(c) { + const live = others(c, null), N = 600; + const res = await Promise.all(Array.from({ length: N }, (_, k) => + fetch(`http://127.0.0.1:${live[k % live.length].port}/set?do=busy${k}&ms=3000`).then((r) => r.status))); + const failed = res.filter((s) => s !== 200).length; + await sleep(10000); + const counts = await Promise.all(Array.from({ length: N }, (_, k) => c.get(live[k % live.length], `state?do=busy${k}`).then((s) => s.fired.length))); + const bad = counts.filter((n) => n !== 1).length; + const busy = c.nodes.some((nd) => nd.log.join("").includes("SQLITE_BUSY")); + if (failed || bad || busy) return `setAlarm failed ${failed}/${N}, not run exactly once ${bad}/${N}, SQLITE_BUSY in logs: ${busy}`; + }, +}; + +const only = process.argv.slice(2); +let failures = 0; +for (const [name, fn] of Object.entries(CASES)) { + if (only.length && !only.some((o) => name.includes(o))) continue; + const c = await cluster(3); + let err; + try { + if (!leaderOf(c)) err = "no alarm leader elected"; + else err = await fn(c); + } catch (e) { err = String(e?.stack ?? e); } + if (err) { failures++; console.log(`FAIL ${name}: ${err}\n${c.log()}`); } else console.log(`PASS ${name}`); + c.stop(); + await sleep(500); +} +console.log(failures ? `${failures} failed` : "all passed"); +process.exit(failures ? 1 : 0); diff --git a/botiverse/alarm-test/config.capnp b/botiverse/alarm-test/config.capnp new file mode 100644 index 00000000000..48a1cb173e9 --- /dev/null +++ b/botiverse/alarm-test/config.capnp @@ -0,0 +1,16 @@ +using Workerd = import "/workerd/workerd.capnp"; +const config :Workerd.Config = ( + services = [ + (name = "main", worker = .w), + (name = "do-storage", disk = (path = "do-storage", writable = true)), + ], + sockets = [ (name = "http", address = "127.0.0.1:8900", http = (), service = "main") ], + cluster = (sharedDirectory = "do-storage", network = "unix", key = "alarm-test-only"), +); +const w :Workerd.Worker = ( + modules = [ (name = "worker.js", esModule = embed "worker.js") ], + compatibilityDate = "2026-09-01", + durableObjectNamespaces = [ (className = "A", uniqueKey = "alarm-test-A") ], + durableObjectStorage = (localDisk = "do-storage"), + bindings = [ (name = "A", durableObjectNamespace = "A"), (name = "NODE", fromEnvironment = "NODE") ], +); diff --git a/botiverse/alarm-test/worker.js b/botiverse/alarm-test/worker.js new file mode 100644 index 00000000000..abe1f5c3093 --- /dev/null +++ b/botiverse/alarm-test/worker.js @@ -0,0 +1,38 @@ +// Alarm semantics test for cluster-mode workerd. Every process runs this same config; NODE (from the +// environment) says which process handled what. +import { DurableObject } from "cloudflare:workers"; + +export class A extends DurableObject { + async fetch(req) { + const u = new URL(req.url), s = this.ctx.storage, p = (k) => u.searchParams.get(k); + if (u.pathname === "/set") { + // rearm=N: the handler sets the next alarm itself, N more times; slow=ms: the handler takes that long + // (before it records anything); del=1: the handler deletes its alarm instead of letting it end. + if (p("rearm")) await s.put("rearm", +p("rearm")); + if (p("slow")) await s.put("slow", +p("slow")); + const at = Date.now() + +p("ms"); + await s.setAlarm(at); + return Response.json({ node: this.env.NODE, set: at }); + } + if (u.pathname === "/del") { await s.deleteAlarm(); return Response.json({ node: this.env.NODE, deleted: true }); } + if (u.pathname === "/throw") { await s.put("throws", +p("n")); return Response.json({ node: this.env.NODE }); } + if (u.pathname === "/state") return Response.json({ node: this.env.NODE, alarm: await s.getAlarm(), fired: (await s.get("fired")) ?? [] }); + return new Response("?", { status: 404 }); + } + async alarm(info) { + const s = this.ctx.storage; + const slow = (await s.get("slow")) ?? 0; + if (slow) await new Promise((r) => setTimeout(r, slow)); + const fired = (await s.get("fired")) ?? [], throws = (await s.get("throws")) ?? 0, rearm = (await s.get("rearm")) ?? 0; + fired.push({ node: this.env.NODE, at: Date.now(), retry: info?.retryCount ?? 0, isRetry: !!info?.isRetry }); + await s.put("fired", fired); + if (throws > 0) { await s.put("throws", throws - 1); throw new Error("planned failure"); } + if (rearm > 0) { await s.put("rearm", rearm - 1); await s.setAlarm(Date.now() + 1000); } + } +} +export default { + async fetch(req, env) { + const u = new URL(req.url), name = u.searchParams.get("do") ?? "x"; + return env.A.get(env.A.idFromName(name)).fetch(req); + }, +}; diff --git a/compile_flags.txt b/compile_flags.txt index 9ffc5c68c3e..fec26b8509d 100644 --- a/compile_flags.txt +++ b/compile_flags.txt @@ -21,7 +21,10 @@ -isystembazel-bin/external/+http+capnp-cpp/src/capnp/compat/_virtual_includes/http-over-capnp -isystembazel-bin/external/+http+capnp-cpp/src/capnp/compat/_virtual_includes/json -isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj --isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj-async +-isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj-async-core +-isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj-async-io +-isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj-async-os-hdrs +-isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj-test -isystembazel-bin/external/+http+capnp-cpp/src/kj/compat/_virtual_includes/kj-brotli -isystembazel-bin/external/+http+capnp-cpp/src/kj/compat/_virtual_includes/kj-gzip -isystembazel-bin/external/+http+capnp-cpp/src/kj/compat/_virtual_includes/kj-http diff --git a/src/workerd/api/restore.c++ b/src/workerd/api/restore.c++ index 142623dcabe..33d404903f8 100644 --- a/src/workerd/api/restore.c++ +++ b/src/workerd/api/restore.c++ @@ -495,46 +495,78 @@ kj::Promise RestoreRpcStubCustomEvent::run } } -kj::Promise RestoreRpcStubCustomEvent::sendRpc( - capnp::HttpOverCapnpFactory& httpOverCapnpFactory, - capnp::ByteStreamFactory& byteStreamFactory, - FrankenvalueHandler& frankenvalueHandler, - rpc::EventDispatcher::Client dispatcher) { - // This contains a lot of similar code to JsRpcSessionCustomEvent::sendRpc() for setting - // up membranes, handling the returned JsRpcSession, etc. +namespace { - auto revokePaf = kj::newPromiseAndFulfiller(); +kj::Promise remoteRpcSessionTask(rpc::JsRpcSession::Client session, + kj::Promise allStubsDropped, + kj::Own> revoker) { + // Revoke all stubs in the session as soon as this task completes or is canceled. Normally the + // server side doesn't end the session while any stubs still exist, so this only makes a + // difference when some sort of error occurred. The stubs are probably already broken in that + // case, but revoking them ensures the underlying transport isn't held open waiting for the JS + // garbage collector to actually collect the JsRpcStub objects. KJ_DEFER({ - if (revokePaf.fulfiller->isWaiting()) { - revokePaf.fulfiller->reject(KJ_EXCEPTION(DISCONNECTED, "JS-RPC-restore session canceled")); + if (revoker->isWaiting()) { + revoker->reject(KJ_EXCEPTION(DISCONNECTED, "JS-RPC session canceled")); } }); - auto req = dispatcher.restoreRpcStubRequest(); - frankenvalueHandler.toCapnp(restoreParams, req.initParams()); - auto sent = req.send(); + try { + // Wait for `session` to resolve to a null capability, which the server does once the session + // is over. (If the server is using the "old approach" of not returning a `session` at all, the + // return itself represents the end of the session and `session` resolves to null just the + // same.) + co_await session.whenResolved().exclusiveJoin(kj::mv(allStubsDropped)); + } catch (...) { + auto e = kj::getCaughtExceptionAsKj(); + if (revoker->isWaiting()) { + revoker->reject(e.clone()); + } + kj::throwFatalException(kj::mv(e)); + } +} - rpc::JsRpcTarget::Client cap = sent.getTarget(); +} // namespace +IoChannelFactory::RpcChannel::Session wrapRemoteRpcSession( + rpc::JsRpcTarget::Client cap, rpc::JsRpcSession::Client session) { + auto revokePaf = kj::newPromiseAndFulfiller(); cap = capnp::membrane(kj::mv(cap), kj::refcounted(kj::mv(revokePaf.promise))); + // When no more stubs exist in the session, proactively end it. This matters in particular when + // the client drops the stub without making any calls at all, e.g. because serializing the + // arguments failed: `cap` may be a pipelined capability on a call that has not returned yet, in + // which case simply dropping it is not detected by the server. auto completionPaf = kj::newPromiseAndFulfiller(); cap = capnp::membrane( kj::mv(cap), kj::refcounted(kj::mv(completionPaf.fulfiller))); - capFulfiller->fulfill(kj::mv(cap)); + return { + .cap = kj::mv(cap), + .task = remoteRpcSessionTask( + kj::mv(session), kj::mv(completionPaf.promise), kj::mv(revokePaf.fulfiller)), + }; +} + +kj::Promise RestoreRpcStubCustomEvent::sendRpc( + capnp::HttpOverCapnpFactory& httpOverCapnpFactory, + capnp::ByteStreamFactory& byteStreamFactory, + FrankenvalueHandler& frankenvalueHandler, + rpc::EventDispatcher::Client dispatcher) { + auto req = dispatcher.restoreRpcStubRequest(); + frankenvalueHandler.toCapnp(restoreParams, req.initParams()); + auto sent = req.send(); + + auto session = wrapRemoteRpcSession(sent.getTarget(), sent.getSession()); - auto session = sent.getSession(); + // `session.task` already propagates any errors from the call, so the call promise itself can be + // dropped. (It would NOT work to use `req.sendForPipeline()` above, since pipelined capabilities + // cannot resolve until the call returns, but `sendForPipeline()` explicitly inhibits the return + // message.) { auto drop = kj::mv(sent); } - try { - co_await session.whenResolved().exclusiveJoin(kj::mv(completionPaf.promise)); - } catch (...) { - auto e = kj::getCaughtExceptionAsKj(); - if (revokePaf.fulfiller->isWaiting()) { - revokePaf.fulfiller->reject(e.clone()); - } - kj::throwFatalException(kj::mv(e)); - } + + capFulfiller->fulfill(kj::mv(session.cap)); + co_await session.task; co_return WorkerInterface::CustomEvent::Result{.outcome = EventOutcome::OK}; } diff --git a/src/workerd/api/restore.h b/src/workerd/api/restore.h index 9a899c90af8..81d9bf3f3fc 100644 --- a/src/workerd/api/restore.h +++ b/src/workerd/api/restore.h @@ -55,6 +55,17 @@ jsg::Promise restoreCurrentEntrypoint(jsg::Lock& js, const jsg::TypeHandler>& fetcherHandler, const jsg::TypeHandler>& rpcStubHandler); +// Wraps the `target` and `session` capabilities returned by an RPC that opened a JS RPC session on +// a remote worker (`EventDispatcher.restoreRpcStub()`, `WorkerdDebugPort.getRpcTargetFromToken()`) +// into an `RpcChannel::Session`. The returned `cap` is membraned so that dropping every stub +// derived from it ends the session promptly, without waiting for the server to notice, and so that +// completing or cancelling the returned `task` revokes any stubs still outstanding, so they cannot +// hold the underlying transport open. `task` completes when `session` resolves (to null, i.e. the +// server ended the session) or all local stubs are dropped, whichever comes first, and fails if the +// session fails. +IoChannelFactory::RpcChannel::Session wrapRemoteRpcSession( + rpc::JsRpcTarget::Client target, rpc::JsRpcSession::Client session); + // Custom event that calls the `[restore]()` method of the entrypoint and expects that the result // is a `Fetcher`. (When the result is expected to be an RpcStub instead, we use // `RestoreRpcStubCustomEvent`.) diff --git a/src/workerd/io/actor-sqlite-test.c++ b/src/workerd/io/actor-sqlite-test.c++ index 502e5df03c0..60dba32d4fe 100644 --- a/src/workerd/io/actor-sqlite-test.c++ +++ b/src/workerd/io/actor-sqlite-test.c++ @@ -2569,42 +2569,36 @@ KJ_TEST("unconfirmed setAlarm failure still breaks output gate") { KJ_EXPECT_THROW_MESSAGE("alarm commit failed", promise.wait(test.ws)); } +// Drives `test` into the broken state through SqliteDatabase's critical-error path: a write that +// fails with SQLITE_NOMEM inside `txn` makes SQLite auto-rollback the transaction, which invokes +// ActorSqlite::onCriticalError(). The global heap limit is restored before returning so that +// connections opened later are unaffected. +void breakViaCriticalError(ActorSqliteTest& test, ActorCacheInterface::Transaction& txn) { + auto heapLimit = test.db.run("PRAGMA hard_heap_limit").getInt64(0); + KJ_DEFER(sqlite3_hard_heap_limit64(heapLimit)); + + test.db.run("PRAGMA hard_heap_limit=8192"); // 8KB limit + + // A value large enough to exceed the limit. Copied once more to convert Array to + // Array. + auto largeData = kj::heapArray(50000, 'X'); + try { + txn.put(kj::str("large_key"), kj::heapArray(largeData.asBytes()), {}, nullptr); + KJ_FAIL_ASSERT("Query should have failed with SQLITE_NOMEM"); + } catch (kj::Exception& e) { + KJ_ASSERT(e.getDescription().contains("SENTRY_DO"), e); + KJ_ASSERT(e.getDescription().contains("out of memory"), e); + } +} + KJ_TEST("sync() throws after critical error in explicit transaction") { ActorSqliteTest test({.monitorOutputGate = false}); - auto heapLimit = [&]() { - auto row = test.db.run("PRAGMA hard_heap_limit"); - return row.getInt(0); - }(); - KJ_DBG(heapLimit); - KJ_DEFER(sqlite3_hard_heap_limit64(heapLimit);); - // Start an explicit transaction + // Start an explicit transaction and do a write within it. auto txn = test.startTransaction(); - - // Do a write within the transaction txn->put(kj::str("foo"), kj::heapArray("bar"_kj.asBytes()), {}, nullptr); - // Trigger a critical error using SQLITE_NOMEM by setting a very low heap limit - // and then trying to insert a large value. - try { - // Set SQLite's memory limit very low to trigger SQLITE_NOMEM - test.db.run("PRAGMA hard_heap_limit=8192"); // 8KB limit - - // Create data that will exceed the memory limit - auto largeData = kj::heapArray(50000, 'X'); // 50KB - - // This should trigger SQLITE_NOMEM, causing SQLite to auto-rollback the transaction, - // which will trigger the critical error handler. - // - // We have to copy it again in order to convert to a Array from an Array. - txn->put(kj::str("large_key"), kj::heapArray(largeData.asBytes()), /*options=*/{}, - nullptr); - KJ_FAIL_ASSERT("Query should have failed with SQLITE_NOMEM"); - } catch (kj::Exception& e) { - // Expected: out of memory error. We catch and ignore this to continue the test. - KJ_ASSERT(e.getDescription().contains("SENTRY_DO")); - KJ_ASSERT(e.getDescription().contains("out of memory")); - } + breakViaCriticalError(test, *txn); // sync() should also throw an exception because the storage is now broken auto syncResult = test.sync(); @@ -2617,6 +2611,112 @@ KJ_TEST("sync() throws after critical error in explicit transaction") { KJ_EXPECT_THROW_MESSAGE("broken", txn->commit()); } +// Asserts that `test.db` has been closed. Direct use fails, and a second connection to the same +// file can take an exclusive lock (which the closed connection's open transaction would have +// blocked) and sees the committed write but not the uncommitted one. +void expectDatabaseClosed(ActorSqliteTest& test) { + KJ_EXPECT_THROW_MESSAGE("database has been closed", test.db.run("SELECT 1")); + + SqliteDatabase fresh(test.vfs, kj::Path({"foo"}), kj::WriteMode::MODIFY); + fresh.run("BEGIN EXCLUSIVE"); + KJ_EXPECT(fresh.run("SELECT count(*) FROM _cf_KV WHERE key = 'committed'").getInt(0) == 1); + KJ_EXPECT(fresh.run("SELECT count(*) FROM _cf_KV WHERE key = 'uncommitted'").getInt(0) == 0); + fresh.run("COMMIT"); +} + +KJ_TEST("shutdown() leaves the connection open; the owner closes it afterwards") { + ActorSqliteTest test({.monitorOutputGate = false}); + + test.put("committed", "yes"); + test.pollAndExpectCalls({"commit"})[0]->fulfill(); + + // Leave a second write open in an implicit transaction whose commit has not run yet. + test.put("uncommitted", "yes"); + + test.actor.shutdown(kj::none); + + // Storage is broken, but shutdown() must not close the connection itself: embedders may call it + // from inside a SQLite callback, where closing is unsafe. The owner closes it later. + KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.actor.getSqliteKv()); + test.db.run("SELECT 1"); + + // The scheduled commit observes the shutdown and breaks the output gate with the shutdown + // exception rather than committing. + KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.gate.onBroken().wait(test.ws)); + test.pollAndExpectCalls({}); + + // Closing with the implicit transaction still open rolls it back and releases the file. + test.db.close(); + expectDatabaseClosed(test); + + // A second shutdown() on a closed database is a no-op. + test.actor.shutdown(kj::none); + KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.actor.getSqliteKv()); + expectDatabaseClosed(test); +} + +KJ_TEST("shutdown() after a critical error keeps the original exception") { + ActorSqliteTest test({.monitorOutputGate = false}); + + test.put("committed", "yes"); + test.pollAndExpectCalls({"commit"})[0]->fulfill(); + + auto txn = test.startTransaction(); + txn->put(kj::str("uncommitted"), kj::heapArray("yes"_kj.asBytes()), {}, nullptr); + breakViaCriticalError(test, *txn); + KJ_EXPECT_THROW_MESSAGE("broken.outputGateBroken", test.actor.getSqliteKv()); + + auto expectOriginalBrokenException = [&]() { + KJ_IF_SOME(e, kj::runCatchingExceptions([&]() { test.actor.getSqliteKv(); })) { + // The first exception wins: the shutdown exception must not replace the storage error. + KJ_EXPECT(e.getDescription().contains("broken.outputGateBroken"), e); + KJ_EXPECT(!e.getDescription().contains(ActorCache::SHUTDOWN_ERROR_MESSAGE), e); + } else { + KJ_FAIL_EXPECT("getSqliteKv() should have thrown"); + } + }; + + // shutdown() runs after the break (the broken output gate aborts the IoContext), then the owner + // closes the connection. + test.actor.shutdown(kj::none); + expectOriginalBrokenException(); + test.db.close(); + expectDatabaseClosed(test); + + test.actor.shutdown(kj::none); + expectOriginalBrokenException(); +} + +KJ_TEST("shutdown() during in-flight commit does not unschedule the persisted alarm") { + ActorSqliteTest test({.monitorOutputGate = false}); + + // Persist an alarm at 2ms, so that the next write moves the alarm earlier and thus starts a + // precommit scheduling request that the commit must wait on. + test.setAlarm(twoMs); + test.pollAndExpectCalls({"scheduleRun(2ms)"})[0]->fulfill(); + test.pollAndExpectCalls({"commit"})[0]->fulfill(); + + // Move the alarm earlier. The local transaction commits, then commitImpl() suspends waiting for + // the scheduler to confirm the 1ms request. + test.setAlarm(oneMs); + auto scheduleFulfiller = kj::mv(test.pollAndExpectCalls({"scheduleRun(1ms)"})[0]); + + // Shut down while that commit is parked. + test.actor.shutdown(kj::none); + + // Resume the commit. Now that storage is broken, it must not consult the database's alarm state + // and must not issue any further scheduler requests. In particular it must not issue + // `scheduleRun(none)`, which would cancel the alarm that the database still persists. + scheduleFulfiller->fulfill(); + test.pollAndExpectCalls({}); + KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.gate.onBroken().wait(test.ws)); + + // The database still holds the 1ms alarm for the next instance to pick up. + SqliteDatabase fresh(test.vfs, kj::Path({"foo"}), kj::WriteMode::MODIFY); + KJ_EXPECT(fresh.run("SELECT value FROM _cf_METADATA WHERE key = 1").getInt64(0) == + (oneMs - kj::UNIX_EPOCH) / kj::NANOSECONDS); +} + KJ_TEST("allowUnconfirmed put in explicit transaction does not block output gate") { ActorSqliteTest test; diff --git a/src/workerd/io/actor-sqlite.c++ b/src/workerd/io/actor-sqlite.c++ index 2d016268911..e0de5c78d38 100644 --- a/src/workerd/io/actor-sqlite.c++ +++ b/src/workerd/io/actor-sqlite.c++ @@ -445,6 +445,9 @@ kj::Promise ActorSqlite::requestScheduledAlarm( } void ActorSqlite::scheduleLaterAlarm(kj::Maybe newAlarmTime, SpanParent parentSpan) { + // A queued update may resume after shutdown, when another instance owns the actor. + if (broken != kj::none) return; + if (alarmLaterIsInFlight) { // There's already a move-later request in-flight. Just store the desired time; the in-flight // request's completion handler will pick it up and start a new request. This overwrites any @@ -524,7 +527,7 @@ kj::Promise ActorSqlite::commitImpl( } commitSpan.setTag("merged_with_pending_commit"_kjc, true); co_await pending.addBranch(); - if (debugAlarmSync) { + if (debugAlarmSync && broken == kj::none) { auto alarmAfterMerge = metadata.getAlarm(); KJ_LOG(WARNING, "NOSENTRY DEBUG_ALARM: Commit merge resumed", logDate(alarmBeforeMerge), logDate(alarmAfterMerge), alarmVersion); @@ -548,6 +551,11 @@ kj::Promise ActorSqlite::commitImpl( auto alarmSpan = commitSpan.newChild("actor_sqlite_alarm_sync"_kjc); haveAlarmForDebug = true; co_await p; + + // We may have become broken (e.g. shutdown() was called) while suspended. Once broken, we no + // longer own the database, and there could be another instance of this actor running + // elsewhere, therefore we should not continue with alarm updates. + requireNotBroken(); } // While the local db state requires an earlier alarm than is known might be scheduled, issue an @@ -577,6 +585,7 @@ kj::Promise ActorSqlite::commitImpl( // future turn of the event loop. That means we're going to suspend, even if the promise is // ready, which means we'd take a performance hit. co_await requestScheduledAlarm(metadata.getAlarm(), kj::READY_NOW); + requireNotBroken(); // See above. syncIterations++; } if (debugAlarmSync && syncIterations > 0) { @@ -615,6 +624,13 @@ kj::Promise ActorSqlite::commitImpl( // Notify any merged commitImpl() requests that the db persistence completed. fulfiller->fulfill(); + // The commit itself is complete. If we became broken while it was in flight, skip the optional + // post-commit alarm sync rather than throwing: the merged waiters' writes did persist, and the + // alarm will self-correct when it fires (see armAlarmHandler()). + if (broken != kj::none) { + co_return; + } + if (debugAlarmSync) { KJ_LOG(WARNING, "NOSENTRY DEBUG_ALARM: Version check", alarmVersionBeforeAsync, alarmVersion, "match", (alarmVersion == alarmVersionBeforeAsync)); @@ -988,6 +1004,12 @@ void ActorSqlite::shutdown(kj::Maybe maybeException) { // `maybeTerminalException` has a value. Any in-flight flushes will continue to run in the // background. Remember that these in-flight flushes may or may not be awaited by the worker, // but they still hold the output lock as long as `allowUnconfirmed` wasn't used. + // + // TODO(cleanup): Instead of letting in-flight commits continue and having every resume point + // in commitImpl() re-check `broken`, consider cancelling them here outright. (Use a + // kj::Canceler for this to make sure all forks are canceled.) This changes the policy + // above (in-flight flushes no longer complete), which embedders with a real commitCallback + // (SRS / internal codebase) may care about -- need to study consequences there. broken.emplace(kj::mv(exception)); // We explicitly do not schedule a flush to break the output gate. This means that if a request diff --git a/src/workerd/io/worker-interface.capnp b/src/workerd/io/worker-interface.capnp index 93e52542e9d..ab630272a84 100644 --- a/src/workerd/io/worker-interface.capnp +++ b/src/workerd/io/worker-interface.capnp @@ -999,7 +999,36 @@ interface WorkerdDebugPort { -> (entrypoint :WorkerdBootstrap); # Get direct access to a stateless entrypoint. - getActor @1 (service :Text, entrypoint :Text, actorId :Text) -> (actor :WorkerdBootstrap); + getActor @1 (service :Text, entrypoint :Text, actorId :Text, actorName :Text) + -> (actor :WorkerdBootstrap); # Get an actor (Durable Object) stub. # The actorId should be a hex string for Durable Objects or a plain string for ephemeral actors. + # `actorName` may optionally be specified for Durable Objects, if `actorId` is derived from the + # name. +} + +interface WorkerdClusterPort { + # Bootstrap interface that workerd exports on its cluster port, which other workerd instances in + # the same cluster connect to in order to reach Durable Objects owned by this instance. See + # ClusterConfig in workerd.capnp for how to configure clustering. The cluster port does not + # implement a regular two-party RPC protocol; `ClusterRegistry` implements the `VatNetwork` used + # for it. Nobody else should try to connect to it. + # + # Every method takes a channel token authenticated with the cluster key, so a peer that does not + # hold the key can do nothing with this interface, and a peer that does gains no authority beyond + # what its tokens already grant. (Contrast WorkerdDebugPort, which lets the client name arbitrary + # services, props, and actors.) + # + # TODO(clustering): Automatically use encryption on non-localhost networks. We have an X25519 + # keypair for each node, no need for certs! + + getChannelFromToken @0 (token :Data) -> (bootstrap :WorkerdBootstrap); + # Resolve a `subrequest`-type channel token (in RPC-usage encoding) on this node and return a + # bootstrap for it. Used to route requests -- including restored stubs whose `[restore]()` chain + # must run on the node that owns the root actor -- to the node that owns the target. + + getRpcTargetFromToken @1 (token :Data) -> (target :JsRpcTarget, session :JsRpcSession); + # Like getChannelFromToken(), but for `rpc`-type tokens. Opens a fresh JS RPC session to the + # restored RpcTarget. `session` has the same semantics as the field of the same name in + # `EventDispatcher.restoreRpcStub()`. } diff --git a/src/workerd/io/worker.c++ b/src/workerd/io/worker.c++ index 465ad1731c7..41172987d62 100644 --- a/src/workerd/io/worker.c++ +++ b/src/workerd/io/worker.c++ @@ -4166,7 +4166,7 @@ bool Worker::Actor::idsEqual(const Id& a, const Id& b) { KJ_UNREACHABLE; } -Worker::Actor::Id Worker::Actor::cloneId(Worker::Actor::Id& id) { +Worker::Actor::Id Worker::Actor::cloneId(const Worker::Actor::Id& id) { KJ_SWITCH_ONEOF(id) { KJ_CASE_ONEOF(coloLocalId, kj::String) { return kj::str(coloLocalId); diff --git a/src/workerd/io/worker.h b/src/workerd/io/worker.h index 9ce9f3f4e77..b9c2a1ac167 100644 --- a/src/workerd/io/worker.h +++ b/src/workerd/io/worker.h @@ -1022,7 +1022,7 @@ class Worker::Actor final: public kj::Refcounted { const Id& getId(); Id cloneId(); - static Id cloneId(Id& id); + static Id cloneId(const Id& id); kj::Maybe> getTransient(Worker::Lock& lock); kj::Maybe getPersistent(); kj::Own getLoopback(); diff --git a/src/workerd/server/BUILD.bazel b/src/workerd/server/BUILD.bazel index 22284af7725..57a66453830 100644 --- a/src/workerd/server/BUILD.bazel +++ b/src/workerd/server/BUILD.bazel @@ -212,6 +212,7 @@ wd_cc_library( ":channel-token_capnp", "//src/workerd/io", "//src/workerd/util:entropy", + "@ssl", ], ) @@ -228,6 +229,9 @@ wd_cc_library( ":alarm-scheduler", ":channel-token", ":channel-token_capnp", + ":cluster-lock", + ":cluster-registry", + ":cluster_capnp", ":container-client", ":facet-tree-index", ":fallback-service", @@ -244,11 +248,45 @@ wd_cc_library( "//src/workerd/jsg", "//src/workerd/util:perfetto", "//src/workerd/util:websocket-error-handler", + "@capnp-cpp//src/capnp:capnp-rpc", "@capnp-cpp//src/kj/compat:kj-gzip", "@capnp-cpp//src/kj/compat:kj-tls", ], ) +wd_capnp_library(src = "cluster.capnp") + +wd_cc_library( + name = "cluster-registry", + srcs = ["cluster-registry.c++"], + hdrs = ["cluster-registry.h"], + visibility = ["//visibility:public"], + deps = [ + ":cluster_capnp", + "//src/workerd/util:ofd-lock", + "@capnp-cpp//src/capnp:capnp-rpc", + "@capnp-cpp//src/kj", + "@capnp-cpp//src/kj:kj-async", + "@ssl", + ], +) + +wd_cc_library( + name = "cluster-lock", + srcs = ["cluster-lock.c++"], + hdrs = ["cluster-lock.h"], + visibility = ["//visibility:public"], + deps = [ + ":cluster-registry", + ":cluster_capnp", + "//src/workerd/io:worker-interface_capnp", + "//src/workerd/util:ofd-lock", + "@capnp-cpp//src/capnp:capnp-rpc", + "@capnp-cpp//src/kj", + "@capnp-cpp//src/kj:kj-async", + ], +) + wd_capnp_library(src = "docker-api.capnp") wd_capnp_library(src = "log-schema.capnp") @@ -422,6 +460,28 @@ kj_test( ], ) +kj_test( + src = "cluster-registry-test.c++", + deps = [ + ":cluster-registry", + "@capnp-cpp//src/capnp:capnp-rpc", + "@capnp-cpp//src/kj", + "@capnp-cpp//src/kj:kj-async", + ], +) + +kj_test( + src = "cluster-lock-test.c++", + deps = [ + ":cluster-lock", + ":cluster-registry", + "//src/workerd/io:worker-interface_capnp", + "@capnp-cpp//src/capnp:capnp-rpc", + "@capnp-cpp//src/kj", + "@capnp-cpp//src/kj:kj-async", + ], +) + kj_test( src = "json-logger-test.c++", deps = [ diff --git a/src/workerd/server/alarm-scheduler.c++ b/src/workerd/server/alarm-scheduler.c++ index 529c12dd8d9..0651528244e 100644 --- a/src/workerd/server/alarm-scheduler.c++ +++ b/src/workerd/server/alarm-scheduler.c++ @@ -291,7 +291,7 @@ kj::Promise AlarmScheduler::makeAlarmTask( entry.value.task = abandonAlarm(actorRef.clone(), scheduledTime); co_return; } - deleteAlarm(actorRef); + finishAlarm(actorRef, scheduledTime); } } catch (...) { auto exception = kj::getCaughtExceptionAsKj(); @@ -300,11 +300,28 @@ kj::Promise AlarmScheduler::makeAlarmTask( } kj::Promise AlarmScheduler::abandonAlarm(kj::Own actor, kj::Date scheduledTime) { + bool notified = false; try { co_await getActor(*actor)->abandonAlarm(scheduledTime); + notified = true; } catch (...) { auto exception = kj::getCaughtExceptionAsKj(); KJ_LOG(WARNING, "abandonAlarm notification failed, keeping alarm in scheduler", exception); + } + if (!notified) { + if (clusterMode) { + // Don't move this task out of its own entry from inside the same turn it was resumed in. + co_await kj::yield(); + // The row is still in the shared table: forget the entry (a FINISHED entry with no task + // would otherwise swallow later times for this actor) and let syncFromDb() deliver it again. + KJ_IF_SOME(entry, alarms.findEntry(*actor)) { + if (entry.value.status == AlarmStatus::FINISHED && + entry.value.scheduledTime == scheduledTime) { + tasks.add(kj::mv(entry.value.task)); + alarms.erase(entry); + } + } + } co_return; } @@ -319,9 +336,95 @@ kj::Promise AlarmScheduler::abandonAlarm(kj::Own actor, kj::Date KJ_IF_SOME(replacement, entry.value.queuedAlarm) { entry.value = scheduleAlarm(clock.now(), kj::mv(entry.value.actor), replacement); } else { - deleteAlarm(*actor); + finishAlarm(*actor, scheduledTime); + } + } +} + +void AlarmScheduler::enableClusterMode() { + clusterMode = true; + // Other instances write the same table (server.c++ ClusterAlarmStore). + db->run("PRAGMA busy_timeout = 2000;"); +} + +void AlarmScheduler::finishAlarm(const ActorKey& actor, kj::Date scheduledTime) { + if (!clusterMode) { + deleteAlarm(actor); + return; + } + // Cluster mode: the scheduler never writes the shared table. The actor owns its row: after a + // successful run ActorSqlite clears its alarm, which reaches ClusterActorSqliteHooks on whichever + // instance holds it, and an alarm set by the handler is written there the same way. Deleting the + // row here could drop that newer alarm if the holder's write failed. A row that is still there is + // delivered again by syncFromDb(), and ActorSqlite::armAlarmHandler() cancels a delivery that no + // longer matches its own alarm state (re-requesting the time it does hold). + recentlyFinished.upsert(kj::str(actor.actorId), + RecentlyFinished{scheduledTime, clock.now() + RECENTLY_FINISHED_GRACE}, + [](RecentlyFinished& existing, RecentlyFinished&& replacement) { existing = replacement; }); + KJ_IF_SOME(entry, alarms.findEntry(actor)) { + KJ_IF_SOME(queued, entry.value.queuedAlarm) { + entry.value = scheduleAlarm(clock.now(), kj::mv(entry.value.actor), queued); + } else if (entry.value.status != AlarmStatus::STARTED) { + alarms.erase(entry); + } + } +} + +void AlarmScheduler::syncFromDb() { + KJ_REQUIRE(clusterMode); + auto now = clock.now(); + kj::HashSet present; + + auto query = db->run(R"( + SELECT actor_id, scheduled_time, actor_name FROM _cf_ALARM; + )"); + while (!query.isDone()) { + auto date = kj::UNIX_EPOCH + (kj::NANOSECONDS * query.getInt64(1)); + ActorKey probe{.actorId = query.getText(0), .name = query.getMaybeText(2)}; + present.insert(kj::str(probe.actorId)); + + KJ_IF_SOME(entry, alarms.findEntry(probe)) { + auto& alarm = entry.value; + if (alarm.status == AlarmStatus::WAITING) { + // Moved earlier or later by the actor: only the latest time counts. + if (alarm.scheduledTime != date) { + alarm = scheduleAlarm(now, kj::mv(alarm.actor), date); + } + } else if (alarm.scheduledTime != date) { + // Running, or waiting to retry: a different time is an alarm set meanwhile, which runs + // after this one (as setAlarm() does in single-process mode). + alarm.queuedAlarm = date; + } + } else { + // The actor deletes the row of an alarm that ran, a little after the run returns: don't + // deliver that same time again while its deletion may still be on its way. + KJ_IF_SOME(f, recentlyFinished.find(probe.actorId)) { + if (f.scheduledTime == date && now < f.until) { + query.nextRow(); + continue; + } + } + // New, or one this process has never seen (another leader before it, or an actor that set + // its alarm on another process): schedule it. It wakes the actor wherever it lives, or + // claims it here if no process holds it. + auto actor = probe.clone(); + auto& actorRef = *actor; + alarms.insert(actorRef, scheduleAlarm(now, kj::mv(actor), date)); + } + query.nextRow(); + } + + // Rows the actors deleted: cancel those still waiting (running ones finish by themselves). + kj::Vector> gone; + for (auto& entry: alarms) { + if (entry.value.status == AlarmStatus::WAITING && !present.contains(entry.key.actorId)) { + gone.add(entry.value.actor->clone()); } } + for (auto& actor: gone) { + alarms.erase(*actor); + } + recentlyFinished.eraseAll([&](auto& key, RecentlyFinished& f) { return f.until <= now; }); } void AlarmScheduler::taskFailed(kj::Exception&& e) { diff --git a/src/workerd/server/alarm-scheduler.h b/src/workerd/server/alarm-scheduler.h index 42a914519bd..58c0b80676b 100644 --- a/src/workerd/server/alarm-scheduler.h +++ b/src/workerd/server/alarm-scheduler.h @@ -83,6 +83,13 @@ class AlarmScheduler final: kj::TaskSet::ErrorHandler { // Cancels all pending alarms and removes them from persistent storage. void deleteAll(); + // Cluster mode (several workerd processes sharing one storage directory). Only one process, the + // alarm leader, runs a scheduler; every process writes alarm times straight into the shared + // `_cf_ALARM` table (ClusterAlarmStore in server.c++). The leader calls syncFromDb() regularly to + // pick up those writes, and never writes the table itself: each actor keeps its own row. + void enableClusterMode(); + void syncFromDb(); + private: enum class AlarmStatus { WAITING, STARTED, FINISHED }; const kj::Clock& clock; @@ -135,12 +142,27 @@ class AlarmScheduler final: kj::TaskSet::ErrorHandler { SqliteDatabase::Statement stmtDeleteAlarm = db->prepare(R"( DELETE FROM _cf_ALARM WHERE actor_id = ? )"); + bool clusterMode = false; + // After an alarm at `scheduledTime` finished for good: forget it, and delete its row (cluster + // mode: leave the row to the actor, see finishAlarm()). + void finishAlarm(const ActorKey& actor, kj::Date scheduledTime); + // Cluster mode: alarms that just ran, so syncFromDb() doesn't deliver them again while the actor's + // deletion of the row is still on its way. + static constexpr auto RECENTLY_FINISHED_GRACE = 5 * kj::SECONDS; + struct RecentlyFinished { + kj::Date scheduledTime; + kj::Date until; + }; + kj::HashMap recentlyFinished; void taskFailed(kj::Exception&& exception) override; int maxJitterMsForDelay(kj::Duration delay); + public: static void ensureInitialized(SqliteDatabase& db); + + private: void loadAlarmsFromDb(); }; diff --git a/src/workerd/server/channel-token.c++ b/src/workerd/server/channel-token.c++ index c9b4c982caa..6d420cc477f 100644 --- a/src/workerd/server/channel-token.c++ +++ b/src/workerd/server/channel-token.c++ @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -22,9 +23,31 @@ namespace workerd::server { -ChannelTokenHandler::ChannelTokenHandler(Resolver& resolver): resolver(resolver) { - getEntropy(tokenKey); +ChannelTokenHandler::ChannelTokenHandler(Resolver& resolver, kj::Maybe clusterKey) + : resolver(resolver) { + KJ_IF_SOME(key, clusterKey) { + // All nodes in the cluster must derive the same key, based on `clusterKey`. We HMAC the + // `clusterKey` with a randomly-chosen salt to derive the channel token key. + + // clang-format off + static constexpr byte SALT[] = { + 0x9a, 0x24, 0x71, 0x39, 0x1b, 0x85, 0xce, 0x97, + 0x6c, 0xf9, 0x1c, 0xdf, 0x93, 0xc0, 0xc6, 0x36, + 0xb9, 0xd2, 0x09, 0xfe, 0x01, 0xde, 0xb1, 0x9a, + 0xda, 0xd3, 0x8e, 0x76, 0xbb, 0xae, 0xeb, 0x89, + }; + // clang-format on + uint outLen = 0; + KJ_ASSERT(HMAC(EVP_sha256(), SALT, sizeof(SALT), key.asBytes().begin(), key.size(), tokenKey, + &outLen) != nullptr); + KJ_ASSERT(outLen == sizeof(tokenKey)); + } else { + // Single-instance, generate a new key for every run. + getEntropy(tokenKey); + } + + // Construct key ID by hashing the key. SHA256_CTX ctx{}; KJ_ASSERT(SHA256_Init(&ctx)); KJ_ASSERT(SHA256_Update(&ctx, tokenKey, sizeof(tokenKey))); @@ -477,8 +500,10 @@ class ChannelTokenHandler::RestoredSubrequestChannel final class ChannelTokenHandler::RestoredRpcChannel final: public IoChannelFactory::RpcChannel { public: // Constructor for the "fresh" path: created by `ctx.restore()`. The vendor is the current - // entrypoint's `ServerSelfTokenFactory`, used only to construct the token. `restore()` is never - // called in this case, because the freshly-created stub is paired with a live capability. + // entrypoint's `ServerSelfTokenFactory`, used to construct the token. The freshly-created stub + // is paired with a live capability, so `restore()` is only reached if the stub is passed + // somewhere that keeps the channel but not the capability (`ctx.props`, a dynamic worker's + // `env`) and then used. RestoredRpcChannel(ChannelTokenHandler& handler, kj::Own vendor, Frankenvalue restoreArg, @@ -503,7 +528,38 @@ class ChannelTokenHandler::RestoredRpcChannel final: public IoChannelFactory::Rp // Unlike RestoredSubrequestChannel, we expect the caller of restore() already caches the // resulting stub, rather than calling again for every request. So, we don't need to cache // our result. - return restoreWith(getVendorChannel()); + KJ_SWITCH_ONEOF(vendor) { + KJ_CASE_ONEOF(channel, kj::Own) { + return restoreWith(kj::addRef(*channel)); + } + KJ_CASE_ONEOF(_, kj::Own) { + // Decode our own token rather than just the vendor's, so that the result is exactly what + // the stub would have been had it arrived as a token -- in particular, so that + // `Resolver::wrapRestoredChannel()` gets to route the whole token to whichever node owns + // the vendor, rather than us trying to replay `[restore]()` over a remote vendor channel. + auto& handlerRef = handler; + kj::Own decoded; + KJ_SWITCH_ONEOF(getTokenMaybeSync(IoChannelFactory::ChannelTokenUsage::RPC)) { + KJ_CASE_ONEOF(token, kj::Array) { + decoded = + handlerRef.decodeRpcChannelToken(IoChannelFactory::ChannelTokenUsage::RPC, token); + } + KJ_CASE_ONEOF(promise, kj::Promise>) { + decoded = newPromisedChannel( + promise.then([&handlerRef](kj::Array token) { + return handlerRef.decodeRpcChannelToken( + IoChannelFactory::ChannelTokenUsage::RPC, token); + })); + } + } + // The decoded channel's session task does not keep the channel itself alive; callers of + // restore() only keep *us* alive. + auto session = decoded->restore(); + session.task = session.task.attach(kj::mv(decoded)); + return session; + } + } + KJ_UNREACHABLE; } void requireAllowsTransfer() override {} @@ -529,34 +585,6 @@ class ChannelTokenHandler::RestoredRpcChannel final: public IoChannelFactory::Rp Frankenvalue restoreArg; Persistent persistent; - // Obtain a callable channel to the vendor, i.e. the entrypoint whose `[restore]()` method must - // be invoked to (re)create the live RPC target. - kj::Own getVendorChannel() { - KJ_SWITCH_ONEOF(vendor) { - KJ_CASE_ONEOF(channel, kj::Own) { - return kj::addRef(*channel); - } - KJ_CASE_ONEOF(factory, kj::Own) { - KJ_SWITCH_ONEOF(factory->getSelfToken(IoChannelFactory::ChannelTokenUsage::RPC)) { - KJ_CASE_ONEOF(token, kj::Array) { - return handler.decodeSubrequestChannelToken( - IoChannelFactory::ChannelTokenUsage::RPC, token); - } - KJ_CASE_ONEOF(promise, kj::Promise>) { - auto& handlerRef = handler; - return newPromisedChannel( - promise.then([&handlerRef](kj::Array token) { - return handlerRef.decodeSubrequestChannelToken( - IoChannelFactory::ChannelTokenUsage::RPC, token); - })); - } - } - KJ_UNREACHABLE; - } - } - KJ_UNREACHABLE; - } - Session restoreWith(kj::Own vendorChannel) { // Just like RestoredSubrequestChannel::ensureRestored(), we must verify that the worker on // which we call `[restore]()` allows persistence, if we are marked persistent. @@ -731,16 +759,24 @@ kj::Own ChannelTokenHandler::decodeChannelTokenImpl auto vendor = decodeSubrequestChannelToken(usage, restored.getVendor()); auto restoreArg = decodeFrankenvalue(usage, restored.getRestoreArg()); + // The restored channel takes ownership of `vendor`, so this reference stays valid for the + // `wrapRestoredChannel()` call below. + auto& vendorRef = *vendor; + kj::Own local; switch (type) { case ChannelToken::Type::SUBREQUEST: - return kj::refcounted( + local = kj::refcounted( *this, kj::mv(vendor), kj::mv(restoreArg), persistent); + break; case ChannelToken::Type::ACTOR_CLASS: KJ_FAIL_REQUIRE("actor class channel tokens cannot have a restore chain"); case ChannelToken::Type::RPC: - return kj::refcounted( + local = kj::refcounted( *this, kj::mv(vendor), kj::mv(restoreArg), persistent); + break; } + + return resolver.wrapRestoredChannel(type, vendorRef, kj::mv(local)); } } diff --git a/src/workerd/server/channel-token.h b/src/workerd/server/channel-token.h index 06943c62e5d..34b005dc827 100644 --- a/src/workerd/server/channel-token.h +++ b/src/workerd/server/channel-token.h @@ -45,6 +45,20 @@ class ChannelTokenHandler { kj::ArrayPtr id, kj::Maybe name, Persistent persistent) = 0; + + // Called after decoding a `restored` token (a token that encodes calling [restore]() on the + // result of some other token). This gives the resolver an opportunity to replace the channel + // with something else. Simply returning `local` is valid when running in single-instance mode. + // When running in clustered mode (Durable Objects distributed over a cluster), the restorer + // checks whether `vendor` refers to a remote actor and, if so, returns a channel that makes + // an RPC to the remote actor's machine, sending the full original token, to be parsed there + // instead. + virtual kj::Own wrapRestoredChannel( + ChannelToken::Type type, + IoChannelFactory::SubrequestChannel& vendor, + kj::Own local) { + return kj::mv(local); + } }; // workerd's implementation of `IoChannelFactory::SelfTokenFactory`. Produces the encoded @@ -58,7 +72,7 @@ class ChannelTokenHandler { IoChannelFactory::ChannelTokenUsage usage) = 0; }; - explicit ChannelTokenHandler(Resolver& resolver); + explicit ChannelTokenHandler(Resolver& resolver, kj::Maybe clusterKey = kj::none); // Helpers to implement `IoChannelFactory::{SubrequestChannel,ActorClassChannel}::getToken()`. // diff --git a/src/workerd/server/cluster-lock-test.c++ b/src/workerd/server/cluster-lock-test.c++ new file mode 100644 index 00000000000..d5356d59008 --- /dev/null +++ b/src/workerd/server/cluster-lock-test.c++ @@ -0,0 +1,355 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "cluster-lock.h" +#include "cluster-registry.h" + +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace workerd { +namespace { + +#if __linux__ + +// Helper to create a temp directory on disk with a `registry/` and `locks/` subdir, mirroring +// the production layout (the cluster registry directory and the per-namespace locks directory +// are always separate). +class TempDir { + public: + TempDir() { + char tmpl[] = "/tmp/cluster-lock-test-XXXXXX"; + KJ_ASSERT(mkdtemp(tmpl) != nullptr); + pathStr = kj::str(tmpl); + disk = kj::newDiskFilesystem(); + root = disk->getRoot().openSubdir(kj::Path::parse(pathStr.slice(1)), // remove leading / + kj::WriteMode::MODIFY); + locksDir = root->openSubdir(kj::Path({"locks"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + } + + ~TempDir() noexcept(false) { + auto p = kj::Path::parse(pathStr.slice(1)); + disk->getRoot().remove(p); + } + + kj::Own registry() { + return root->openSubdir(kj::Path({"registry"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + } + + kj::Own locks() { + return locksDir->clone(); + } + + // Read the content of a lock file directly (for asserting that the file is in the expected state). + kj::Array readLockFile(kj::StringPtr name) { + auto file = KJ_ASSERT_NONNULL(locksDir->tryOpenFile(kj::Path({name}))); + return file->readAllBytes(); + } + + // Write raw bytes to a lock file (for setting up test scenarios). + void writeLockFile(kj::StringPtr name, kj::ArrayPtr data) { + auto file = locksDir->openFile(kj::Path({name}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + file->writeAll(data); + } + + private: + kj::String pathStr; + kj::Own disk; + kj::Own root; + kj::Own locksDir; +}; + +// Stub WorkerdClusterPort server used as a per-node bootstrap. We don't actually invoke methods on +// it from these tests; we just need a capability to hand to RpcSystem. +class StubClusterPort final: public rpc::WorkerdClusterPort::Server {}; + +// Make a valid actor ID (64 digits) given some shorter base. +kj::StringPtr testId(kj::StringPtr base) { + static kj::HashMap cache; + return cache.findOrCreate(base, [&]() -> decltype(cache)::Entry { + return {base, kj::str(base, kj::repeat('0', 64 - base.size()))}; + }); +} + +KJ_TEST("ClusterLockManager: unowned (empty file) -> acquire succeeds") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + auto result = lockManager.acquireOrRoute(testId("a1")).wait(io.waitScope); + KJ_ASSERT(result.is()); + + // The lock file should now contain our key. + auto content = tmpDir.readLockFile(testId("a1")); + KJ_EXPECT(content == reg.getPublicKey().bytes.asPtr()); +} + +KJ_TEST("ClusterLockManager: owned by self -> returns self-bootstrap, no wire traffic") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + // First acquire to populate the lock file with our own key. + auto ownedResult = lockManager.acquireOrRoute(testId("a2")).wait(io.waitScope); + KJ_ASSERT(ownedResult.is()); + + // Now simulate someone else also calling acquireOrRoute on the same actor (i.e. a forwarding + // path). With the OwnedLock still held, the lock file is non-empty and our key is the owner. + // acquireOrRoute should return a bootstrap client (routing to ourselves). + auto routeResult = lockManager.acquireOrRoute(testId("a2")).wait(io.waitScope); + KJ_ASSERT(routeResult.is()); +} + +KJ_TEST("ClusterLockManager: stale owner (peer dead) -> claim path runs") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + // Manually create a lock file naming a non-existent peer as owner. Since `deadKey`'s registry + // file doesn't exist in tmpDir, isPeerDead(deadKey) returns true and the claim path should run. + X25519PublicKey deadKey; + deadKey.bytes.fill(0xAB); + + tmpDir.writeLockFile(testId("a3"), deadKey.bytes); + + // Confirm isPeerDead returns true for the dead key. + KJ_EXPECT(reg.isPeerDead(deadKey)); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + // acquireOrRoute should run the claim path and succeed. + auto result = lockManager.acquireOrRoute(testId("a3")).wait(io.waitScope); + KJ_ASSERT(result.is()); + + // The lock file should now contain *our* key. + auto content = tmpDir.readLockFile(testId("a3")); + KJ_EXPECT(content == reg.getPublicKey().bytes.asPtr()); +} + +KJ_TEST("ClusterLockManager: wrong-sized stale content + writer dead -> claim immediately") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + // Write a lock file of unexpected size (e.g. a crashed writer left a half-written file). + // Nobody holds the lock, so the claim path should succeed. + kj::byte garbage[10]; + memset(garbage, 0xFF, sizeof(garbage)); + tmpDir.writeLockFile(testId("a4"), kj::arrayPtr(garbage, sizeof(garbage))); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + auto result = lockManager.acquireOrRoute(testId("a4")).wait(io.waitScope); + KJ_ASSERT(result.is()); + + // The lock file should now contain our key. + auto content = tmpDir.readLockFile(testId("a4")); + KJ_EXPECT(content == reg.getPublicKey().bytes.asPtr()); +} + +KJ_TEST("ClusterLockManager: OwnedLock destructor truncates and releases") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + { + auto result = lockManager.acquireOrRoute(testId("a5")).wait(io.waitScope); + KJ_ASSERT(result.is()); + // Lock file should have our key. + auto content = tmpDir.readLockFile(testId("a5")); + KJ_ASSERT(content.size() == 32); + } + + // After OwnedLock destructor, the file should be truncated to 0 bytes (and the OFD lock + // released). + auto content = tmpDir.readLockFile(testId("a5")); + KJ_EXPECT(content.size() == 0); + + // A second acquire should now succeed via the claim path. + auto result2 = lockManager.acquireOrRoute(testId("a5")).wait(io.waitScope); + KJ_ASSERT(result2.is()); +} + +KJ_TEST("ClusterLockManager: stale file naming self with no lock held -> reclaims") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + // Simulate a claim or release that published our key but failed before clearing it: the file + // names this node, but no OwnedLock (and hence no OFD lock) exists. Routing to ourselves here + // would loop forever; we must reclaim instead. + tmpDir.writeLockFile(testId("a6"), reg.getPublicKey().bytes); + + auto result = lockManager.acquireOrRoute(testId("a6")).wait(io.waitScope); + KJ_ASSERT(result.is()); + + auto content = tmpDir.readLockFile(testId("a6")); + KJ_EXPECT(content == reg.getPublicKey().bytes.asPtr()); + + // With the OwnedLock now held, a further lookup routes to ourselves rather than reclaiming. + auto routeResult = lockManager.acquireOrRoute(testId("a6")).wait(io.waitScope); + KJ_ASSERT(routeResult.is()); +} + +KJ_TEST("ClusterLockManager: owned by live peer -> returns bootstrap client") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + // Stand up two registries in the same dir (this creates registry entries for both, so neither + // is "dead"). + ClusterRegistry reg1( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + ClusterRegistry reg2( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc1 = capnp::makeRpcServer(reg1, capnp::Capability::Client(kj::heap())); + + // reg1 owns a ClusterLockManager; we'll set up a lock file naming reg2 as the owner. + // reg2 is alive (its registry file exists in tmpDir), so isPeerDead(reg2) returns false, + // and acquireOrRoute should return a bootstrap client without falling through to claim. + // + // Note: actor IDs aren't hex of 64 chars, so they won't collide with registry filenames. + tmpDir.writeLockFile(testId("a7"), reg2.getPublicKey().bytes); + + KJ_EXPECT(!reg1.isPeerDead(reg2.getPublicKey())); + + ClusterLockManager lockManager(tmpDir.locks(), reg1, rpc1, io.provider->getTimer()); + + auto result = lockManager.acquireOrRoute(testId("a7")).wait(io.waitScope); + KJ_ASSERT(result.is()); + + // The lock file should NOT have been modified (we did not claim). + auto content = tmpDir.readLockFile(testId("a7")); + KJ_EXPECT(content == reg2.getPublicKey().bytes.asPtr()); +} + +KJ_TEST("ClusterLockManager: writer alive -> retries until writer releases") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + // Open the lock file from outside the ClusterLockManager and take an exclusive lock. This + // simulates a writer that holds the lock but hasn't yet flushed its key. The file is left + // empty (matching the freshly-truncated state during the writer's claim path). + auto externalFile = tmpDir.locks()->openFile( + kj::Path({testId("a8")}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + int externalFd = KJ_REQUIRE_NONNULL(externalFile->getFd()); + + auto externalLock = KJ_ASSERT_NONNULL(OfdLock::tryLock(externalFd, OfdLock::EXCLUSIVE)); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + // acquireOrRoute should see an empty file, attempt to lock, fail, then retry with backoff. + auto promise = lockManager.acquireOrRoute(testId("a8")); + + bool completed = false; + auto trackedPromise = promise.then( + [&](kj::OneOf&& r) { + completed = true; + return kj::mv(r); + }); + + // Pump the event loop briefly; the implementation should be stuck in its backoff loop. + io.provider->getTimer().afterDelay(50 * kj::MILLISECONDS).wait(io.waitScope); + KJ_EXPECT(!completed, "acquireOrRoute should still be waiting for the external lock"); + + // Now the "external writer" finishes publishing its key (our own, for simplicity) while still + // holding the lock, as a real owner does. acquireOrRoute, on its next iteration, should see the + // key, find the OFD lock held, and return a bootstrap client routing to the owner. (Releasing + // the lock instead would leave a self-naming file with no holder, which acquireOrRoute reclaims; + // see the "stale file naming self" test.) + externalFile->write(0, reg.getPublicKey().bytes); + + auto result = trackedPromise.wait(io.waitScope); + KJ_EXPECT(completed); + KJ_ASSERT(result.is()); +} + +KJ_TEST("ClusterLockManager: concurrent acquisitions, exactly one wins") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + // Launch several concurrent calls. + constexpr uint N = 5; + kj::Vector>> + promises; + for (uint i = 0; i < N; ++i) { + promises.add(lockManager.acquireOrRoute(testId("a9"))); + } + + auto results = kj::joinPromises(promises.releaseAsArray()).wait(io.waitScope); + + uint ownedCount = 0; + uint routedCount = 0; + for (auto& r: results) { + if (r.is()) { + ++ownedCount; + } else { + ++routedCount; + } + } + // Exactly one should have won. + KJ_EXPECT(ownedCount == 1); + KJ_EXPECT(routedCount == N - 1); +} + +#else // #if __linux__ + +KJ_TEST("dummy test: platform not supported") {} + +#endif // #if __linux__, #else + +} // namespace +} // namespace workerd diff --git a/src/workerd/server/cluster-lock.c++ b/src/workerd/server/cluster-lock.c++ new file mode 100644 index 00000000000..4c0648cffc6 --- /dev/null +++ b/src/workerd/server/cluster-lock.c++ @@ -0,0 +1,192 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "cluster-lock.h" + +#include +#include +#include + +namespace workerd { + +namespace { + +// The lock file contains exactly one X25519 public key (the owner). No checksum is needed: a +// single write of `sizeof(X25519PublicKey::bytes)` bytes is atomic with respect to concurrent +// readers on Linux (and likewise on NFS, where it's a single RPC to the server). The writer's +// protocol is truncate(0) → write(full key) → sync, so a reader interleaving sees either size +// 0 (treat as unowned, run claim path) or the full key. + +// Build a `cluster::VatId` Reader for a given public key, backed by an internal `MessageBuilder`. +class VatIdHolder { + public: + explicit VatIdHolder(const X25519PublicKey& key): msg(scratch) { + auto vatId = msg.initRoot(); + vatId.setPublicKey(key.bytes.asPtr()); + } + + cluster::VatId::Reader getReader() { + return msg.getRoot().asReader(); + } + + private: + capnp::word scratch[8]{}; + capnp::MallocMessageBuilder msg; +}; + +constexpr kj::Duration INITIAL_BACKOFF = 10 * kj::MILLISECONDS; +constexpr kj::Duration MAX_BACKOFF = 1 * kj::SECONDS; + +// Compute the next backoff delay with ±25% jitter. `attempt` starts at 0. +kj::Duration computeBackoff(uint attempt) { + // Exponential: 10ms, 20ms, 40ms, ..., capped at 1s. + kj::Duration base = INITIAL_BACKOFF; + for (uint i = 0; i < attempt && base < MAX_BACKOFF; ++i) { + base = base * 2; + } + if (base > MAX_BACKOFF) base = MAX_BACKOFF; + + // Jitter: multiply by a random value in [0.75, 1.25]. Use a cheap pseudo-random source. + auto nanos = + (kj::systemPreciseMonotonicClock().now() - kj::origin()) / kj::NANOSECONDS; + uint32_t r = kj::hashCode(nanos); + constexpr decltype(r) MAX_R = kj::maxValue; + double jitter = 0.75 + (r / static_cast(MAX_R)) * 0.5; + return base * static_cast(jitter * 1000) / 1000; +} + +} // namespace + +// ======================================================================================= +// ClusterLockManager::OwnedLock + +ClusterLockManager::OwnedLock::~OwnedLock() noexcept(false) { + if (file.get() != nullptr) { + // Truncate the file back to zero so the next claimant sees an empty file. We then drop the + // OFD lock (via OfdLock's destructor) which makes the file available for other nodes to claim. + // + // Best-effort: don't throw from the destructor. If this fails, the file keeps naming this node + // with no lock held. That's benign: the next acquireOrRoute() on this node sees its own key, + // takes the OFD lock, and reclaims (remote nodes route here, so they land on that path too). + KJ_TRY { + file->truncate(0); + } + KJ_CATCH(e) { + KJ_LOG(WARNING, "truncate(0) on cluster lock file failed during release", e); + } + } +} + +// ======================================================================================= +// ClusterLockManager + +ClusterLockManager::ClusterLockManager(kj::Own dir, + ClusterRegistry& registry, + capnp::RpcSystem& clusterRpc, + kj::Timer& timer) + : dir(kj::mv(dir)), + registry(registry), + clusterRpc(clusterRpc), + timer(timer) {} + +kj::Promise> +ClusterLockManager::acquireOrRoute(kj::StringPtr actorId) { + // Validate that the actor ID is a hex string. This is the only form of actor ID we expect on + // this path (Durable Objects named by a 64-char hex SHA-256 hash, or by `idFromName` which also + // produces hex). Enforcing hex incidentally guarantees the ID is safe to use as a filename. + KJ_REQUIRE(actorId.size() == 64, "invalid actor ID", actorId); + for (char c: actorId) { + KJ_REQUIRE((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'), + "invalid actor ID", actorId); + } + + kj::Path path({actorId}); + + for (uint attempt = 0;; ++attempt) { + if (attempt > 0) { + co_await timer.afterDelay(computeBackoff(attempt - 1)); + } + + // Open or create the lock file. + auto file = dir->openFile(path, kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + + // Read the owner key directly. read() returns: + // 0 → file is empty (freshly created, or previous owner cleanly + // released) + // sizeof(ownerKey.bytes) → full key present + // anything else → unexpected partial state (e.g. writer crashed mid-protocol) + // + // Since the key has a fixed width, we can assume that if we read the entire key, it is the + // complete and valid key. If the key is not done being written, we could get a short read + // (unlikely, since you'd expect a 32-byte read to always be atomic, but it's not guaranteed). + X25519PublicKey ownerKey; + auto n = file->read(0, ownerKey.bytes); + + const auto& myKey = registry.getPublicKey(); + bool namesSelf = n == sizeof(ownerKey.bytes) && ownerKey == myKey; + + if (n == sizeof(ownerKey.bytes)) { + if (namesSelf) { + // The file names this node. That is authoritative only if an OwnedLock on this node still + // holds the OFD lock; otherwise a claim or release failed partway and left our key behind. + // Routing to ourselves in the latter case would loop forever (resolveNode() finds no + // container and lands back here), so the OFD lock decides: fall through to tryLock(). + } else if (!registry.isPeerDead(ownerKey)) { + // Trust the lock file. Build a VatId and ask the RpcSystem to bootstrap. + // + // If the owner is actually unreachable, the bootstrap call will fail later. Internally, + // ClusterRegistry::ConnectionImpl::unregister() runs the dead-peer cleanup probe; + // if the peer's registry file is unlocked, the file is unlinked. Whoever retries + // acquireOrRoute() will then see isPeerDead(ownerKey) return true and fall through to + // the claim path. + VatIdHolder holder(ownerKey); + co_return clusterRpc.bootstrap(holder.getReader()).castAs(); + } + + // else: owner is confirmed dead. Fall through to the claim path. + } + // else: n == 0 (empty file) or n is some unexpected partial size. In either case, the claim + // path is the right next step: if a live writer is currently producing the file, they + // hold the exclusive lock and our tryLock will fail; if no one holds the lock, we'll + // succeed and overwrite whatever was there. + + // Claim path. Try to acquire an exclusive OFD lock. + int fd = KJ_REQUIRE_NONNULL(file->getFd(), "lock directory must be disk-backed"); + KJ_IF_SOME(lock, OfdLock::tryLock(fd, OfdLock::EXCLUSIVE)) { + // We got the exclusive lock. Clear any stale content and write our identity. Construct the + // OwnedLock first so that if any of these operations throws, unwinding runs ~OwnedLock, + // which clears the file (best-effort) before the OFD lock is dropped. + OwnedLock owned(kj::mv(file), kj::mv(lock)); + + // Holding the lock proves the actor is unowned (a file naming this node was stale). A + // draining node must not become its owner: its cluster listener no longer accepts + // connections, so no other node could reach the actor until this process exits. The check + // comes after tryLock() because the lock is what distinguishes "unowned" from "owned here", + // and the latter must still route to ourselves below. Unwinding runs ~OwnedLock, which + // leaves the file empty and unlocked for a live node to claim. + if (registry.isDraining()) { + kj::throwFatalException( + KJ_EXCEPTION(DISCONNECTED, "cannot claim Durable Object: this node is shutting down")); + } + + owned.file->truncate(0); + owned.file->write(0, myKey.bytes); + owned.file->sync(); + + co_return kj::mv(owned); + } + + if (namesSelf) { + // The file names this node and an OwnedLock on this node holds the OFD lock, so the actor's + // container exists here (see ActorContainer::ownershipLock). Route to ourselves; the + // RpcSystem returns the local bootstrap directly. + VatIdHolder holder(ownerKey); + co_return clusterRpc.bootstrap(holder.getReader()).castAs(); + } + + // tryLock failed — someone else owns it (or is racing us to claim it). Retry. + } +} + +} // namespace workerd diff --git a/src/workerd/server/cluster-lock.h b/src/workerd/server/cluster-lock.h new file mode 100644 index 00000000000..90c5c4e5ac0 --- /dev/null +++ b/src/workerd/server/cluster-lock.h @@ -0,0 +1,81 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace workerd { + +// Manages a directory of lock files for DO ownership. Each lock file is named after the actor ID +// and contains the owning node's public key (or is empty if unowned). +// +// The ownership protocol: +// 1. Open/create the lock file. +// 2. Read it. If it holds a full key, that's the owner. +// - If it's another node and !registry.isPeerDead(ownerKey): route via +// clusterRpc.bootstrap(vatIdFor(ownerKey)). +// - If it's another node and registry.isPeerDead(ownerKey): fall through to the claim path. +// - If it's this node: the file alone is not authoritative (a failed claim or release can +// leave our key behind). Fall through to the claim path; if tryLock() is refused, an +// OwnedLock on this node holds the actor and we route to ourselves. +// 3. Claim path: tryLock(EXCLUSIVE). If granted, ftruncate, write our key, fsync, +// return OwnedLock. If refused, retry with backoff. +class ClusterLockManager { + public: + ClusterLockManager(kj::Own dir, + ClusterRegistry& registry, + capnp::RpcSystem& clusterRpc, + kj::Timer& timer); + + // RAII ownership handle. Holding this object means this node owns the DO: it holds the + // exclusive OFD lock on the lock file. Destruction truncates the lock file (best-effort) and + // releases the OFD lock, making the DO available for other nodes to claim. + class OwnedLock { + public: + ~OwnedLock() noexcept(false); + OwnedLock(OwnedLock&&) = default; + OwnedLock& operator=(OwnedLock&&) = default; + KJ_DISALLOW_COPY(OwnedLock); + + private: + OwnedLock(kj::Own file, OfdLock lock): file(kj::mv(file)), lock(kj::mv(lock)) {} + + kj::Own file; + OfdLock lock; + friend class ClusterLockManager; + }; + + // The core routing operation. Either acquires local ownership or returns a capability for + // reaching the current owner. + // + // Encapsulates the full protocol including retries with backoff, registry liveness + // cross-referencing, and capability resolution (with retry on stale owners). + // + // Returns: + // OwnedLock — this node has newly acquired ownership (caller should start the DO). + // rpc::WorkerdClusterPort::Client — the DO has an existing owner. The caller sends the + // actor's channel token to this to reach the actor. The owner might be this node, in + // which case the RpcSystem returns the local self-bootstrap directly (no wire traffic). + // The caller does not need to distinguish local vs remote. + kj::Promise> acquireOrRoute( + kj::StringPtr actorId); + + private: + kj::Own dir; + ClusterRegistry& registry; + capnp::RpcSystem& clusterRpc; + kj::Timer& timer; +}; + +} // namespace workerd diff --git a/src/workerd/server/cluster-registry-test.c++ b/src/workerd/server/cluster-registry-test.c++ new file mode 100644 index 00000000000..f4adb7dcb90 --- /dev/null +++ b/src/workerd/server/cluster-registry-test.c++ @@ -0,0 +1,401 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "cluster-registry.h" + +#include + +#if !_WIN32 +#include +#include +#endif + +#include +#include +#include +#include +#include +#include + +namespace workerd { +namespace { + +#if __linux__ + +// A trivial capability server that counts calls. +class CallCountServer final: public capnp::Capability::Server { + public: + int callCount = 0; + + DispatchCallResult dispatchCall(uint64_t interfaceId, + uint16_t methodId, + capnp::CallContext context) override { + ++callCount; + return {kj::READY_NOW, false, true}; + } +}; + +// Helper to create a temp directory on disk. +class TempDir { + public: + TempDir() { + char tmpl[] = "/tmp/cluster-registry-test-XXXXXX"; + KJ_ASSERT(mkdtemp(tmpl) != nullptr); + pathStr = kj::str(tmpl); + disk = kj::newDiskFilesystem(); + dir = disk->getRoot().openSubdir(kj::Path::parse(pathStr.slice(1)), // remove leading / + kj::WriteMode::MODIFY); + } + + ~TempDir() noexcept(false) { + auto p = kj::Path::parse(pathStr.slice(1)); + disk->getRoot().remove(p); + } + + kj::Own get() { + return dir->clone(); + } + + private: + kj::String pathStr; + kj::Own disk; + kj::Own dir; +}; + +// Helper to make a simple RPC call on a capability client. +void makeCall(capnp::Capability::Client& client, kj::WaitScope& ws) { + auto req = client.typelessRequest( + 0, 0, capnp::MessageSize{4, 0}, capnp::Capability::Client::CallHints()); + req.send().wait(ws); +} + +// A kj::Network wrapper whose outbound connects never complete, simulating a peer address that +// blackholes our SYNs. Listening is delegated unchanged. +class BlackholeNetwork final: public kj::Network { + public: + BlackholeNetwork(kj::Network& inner): inner(inner) {} + + kj::Promise> parseAddress( + kj::StringPtr addr, uint portHint) override { + return inner.parseAddress(addr, portHint) + .then([](kj::Own a) -> kj::Own { + return kj::heap
(kj::mv(a)); + }); + } + + kj::Own getSockaddr(const void* sockaddr, uint len) override { + return kj::heap
(inner.getSockaddr(sockaddr, len)); + } + + kj::Own restrictPeers( + kj::ArrayPtr allow, kj::ArrayPtr deny) override { + KJ_UNIMPLEMENTED("not needed in test"); + } + + private: + class Address final: public kj::NetworkAddress { + public: + Address(kj::Own inner): inner(kj::mv(inner)) {} + + kj::Promise> connect() override { + return kj::NEVER_DONE; + } + kj::Own listen() override { + return inner->listen(); + } + kj::Own clone() override { + return kj::heap
(inner->clone()); + } + kj::String toString() override { + return inner->toString(); + } + + private: + kj::Own inner; + }; + + kj::Network& inner; +}; + +KJ_TEST("X25519PublicKey: hex roundtrip") { + X25519PublicKey key; + key.bytes.fill(0xAB); + auto hex = key.toHex(); + KJ_EXPECT(hex.size() == 64); + auto decoded = X25519PublicKey::fromHex(hex); + KJ_EXPECT(decoded == key); +} + +KJ_TEST("X25519PublicKey: equality") { + X25519PublicKey a, b; + a.bytes.fill(1); + b.bytes.fill(1); + KJ_EXPECT(a == b); + + b.bytes[0] = 2; + KJ_EXPECT(!(a == b)); +} + +KJ_TEST("ClusterRegistry: self is never dead") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + KJ_EXPECT(!reg.isPeerDead(reg.getPublicKey())); +} + +KJ_TEST("ClusterRegistry: self-connect returns none") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(reg.getPublicKey().bytes.asPtr()); + + KJ_EXPECT(reg.connect(vatId.asReader()) == kj::none); +} + +KJ_TEST("ClusterRegistry: self-bootstrap returns local capability") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto server = kj::heap(); + auto& serverRef = *server; + auto bootstrap = capnp::Capability::Client(kj::mv(server)); + auto rpc = capnp::makeRpcServer(reg, kj::mv(bootstrap)); + + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(reg.getPublicKey().bytes.asPtr()); + + auto client = rpc.bootstrap(vatId); + makeCall(client, io.waitScope); + KJ_EXPECT(serverRef.callCount == 1); +} + +KJ_TEST("ClusterRegistry: peer discovery via directory scan") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg1(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + ClusterRegistry reg2(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + // In unix mode, socket files in the directory are registry entries. + KJ_EXPECT(!reg1.isPeerDead(reg2.getPublicKey())); + KJ_EXPECT(!reg2.isPeerDead(reg1.getPublicKey())); +} + +KJ_TEST("ClusterRegistry: isPeerDead caches live peers briefly") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg1(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + ClusterRegistry reg2(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto peerPath = kj::Path({reg2.getPublicKey().toHex()}); + KJ_EXPECT(!reg1.isPeerDead(reg2.getPublicKey())); + + tmpDir.get()->tryRemove(peerPath); + KJ_EXPECT(!reg1.isPeerDead(reg2.getPublicKey())); +} + +KJ_TEST("ClusterRegistry: isPeerDead caches dead peers permanently") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + X25519PublicKey peerKey; + peerKey.bytes.fill(0xCA); + auto peerPath = kj::Path({peerKey.toHex()}); + + KJ_EXPECT(reg.isPeerDead(peerKey)); + tmpDir.get()->openFile(peerPath, kj::WriteMode::CREATE)->writeAll("placeholder"); + KJ_EXPECT(reg.isPeerDead(peerKey)); +} + +KJ_TEST("ClusterRegistry: end-to-end RPC between two registries") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg1(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + ClusterRegistry reg2(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto maint1 = reg1.runMaintenance(); + auto maint2 = reg2.runMaintenance(); + + auto server1 = kj::heap(); + auto& server1Ref = *server1; + auto server2 = kj::heap(); + auto& server2Ref = *server2; + + auto rpc1 = capnp::makeRpcServer(reg1, capnp::Capability::Client(kj::mv(server1))); + auto rpc2 = capnp::makeRpcServer(reg2, capnp::Capability::Client(kj::mv(server2))); + + auto run1 = rpc1.run(); + auto run2 = rpc2.run(); + + // reg1 -> reg2 + { + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(reg2.getPublicKey().bytes.asPtr()); + + auto client = rpc1.bootstrap(vatId); + makeCall(client, io.waitScope); + KJ_EXPECT(server2Ref.callCount == 1); + } + + // reg2 -> reg1 + { + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(reg1.getPublicKey().bytes.asPtr()); + + auto client = rpc2.bootstrap(vatId); + makeCall(client, io.waitScope); + KJ_EXPECT(server1Ref.callCount == 1); + } +} + +KJ_TEST("ClusterRegistry: end-to-end RPC with binary IP registry entries") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg1( + tmpDir.get(), "127.0.0.0/8"_kj, io.provider->getNetwork(), io.provider->getTimer()); + ClusterRegistry reg2( + tmpDir.get(), "127.0.0.0/8"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto maint1 = reg1.runMaintenance(); + auto maint2 = reg2.runMaintenance(); + + auto server2 = kj::heap(); + auto& server2Ref = *server2; + + auto rpc1 = capnp::makeRpcServer(reg1, capnp::Capability::Client(kj::heap())); + auto rpc2 = capnp::makeRpcServer(reg2, capnp::Capability::Client(kj::mv(server2))); + + auto run1 = rpc1.run(); + auto run2 = rpc2.run(); + + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(reg2.getPublicKey().bytes.asPtr()); + + auto client = rpc1.bootstrap(vatId); + makeCall(client, io.waitScope); + KJ_EXPECT(server2Ref.callCount == 1); +} + +KJ_TEST("ClusterRegistry: idle timeout closes connections") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + auto shortTimeout = 100 * kj::MILLISECONDS; + + ClusterRegistry reg1( + tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer(), shortTimeout); + ClusterRegistry reg2( + tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer(), shortTimeout); + + auto server2 = kj::heap(); + auto& server2Ref = *server2; + + auto rpc1 = capnp::makeRpcServer(reg1, capnp::Capability::Client(kj::heap())); + auto rpc2 = capnp::makeRpcServer(reg2, capnp::Capability::Client(kj::mv(server2))); + + auto run1 = rpc1.run(); + auto run2 = rpc2.run(); + + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(reg2.getPublicKey().bytes.asPtr()); + + { + auto client = rpc1.bootstrap(vatId); + makeCall(client, io.waitScope); + KJ_EXPECT(server2Ref.callCount == 1); + } + // Client dropped — connection should become idle. + + // Wait for the idle timeout to fire. + io.provider->getTimer().afterDelay(shortTimeout + 200 * kj::MILLISECONDS).wait(io.waitScope); + + // After idle close, a new bootstrap should still work (new connection). + auto client2 = rpc1.bootstrap(vatId); + makeCall(client2, io.waitScope); + KJ_EXPECT(server2Ref.callCount == 2); +} + +KJ_TEST("ClusterRegistry: hung connect is torn down and dead peer cleaned up within timeout") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + auto connectTimeout = 100 * kj::MILLISECONDS; + BlackholeNetwork network(io.provider->getNetwork()); + + ClusterRegistry reg(tmpDir.get(), "127.0.0.0/8"_kj, network, io.provider->getTimer(), + 60 * kj::SECONDS, connectTimeout); + auto maint = reg.runMaintenance(); + + // Forge a registry entry for a peer that is gone: a valid address (copied from our own entry), + // not locked by anyone, and old enough to be past the new-entry grace period. + X25519PublicKey peerKey; + peerKey.bytes.fill(0xBE); + auto peerPath = kj::Path({peerKey.toHex()}); + { + auto dir = tmpDir.get(); + auto ownEntry = dir->openFile(kj::Path({reg.getPublicKey().toHex()}))->readAllBytes(); + auto peerFile = dir->openFile(peerPath, kj::WriteMode::CREATE); + peerFile->writeAll(ownEntry); + timespec old[2] = {{.tv_sec = 1, .tv_nsec = 0}, {.tv_sec = 1, .tv_nsec = 0}}; + KJ_SYSCALL(futimens(KJ_ASSERT_NONNULL(peerFile->getFd()), old)); + } + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + auto run = rpc.run(); + + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(peerKey.bytes.asPtr()); + + auto& timer = io.provider->getTimer(); + auto start = timer.now(); + { + auto client = rpc.bootstrap(vatId); + KJ_EXPECT_THROW_MESSAGE("timed out", makeCall(client, io.waitScope)); + } + + // Give the RPC system a chance to finish dropping the connection. + timer.afterDelay(connectTimeout).wait(io.waitScope); + KJ_EXPECT(timer.now() - start < 10 * connectTimeout); + + // The dead-peer probe must have run: the unlocked, stale entry is removed and the peer is dead. + KJ_EXPECT(!tmpDir.get()->exists(peerPath)); + KJ_EXPECT(reg.isPeerDead(peerKey)); +} + +KJ_TEST("ClusterRegistry: verifyNfsLease() succeeds when lock is held") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + reg.verifyNfsLease(); +} + +#else // #if __linux__ + +KJ_TEST("dummy test: platform not supported") {} + +#endif // #if __linux__, #else + +} // namespace +} // namespace workerd diff --git a/src/workerd/server/cluster-registry.c++ b/src/workerd/server/cluster-registry.c++ new file mode 100644 index 00000000000..60be9c496ef --- /dev/null +++ b/src/workerd/server/cluster-registry.c++ @@ -0,0 +1,830 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "cluster-registry.h" + +#if !_WIN32 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#include +#include +#include +#include +#include + +namespace workerd { + +#if __linux__ + +namespace { + +constexpr kj::Duration REGISTRY_MAINTENANCE_INTERVAL = 60 * kj::SECONDS; +constexpr kj::Duration DIRECTORY_SCAN_INTERVAL = 30 * kj::SECONDS; +constexpr kj::Duration NEW_REGISTRY_ENTRY_GRACE_PERIOD = 15 * kj::SECONDS; +constexpr kj::Duration FAILED_PROBE_THROTTLE = 5 * kj::SECONDS; +constexpr kj::Duration SUSPECT_PEER_AVOIDANCE_PERIOD = 15 * kj::SECONDS; +constexpr kj::Duration PEER_LIVENESS_CACHE_TTL = 15 * kj::SECONDS; + +} // namespace + +// ======================================================================================= +// X25519PublicKey + +kj::String X25519PublicKey::toHex() const { + return kj::encodeHex(bytes); +} + +X25519PublicKey X25519PublicKey::fromHex(kj::StringPtr hex) { + KJ_REQUIRE(hex.size() == 64, "X25519 public key hex must be 64 characters", hex.size()); + auto decoded = kj::decodeHex(hex.asArray()); + KJ_REQUIRE(!decoded.hadErrors, "invalid hex in X25519 public key"); + KJ_ASSERT(decoded.size() == 32); + X25519PublicKey result; + result.bytes.asPtr().copyFrom(decoded); + return result; +} + +// ======================================================================================= +// ClusterRegistry::NativeAddress + +void ClusterRegistry::NativeAddress::setPort(uint port) { + KJ_REQUIRE(port <= static_cast(kj::maxValue), "invalid port", port); + + switch (storage.ss_family) { + case AF_INET: { + KJ_REQUIRE(length == sizeof(sockaddr_in), "invalid IPv4 sockaddr length", length); + auto& sin = *reinterpret_cast(&storage); + sin.sin_port = htons(port); + break; + } + case AF_INET6: { + KJ_REQUIRE(length == sizeof(sockaddr_in6), "invalid IPv6 sockaddr length", length); + auto& sin6 = *reinterpret_cast(&storage); + sin6.sin6_port = htons(port); + break; + } + default: + KJ_FAIL_REQUIRE("address does not have a TCP port", storage.ss_family); + } +} + +ClusterRegistry::NativeAddress ClusterRegistry::NativeAddress::fromSockaddr(const sockaddr* addr) { + KJ_REQUIRE(addr != nullptr, "missing sockaddr"); + + NativeAddress result; + switch (addr->sa_family) { + case AF_INET: + result.length = sizeof(sockaddr_in); + break; + case AF_INET6: + result.length = sizeof(sockaddr_in6); + break; + default: + KJ_FAIL_REQUIRE("sockaddr is not an IP address", addr->sa_family); + } + + memcpy(&result.storage, addr, result.length); + return result; +} + +ClusterRegistry::NativeAddress ClusterRegistry::NativeAddress::parse( + kj::ArrayPtr bytes) { + KJ_REQUIRE( + bytes.size() >= sizeof(sa_family_t), "peer registry address is too short", bytes.size()); + KJ_REQUIRE( + bytes.size() <= sizeof(sockaddr_storage), "peer registry address is too long", bytes.size()); + + NativeAddress result; + result.length = bytes.size(); + memcpy(&result.storage, bytes.begin(), bytes.size()); + + switch (result.storage.ss_family) { + case AF_INET: + KJ_REQUIRE( + result.length == sizeof(sockaddr_in), "invalid IPv4 sockaddr length", result.length); + break; + case AF_INET6: + KJ_REQUIRE( + result.length == sizeof(sockaddr_in6), "invalid IPv6 sockaddr length", result.length); + break; + default: + KJ_FAIL_REQUIRE("peer registry address is not an IP sockaddr", result.storage.ss_family); + } + + return result; +} + +// ======================================================================================= +// ClusterRegistry::ConnectionImpl + +class ClusterRegistry::ConnectionImpl final: public ClusterVatNetworkBase::Connection, + public kj::Refcounted { + public: + ConnectionImpl(ClusterRegistry& registry, + kj::Own stream, + kj::Maybe peerKey, + kj::Maybe> wasRefused) + : registry(registry), + stream(kj::mv(stream)), + msgStream(*this->stream, capnp::IncomingRpcMessage::getShortLivedCallback()), + peerKey(peerKey), + wasRefused(kj::mv(wasRefused)), + outbound(peerKey != kj::none) { + // We always need to pump immediately to send the handshake. + ensurePumping(); + } + + ~ConnectionImpl() noexcept(false) { + unregister(); + } + + cluster::VatId::Reader getPeerVatId() override { + // We can safely expect the RPC system won't call this method until there is actually some + // activity on the stream that causes it to be needed. If this is an inbound stream, there + // is necessarily no activity until we receive some messages, and we must have received the + // peer key before that. + // + // This is convenient because otherwise we'd have to figure out what to do with this call + // if we haven't received the handshake yet, and therefore don't actually know who the peer + // is. + auto pk = KJ_ASSERT_NONNULL(peerKey, "getPeerVatId() called too early"); + + // Build `peerVatId` on first use. + KJ_IF_SOME(p, peerVatId) { + return p.getRoot().asReader(); + } else { + auto builder = peerVatId.emplace(peerVatIdScratch).getRoot(); + builder.setPublicKey(pk.bytes.asPtr()); + return builder.asReader(); + } + } + + kj::Own newOutgoingMessage(uint firstSegmentWordSize) override; + kj::Promise>> receiveIncomingMessage() override; + kj::Promise shutdown() override; + void setIdle(bool newIdle) override; + + private: + class OutgoingMessageImpl; + class IncomingMessageImpl; + + ClusterRegistry& registry; + kj::Own stream; + capnp::BufferedMessageStream msgStream; + kj::Maybe peerKey; + kj::Maybe> wasRefused; + bool outbound; // is this an outbound or inbound connection? + + capnp::word peerVatIdScratch[8]{}; + kj::Maybe peerVatId; + + kj::Vector> writeQueue; + kj::Maybe> pumpTask; + kj::Maybe brokenReason; + + bool pumping = false; // is pump() running? + bool isShutdown = false; // has shutdown() been called? + + bool idle = false; // tracks last value passed to setIdle() + bool idleTimedOut = false; + + bool sentHandshake = false; + bool receivedHandshake = false; + + kj::Maybe> idleTimer; + kj::Maybe>>>> + interruptReceiveFulfiller; + + void ensurePumping(); + kj::Promise pump(); + + void requireOpen(); + void unregister(); + + friend class ClusterRegistry; +}; + +class ClusterRegistry::ConnectionImpl::OutgoingMessageImpl final: public capnp::OutgoingRpcMessage, + public kj::Refcounted { + public: + OutgoingMessageImpl(ConnectionImpl& conn, uint firstSegmentWordSize) + : conn(conn), + message(firstSegmentWordSize == 0 ? capnp::SUGGESTED_FIRST_SEGMENT_WORDS + : firstSegmentWordSize) {} + + capnp::AnyPointer::Builder getBody() override { + return message.getRoot(); + } + + void send() override { + conn.requireOpen(); + KJ_REQUIRE(!conn.idle, "bug in RpcSystem: trying to send a message while idle"); + KJ_REQUIRE(!conn.idleTimedOut, "send() after idle period timed out"); + + conn.writeQueue.add(kj::addRef(*this)); + conn.ensurePumping(); + } + + size_t sizeInWords() override { + return message.sizeInWords(); + } + + private: + ConnectionImpl& conn; + capnp::MallocMessageBuilder message; + + friend class ConnectionImpl; +}; + +class ClusterRegistry::ConnectionImpl::IncomingMessageImpl final: public capnp::IncomingRpcMessage { + public: + IncomingMessageImpl(kj::Own message): message(kj::mv(message)) {} + + capnp::AnyPointer::Reader getBody() override { + return message->getRoot(); + } + + size_t sizeInWords() override { + return message->sizeInWords(); + } + + private: + kj::Own message; +}; + +kj::Own ClusterRegistry::ConnectionImpl::newOutgoingMessage( + uint firstSegmentWordSize) { + KJ_REQUIRE(!idle, "bug in RpcSystem: trying to send a message while idle"); + return kj::refcounted(*this, firstSegmentWordSize); +} + +kj::Promise>> ClusterRegistry::ConnectionImpl:: + receiveIncomingMessage() { + KJ_IF_SOME(b, brokenReason) { + kj::throwFatalException(b.clone()); + } + if (idleTimedOut) { + co_return kj::none; + } + + if (!receivedHandshake) { + X25519PublicKey receivedKey; + auto readHandshake = stream->read(receivedKey.bytes); + if (outbound && registry.nfsMode()) { + co_await registry.timer.timeoutAfter(registry.connectTimeout, kj::mv(readHandshake)); + } else { + co_await readHandshake; + } + KJ_IF_SOME(expectedKey, peerKey) { + KJ_REQUIRE(receivedKey == expectedKey, "peer did not send expected handshake bytes"); + } else { + peerKey = receivedKey; + } + receivedHandshake = true; + if (outbound) { + KJ_IF_SOME(entry, registry.peers.find(KJ_ASSERT_NONNULL(peerKey))) { + entry.suspect = false; + entry.suspectSince = kj::none; + } + } + } + + auto paf = kj::newPromiseAndFulfiller>>(); + interruptReceiveFulfiller = kj::mv(paf.fulfiller); + + auto message = co_await msgStream.tryReadMessage().exclusiveJoin(kj::mv(paf.promise)); + + KJ_IF_SOME(m, message) { + co_return kj::heap(kj::mv(m)); + } else { + co_return kj::none; + } +} + +kj::Promise ClusterRegistry::ConnectionImpl::shutdown() { + requireOpen(); + isShutdown = true; + ensurePumping(); + return KJ_ASSERT_NONNULL(kj::mv(pumpTask)); +} + +void ClusterRegistry::ConnectionImpl::setIdle(bool newIdle) { + if (newIdle == idle) return; // Already in the desired state. + idle = newIdle; + + // Only outbound connections should potentially close themselves when idle. Inbound should + // be closed at the other end. + if (outbound) { + if (idle) { + // Wait for the idle timeout. + idleTimer = registry.timer.afterDelay(registry.idleTimeout) + .then([this]() { + // Wait until the event queue is empty, just in case we're concurrently receiving a + // message that causes us to become non-idle. (But that shouldn't happen since this is + // an outbound connection. Once idle, it should only be revived from our end. But just to + // be safe.) + return kj::evalLast([this]() { + // We officially timed out. Make sure all future calls to receiveIncomingMessage() return + // kj::none. + idleTimedOut = true; + + // Cancel the current receiveIncomingMessage() by fulfilling the promise that is + // joined with the `tryReadMessage()` promise. + KJ_IF_SOME(i, interruptReceiveFulfiller) { + i->fulfill(kj::none); + } + + // Make sure a new call to connect() can't return this again. + unregister(); + }); + }).eagerlyEvaluate(nullptr); + } else { + // Cancel the last idle wait. + idleTimer = kj::none; + } + } +} + +void ClusterRegistry::ConnectionImpl::ensurePumping() { + if (!pumping) { + pumping = true; + pumpTask = pump(); + } +} + +kj::Promise ClusterRegistry::ConnectionImpl::pump() { + KJ_TRY { + if (!sentHandshake) { + co_await stream->write(registry.publicKey.bytes); + sentHandshake = true; + } + + // Give a chance for messages to accumulate to be sent all at once. + co_await kj::yieldUntilQueueEmpty(); + + while (!writeQueue.empty()) { + // Take ownership of all messages currently pending. `writeQueue` is left empty, but may + // be repopulated concurrently. + auto ownMessages = kj::mv(writeQueue); + + // Write them all at once. + auto messageSegments = + KJ_MAP(msg, ownMessages) { return msg->message.getSegmentsForOutput(); }; + co_await msgStream.writeMessages(messageSegments); + } + + if (isShutdown) { + co_await msgStream.end(); + } + + pumping = false; + } + KJ_CATCH(exception) { + brokenReason = exception.clone(); + KJ_IF_SOME(i, interruptReceiveFulfiller) { + i->reject(exception.clone()); + } + kj::throwFatalException(kj::mv(exception)); + } +} + +void ClusterRegistry::ConnectionImpl::requireOpen() { + KJ_IF_SOME(e, brokenReason) { + kj::throwFatalException(e.clone()); + } + KJ_REQUIRE(!isShutdown, "bug in RpcSystem: can't send() after shutdown()"); +} + +void ClusterRegistry::ConnectionImpl::unregister() { + if (outbound) { + auto key = KJ_ASSERT_NONNULL(peerKey); // peerKey is always set for outbounds + + KJ_TRY { + KJ_IF_SOME(entry, registry.peers.findEntry(key)) { + KJ_IF_SOME(conn, entry.value.outboundConnection) { + if (&conn == this) { + entry.value.outboundConnection = kj::none; + } + } + + // If we failed to receive the peer handshake, mark the peer as suspect and run the + // dead-peer cleanup path. We may have failed to connect entirely, or we may have + // connected to a server that wasn't the peer we expected. + if (!receivedHandshake) { + bool wasRefusedUnix = false; + KJ_IF_SOME(w, wasRefused) { + wasRefusedUnix = *w; + } + registry.cleanupFailedPeer(entry, wasRefusedUnix); + } + } + } + KJ_CATCH(exception) { + KJ_LOG(WARNING, "failed to clean up cluster peer after connection failure", key.toHex(), + exception); + } + } +} + +// ======================================================================================= +// ClusterRegistry constructor + +ClusterRegistry::ClusterRegistry(kj::Own registryDirParam, + kj::StringPtr networkConfig, + kj::Network& kjNetwork, + kj::Timer& timer, + kj::Duration idleTimeout, + kj::Duration connectTimeout) + : registryDir(kj::mv(registryDirParam)), + dirFd(KJ_REQUIRE_NONNULL( + registryDir->getFd(), "cluster registry directory must be a disk-backed directory")), + timer(timer), + network(kjNetwork), + idleTimeout(idleTimeout), + connectTimeout(connectTimeout) { + // Generate X25519 keypair. + X25519_keypair(publicKey.bytes.begin(), privateKey.begin()); + publicKeyHex = publicKey.toHex(); + + if (networkConfig == "unix") { + // Unix mode: bind a Unix domain socket at /. + // Use /proc/self/fd// as the socket path. + auto socketPath = kj::str("/proc/self/fd/", dirFd, "/", publicKeyHex); + sockaddr_un addr{}; + KJ_REQUIRE( + socketPath.size() < sizeof(addr.sun_path), "Unix socket path is too long", socketPath); + addr.sun_family = AF_UNIX; + memcpy(addr.sun_path, socketPath.begin(), socketPath.size()); + listener = kjNetwork.getSockaddr(&addr, sizeof(addr))->listen(); + } else { + // CIDR/IP/NFS mode: find a matching local IP and bind an ephemeral port. + auto addr = findMatchingIp(networkConfig); + listener = kjNetwork.getSockaddr(addr.asSockaddr(), addr.length)->listen(); + auto port = listener->getPort(); + addr.setPort(port); + boundAddress = addr; + } +} + +ClusterRegistry::~ClusterRegistry() noexcept(false) { + registryDir->tryRemove(kj::Path({publicKeyHex})); +} + +// Search all IP addresses for all interfaces available in the network namespace to find one that +// matches the given CIDR. +ClusterRegistry::NativeAddress ClusterRegistry::findMatchingIp(kj::StringPtr cidr) { + kj::CidrRange range(cidr); + + struct ifaddrs* ifap = nullptr; + KJ_SYSCALL(getifaddrs(&ifap)); + KJ_DEFER(freeifaddrs(ifap)); + + for (auto* ifa = ifap; ifa != nullptr; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == nullptr) continue; + if (range.matches(ifa->ifa_addr)) { + auto result = NativeAddress::fromSockaddr(ifa->ifa_addr); + result.setPort(0); + return result; + } + } + + KJ_FAIL_REQUIRE("no local interface matches CIDR", cidr); +} + +// ======================================================================================= +// Registry maintenance + +kj::Promise ClusterRegistry::runMaintenance() { + KJ_IF_SOME(addr, boundAddress) { + // CIDR/IP mode: create the registry file, lock it, write our address, then periodically + // verify that our registry entry and NFS lock lease are still valid. + + auto registryFile = registryDir->openFile(kj::Path({publicKeyHex}), kj::WriteMode::CREATE); + + // Acquire an exclusive OFD lock. + auto lock = KJ_ASSERT_NONNULL( + OfdLock::tryLock(KJ_ASSERT_NONNULL(registryFile->getFd()), OfdLock::EXCLUSIVE), + "our registry entry is already locked?"); + + // Write our address. + registryFile->write(0, addr.asBytes()); + registryFile->sync(); + + return maintenanceLoopCanceler.wrap(maintenanceLoop(registration.emplace(Registration{ + .file = kj::mv(registryFile), + .lock = kj::mv(lock), + }))); + } else { + // Unix socket mode: the listener socket's lifetime IS the registry entry. Nothing to maintain. + return kj::NEVER_DONE; + } +} + +kj::Promise ClusterRegistry::maintenanceLoop(Registration& registration) { + for (;;) { + co_await timer.afterDelay(REGISTRY_MAINTENANCE_INTERVAL); + + // Verify nlink > 0 (not unlinked by another node). + KJ_REQUIRE(registration.file->stat().linkCount > 0, + "registry file was unlinked by another node; this instance should shut down"); + + // Verify the OFD lock is still held (NFSv4 lease check). + registration.lock.verifyHeld(); + } +} + +void ClusterRegistry::verifyNfsLease() { + KJ_IF_SOME(r, registration) { + KJ_TRY { + r.lock.verifyHeld(); + } + KJ_CATCH(exception) { + // Might as well cancel the maintenance loop immediately. + maintenanceLoopCanceler.cancel(exception); + kj::throwFatalException(kj::mv(exception)); + } + } +} + +// ======================================================================================= +// Peer discovery + +void ClusterRegistry::scanDirectory() { + // List the registry to discover new peers and add them to the `peers` map, so that they can + // be considered by `pickRandomPeer()`. Keep in mind that on NFS, directory listings tend to be + // cached, and therefore we can's assume the existence or absence of a file really tells us + // anything about whether the peer is currently alive or dead. The best we can do is populate the + // map entries so that we know that peers exist at all, and then we'll need to probe them for + // liveness as usual later on. + + auto names = registryDir->listNames(); + + for (auto& name: names) { + if (name.size() != 64) continue; + if (name == publicKeyHex) continue; + + auto decoded = kj::decodeHex(name.asArray()); + if (decoded.hadErrors || decoded.size() != 32) continue; + + X25519PublicKey key; + key.bytes.asPtr().copyFrom(decoded); + + peers.findOrCreate(key, [&]() -> decltype(peers)::Entry { return {.key = key}; }); + } +} + +void ClusterRegistry::scanDirectoryIfStale(kj::TimePoint now) { + KJ_IF_SOME(lastScan, lastDirectoryScan) { + if (now - lastScan < DIRECTORY_SCAN_INTERVAL) return; + } + + scanDirectory(); + lastDirectoryScan = now; +} + +bool ClusterRegistry::isPeerDead(const X25519PublicKey& key) { + if (key == publicKey) return false; + + auto& entry = + peers.findOrCreateEntry(key, [&]() -> decltype(peers)::Entry { return {.key = key}; }); + + if (entry.value.knownDead) return true; + + auto now = timer.now(); + KJ_IF_SOME(lastLive, entry.value.lastConfirmedLiveTime) { + if (now - lastLive < PEER_LIVENESS_CACHE_TTL) return false; + } + + if (registryDir->exists(kj::Path({key.toHex()}))) { + entry.value.lastConfirmedLiveTime = now; + return false; + } + + entry.value.knownDead = true; + return true; +} + +kj::Maybe ClusterRegistry::pickRandomPeer() { + kj::Vector candidates; + auto now = timer.now(); + scanDirectoryIfStale(now); + + for (auto& peer: peers) { + if (peer.key == publicKey) continue; + if (peer.value.knownDead) continue; + if (peer.value.suspect) { + KJ_IF_SOME(since, peer.value.suspectSince) { + if (now - since < SUSPECT_PEER_AVOIDANCE_PERIOD) continue; + } + } + candidates.add(peer.key); + } + + if (candidates.empty()) return kj::none; + + // Simple pseudo-random selection using the monotonic clock as a cheap source. (Don't use + // timer.now() since it only updates when the event loop waits for I/O.) + auto nanos = + (kj::systemPreciseMonotonicClock().now() - kj::origin()) / kj::NANOSECONDS; + auto idx = kj::hashCode(nanos) % candidates.size(); + return candidates[idx]; +} + +void ClusterRegistry::cleanupFailedPeer(decltype(peers)::Entry& entry, bool wasRefusedUnix) { + auto now = timer.now(); + + entry.value.suspect = true; + entry.value.suspectSince = now; + + auto path = kj::Path({entry.key.toHex()}); + + if (!nfsMode()) { + if (wasRefusedUnix) { + // We failed with ECONNREFUSED, which means the listener is gone (process exited or closed + // the listen socket), in which case we can certainly clean up the socket from disk. + registryDir->tryRemove(path); + entry.value.knownDead = true; + } else { + // We failed for some other reason. Check if the file has disappeared from disk. + if (registryDir->exists(path)) { + // File still exists. We may have failed with EAGAIN (the peer's listen queue is full) or + // some other error where the peer is not necessarily gone. We've already marked it + // suspect, but we should not actually erase it. + } else { + // File is gone from disk. This may be exactly why the connection failed, but regardless, + // we should now mark the peer dead in our cache. + entry.value.knownDead = true; + } + } + + return; + } + + KJ_IF_SOME(lastProbe, entry.value.lastFailedProbeTime) { + if (now - lastProbe < FAILED_PROBE_THROTTLE) return; + } + entry.value.lastFailedProbeTime = now; + + KJ_IF_SOME(file, registryDir->tryOpenFile(path)) { + auto meta = file->stat(); + auto calendarNow = kj::systemCoarseCalendarClock().now(); + if (calendarNow - meta.lastModified < NEW_REGISTRY_ENTRY_GRACE_PERIOD) return; + + if (OfdLock::tryLock(KJ_ASSERT_NONNULL(file->getFd()), OfdLock::SHARED) != kj::none) { + registryDir->tryRemove(path); + entry.value.knownDead = true; + } + } else { + entry.value.knownDead = true; + } +} + +// ======================================================================================= +// Peer address lookup + +ClusterRegistry::NativeAddress ClusterRegistry::readPeerAddress(const X25519PublicKey& key) { + auto path = kj::Path({key.toHex()}); + auto file = + KJ_REQUIRE_NONNULL(registryDir->tryOpenFile(path), "peer not found in registry", key.toHex()); + auto content = file->readAllBytes(); + KJ_REQUIRE(content.size() > 0, "peer registry file is empty", key.toHex()); + return NativeAddress::parse(content); +} + +// ======================================================================================= +// VatNetwork: connect() + +kj::Maybe> ClusterRegistry::connect( + cluster::VatId::Reader peer) { + auto keyData = peer.getPublicKey(); + KJ_REQUIRE(keyData.size() == 32, "invalid VatId: publicKey must be 32 bytes"); + + X25519PublicKey peerKey; + peerKey.bytes.asPtr().copyFrom(keyData); + + // Self-connect returns kj::none per VatNetwork contract. + if (peerKey == publicKey) { + return kj::none; + } + + // Check for cached outbound connection. + KJ_IF_SOME(entry, peers.find(peerKey)) { + KJ_IF_SOME(conn, entry.outboundConnection) { + return kj::addRef(conn); + } + } + + auto& entry = peers.findOrCreateEntry( + peerKey, [&]() -> decltype(peers)::Entry { return {.key = peerKey}; }); + + KJ_REQUIRE(!entry.value.knownDead, "peer is known to be dead", peerKey.toHex()); + + // Need to establish a new connection. Look up the peer's address. + kj::Own networkAddress; + if (nfsMode()) { + NativeAddress address; + try { + address = readPeerAddress(peerKey); + } catch (...) { + cleanupFailedPeer(entry, false); + throw; + } + networkAddress = network.getSockaddr(address.asSockaddr(), address.length); + } else { + auto path = kj::str("/proc/self/fd/", dirFd, "/", peerKey.toHex()); + sockaddr_un addr{}; + KJ_REQUIRE(path.size() < sizeof(addr.sun_path), "Unix socket path is too long", path); + addr.sun_family = AF_UNIX; + memcpy(addr.sun_path, path.begin(), path.size()); + networkAddress = network.getSockaddr(&addr, sizeof(addr)); + } + + auto connectPromise = networkAddress->connect(); + if (nfsMode()) { + // Enforce timeout when connecting over the network. (When not in nfsMode(), we're connecting + // to a local unix socket, in which case no timeout is needed.) + connectPromise = timer.timeoutAfter(connectTimeout, kj::mv(connectPromise)); + } + + kj::Maybe> wasRefused; + if (!nfsMode()) { + auto wrapper = kj::refcountedWrapper(false); + wasRefused = wrapper->addWrappedRef(); + connectPromise = connectPromise.catch_( + [wrapper = kj::mv(wrapper)]( + kj::Exception&& e) mutable -> kj::Promise> { + if (e.getType() == kj::Exception::Type::DISCONNECTED) { + // If connect() failed with DISCONNECTED on a unix socket, this likely means we got + // ECONNREFUSED. Record this. + wrapper->getWrapped() = true; + } + kj::throwFatalException(kj::mv(e)); + }); + } + + auto promisedStream = kj::newPromisedStream(connectPromise.attach(kj::mv(networkAddress))); + + auto conn = + kj::refcounted(*this, kj::mv(promisedStream), peerKey, kj::mv(wasRefused)); + + // Cache the outbound connection. + entry.value.outboundConnection = *conn; + + return kj::Own(kj::addRef(*conn)); +} + +// ======================================================================================= +// VatNetwork: accept() + +kj::Promise> ClusterRegistry::accept() { + auto stream = co_await listener->accept(); + co_return kj::refcounted(*this, kj::mv(stream), kj::none, kj::none); +} + +// ======================================================================================= + +#else // #if __linux__ + +ClusterRegistry::ClusterRegistry(kj::Own registryDir, + kj::StringPtr network, + kj::Network& kjNetwork, + kj::Timer& timer, + kj::Duration idleTimeout, + kj::Duration connectTimeout) { + KJ_UNIMPLEMENTED("cluster mode is only implemented on linux"); +} +ClusterRegistry::~ClusterRegistry() noexcept(false) {} + +kj::Promise ClusterRegistry::runMaintenance() { + KJ_UNREACHABLE; +} +void ClusterRegistry::verifyNfsLease() { + KJ_UNREACHABLE; +} +bool ClusterRegistry::isPeerDead(const X25519PublicKey& key) { + KJ_UNREACHABLE; +} +kj::Maybe ClusterRegistry::pickRandomPeer() { + KJ_UNREACHABLE; +} +kj::Maybe> ClusterRegistry::connect( + cluster::VatId::Reader peer) { + KJ_UNREACHABLE; +} +kj::Promise> ClusterRegistry::accept() { + KJ_UNREACHABLE; +} + +#endif // #if __linux__, #else + +} // namespace workerd diff --git a/src/workerd/server/cluster-registry.h b/src/workerd/server/cluster-registry.h new file mode 100644 index 00000000000..11494af78d6 --- /dev/null +++ b/src/workerd/server/cluster-registry.h @@ -0,0 +1,185 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once + +#include +#include + +#if __linux__ +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +namespace workerd { + +using kj::byte; +using kj::uint; + +struct X25519PublicKey { + kj::FixedArray bytes; + + kj::String toHex() const; + static X25519PublicKey fromHex(kj::StringPtr hex); + + bool operator==(const X25519PublicKey& other) const { + return bytes.asPtr() == other.bytes.asPtr(); + } + kj::uint hashCode() const { + return kj::hashCode(bytes.asPtr()); + } +}; + +// Type alias for the VatNetwork base that ClusterRegistry implements. +using ClusterVatNetworkBase = capnp::VatNetwork; + +class ClusterRegistry final: public ClusterVatNetworkBase { + // Manages this node's cluster identity, registry presence, peer discovery, and the + // VatNetwork that the cluster's RpcSystem sits on top of. + public: + ClusterRegistry(kj::Own registryDir, + kj::StringPtr network, + kj::Network& kjNetwork, + kj::Timer& timer, + kj::Duration idleTimeout = 60 * kj::SECONDS, + kj::Duration connectTimeout = 1 * kj::SECONDS); + ~ClusterRegistry() noexcept(false); + + const X25519PublicKey& getPublicKey() { + return publicKey; + } + + // Run the registry maintenance loop. Returns a promise that must be held alive for + // the lifetime of the server. + kj::Promise runMaintenance(); + + // Verify that the NFS lease is still valid on the registration file, or throw an exception if + // not (no-op if not in NFS mode). + // + // More specifically: If you call this immediately after open()ing a file, and it returns + // successfully, then the newly-opened file is guaranteed to be on the same lease as the + // registration. (Technically, the lease could have been lost already, but if so, the + // newly-opened file will not be writable.) + void verifyNfsLease(); + + // Definitive proof-of-death query. Returns true only when the peer's registry entry is absent. + bool isPeerDead(const X25519PublicKey& key); + + // Pick a random peer's key from the cached peer list, biased away from recent failures. + kj::Maybe pickRandomPeer(); + + // Marks this node as leaving the cluster. Once the server begins draining, its cluster listener + // stops accepting connections, so an actor claimed here would be unreachable from every other + // node until this process exits. ClusterLockManager::acquireOrRoute() consults isDraining() and + // refuses to claim unowned actors; routing to an existing owner is unaffected, since outbound + // connections keep working for in-flight requests. + void startDraining() { + draining = true; + } + bool isDraining() { + return draining; + } + + // Returns true if this registry uses IP sockets (not unix sockets) and is therefore in + // "NFS mode" where it tries to be NFS-friendly. + bool nfsMode() { + return boundAddress != kj::none; + } + + // implements VatNetwork ----------------------------------------------------- + kj::Maybe> connect(cluster::VatId::Reader peer) override; + kj::Promise> accept() override; + + private: + bool draining = false; + +#if __linux__ + class ConnectionImpl; + + struct NativeAddress { + sockaddr_storage storage{}; + socklen_t length = 0; + + const sockaddr* asSockaddr() const { + return reinterpret_cast(&storage); + } + + kj::ArrayPtr asBytes() const { + return kj::arrayPtr(reinterpret_cast(&storage), length); + } + + void setPort(uint port); + + static NativeAddress fromSockaddr(const sockaddr* addr); + static NativeAddress parse(kj::ArrayPtr bytes); + }; + + kj::Own registryDir; + int dirFd; // raw fd of registryDir, for POSIX ops + kj::Timer& timer; + kj::Network& network; + kj::Duration idleTimeout; + kj::Duration connectTimeout; + + // Identity + kj::FixedArray privateKey; + X25519PublicKey publicKey; + kj::String publicKeyHex; + + // Address we wrote to the registry file (CIDR/IP/NFS mode). Null in Unix socket mode. + kj::Maybe boundAddress; + + struct Registration { + kj::Own file; + OfdLock lock; + }; + kj::Maybe registration; // only in NFS mode + + kj::Canceler maintenanceLoopCanceler; + + // Listener + kj::Own listener; + + // Peer cache + struct PeerInfo { + kj::Maybe outboundConnection; + kj::Maybe lastConfirmedLiveTime; + kj::Maybe lastFailedProbeTime; + bool suspect = false; + kj::Maybe suspectSince; + bool knownDead = false; + }; + kj::HashMap peers; + kj::Maybe lastDirectoryScan; + + void scanDirectory(); + void scanDirectoryIfStale(kj::TimePoint now); + kj::Promise maintenanceLoop(Registration& registration); + void cleanupFailedPeer(decltype(peers)::Entry& entry, bool wasRefusedUnix); + + NativeAddress findMatchingIp(kj::StringPtr cidr); + + // Read the address for a peer from its registry file. CIDR/IP mode only. + NativeAddress readPeerAddress(const X25519PublicKey& key); + +#else // #if __linux__ + // Just so inline methods compile + X25519PublicKey publicKey; + kj::Maybe boundAddress; + +#endif // #if __linux__, #else +}; + +} // namespace workerd diff --git a/src/workerd/server/cluster.capnp b/src/workerd/server/cluster.capnp new file mode 100644 index 00000000000..35532c83478 --- /dev/null +++ b/src/workerd/server/cluster.capnp @@ -0,0 +1,25 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +@0xbd0c09739255a698; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("workerd::cluster"); +$Cxx.allowCancellation; + +struct VatId { + publicKey @0 :Data; + # 32-byte raw X25519 public key. +} + +# The following are stubs for v1. They are not used because canIntroduceTo() returns +# false (the default), so the RPC system falls back to proxying for any cross-peer +# capability passing. Cap'n Proto itself has not yet fully implemented Level 3, so +# leaving these empty is fine. +struct ThirdPartyToContact {} +struct ThirdPartyToAwait {} +struct ThirdPartyCompletion {} + +# Level 4 (Join) is not supported. +struct JoinResult {} diff --git a/src/workerd/server/server-test.c++ b/src/workerd/server/server-test.c++ index 2be79f35d54..b02d529d824 100644 --- a/src/workerd/server/server-test.c++ +++ b/src/workerd/server/server-test.c++ @@ -7132,6 +7132,55 @@ KJ_TEST("Server: debug port RPC calls") { auto jsonResult = deserializeV8ToJson(resultData); KJ_EXPECT(jsonResult == "8", jsonResult, "Expected result to be 8"); } + + // Test 7: `startEvent(fromPersistentStub = true)` is propagated to the target for non-HTTP + // events. `rpc-service` lacks `allow_irrevocable_stub_storage`, so the target must reject the + // event; this is the only observable that proves the flag reached WorkerEntrypoint::construct(). + { + auto startSession = [&](bool fromPersistentStub) { + auto bootstrap = + getBootstrap("rpc-service", kj::none, [](auto& props) { props.setEmptyObject(); }); + auto eventReq = bootstrap.startEventRequest(); + eventReq.setFromPersistentStub(fromPersistentStub); + auto dispatcher = eventReq.send().wait(test.ws).getDispatcher(); + return dispatcher.jsRpcSessionRequest().send(); + }; + + auto callAdd = [&](rpc::JsRpcTarget::Client rpcTarget) { + auto v8SerializedArgs = serializeJsArguments({[](jsg::Lock& js) { + return jsg::JsValue(js.num(2)); + }, [](jsg::Lock& js) { return jsg::JsValue(js.num(3)); }}); + auto callReq = rpcTarget.callRequest(); + callReq.setMethodName("add"); + callReq.initOperation().initCallWithArgs().setV8Serialized(v8SerializedArgs); + return callReq.send(); + }; + + // Control: without the flag, the session works. + { + auto sessionPromise = startSession(false); + auto callResp = callAdd(sessionPromise.getTopLevel()).wait(test.ws); + KJ_EXPECT(deserializeV8ToJson(callResp.getResult().getV8Serialized()) == "5"); + } + + // With the flag, the target rejects the RPC session. + { + auto sessionPromise = startSession(true); + KJ_EXPECT_THROW_MESSAGE("no longer has the allow_irrevocable_stub_storage", + callAdd(sessionPromise.getTopLevel()).wait(test.ws)); + } + + // Control: the HTTP path rejects it too. + { + auto bootstrap = + getBootstrap("rpc-service", kj::none, [](auto& props) { props.setEmptyObject(); }); + auto eventReq = bootstrap.startEventRequest(); + eventReq.setFromPersistentStub(true); + auto dispatcher = eventReq.send().wait(test.ws).getDispatcher(); + KJ_EXPECT_THROW_MESSAGE("no longer has the allow_irrevocable_stub_storage", + makeHttpRequestFromDispatcher(kj::mv(dispatcher), "/")); + } + } } KJ_TEST("Server: workerdDebugPort binding current process test") { diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index c7e137fde87..5c143f59f83 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -5,6 +5,7 @@ #include "server.h" #include "alarm-scheduler.h" +#include "cluster-lock.h" #include "container-client.h" #include "pyodide.h" #include "workerd-api.h" @@ -12,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -45,10 +47,12 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -58,6 +62,7 @@ #include #include +#include #include namespace workerd::server { @@ -238,7 +243,6 @@ Server::Server(kj::Filesystem& fs, reportConfigWarning(kj::mv(reportConfigWarning)), loggingOptions(loggingOptions), memoryCacheProvider(kj::heap(timer)), - channelTokenHandler(*this), tasks(*this) {} struct Server::GlobalContext { @@ -340,6 +344,14 @@ Server::~Server() noexcept { abortAllActors(KJ_EXCEPTION(DISCONNECTED, "Server shutting down.")); tasks.clear(); + // Tear down the cluster RPC system now that nothing can issue requests through it. This must + // happen after `tasks.clear()` (the accept loop task references the RpcSystem) and before the + // services are unlinked: open cluster connections export capabilities that hold references into + // services, and dropping those connections is what lets the unlink below eliminate all + // references. The ClusterLockManagers that reference `clusterRpc` live in ActorNamespaces, which + // are destroyed by WorkerService::unlink(). + clusterRpc = kj::none; + // Unlink all the services, which should remove all refcount cycles. unlinkWorkerLoaders(); for (auto& service: services) { @@ -366,6 +378,7 @@ class Server::ActorNamespace final { const kj::Clock& clock, kj::Timer& timer, capnp::ByteStreamFactory& byteStreamFactory, + capnp::HttpOverCapnpFactory& httpOverCapnpFactory, ChannelTokenHandler& channelTokenHandler, kj::Network& dockerNetwork, kj::Maybe dockerPath, @@ -377,6 +390,7 @@ class Server::ActorNamespace final { clock(clock), timer(timer), byteStreamFactory(byteStreamFactory), + httpOverCapnpFactory(httpOverCapnpFactory), channelTokenHandler(channelTokenHandler), dockerNetwork(dockerNetwork), dockerPath(dockerPath), @@ -384,15 +398,48 @@ class Server::ActorNamespace final { waitUntilTasks(waitUntilTasks), selfTokensArePersistent(selfTokensArePersistent) {} - void link(kj::Maybe serviceActorStorage) { + void link(kj::Maybe serviceActorStorage, + kj::Maybe clusterRegistry, + kj::Maybe&> clusterRpc) { KJ_IF_SOME(dir, serviceActorStorage) { KJ_IF_SOME(d, config.tryGet()) { this->actorStorage.emplace( - dir.openSubdir(kj::Path({d.uniqueKey}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY)); + dir.openSubdir(kj::Path({d.uniqueKey}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY), + *this); + KJ_IF_SOME(registry, clusterRegistry) { + auto& rpc = KJ_ASSERT_NONNULL(clusterRpc); + this->clusterRegistry = registry; + auto lockDirectory = KJ_ASSERT_NONNULL(this->actorStorage) + .directory->openSubdir(kj::Path({"locks"}), + kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + this->lockManager.emplace(kj::mv(lockDirectory), registry, rpc, timer); + } } } KJ_IF_SOME(d, config.tryGet()) { + if (lockManager != kj::none) { + if (KJ_ASSERT_NONNULL(clusterRegistry).nfsMode()) { + // Across machines (NFS) a shared SQLite file in WAL mode isn't safe: no alarms there. + // ClusterActorSqliteHooks refuses them, as before. + return; + } + // Cluster mode on one machine: every instance writes alarm times into the shared + // metadata.sqlite (the native VFS takes real file locks, so several processes can share it + // on a local disk); exactly one instance, the alarm leader, runs the AlarmScheduler. + // Leadership is an OFD lock on `alarm-leader`, so it moves to another instance when the + // leader goes away. + auto& as = KJ_ASSERT_NONNULL(this->actorStorage); + clusterAlarmStore = kj::heap(as.vfs); + alarmLeaderFile = as.directory->openFile( + kj::Path({"alarm-leader"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + alarmLeaderTask = + alarmLeaderLoop(kj::str(d.uniqueKey)).eagerlyEvaluate([](kj::Exception&& e) { + KJ_LOG(ERROR, "cluster alarm leader loop failed; alarms stop on this instance", e); + }); + return; + } + auto idFactory = kj::heap(d.uniqueKey); AlarmScheduler::GetActorFn getActor = [this, idFactory = kj::mv(idFactory)]( @@ -439,6 +486,8 @@ class Server::ActorNamespace final { return result; } + // Get the ActorChannel for the given actor ID, routing to the appropriate workerd instance + // if needed. kj::Own getActorChannel( Worker::Actor::Id id, Persistent persistent = Persistent::NO) { KJ_IF_SOME(doId, id.tryGet>()) { @@ -452,7 +501,13 @@ class Server::ActorNamespace final { } } - return kj::refcounted(getActorContainer(kj::mv(id)), persistent); + if (lockManager != kj::none) { + // Cluster mode: resolution of the owning node is deferred until the channel is first used, + // so that merely obtaining a stub never takes cluster locks or wakes actors. + return kj::refcounted(*this, kj::mv(id), persistent); + } else { + return kj::refcounted(getActorContainer(kj::mv(id)), persistent); + } } class ActorContainer; @@ -485,8 +540,10 @@ class Server::ActorNamespace final { ActorNamespace& ns, kj::Maybe parent, kj::OneOf> classAndIdParam, - kj::Timer& timer) - : key(kj::mv(key)), + kj::Timer& timer, + kj::Maybe ownershipLock = kj::none) + : ownershipLock(kj::mv(ownershipLock)), + key(kj::mv(key)), tracker(kj::refcounted(*this)), ns(ns), root(parent.map([](ActorContainer& p) -> ActorContainer& { return p.root; }) @@ -532,6 +589,7 @@ class Server::ActorNamespace final { auto reason = 0; a->shutdown(reason); } + closeStorage(); // Drop the container client reference // If setInactivityTimeout() was called, there's still a timer holding a reference @@ -671,7 +729,9 @@ class Server::ActorNamespace final { // removed from any maps that would cause it to receive further traffic, since any further // requests will be expected to fail. abort() does NOT attempt to remove the ActorContainer // from the parent facet map since at most call sites it makes more sense to handle this - // directly. + // directly. Callers that remove the container from `ns.actors` must do so in the same + // synchronous continuation as this call: abort() releases the ownership lock (see + // `ownershipLock`). void abort(kj::Maybe reason) { if (brokenReason != kj::none) return; @@ -692,9 +752,16 @@ class Server::ActorNamespace final { shutdownTask = kj::none; manager = kj::none; tracker->shutdown(); + closeStorage(); actor = kj::none; containerClient = kj::none; + // The actor (and every facet) shut down above and closeStorage() closed the database, so + // nothing on this node holds a handle into the actor's storage anymore. Drop ours and let + // another node (or a fresh container on this one) claim the actor. + facetTreeIndex = kj::none; + ownershipLock = kj::none; + KJ_IF_SOME(r, reason) { brokenReason = r.clone(); } else { @@ -703,17 +770,31 @@ class Server::ActorNamespace final { } // Resets the actor's SQLite database while the connection is still open, - // avoiding file-locking issues on Windows. + // avoiding file-locking issues on Windows. Storage that is already closed is skipped + // (reset() refuses to reopen a closed database). void resetStorage() { KJ_IF_SOME(a, actor) { KJ_IF_SOME(cache, a->getPersistent()) { - KJ_IF_SOME(db, cache.getSqliteDatabase()) { - kj::runCatchingExceptions([&]() { db.reset(); }); - } + kj::runCatchingExceptions([&]() { + KJ_IF_SOME(db, cache.getSqliteDatabase()) { + db.reset(); + } + }); } } } + // Closes the actor's SQLite connection so the database file is fully and synchronously + // released (WAL checkpointed and removed; in NFS mode the EXCLUSIVE lock dropped). Must run + // after the actor has been shut down (so ActorSqlite is `broken` and will not touch the + // connection again) and before `actor` is dropped (see `sqliteDb`). Idempotent. + void closeStorage() { + KJ_IF_SOME(db, sqliteDb) { + db.close(); + } + sqliteDb = kj::none; + } + kj::Own getFacetContainer( kj::String childKey, kj::Function()> getStartInfo) { auto makeContainer = [&]() { @@ -825,9 +906,38 @@ class Server::ActorNamespace final { } private: + // In cluster mode, the root container's claim on the actor. Declared first so that on plain + // destruction it is released only after everything below it -- the Worker::Actor, facets, + // hibernation manager, and facet tree index -- has been torn down. + // + // Invariant: an OwnedLock for the actor is alive on this node if and only if its container is + // in `ns.actors`. ClusterLockManager::acquireOrRoute() routes to this node whenever the lock + // file names it and the OFD lock is held, so an OwnedLock with no map entry would make every + // request for the actor loop through self-bootstrap (see ActorNamespace::resolveNode()). The + // lock is therefore released at the moment the container leaves the map (abort(), + // monitorOnBroken()), not when the last reference to the container is dropped. Releasing it + // there is only safe because closeStorage() runs first: by the time either path hollows the + // container, every SQLite handle under the actor and its facets is already closed, so no handle + // into the actor's storage outlives the claim even if an in-flight request still holds the + // Worker::Actor. + // + // The lock file's contents alone carry no such guarantee: a failed claim or release can leave + // this node's key behind with no OwnedLock held. acquireOrRoute() detects that (the OFD lock + // is free) and reclaims, so it needs no handling here. + kj::Maybe ownershipLock; + // The actor is constructed after the ActorContainer so it starts off empty. kj::Maybe> actor; + // The actor's SQLite connection, owned by the ActorSqlite inside `actor`. Set by start() (the + // actor constructs its cache synchronously) and cleared by closeStorage(), which every path + // that drops `actor` calls first, so this never outlives the connection it refers to. + // + // The connection is closed here rather than in ActorSqlite::shutdown() because shutdown() can + // run synchronously inside a SQLite callback (e.g. a VFS write that hits a storage limit), + // where closing the handle is unsafe. The container only drops the actor from the event loop. + kj::Maybe sqliteDb; + kj::String key; kj::Own tracker; ActorNamespace& ns; @@ -883,6 +993,7 @@ class Server::ActorNamespace final { ns.actorStorage, "can't call getFacetId() when there's no backing storage"); auto indexFile = as.directory->openFile( kj::Path({kj::str(key, ".facets")}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + ns.verifyRegistration(); return *facetTreeIndex.emplace(kj::heap(kj::mv(indexFile))); } } @@ -900,6 +1011,7 @@ class Server::ActorNamespace final { auto indexFile = KJ_UNWRAP_OR( as.directory->tryOpenFile(kj::Path({kj::str(key, ".facets")}), kj::WriteMode::MODIFY), return kj::none); + ns.verifyRegistration(); return *facetTreeIndex.emplace(kj::heap(kj::mv(indexFile))); } } @@ -1021,6 +1133,15 @@ class Server::ActorNamespace final { tracker->shutdown(); auto managerToDrop = kj::mv(manager); + // onBroken fired only after IoContext::abort() shut down the actor cache, and the facets + // were aborted above, so closing the database now is safe and leaves every database under + // this actor closed. Release the claim in the same synchronous continuation as the erase() + // below (see `ownershipLock`). Any stub still pinning this container after the erase keeps + // only a hollow shell. + closeStorage(); + facetTreeIndex = kj::none; + ownershipLock = kj::none; + // Note that we remove the entire ActorContainer from the map -- this drops the // HibernationManager so any connected hibernatable websockets will be disconnected. KJ_IF_SOME(p, parent) { @@ -1072,6 +1193,7 @@ class Server::ActorNamespace final { a->shutdown(0, KJ_EXCEPTION(DISCONNECTED, "broken.dropped; Actor freed due to inactivity")); } // Destroy the last strong Worker::Actor reference. + closeStorage(); actor = kj::none; // Drop our reference to the ContainerClient @@ -1140,6 +1262,7 @@ class Server::ActorNamespace final { onBrokenTask = kj::none; // Destroy the last strong Worker::Actor reference. + closeStorage(); actor = kj::none; if (webSocketMode == IoChannelFactory::EvictWebSocketMode::CLOSE) { @@ -1245,7 +1368,15 @@ class Server::ActorNamespace final { KJ_IF_SOME(as, ns.actorStorage) { kj::Own sqliteHooks; if (parent == kj::none) { - KJ_IF_SOME(a, ns.alarmScheduler) { + if (ns.lockManager != kj::none) { + auto actorKey = ActorKey{.actorId = key, .name = actorName.map([](kj::String& n) { + return n.asPtr(); + })}.clone(); + sqliteHooks = kj::heap( + ns.clusterAlarmStore.map( + [](kj::Own& s) -> ClusterAlarmStore& { return *s; }), + kj::mv(actorKey)); + } else KJ_IF_SOME(a, ns.alarmScheduler) { // clone() copies the id and name into storage owned by the returned ActorKey, so // the temporary StringPtrs below only need to outlive this call. auto actorKey = ActorKey{.actorId = key, .name = actorName.map([](kj::String& n) { @@ -1268,9 +1399,15 @@ class Server::ActorNamespace final { // Before we do anything, make sure the database is in WAL mode. We also need to // do this after reset() is used, so register a callback for that. + if (ns.nfsMode()) { + db->run("PRAGMA locking_mode=EXCLUSIVE;"); + } db->run("PRAGMA journal_mode=WAL;"); db->afterReset([this, &dir = *as.directory, selfId](SqliteDatabase& db) { + if (ns.nfsMode()) { + db.run("PRAGMA locking_mode=EXCLUSIVE;"); + } db.run("PRAGMA journal_mode=WAL;"); // reset() is used when the app called deleteAll(), in which case we also want to @@ -1284,6 +1421,8 @@ class Server::ActorNamespace final { deleteDescendantStorage(dir, selfId); }); + KJ_ASSERT(sqliteDb == kj::none); + sqliteDb = *db; return kj::heap(kj::mv(db), outputGate, [](SpanParent) -> kj::Promise { return kj::READY_NOW; }, *sqliteHooks) .attach(kj::mv(sqliteHooks)); @@ -1371,6 +1510,9 @@ class Server::ActorNamespace final { kj::mv(privileges)); } + // If actor construction throws after makeActorCache ran, the database it recorded is + // destroyed along with the half-built actor; don't leave `sqliteDb` pointing at it. + KJ_ON_SCOPE_FAILURE(sqliteDb = kj::none); auto actor = actorClass->newActor(getTracker(), Worker::Actor::cloneId(id), kj::mv(makeActorCache), kj::mv(makeStorage), kj::mv(loopback), tryGetManagerRef(), kj::mv(container), kj::mv(containerImages), *this); @@ -1390,28 +1532,29 @@ class Server::ActorNamespace final { kj::Array getChannelTokenImpl(IoChannelFactory::ChannelTokenUsage usage, const Worker::Actor::Id& id, Persistent persistent) { - kj::StringPtr uniqueKey = KJ_ASSERT_NONNULL(ns.getConfig().tryGet()).uniqueKey; - auto& abstractId = *KJ_ASSERT_NONNULL(id.tryGet>()); - auto& idImpl = - KJ_ASSERT_NONNULL(kj::tryDowncast(abstractId)); - return ns.channelTokenHandler.encodeActorChannelToken( - usage, uniqueKey, idImpl.getRaw(), idImpl.getName(), persistent); + return ns.encodeActorChannelToken(usage, id, persistent); } }; - kj::Own getActorContainer(Worker::Actor::Id id) { - kj::String key; - + kj::String actorKey(const Worker::Actor::Id& id) { KJ_SWITCH_ONEOF(id) { KJ_CASE_ONEOF(obj, kj::Own) { KJ_REQUIRE(config.is()); - key = obj->toString(); + return obj->toString(); } KJ_CASE_ONEOF(str, kj::String) { KJ_REQUIRE(config.is()); - key = kj::str(str); + return kj::str(str); } } + KJ_UNREACHABLE; + } + + // Get or create an ActorContainer. Use only when NOT in cluster mode. + kj::Own getActorContainer(Worker::Actor::Id id) { + KJ_REQUIRE(lockManager == kj::none); + + kj::String key = actorKey(id); return actors .findOrCreate(key, [&]() mutable { @@ -1423,6 +1566,65 @@ class Server::ActorNamespace final { })->addRef(); } + // Create an ActorContainer in cluster mode. Must be called in the same synchronous continuation + // that received `ownership` from ClusterLockManager::acquireOrRoute(): an OwnedLock alive on + // this node while `actors` has no entry for the key would make requests for the actor loop + // through self-bootstrap until the entry appears (see resolveNode()). + kj::Own createActorContainer( + kj::String key, Worker::Actor::Id id, ClusterLockManager::OwnedLock ownership) { + // Invariant: an OwnedLock is alive on this node iff a container holding it is in `actors`. We + // hold the lock and are about to insert; nothing else may already be mapped. The other + // direction is upheld by ActorContainer::abort() and monitorOnBroken(), which release the + // lock as the container leaves the map. + KJ_ASSERT(actors.find(key) == kj::none); + + auto container = kj::refcounted(kj::str(key), *this, kj::none, + ActorContainer::ClassAndId(kj::addRef(*actorClass), kj::mv(id)), timer, kj::mv(ownership)); + actors.insert(container->getKey(), container->addRef()); + return container; + } + + // Which node currently owns the actor with the given key: a container on this node (created + // here if we just claimed the lock), or the owning node's cluster port. Not cached; callers + // cache what they derive from it. Every cluster-routed channel type resolves through this method + // so that the lock protocol lives in exactly one place. `key` and `id` must remain valid until + // the returned promise settles. Cluster mode only. + using Node = kj::OneOf, rpc::WorkerdClusterPort::Client>; + kj::Promise resolveNode(kj::StringPtr key, const Worker::Actor::Id& id) { + // Local-first. This lookup is what terminates the routing loop: when an OwnedLock on this + // node holds the actor, acquireOrRoute() returns our own cluster port with no backoff, and the + // resulting getChannelFromToken() decodes the token into a channel that lands right back here. + // That is only correct because "OwnedLock alive on this node" implies "container is in + // `actors`" -- see ActorContainer::ownershipLock. + KJ_IF_SOME(container, actors.find(key)) { + co_return container->addRef(); + } + + auto lockOrClient = co_await KJ_ASSERT_NONNULL(lockManager).acquireOrRoute(key); + KJ_SWITCH_ONEOF(lockOrClient) { + KJ_CASE_ONEOF(ownership, ClusterLockManager::OwnedLock) { + co_return createActorContainer(kj::str(key), Worker::Actor::cloneId(id), kj::mv(ownership)); + } + KJ_CASE_ONEOF(clusterPort, rpc::WorkerdClusterPort::Client) { + co_return kj::mv(clusterPort); + } + } + KJ_UNREACHABLE; + } + + // Encode a channel token pointing at the given actor in this namespace. The caller is + // responsible for having checked transferrability. + kj::Array encodeActorChannelToken(IoChannelFactory::ChannelTokenUsage usage, + const Worker::Actor::Id& id, + Persistent persistent) { + kj::StringPtr uniqueKey = KJ_ASSERT_NONNULL(config.tryGet()).uniqueKey; + auto& abstractId = *KJ_ASSERT_NONNULL(id.tryGet>()); + auto& idImpl = + KJ_ASSERT_NONNULL(kj::tryDowncast(abstractId)); + return channelTokenHandler.encodeActorChannelToken( + usage, uniqueKey, idImpl.getRaw(), idImpl.getName(), persistent); + } + kj::Own getContainerClient( kj::StringPtr containerId, kj::StringPtr imageName, ContainerPrivileges privileges) { KJ_IF_SOME(existingClient, containerClients.find(containerId)) { @@ -1528,8 +1730,12 @@ class Server::ActorNamespace final { abortAll(reason); - KJ_IF_SOME(scheduler, ownAlarmScheduler) { - scheduler->deleteAll(); + // In cluster mode the alarm table is shared by every instance, and this only resets this + // instance's actors: leave it (each reset actor cancels its stale alarm when it is delivered). + if (lockManager == kj::none) { + KJ_IF_SOME(scheduler, ownAlarmScheduler) { + scheduler->deleteAll(); + } } } @@ -1542,9 +1748,12 @@ class Server::ActorNamespace final { kj::Own directory; SqliteDatabase::Vfs vfs; - ActorStorage(kj::Own directoryParam) + ActorStorage(kj::Own directoryParam, ActorNamespace& ns) : directory(kj::mv(directoryParam)), - vfs(*directory) {} + vfs(*directory, + SqliteDatabase::VfsOptions{ + .afterOpen = kj::Function(KJ_BIND_METHOD(ns, verifyRegistration)), + }) {} }; // Note: The Vfs, actorStorage, and ownAlarmScheduler must not be torn down until all actors @@ -1552,6 +1761,15 @@ class Server::ActorNamespace final { kj::Maybe actorStorage; kj::Maybe> ownAlarmScheduler; + kj::Maybe clusterRegistry; + kj::Maybe lockManager; + + // Cluster mode alarms (see alarmLeaderLoop()). + class ClusterAlarmStore; + kj::Maybe> clusterAlarmStore; + kj::Maybe> alarmLeaderFile; + kj::Maybe alarmLeaderLock; + // Tracks the canceler and cleanup promise for a Docker container's lifecycle cleanup. // Useful to await on async calls of a ContainerClient destructor when the new // one appears before they've been resolved. @@ -1585,8 +1803,11 @@ class Server::ActorNamespace final { ActorMap actors; kj::Maybe> cleanupTask; + // Declared after `actors` and the scheduler so it is torn down first. + kj::Maybe> alarmLeaderTask; kj::Timer& timer; capnp::ByteStreamFactory& byteStreamFactory; + capnp::HttpOverCapnpFactory& httpOverCapnpFactory; ChannelTokenHandler& channelTokenHandler; kj::Network& dockerNetwork; kj::Maybe dockerPath; @@ -1635,6 +1856,23 @@ class Server::ActorNamespace final { } } + bool nfsMode() { + KJ_IF_SOME(r, clusterRegistry) { + return r.nfsMode(); + } else { + return false; + } + } + + // Callback called immediately after any actor storage file is opened to verify that the + // registry lock is still held, and therefore the new open must be on the same NFS lease as + // the registry lock. + void verifyRegistration() { + KJ_IF_SOME(r, clusterRegistry) { + r.verifyNfsLease(); + } + } + class ActorChannelImpl final: public IoChannelFactory::ActorChannel { public: ActorChannelImpl(kj::Own actorContainer, Persistent persistent) @@ -1755,6 +1993,434 @@ class Server::ActorNamespace final { JSG_FAIL_REQUIRE(Error, "Facets currently cannot set alarms."); } }; + + // Cluster mode: an actor's alarm time goes into the shared `_cf_ALARM` table, wherever the actor + // runs. The alarm leader's AlarmScheduler picks it up (AlarmScheduler::syncFromDb()). + class ClusterAlarmStore { + public: + explicit ClusterAlarmStore(const SqliteDatabase::Vfs& vfs) + : db([&] { + auto db = kj::heap(vfs, kj::Path({"metadata.sqlite"}), + kj::WriteMode::CREATE | kj::WriteMode::MODIFY | kj::WriteMode::CREATE_PARENT); + // Other instances write the same file: wait for their (tiny) transactions. + db->run("PRAGMA busy_timeout = 2000;"); + // Instances starting together race to create the table and switch it to WAL; SQLite + // answers SQLITE_BUSY at once in that case (busy_timeout does not cover it), so retry. + for (uint attempt = 0;; ++attempt) { + try { + AlarmScheduler::ensureInitialized(*db); + break; + } catch (...) { + auto e = kj::getCaughtExceptionAsKj(); + if (attempt >= 100 || strstr(e.getDescription().cStr(), "SQLITE_BUSY") == nullptr) { + kj::throwFatalException(kj::mv(e)); + } + usleep(20 * 1000); + } + } + return kj::mv(db); + }()) {} + + void set(const ActorKey& actor, kj::Date scheduledTime) { + int64_t scheduledTimeNs = (scheduledTime - kj::UNIX_EPOCH) / kj::NANOSECONDS; + SqliteDatabase::Query::ValuePtr nameParam = nullptr; + KJ_IF_SOME(n, actor.name) { + nameParam = n; + } + stmtSet.run(actor.actorId, scheduledTimeNs, nameParam); + } + void remove(const ActorKey& actor) { + stmtDelete.run(actor.actorId); + } + + private: + kj::Own db; + SqliteDatabase::Statement stmtSet = db->prepare(R"( + INSERT INTO _cf_ALARM VALUES(?, ?, ?) + ON CONFLICT DO UPDATE SET scheduled_time = excluded.scheduled_time, + actor_name = COALESCE(excluded.actor_name, _cf_ALARM.actor_name); + )"); + SqliteDatabase::Statement stmtDelete = db->prepare(R"( + DELETE FROM _cf_ALARM WHERE actor_id = ? + )"); + }; + + class ClusterActorSqliteHooks final: public ActorSqlite::Hooks { + public: + // `store` is none across machines (NFS mode), where alarms are not supported. + ClusterActorSqliteHooks(kj::Maybe store, kj::Own actor) + : store(store), + actor(kj::mv(actor)) {} + + kj::Promise scheduleRun( + kj::Maybe newAlarmTime, kj::Promise priorTask) override { + auto& s = JSG_REQUIRE_NONNULL(store, Error, + "Durable Object alarms are not yet supported in cluster mode across machines (NFS); " + "they are supported when all instances share a local disk (network = \"unix\")"); + KJ_IF_SOME(scheduledTime, newAlarmTime) { + s.set(*actor, scheduledTime); + } else { + s.remove(*actor); + } + return kj::READY_NOW; + } + + private: + kj::Maybe store; + kj::Own actor; + }; + + // Cluster mode: try to become the alarm leader every second; once leader, run the + // AlarmScheduler and pick up alarm times the other instances wrote, every 250 ms. A due alarm is + // delivered like any request to the actor (ClusterActorChannel): to the instance that holds it, or + // claimed by this one when no instance does. + kj::Promise alarmLeaderLoop(kj::String uniqueKey) { + auto& as = KJ_ASSERT_NONNULL(actorStorage); + for (;;) { + if (KJ_ASSERT_NONNULL(clusterRegistry).isDraining()) { + // A draining instance can't claim actors any more, so it can't deliver alarms either: hand + // leadership to another instance (dropping the scheduler cancels its pending deliveries; the + // next leader re-reads them from the table). + if (ownAlarmScheduler != kj::none) { + KJ_LOG(INFO, "draining: giving up alarm leadership", uniqueKey); + } + ownAlarmScheduler = kj::none; + alarmLeaderLock = kj::none; + co_return; + } + if (ownAlarmScheduler == kj::none) { + auto fd = KJ_ASSERT_NONNULL(KJ_ASSERT_NONNULL(alarmLeaderFile)->getFd()); + KJ_IF_SOME(lock, OfdLock::tryLock(fd, OfdLock::EXCLUSIVE)) { + alarmLeaderLock = kj::mv(lock); + AlarmScheduler::GetActorFn getActor = + [this, idFactory = kj::heap(uniqueKey)]( + const ActorKey& actor) mutable -> kj::Own { + Worker::Actor::Id id = idFactory->idFromStringNamed( + kj::str(actor.actorId), actor.name.map([](kj::StringPtr n) { return kj::str(n); })); + return getActorChannel(kj::mv(id))->startRequest({}); + }; + try { + auto scheduler = kj::heap( + clock, timer, as.vfs, kj::Path({"metadata.sqlite"}), kj::mv(getActor)); + scheduler->enableClusterMode(); + ownAlarmScheduler = kj::mv(scheduler); + KJ_LOG(INFO, "this instance is now the alarm leader", uniqueKey); + } catch (...) { + // e.g. SQLITE_BUSY while another instance initializes the table: give up the lock and + // try again next round. + KJ_LOG(WARNING, "could not start the alarm scheduler; retrying", + kj::getCaughtExceptionAsKj()); + alarmLeaderLock = kj::none; + } + } + } + KJ_IF_SOME(scheduler, ownAlarmScheduler) { + try { + scheduler->syncFromDb(); + } catch (...) { + KJ_LOG(WARNING, "cluster alarm sync failed; retrying", kj::getCaughtExceptionAsKj()); + } + co_await timer.afterDelay(250 * kj::MILLISECONDS); + } else { + co_await timer.afterDelay(1 * kj::SECONDS); + } + } + } + + // Base class for SubrequestChannels in cluster mode whose requests are routed to whichever node + // currently owns some root actor in this namespace: plain actor stubs (ClusterActorChannel) and + // restored stubs whose vendor chain bottoms out at such an actor + // (ClusterRestoredSubrequestChannel). Resolution of the owning node is deferred until first use, + // so that obtaining or deserializing a stub does not take cluster locks or wake remote actors. + // + // Server::wrapRestoredChannel() recognizes a cluster-routed vendor by this type and copies the + // root actor's identity out of it so that the wrapper can resolve that node for itself. No + // resolution state is shared between channels; each caches its own `target`. + class ClusterRoutedChannel: public IoChannelFactory::SubrequestChannel { + public: + ClusterRoutedChannel(ActorNamespace& ns, Worker::Actor::Id id) + : ns(ns), + id(kj::mv(id)), + key(ns.actorKey(this->id)) {} + ~ClusterRoutedChannel() noexcept(false) { + KJ_IF_SOME(t, target) { + KJ_IF_SOME(container, t.tryGet>()) { + container->updateAccessTime(); + } + } + } + + ActorNamespace& getRootNamespace() { + return ns; + } + const Worker::Actor::Id& getRootId() { + return id; + } + + kj::Own startRequest(IoChannelFactory::SubrequestMetadata metadata) override { + return newPromisedWorkerInterface(startRequestImpl(kj::addRef(*this), kj::mv(metadata))); + } + + protected: + ActorNamespace& ns; + Worker::Actor::Id id; + kj::String key; // actorKey(id), computed once + + // Where requests on this channel go, resolved on first startRequest() and retained for the + // life of the channel. Local: the container, pinned exactly as ActorChannelImpl pins it (and + // therefore the ownership lock). Remote: a WorkerdBootstrap for this channel's target on the + // owning node, obtained by sending the channel's own token to that node's + // WorkerdClusterPort.getChannelFromToken(). Retaining the target gives remote stubs the same + // failure semantics as local ones: a local channel keeps throwing `brokenReason` once its + // container breaks; a retained bootstrap likewise keeps failing once the far end breaks, rather + // than silently reaching a freshly re-instantiated actor on the next call. + using Target = kj::OneOf, rpc::WorkerdBootstrap::Client>; + kj::Maybe target; + + // Resolves `target`, caching the result. Safe to call concurrently; all callers share one + // ns.resolveNode() attempt. A failed resolution is retained too, so every later request on the + // channel fails with the same exception. + kj::Promise resolveTarget() { + if (target != kj::none) return kj::READY_NOW; + KJ_IF_SOME(r, resolving) return r.addBranch(); + return resolving.emplace(resolveTargetImpl().fork()).addBranch(); + } + + // Start a request when `target` resolved to a container on this node. + virtual kj::Promise> startLocalRequest( + ActorContainer& container, IoChannelFactory::SubrequestMetadata metadata) = 0; + + private: + // Set once the first resolution starts. Its branches are only ever held by coroutines that + // also hold a reference to this channel, so the underlying resolution never outlives `this`. + kj::Maybe> resolving; + + static rpc::WorkerdBootstrap::Client bootstrapFromToken( + rpc::WorkerdClusterPort::Client& clusterPort, kj::ArrayPtr token) { + auto req = clusterPort.getChannelFromTokenRequest( + capnp::MessageSize{token.size() / sizeof(capnp::word) + 4, 0}); + req.setToken(token); + return req.send().getBootstrap(); + } + + kj::Promise resolveTargetImpl() { + auto node = co_await ns.resolveNode(key, id); + KJ_SWITCH_ONEOF(node) { + KJ_CASE_ONEOF(container, kj::Own) { + target = kj::mv(container); + } + KJ_CASE_ONEOF(clusterPort, rpc::WorkerdClusterPort::Client) { + // Convert the owner's cluster port into a bootstrap for this channel's target by sending + // it our own token. Nothing else on this channel needs the cluster port, so only the + // (pipelined) bootstrap is retained: if getChannelFromToken() fails, the bootstrap is + // broken and every later startEvent() fails with that exception. A restored token may + // be asynchronous (when its restoreArg holds channels with asynchronous tokens); a + // promise capability covers that case with the same semantics. + KJ_SWITCH_ONEOF(getTokenMaybeSync(IoChannelFactory::ChannelTokenUsage::RPC)) { + KJ_CASE_ONEOF(token, kj::Array) { + target = bootstrapFromToken(clusterPort, token); + } + KJ_CASE_ONEOF(promise, kj::Promise>) { + target = rpc::WorkerdBootstrap::Client( + promise.then([clusterPort = kj::mv(clusterPort)](kj::Array token) mutable { + return bootstrapFromToken(clusterPort, token); + })); + } + } + } + } + } + + static kj::Promise> startRequestImpl( + kj::Own self, IoChannelFactory::SubrequestMetadata metadata) { + co_await self->resolveTarget(); + + KJ_SWITCH_ONEOF(KJ_ASSERT_NONNULL(self->target)) { + KJ_CASE_ONEOF(container, kj::Own) { + co_return co_await self->startLocalRequest(*container, kj::mv(metadata)); + } + KJ_CASE_ONEOF(bootstrap, rpc::WorkerdBootstrap::Client) { + capnp::MessageSize sizeHint{4, 0}; + KJ_IF_SOME(cf, metadata.cfBlobJson) { + sizeHint.wordCount += cf.size() / sizeof(capnp::word) + 1; + } + + // The remote instance reconstructs SubrequestMetadata from these params. + auto req = bootstrap.startEventRequest(sizeHint); + KJ_IF_SOME(cf, metadata.cfBlobJson) { + req.setCfBlobJson(cf); + } + req.setFromPersistentStub(metadata.fromPersistentStub.toBool()); + + auto dispatcher = req.sendForPipeline().getDispatcher(); + // Restore events never cross the cluster -- restored stubs are forwarded as whole + // tokens instead (see ClusterRestoredSubrequestChannel) -- so no FrankenvalueHandler is + // needed to marshal restore params here. + co_return kj::heap(self->ns.httpOverCapnpFactory, + self->ns.byteStreamFactory, getUnsupportedFrankenvalueHandler(), kj::mv(dispatcher)); + } + } + KJ_UNREACHABLE; + } + }; + + // Channel to a root actor in this namespace in cluster mode. + class ClusterActorChannel final: public ClusterRoutedChannel { + public: + ClusterActorChannel(ActorNamespace& ns, Worker::Actor::Id id, Persistent persistent) + : ClusterRoutedChannel(ns, kj::mv(id)), + persistent(persistent) {} + + kj::Own startRequest(IoChannelFactory::SubrequestMetadata metadata) override { + // If this channel was reconstructed from a persistent (stored) stub, signal the target so it + // can re-verify that it still allows persistent stubs. + metadata.fromPersistentStub = metadata.fromPersistentStub || persistent; + return ClusterRoutedChannel::startRequest(kj::mv(metadata)); + } + + kj::Promise evictForTest(IoChannelFactory::EvictWebSocketMode webSocketMode) override { + KJ_IF_SOME(t, target) { + KJ_SWITCH_ONEOF(t) { + KJ_CASE_ONEOF(container, kj::Own) { + return container->evictForTest(webSocketMode).attach(kj::addRef(*this)); + } + KJ_CASE_ONEOF(_, rpc::WorkerdBootstrap::Client) { + JSG_FAIL_REQUIRE(Error, + "Cannot evict Durable Object: it is owned by another workerd instance in the " + "cluster."); + } + } + } + + // Not resolved yet. If the actor is running on this node, evict it without pinning it to + // this channel. + KJ_IF_SOME(container, ns.actors.find(key)) { + return container->evictForTest(webSocketMode).attach(container->addRef()); + } + JSG_FAIL_REQUIRE(Error, "Cannot evict Durable Object: it is not currently running."); + } + + void requireAllowsTransfer() override { + // Cluster mode only routes root actors of durable namespaces, which are always + // transferrable via channel tokens. + } + + kj::OneOf, kj::Promise>> getTokenMaybeSync( + IoChannelFactory::ChannelTokenUsage usage) override { + return ns.encodeActorChannelToken(usage, id, persistent); + } + + private: + Persistent persistent; + + kj::Promise> startLocalRequest( + ActorContainer& container, IoChannelFactory::SubrequestMetadata metadata) override { + return container.startRequest(kj::mv(metadata)).attach(container.addRef()); + } + }; + + // Wrapper around a decoded `restored` SubrequestChannel whose vendor chain bottoms out at a + // cluster-routed actor. Rather than replaying `[restore]()` hop-by-hop across the network (which + // could not carry `restoredSelfTokenFactory`, so a remotely-restored facet could not itself call + // `ctx.restore()`), the whole token is either run locally (this node owns the root actor) or + // shipped to the owning node, which decodes it and runs the entire restore chain in-process. + // + // The wrapped `local` tree already knows how to re-encode itself for the wire, including + // converting STORAGE-usage tokens to RPC-usage at every nesting level. For a multiply-nested + // token each level is wrapped; only the outermost wrapper is ever exercised on the remote path, + // and on the local path each level's resolution finds the container the outer one created or + // found (see ActorNamespace::resolveNode()). + class ClusterRestoredSubrequestChannel final: public ClusterRoutedChannel { + public: + ClusterRestoredSubrequestChannel( + ClusterRoutedChannel& vendor, kj::Own local) + : ClusterRoutedChannel( + vendor.getRootNamespace(), Worker::Actor::cloneId(vendor.getRootId())), + local(kj::mv(local)) {} + + kj::Promise evictForTest(IoChannelFactory::EvictWebSocketMode webSocketMode) override { + return local->evictForTest(webSocketMode); + } + + void requireAllowsTransfer() override { + local->requireAllowsTransfer(); + } + + kj::OneOf, kj::Promise>> getTokenMaybeSync( + IoChannelFactory::ChannelTokenUsage usage) override { + return local->getTokenMaybeSync(usage); + } + + private: + kj::Own local; + + kj::Promise> startLocalRequest( + ActorContainer& container, IoChannelFactory::SubrequestMetadata metadata) override { + // `local` caches its restored channel across requests, as it does outside cluster mode. The + // root ClusterActorChannel inside it resolves to the container we just found or created. + return local->startRequest(kj::mv(metadata)); + } + }; + + // RpcChannel counterpart of ClusterRestoredSubrequestChannel. `restore()` is a fresh-session + // operation by contract, so nothing is retained across calls here: each call resolves the owning + // node anew and either restores locally or opens a session on the owner via + // WorkerdClusterPort.getRpcTargetFromToken(). + class ClusterRestoredRpcChannel final: public IoChannelFactory::RpcChannel { + public: + ClusterRestoredRpcChannel( + ClusterRoutedChannel& vendor, kj::Own local) + : ns(vendor.getRootNamespace()), + id(Worker::Actor::cloneId(vendor.getRootId())), + key(ns.actorKey(id)), + local(kj::mv(local)) {} + + Session restore() override { + auto split = restoreImpl(kj::addRef(*this)).split(); + return { + .cap = kj::mv(kj::get<0>(split)), + .task = kj::mv(kj::get<1>(split)), + }; + } + + void requireAllowsTransfer() override { + local->requireAllowsTransfer(); + } + + kj::OneOf, kj::Promise>> getTokenMaybeSync( + IoChannelFactory::ChannelTokenUsage usage) override { + return local->getTokenMaybeSync(usage); + } + + private: + ActorNamespace& ns; + Worker::Actor::Id id; + kj::String key; + kj::Own local; + + static kj::Promise>> restoreImpl( + kj::Own self) { + auto node = co_await self->ns.resolveNode(self->key, self->id); + KJ_SWITCH_ONEOF(node) { + KJ_CASE_ONEOF(container, kj::Own) { + // The root ClusterActorChannel inside `local` resolves to the container we just found or + // created and pins it for the session. + auto session = self->local->restore(); + co_return kj::tuple(kj::mv(session.cap), session.task.attach(kj::mv(self))); + } + KJ_CASE_ONEOF(clusterPort, rpc::WorkerdClusterPort::Client) { + auto token = co_await self->local->getToken(IoChannelFactory::ChannelTokenUsage::RPC); + auto req = clusterPort.getRpcTargetFromTokenRequest( + capnp::MessageSize{token.size() / sizeof(capnp::word) + 4, 0}); + req.setToken(token); + auto sent = req.send(); + auto session = api::wrapRemoteRpcSession(sent.getTarget(), sent.getSession()); + co_return kj::tuple(kj::mv(session.cap), session.task.attach(kj::mv(self))); + } + } + KJ_UNREACHABLE; + } + }; }; // ======================================================================================= @@ -3432,6 +4098,8 @@ class Server::WorkerService final: public Service, kj::Array> workerLoaders; kj::Maybe workerdDebugPortNetwork; kj::Maybe workerdDebugPortServer; + kj::Maybe clusterRegistry; + kj::Maybe&> clusterRpc; }; using LinkCallback = kj::Function; @@ -3493,8 +4161,9 @@ class Server::WorkerService final: public Service, auto actorClass = kj::refcounted(*this, entry.key, Frankenvalue()); auto ns = kj::heap(kj::mv(actorClass), entry.value, kj::systemPreciseCalendarClock(), threadContext.getUnsafeTimer(), - threadContext.getByteStreamFactory(), channelTokenHandler, network, dockerPath, - containerEgressInterceptorImage, waitUntilTasks, selfTokensArePersistent()); + threadContext.getByteStreamFactory(), threadContext.getHttpOverCapnpFactory(), + channelTokenHandler, network, dockerPath, containerEgressInterceptorImage, waitUntilTasks, + selfTokensArePersistent()); KJ_IF_SOME(d, entry.value.tryGet()) { actorNamespacesByUniqueKey.insert(d.uniqueKey, ns.get()); } @@ -3627,7 +4296,7 @@ class Server::WorkerService final: public Service, auto linked = callback(*this, errorReporter); for (auto& ns: actorNamespaces) { - ns.value->link(linked.actorStorage); + ns.value->link(linked.actorStorage, linked.clusterRegistry, linked.clusterRpc); } ioChannels = kj::mv(linked); @@ -5920,6 +6589,10 @@ kj::Promise> Server::makeWorkerImpl(kj::StringPtr auto linkCallback = [this, def = kj::mv(def), totalActorChannels](WorkerService& workerService, Worker::ValidationErrorReporter& errorReporter) mutable { WorkerService::LinkedIoChannels result; + KJ_IF_SOME(registry, this->clusterRegistry) { + result.clusterRegistry = *registry; + result.clusterRpc = KJ_ASSERT_NONNULL(this->clusterRpc); + } auto entrypointNames = workerService.getEntrypointNames(); auto actorClassNames = workerService.getActorClassNames(); @@ -6097,7 +6770,7 @@ kj::Promise> Server::makeWorkerImpl(kj::StringPtr kj::Maybe serviceName; if (!def.isDynamic) serviceName = name; - auto result = kj::refcounted(channelTokenHandler, serviceName, + auto result = kj::refcounted(KJ_ASSERT_NONNULL(channelTokenHandler), serviceName, globalContext->threadContext, monotonicClock, kj::mv(worker), kj::mv(errorReporter.defaultEntrypoint), kj::mv(errorReporter.namedEntrypoints), kj::mv(errorReporter.actorClasses), kj::mv(linkCallback), @@ -6302,6 +6975,28 @@ kj::Own Server::resolveActor(kj::StringPtr names return ns.getActorChannel(kj::mv(idObj), persistent); } +kj::Own Server::wrapRestoredChannel(ChannelToken::Type type, + IoChannelFactory::SubrequestChannel& vendor, + kj::Own local) { + KJ_IF_SOME(routed, kj::tryDowncast(vendor)) { + switch (type) { + case ChannelToken::Type::SUBREQUEST: + return kj::refcounted( + routed, kj::mv(local).downcast()); + case ChannelToken::Type::RPC: + return kj::refcounted( + routed, kj::mv(local).downcast()); + case ChannelToken::Type::ACTOR_CLASS: + KJ_UNREACHABLE; // decodeChannelTokenImpl() already rejected this + } + KJ_UNREACHABLE; + } + + // The vendor is a static service, or we are not in cluster mode (resolveActor() then returns a + // plain ActorChannelImpl). Nothing to route. + return kj::mv(local); +} + // ======================================================================================= class Server::WorkerdBootstrapImpl final: public rpc::WorkerdBootstrap::Server { @@ -6372,8 +7067,30 @@ class Server::WorkerdBootstrapImpl final: public rpc::WorkerdBootstrap::Server { throwUnsupported(); } + // Cluster mode: the alarm leader delivers a due alarm to the instance that holds the actor + // (AlarmScheduler -> ClusterActorChannel -> here). kj::Promise runAlarm(RunAlarmContext context) override { - throwUnsupported(); + auto params = context.getParams(); + auto scheduledTime = kj::UNIX_EPOCH + params.getScheduledTime() * kj::MILLISECONDS; + auto worker = getWorker(); + auto result = co_await worker->runAlarm(scheduledTime, params.getRetryCount()); + auto resp = context.getResults().initResult(); + resp.setOutcome(result.outcome); + resp.setRetry(result.retry); + resp.setRetryCountsAgainstLimit(result.retryCountsAgainstLimit); + KJ_IF_SOME(d, result.errorDescription) { + resp.setErrorDescription(d); + } + } + + kj::Promise abandonAlarm(AbandonAlarmContext context) override { + auto scheduledTime = + kj::UNIX_EPOCH + context.getParams().getScheduledTimeMs() * kj::MILLISECONDS; + auto worker = getWorker(); + auto stored = co_await worker->abandonAlarm(scheduledTime); + KJ_IF_SOME(t, stored) { + context.getResults().setStoredAlarmTimeMs((t - kj::UNIX_EPOCH) / kj::MILLISECONDS); + } } kj::Promise queue(QueueContext context) override { @@ -6412,9 +7129,12 @@ class Server::WorkerdBootstrapImpl final: public rpc::WorkerdBootstrap::Server { } kj::Own getWorker() { - // For non-HTTP events (RPC, traces, etc.), create WorkerInterface with - // empty metadata since there's no HTTP request to extract cf from. - return getService()->startRequest({}); + // For non-HTTP events (RPC, traces, etc.), there's no HTTP request to extract cf from, so + // cfBlobJson is left unset. fromPersistentStub must still be propagated so that the target + // re-verifies allow_irrevocable_stub_storage. + IoChannelFactory::SubrequestMetadata metadata; + metadata.fromPersistentStub = fromPersistentStub; + return getService()->startRequest(kj::mv(metadata)); } [[noreturn]] void throwUnsupported() { @@ -6791,8 +7511,12 @@ class Server::WorkerdDebugPortImpl final: public rpc::WorkerdDebugPort::Server { auto decoded = kj::decodeHex(actorIdStr); KJ_REQUIRE(decoded.size() == SHA256_DIGEST_LENGTH, "Invalid Durable Object ID: expected 64 hex characters (32 bytes)", decoded.size()); + kj::Maybe name; + if (params.hasActorName()) { + name = params.getActorName().clone(); + } kj::Own id = - kj::heap(decoded.begin(), kj::none); + kj::heap(decoded.begin(), kj::mv(name)); actorId = kj::mv(id); } KJ_CASE_ONEOF(c, Ephemeral) { @@ -6813,6 +7537,48 @@ class Server::WorkerdDebugPortImpl final: public rpc::WorkerdDebugPort::Server { capnp::HttpOverCapnpFactory& httpOverCapnpFactory; }; +// Bootstrap served to cluster peers. Every method takes a channel token, so this grants nothing +// beyond what the caller's tokens already do; see WorkerdClusterPort in worker-interface.capnp. +class Server::WorkerdClusterPortImpl final: public rpc::WorkerdClusterPort::Server { + public: + WorkerdClusterPortImpl( + workerd::server::Server& srv, capnp::HttpOverCapnpFactory& httpOverCapnpFactory) + : srv(srv), + httpOverCapnpFactory(httpOverCapnpFactory) {} + + kj::Promise getChannelFromToken(GetChannelFromTokenContext context) override { + // Decoding on this node yields the same lazy channel tree the sender has. If we own the root + // actor, its ClusterActorChannel resolves locally and any restore chain runs in-process with + // correct self-tokens; if we have just lost ownership, it re-claims or forwards to the new + // owner. Either way this is bounded by the invariant described at ActorContainer::ownershipLock. + auto channel = KJ_ASSERT_NONNULL(srv.channelTokenHandler) + .decodeSubrequestChannelToken(IoChannelFactory::ChannelTokenUsage::RPC, + context.getParams().getToken()); + context.initResults(capnp::MessageSize{4, 1}) + .setBootstrap(kj::heap(kj::mv(channel), httpOverCapnpFactory)); + return kj::READY_NOW; + } + + kj::Promise getRpcTargetFromToken(GetRpcTargetFromTokenContext context) override { + auto channel = KJ_ASSERT_NONNULL(srv.channelTokenHandler) + .decodeRpcChannelToken(IoChannelFactory::ChannelTokenUsage::RPC, + context.getParams().getToken()); + auto session = channel->restore(); + + auto results = context.getResults(capnp::MessageSize{4, 2}); + results.setTarget(kj::mv(session.cap)); + // The returned session capability keeps the session alive; dropping it cancels the task. Same + // shape as RestoreRpcStubCustomEvent::receiveRpc(). + results.setSession(session.task.then( + [channel = kj::mv(channel)]() { return rpc::JsRpcSession::Client(nullptr); })); + return kj::READY_NOW; + } + + private: + workerd::server::Server& srv; + capnp::HttpOverCapnpFactory& httpOverCapnpFactory; +}; + class Server::DebugPortListener { public: DebugPortListener(Server& owner, kj::Own listener) @@ -6857,6 +7623,29 @@ kj::Promise Server::handleDrain(kj::Promise drainWhen) { // doc comment, we instead add the promise to `tasks` to be safe. tasks.add(httpServer.httpServer.drain()); } + + KJ_IF_SOME(registry, clusterRegistry) { + // When draining begins, we stop accepting cluster RPC connections (`clusterRpc`'s accept + // loop is joined with `forkedDrainWhen` -- see `startServices()`). As such, as soon as we + // start draining, all actors hosted here become unreachable from the rest of the cluster. We + // MUST, therefore, abort them, and refuse to claim any more: an in-flight local request that + // reaches an unowned actor during the drain would otherwise make this node its owner (see + // ClusterLockManager::acquireOrRoute()). + // + // TODO(clustering): This is a harsh ending for actors on this machine. We should really evict + // each one at a time that is convenient, and wait some time for RPC requests to drain. + // As of this writing, though, we don't really have any machinery for determining what time + // is "convenient" nor for doing any sort of clean hand-off between instances of an actor. + // The Cloudflare production behavior is widely considered not great: draining instances lose + // access to storage but are allowed to finish in-flight requests. The DO team has plans to + // replace this behavior with something better in the not-so-distant future (maybe), so for + // now we don't try to recreate it here -- we just hard-abort actors when drain starts. Note + // that it is important that we abort at drain start rather than continue accepting cluster + // connections until fully drained since we want actor locks to be released cleanly; a stale + // lock takes more effort to clean up. + registry->startDraining(); + abortAllActors(kj::none); + } } kj::Promise Server::run( @@ -6877,6 +7666,8 @@ kj::Promise Server::run( loggingOptions.structuredLogging = StructuredLogging(config.getStructuredLogging()); } + setupChannelTokenHandler(config); + kj::HttpHeaderTable::Builder headerTableBuilder; globalContext = kj::heap(*this, v8System, headerTableBuilder); invalidConfigServiceSingleton = kj::refcounted(); @@ -6899,6 +7690,15 @@ kj::Promise Server::run( co_return co_await listenPromise.exclusiveJoin(kj::mv(fatalPromise)); } +void Server::setupChannelTokenHandler(config::Config::Reader config) { + ChannelTokenHandler::Resolver& tokenResolver = *this; + kj::Maybe channelTokenKey; + if (config.hasCluster()) { + channelTokenKey = config.getCluster().getKey(); + } + channelTokenHandler.emplace(tokenResolver, channelTokenKey); +} + // Configure and start the inspector socket, returning the port the socket started on. uint startInspector( kj::StringPtr inspectorAddress, Server::InspectorServiceIsolateRegistrar& registrar) { @@ -7024,7 +7824,22 @@ kj::Promise Server::startServices(jsg::V8System& v8System, } goto validDurableObjectStorage; case config::Worker::DurableObjectStorage::IN_MEMORY: + if (config.hasCluster() && hadDurable) { + reportConfigError(kj::str("Worker service \"", name, + "\" uses in-memory Durable Object storage, which is not supported in cluster " + "mode.")); + } + goto validDurableObjectStorage; case config::Worker::DurableObjectStorage::LOCAL_DISK: + if (config.hasCluster() && hadDurable && + workerConf.getDurableObjectStorage().getLocalDisk() != + config.getCluster().getSharedDirectory()) { + reportConfigError( + kj::str("Worker service \"", name, "\" uses Durable Object storage disk service \"", + workerConf.getDurableObjectStorage().getLocalDisk(), + "\", but cluster.sharedDirectory is \"", + config.getCluster().getSharedDirectory(), "\".")); + } goto validDurableObjectStorage; } reportConfigError(kj::str("Encountered unknown durableObjectStorage type in service \"", name, @@ -7087,6 +7902,49 @@ kj::Promise Server::startServices(jsg::V8System& v8System, return decltype(services)::Entry{kj::str("internet"_kj), kj::mv(service)}; }); + if (config.hasCluster()) { + if (!experimental) { + reportConfigError( + "Cluster mode is experimental and subject to change. You must run workerd with " + "`--experimental` to use it."_kj.clone()); + } +#ifndef __linux__ + reportConfigError("Cluster mode is currently implemented only on Linux."_kj.clone()); +#endif + + auto cluster = config.getCluster(); + KJ_IF_SOME(sharedDirService, services.find(cluster.getSharedDirectory())) { + auto diskSvc = dynamic_cast(sharedDirService.get()); + if (diskSvc == nullptr) { + reportConfigError(kj::str("cluster.sharedDirectory refers to service \"", + cluster.getSharedDirectory(), "\", but that service is not a disk directory.")); + } else KJ_IF_SOME(dir, diskSvc->getWritable()) { + auto registryDir = dir.openSubdir( + kj::Path({cluster.getRegistrySubdir()}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + auto registry = + kj::heap(kj::mv(registryDir), cluster.getNetwork(), network, timer); + tasks.add(registry->runMaintenance().exclusiveJoin(forkedDrainWhen.addBranch())); + auto& reg = *clusterRegistry.emplace(kj::mv(registry)); + clusterRpc.emplace(capnp::makeRpcServer(reg, + rpc::WorkerdClusterPort::Client( + kj::heap(*this, globalContext->httpOverCapnpFactory)))); + + // TODO(clustering): We really shouldn't start accepting connections until the server + // is totally up, but the RPC system actually starts accepting immediately on + // construction, even if you don't call run(). + KJ_IF_SOME(rpc, clusterRpc) { + tasks.add(rpc.run().exclusiveJoin(forkedDrainWhen.addBranch())); + } + } else { + reportConfigError(kj::str("cluster.sharedDirectory refers to disk service \"", + cluster.getSharedDirectory(), "\", but that service is defined read-only.")); + } + } else { + reportConfigError(kj::str("cluster.sharedDirectory refers to service \"", + cluster.getSharedDirectory(), "\", but no such service is defined.")); + } + } + // Third pass: Cross-link services. for (auto& service: services) { ConfigErrorReporter errorReporter(*this, service.key); @@ -7304,6 +8162,8 @@ kj::Promise Server::test(jsg::V8System& v8System, loggingOptions.structuredLogging = StructuredLogging(config.getStructuredLogging()); } + setupChannelTokenHandler(config); + kj::HttpHeaderTable::Builder headerTableBuilder; globalContext = kj::heap(*this, v8System, headerTableBuilder); invalidConfigServiceSingleton = kj::refcounted(); diff --git a/src/workerd/server/server.h b/src/workerd/server/server.h index 83364059f2e..ba4982e8919 100644 --- a/src/workerd/server/server.h +++ b/src/workerd/server/server.h @@ -9,8 +9,10 @@ #include #include #include +#include #include +#include #include #include #include @@ -162,7 +164,10 @@ class Server final: private kj::TaskSet::ErrorHandler, private ChannelTokenHandl kj::Own memoryCacheProvider; - ChannelTokenHandler channelTokenHandler; + kj::Maybe channelTokenHandler; + + kj::Maybe> clusterRegistry; + kj::Maybe> clusterRpc; kj::HashMap>> socketOverrides; kj::HashMap directoryOverrides; @@ -290,6 +295,9 @@ class Server final: private kj::TaskSet::ErrorHandler, private ChannelTokenHandl kj::ArrayPtr id, kj::Maybe name, Persistent persistent) override; + kj::Own wrapRestoredChannel(ChannelToken::Type type, + IoChannelFactory::SubrequestChannel& vendor, + kj::Own local) override; kj::Array encodeChannelToken(IoChannelFactory::ChannelTokenUsage usage, kj::StringPtr serviceName, @@ -326,6 +334,7 @@ class Server final: private kj::TaskSet::ErrorHandler, private ChannelTokenHandl class TcpListener; class DebugPortListener; class WorkerdDebugPortImpl; + class WorkerdClusterPortImpl; struct ErrorReporter; struct ConfigErrorReporter; @@ -358,6 +367,8 @@ class Server final: private kj::TaskSet::ErrorHandler, private ChannelTokenHandl void unlinkWorkerLoaders(); + void setupChannelTokenHandler(config::Config::Reader config); + kj::Promise preloadPython( kj::StringPtr workerName, const WorkerDef& workerDef, ErrorReporter& errorReporter); diff --git a/src/workerd/server/tests/cluster/BUILD.bazel b/src/workerd/server/tests/cluster/BUILD.bazel new file mode 100644 index 00000000000..250d88507f0 --- /dev/null +++ b/src/workerd/server/tests/cluster/BUILD.bazel @@ -0,0 +1,45 @@ +load("@aspect_rules_js//js:defs.bzl", "js_test") + +js_test( + name = "cluster-test-unix", + size = "large", + data = [ + ":cluster-test.wd-test", + ":cluster-worker.js", + "//src/workerd/server:workerd", + ], + entry_point = "cluster-test.mjs", + env = { + "WORKERD_BINARY": "$(rootpath //src/workerd/server:workerd)", + "WD_TEST_CONFIG": "$(rootpath :cluster-test.wd-test)", + }, + tags = ["js-test"], + target_compatible_with = select({ + "@platforms//os:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) + +js_test( + name = "cluster-test-cidr", + # The CIDR variant exercises the dead-peer detection path, which has to + # wait out the new-entry grace period before unlinking the dead peer's + # registry file. That makes the takeover test ~15s on its own, so the + # combined run needs more than the default "medium" budget. + size = "large", + data = [ + ":cluster-test-cidr.wd-test", + ":cluster-worker.js", + "//src/workerd/server:workerd", + ], + entry_point = "cluster-test.mjs", + env = { + "WORKERD_BINARY": "$(rootpath //src/workerd/server:workerd)", + "WD_TEST_CONFIG": "$(rootpath :cluster-test-cidr.wd-test)", + }, + tags = ["js-test"], + target_compatible_with = select({ + "@platforms//os:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) diff --git a/src/workerd/server/tests/cluster/cluster-test-cidr.wd-test b/src/workerd/server/tests/cluster/cluster-test-cidr.wd-test new file mode 100644 index 00000000000..837070346ae --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-test-cidr.wd-test @@ -0,0 +1,49 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +# CIDR/IP variant of cluster-test.wd-test. Uses 127.0.0.0/8 (the loopback) so +# the registry-file locking, TCP peer connections, new-entry grace period, and +# dead-peer lock probes are exercised independently of the unix-socket variant. + +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + ( name = "shared", + disk = ( + writable = true, + allowDotfiles = true, + ) + ), + + ( name = "main", + worker = ( + modules = [ + ( name = "cluster-worker.js", esModule = embed "cluster-worker.js" ), + ], + compatibilityDate = "2026-01-01", + compatibilityFlags = ["nodejs_compat", "allow_irrevocable_stub_storage"], + bindings = [ + ( name = "COUNTER", durableObjectNamespace = "Counter" ), + ( name = "NODE_ID", fromEnvironment = "NODE_ID" ), + ], + durableObjectNamespaces = [ + ( className = "Counter", uniqueKey = "counter-test-namespace" ), + ], + durableObjectStorage = ( localDisk = "shared" ), + ) + ), + ], + + sockets = [ + ( name = "http", address = "*:0", http = (), service = "main" ), + ], + + cluster = ( + sharedDirectory = "shared", + registrySubdir = "workerd-registry", + network = "127.0.0.0/8", + key = "test-cluster-shared-secret", + ), +); diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs new file mode 100644 index 00000000000..6f6a21d0933 --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -0,0 +1,1082 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +// +// Integration test for the cluster mode described in scalable-durable-objects.md. +// +// The test spawns several workerd instances pointing at the same shared +// directory, then exercises: +// 1. Consistent DO state across instances (single-writer correctness). +// 2. Transparent forwarding when a request arrives at the "wrong" instance. +// 3. Takeover after a node is killed. +// 4. Unix variant: an alarm runs exactly once, wherever the object lives. +// CIDR (NFS) variant: alarms are rejected with a clear error, and +// 5. no `metadata.sqlite` is ever created (no AlarmScheduler across machines). +// 6. Storing/loading a DO stub without calling it takes no ownership lock. +// 7. A broken DO releases its ownership lock even while stubs still pin it, +// so it can be re-instantiated (on any node) instead of hanging requests. +// 8. Persistent stubs to restored targets (facets, RpcTargets, and chains of +// them) rooted at a DO on one node can be stored and redeemed from a DO on +// another node; the restore chain runs on the root DO's owner. This holds +// whether the stub arrives as a token or was minted in-process and passed +// through props. +// 9. A stub that has resolved to a remote DO keeps failing once that DO +// breaks, while a fresh stub reaches the re-instantiated DO. +// +// Two variants are run: the unix-socket cluster network mode (default) and a +// localhost CIDR (`127.0.0.0/8`) IP-socket mode that exercises the registry +// file locking and TCP peer connections separately. The variant is selected +// by the WD_TEST_CONFIG environment variable supplied by the BUILD target. + +import { spawn } from 'node:child_process'; +import { mkdtemp, readdir, readFile, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { env } from 'node:process'; +import { test } from 'node:test'; +import assert from 'node:assert'; +import { setTimeout as sleep } from 'node:timers/promises'; + +assert( + env.WORKERD_BINARY !== undefined, + 'You must set the WORKERD_BINARY environment variable.' +); +assert( + env.WD_TEST_CONFIG !== undefined, + 'You must set the WD_TEST_CONFIG environment variable.' +); + +const CONTROL_FD = 3; + +// The CIDR variant stands in for a cluster across machines (NFS mode), where alarms stay unsupported. +const NFS_VARIANT = env.WD_TEST_CONFIG.includes('cidr'); + +// A minimal harness specialized for the cluster test. Unlike server-harness.mjs +// it: +// - allows overriding bindings via process environment (NODE_ID), +// - permits multiple instances to share a directory, +// - supports SIGKILL in addition to SIGTERM for the takeover test. +class ClusterNode { + #binary; + #config; + #sharedPath; + #nodeId; + #child = null; + #httpPort = null; + #closed = null; + #stderrBuffer = ''; + + constructor({ binary, config, sharedPath, nodeId }) { + this.#binary = binary; + this.#config = config; + this.#sharedPath = sharedPath; + this.#nodeId = nodeId; + } + + get nodeId() { + return this.#nodeId; + } + + get httpPort() { + assert(this.#httpPort !== null, 'node not started'); + return this.#httpPort; + } + + // Returns the contents of the node's stderr observed so far. Useful for + // verifying that internal error messages (e.g. the cluster alarm rejection) + // were emitted. + get stderr() { + return this.#stderrBuffer; + } + + async start() { + assert.strictEqual(this.#child, null); + + const args = [ + 'serve', + this.#config, + '--experimental', + '--verbose', + `--control-fd=${CONTROL_FD}`, + `--directory-path=shared=${this.#sharedPath}`, + `--socket-addr=http=127.0.0.1:0`, + ]; + + const child = spawn(this.#binary, args, { + stdio: ['ignore', 'inherit', 'pipe', 'pipe'], + env: { ...env, NODE_ID: this.#nodeId }, + }); + this.#child = child; + + child.stderr.on('data', (data) => { + const chunk = data.toString('utf8'); + this.#stderrBuffer += chunk; + // Mirror to our own stderr so test output remains useful. + process.stderr.write(`[${this.#nodeId}] ${chunk}`); + }); + + // Watch the control fd for the assigned http port. + const portPromise = new Promise((resolve, reject) => { + let buffer = ''; + const onData = (data) => { + buffer += data.toString('utf8'); + let nl; + while ((nl = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, nl).trim(); + buffer = buffer.slice(nl + 1); + if (!line) continue; + try { + const parsed = JSON.parse(line); + if (parsed.event === 'listen' && parsed.socket === 'http') { + child.stdio[CONTROL_FD].off('data', onData); + resolve(parsed.port); + return; + } + } catch (err) { + reject( + new Error( + `Failed to parse control message from node ${this.#nodeId}: ${line}` + ) + ); + return; + } + } + }; + child.stdio[CONTROL_FD].on('data', onData); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (this.#httpPort === null) { + reject( + new Error( + `Node ${this.#nodeId} exited before listening (code=${code}, signal=${signal})` + ) + ); + } + }); + }); + + this.#closed = new Promise((resolve) => { + child.once('exit', (code, signal) => resolve({ code, signal })); + }); + + await new Promise((resolve, reject) => { + child.once('spawn', resolve).once('error', reject); + }); + + this.#httpPort = await portPromise; + } + + async stop({ signal = 'SIGTERM', timeoutMs = 10_000 } = {}) { + if (this.#child === null) return null; + const child = this.#child; + this.#child = null; + child.kill(signal); + + const killTimer = setTimeout(() => { + try { + child.kill('SIGKILL'); + } catch (_) { + // process may already be gone. + } + }, timeoutMs); + + const result = await this.#closed; + clearTimeout(killTimer); + this.#httpPort = null; + return result; + } +} + +// Fetches `path` from the given node and parses the JSON response. When +// `timeoutMs` is given, a request that does not complete in time fails with a +// clear error instead of hanging the test (used where a bug would manifest as +// a request that never completes). +async function fetchJson(port, path, { timeoutMs } = {}) { + const url = `http://127.0.0.1:${port}${path}`; + let res; + try { + res = await fetch(url, { + signal: + timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs), + }); + } catch (err) { + if (err.name === 'TimeoutError') { + throw new Error( + `request to ${url} did not complete within ${timeoutMs}ms` + ); + } + throw err; + } + const text = await res.text(); + let body; + try { + body = JSON.parse(text); + } catch (err) { + throw new Error( + `Non-JSON response from ${url} (status ${res.status}): ${text}` + ); + } + return { status: res.status, body }; +} + +// Returns the hex-encoded public key of the node that currently owns the lock +// file at `path` (the node's registry entry name), or null if the file is +// absent or empty. +async function readLockOwner(path) { + let content; + try { + content = await readFile(path); + } catch (err) { + if (err.code === 'ENOENT') return null; + throw err; + } + return content.length === 0 ? null : content.toString('hex'); +} + +// Starts `numNodes` workerd instances sharing one directory and runs `fn` with: +// nodes the ClusterNode instances, in start order +// sharedPath the shared directory +// nodeKeys Map from nodeId to the node's registry key (64-char hex), which +// is also what ownership lock files contain +async function withCluster(numNodes, fn) { + // Each test run gets its own shared directory, so the registry/lock-file + // state doesn't leak between tests. + const sharedPath = await mkdtemp(join(tmpdir(), 'workerd-cluster-')); + const registryDir = join(sharedPath, 'workerd-registry'); + const nodes = []; + const nodeKeys = new Map(); + try { + for (let i = 0; i < numNodes; i++) { + const node = new ClusterNode({ + binary: env.WORKERD_BINARY, + config: env.WD_TEST_CONFIG, + sharedPath, + nodeId: `node${i}`, + }); + await node.start(); + nodes.push(node); + + // Each node writes its registry entry before it reports its listening + // socket, but allow a small grace period for filesystem visibility. Since + // nodes start one at a time, the entry that is new is this node's. + const known = new Set(nodeKeys.values()); + const deadline = Date.now() + 5000; + let fresh = []; + while (Date.now() < deadline) { + let entries = []; + try { + entries = await readdir(registryDir); + } catch (_) { + // Not created yet. + } + fresh = entries.filter((name) => !known.has(name)); + if (fresh.length > 0) break; + await sleep(50); + } + assert.strictEqual( + fresh.length, + 1, + `expected exactly one new registry entry after starting ${node.nodeId}, got: [${fresh.join(', ')}]` + ); + nodeKeys.set(node.nodeId, fresh[0]); + } + await fn({ nodes, sharedPath, nodeKeys }); + } finally { + // Stop all nodes (ignore errors -- some may already be stopped by the + // test itself). + for (const node of nodes) { + try { + await node.stop({ timeoutMs: 5000 }); + } catch (_) { + // ignore + } + } + await rm(sharedPath, { recursive: true, force: true }); + } +} + +// --------------------------------------------------------------------------- +// Tests + +test('cluster: DO state is consistent across instances (single writer)', async () => { + await withCluster(3, async ({ nodes }) => { + // Send a series of /increment requests to a round-robin of nodes, all for + // the same DO name. Track the maximum returned count -- it must be a + // strictly-increasing sequence (1, 2, 3, ...) because only one instance is + // the writer at any given time and the DO state is persistent. + const totalRequests = 12; + const expected = []; + for (let i = 1; i <= totalRequests; i++) expected.push(i); + + const observed = []; + const ownerIds = new Set(); + for (let i = 0; i < totalRequests; i++) { + const node = nodes[i % nodes.length]; + const { status, body } = await fetchJson( + node.httpPort, + '/increment?name=consistency' + ); + assert.strictEqual( + status, + 200, + `request to ${node.nodeId} failed: ${JSON.stringify(body)}` + ); + observed.push(body.count); + ownerIds.add(body.nodeId); + } + observed.sort((a, b) => a - b); + assert.deepStrictEqual( + observed, + expected, + `each /increment must be a unique strictly-increasing count` + ); + + // The DO is owned by exactly one node at a time. Even though we directed + // requests at all 3 nodes, every observed response should report the same + // nodeId -- the actual owner -- because incoming requests are forwarded. + assert.strictEqual( + ownerIds.size, + 1, + `expected exactly one DO owner across ${totalRequests} requests, got: ${[...ownerIds].join(', ')}` + ); + }); +}); + +test('cluster: requests to non-owner are transparently forwarded', async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO by sending an increment to node 0. That node should claim + // ownership and respond. Then send a /get to node 1. Node 1 should forward + // to node 0 (the owner) and return the same state, and node 0 should be + // identified as the responder. + const first = await fetchJson( + nodes[0].httpPort, + '/increment?name=forwarding' + ); + assert.strictEqual(first.status, 200); + assert.strictEqual(first.body.count, 1); + const owner = first.body.nodeId; + + const second = await fetchJson(nodes[1].httpPort, '/get?name=forwarding'); + assert.strictEqual(second.status, 200); + assert.strictEqual( + second.body.count, + 1, + `forwarded /get must see prior increment` + ); + assert.strictEqual( + second.body.nodeId, + owner, + `forwarded request must be served by the original owner (${owner}), not the forwarding node (${nodes[1].nodeId})` + ); + + // Same id must be reported for both calls (sanity check for idFromName + // determinism across instances). + assert.strictEqual(first.body.id, second.body.id); + }); +}); + +test('cluster: killing the owner allows another node to take over', async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO on whichever node ends up owning it. + const initial = await fetchJson( + nodes[0].httpPort, + '/increment?name=takeover' + ); + assert.strictEqual(initial.status, 200); + assert.strictEqual(initial.body.count, 1); + const originalOwner = initial.body.nodeId; + const ownerIndex = nodes.findIndex((n) => n.nodeId === originalOwner); + assert.notStrictEqual(ownerIndex, -1); + const survivorIndex = ownerIndex === 0 ? 1 : 0; + const survivor = nodes[survivorIndex]; + + // Hard-kill the owner. SIGKILL leaves the registry file in place, so the + // survivor has to detect death via a failed RPC + dead-peer cleanup probe. + await nodes[ownerIndex].stop({ signal: 'SIGKILL', timeoutMs: 2000 }); + + // Hit the survivor repeatedly. The first request may fail or take time + // while the survivor's RPC attempt to the dead owner is in flight; once + // the dead-peer cleanup completes, subsequent requests succeed and the + // survivor takes ownership. The persisted count must be preserved. + let response = null; + let lastErr = null; + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + response = await fetchJson( + survivor.httpPort, + '/increment?name=takeover' + ); + if (response.status === 200) break; + } catch (err) { + lastErr = err; + } + await sleep(200); + } + assert( + response !== null && response.status === 200, + `survivor never succeeded; last error: ${lastErr}` + ); + assert.strictEqual( + response.body.nodeId, + survivor.nodeId, + `survivor (${survivor.nodeId}) must own the DO after takeover, got ${response.body.nodeId}` + ); + assert.strictEqual( + response.body.count, + 2, + `persisted state must be preserved across takeover` + ); + + // Further requests to the survivor continue to work and increment. + const followup = await fetchJson( + survivor.httpPort, + '/increment?name=takeover' + ); + assert.strictEqual(followup.status, 200); + assert.strictEqual(followup.body.count, 3); + assert.strictEqual(followup.body.nodeId, survivor.nodeId); + }); +}); + +test('cluster: draining node still routes in-flight requests to a remote owner', async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO so that exactly one node owns it. + const initial = await fetchJson(nodes[0].httpPort, '/increment?name=drain'); + assert.strictEqual(initial.status, 200); + assert.strictEqual(initial.body.count, 1); + const owner = nodes.find((n) => n.nodeId === initial.body.nodeId); + const nonOwner = nodes.find((n) => n.nodeId !== initial.body.nodeId); + assert(owner !== undefined && nonOwner !== undefined); + + // Put a request in flight on the non-owner whose stateless handler waits + // before calling the DO, then SIGTERM the non-owner while it is waiting. + // Draining lets in-flight requests finish, so this request must still be + // able to route to the owner over cluster RPC after draining began. + const requestPromise = fetchJson( + nonOwner.httpPort, + '/increment?name=drain&delay=1000', + { timeoutMs: 10_000 } + ); + await sleep(250); + const exitPromise = nonOwner.stop({ signal: 'SIGTERM', timeoutMs: 10_000 }); + + const response = await requestPromise; + assert.strictEqual( + response.status, + 200, + `in-flight request on draining node failed: ${JSON.stringify(response.body)}` + ); + assert.strictEqual(response.body.nodeId, owner.nodeId); + assert.strictEqual(response.body.count, 2); + + // The draining node must have exited cleanly once the request finished. + const exit = await exitPromise; + assert.deepStrictEqual(exit, { code: 0, signal: null }); + }); +}); + +test('cluster: draining node refuses to claim an unowned DO', async () => { + await withCluster(2, async ({ nodes, sharedPath, nodeKeys }) => { + // Put a request in flight on node 0 for a DO that nobody owns yet, then + // SIGTERM node 0 while its stateless handler is still waiting. By the time + // the handler calls the DO, node 0 is draining: its cluster listener no + // longer accepts connections, so claiming the DO there would leave it + // unreachable from the rest of the cluster until node 0 exits. The claim + // must be refused instead. + const requestPromise = fetchJson( + nodes[0].httpPort, + '/increment?name=lame-duck&delay=1000', + { timeoutMs: 10_000 } + ); + await sleep(250); + const exitPromise = nodes[0].stop({ signal: 'SIGTERM', timeoutMs: 10_000 }); + + const response = await requestPromise; + assert.strictEqual( + response.status, + 500, + `draining node must not claim the DO: ${JSON.stringify(response.body)}` + ); + // The refusal is a DISCONNECTED exception, which JS sees as a generic + // connection-loss error. + assert.match(response.body.error, /Network connection lost/); + + const exit = await exitPromise; + assert.deepStrictEqual(exit, { code: 0, signal: null }); + + // The DO was never instantiated anywhere: the survivor claims it fresh. + const survivor = await fetchJson( + nodes[1].httpPort, + '/increment?name=lame-duck' + ); + assert.strictEqual(survivor.status, 200, JSON.stringify(survivor.body)); + assert.strictEqual(survivor.body.count, 1); + assert.strictEqual(survivor.body.nodeId, nodes[1].nodeId); + assert.strictEqual( + await readLockOwner(lockFilePath(sharedPath, survivor.body.id)), + nodeKeys.get(nodes[1].nodeId) + ); + }); +}); + +test( + 'cluster: alarms are rejected across machines (CIDR/NFS variant)', + { skip: !NFS_VARIANT }, + async () => { + await withCluster(1, async ({ nodes, sharedPath }) => { + // Fire the request. In cluster mode `setAlarm()` does not throw + // synchronously to the JS code -- the rejection happens during output-gate + // flush, after the worker's fetch handler has already returned. The + // observable behaviour is therefore a non-2xx HTTP status, while the + // canonical error message is logged to stderr where the test can detect + // it. (The spec mandates the message "Durable Object alarms are not yet + // supported in cluster mode".) + let response; + try { + response = await fetch( + `http://127.0.0.1:${nodes[0].httpPort}/set-alarm?name=alarm-test` + ); + } catch (err) { + // A network-level failure is acceptable too: it confirms that the alarm + // path did not silently succeed. + response = null; + } + if (response !== null) { + // Drain the body to allow logging to flush. + try { + await response.text(); + } catch (_) { + // ignore + } + assert.notStrictEqual( + Math.floor(response.status / 100), + 2, + `expected non-2xx for set-alarm in cluster mode, got ${response.status}` + ); + } + + // Wait briefly for stderr to flush the error message. + const deadline = Date.now() + 5000; + const expectedMessage = + 'Durable Object alarms are not yet supported in cluster mode'; + while ( + Date.now() < deadline && + !nodes[0].stderr.includes(expectedMessage) + ) { + await sleep(50); + } + assert( + nodes[0].stderr.includes(expectedMessage), + `expected stderr to contain the cluster alarm rejection message ` + + `("${expectedMessage}"); stderr so far:\n${nodes[0].stderr}` + ); + + // No per-namespace metadata.sqlite should exist anywhere under the + // shared directory -- across machines the AlarmScheduler is never + // constructed. + const nsDir = join(sharedPath, 'counter-test-namespace'); + let entries = []; + try { + entries = await readdir(nsDir); + } catch (err) { + // Namespace dir might not exist if the DO was never created on disk + // because the request failed before storage was opened. That is a + // valid (and even stronger) signal that no metadata.sqlite was made. + } + assert( + !entries.some((name) => name.startsWith('metadata.sqlite')), + `cluster mode must not create metadata.sqlite; found: ${entries.join(', ')}` + ); + }); + } +); + +test( + 'cluster: an alarm runs once, delivered across instances (unix variant)', + { skip: NFS_VARIANT }, + async () => { + await withCluster(2, async ({ nodes, sharedPath }) => { + // Set the alarm through node 0 (which then holds the object), read it back + // through node 1 (forwarded to the holder) once it is due. + const set = await fetch( + `http://127.0.0.1:${nodes[0].httpPort}/set-alarm?name=alarm-test&ms=1000` + ); + const setBody = await set.json(); + assert.strictEqual(set.status, 200, JSON.stringify(setBody)); + + const deadline = Date.now() + 15000; + let state; + while (Date.now() < deadline) { + const r = await fetch( + `http://127.0.0.1:${nodes[1].httpPort}/alarm-state?name=alarm-test` + ); + state = await r.json(); + if (state.fired.length > 0) break; + await sleep(200); + } + assert.strictEqual( + state.fired.length, + 1, + `alarm should run once: ${JSON.stringify(state)}` + ); + assert.strictEqual( + state.alarm, + null, + `no alarm left after the run: ${JSON.stringify(state)}` + ); + + // It must not run a second time (a stale table row is not re-run). + await sleep(3000); + const after = await ( + await fetch( + `http://127.0.0.1:${nodes[1].httpPort}/alarm-state?name=alarm-test` + ) + ).json(); + assert.strictEqual( + after.fired.length, + 1, + `alarm ran again: ${JSON.stringify(after)}` + ); + + // The shared alarm table exists (one per namespace, shared by all instances). + const entries = await readdir(join(sharedPath, 'counter-test-namespace')); + assert( + entries.some((name) => name === 'metadata.sqlite'), + `expected metadata.sqlite; found: ${entries.join(', ')}` + ); + }); + } +); + +test('cluster: registry directory is populated for each running instance', async () => { + await withCluster(2, async ({ nodes, sharedPath }) => { + // Give the instances a brief moment to settle. Both should have written + // their registry entries before they reported their listening sockets, + // but allow a small grace period for filesystem visibility. + const registryDir = join(sharedPath, 'workerd-registry'); + let entries = []; + const deadline = Date.now() + 5000; + while (Date.now() < deadline) { + try { + entries = await readdir(registryDir); + } catch (_) { + entries = []; + } + if (entries.length >= nodes.length) break; + await sleep(100); + } + assert.strictEqual( + entries.length, + nodes.length, + `expected ${nodes.length} registry entries, got ${entries.length}: [${entries.join(', ')}]` + ); + // Each entry should be a 64-char hex public key. + for (const name of entries) { + assert.match( + name, + /^[0-9a-f]{64}$/, + `registry entry name should be 64-char hex, got: ${name}` + ); + } + }); +}); + +async function exists(path) { + try { + await stat(path); + return true; + } catch (err) { + if (err.code === 'ENOENT') return false; + throw err; + } +} + +test('cluster: storing and loading a stub does not take the ownership lock', async () => { + await withCluster(2, async ({ nodes, sharedPath }) => { + const nsDir = join(sharedPath, 'counter-test-namespace'); + const locksDir = join(nsDir, 'locks'); + + // DO "holder" (on whichever node claims it) stores a stub to DO "lazy" and + // reads it back. Serializing and deserializing the stub must not claim + // "lazy": no lock file, and no storage for it. + const stored = await fetchJson( + nodes[1].httpPort, + '/store-stub?name=holder&target=lazy' + ); + assert.strictEqual(stored.status, 200, JSON.stringify(stored.body)); + assert.strictEqual(stored.body.hasStub, true); + const holderId = stored.body.id; + const targetId = stored.body.targetId; + assert.notStrictEqual(holderId, targetId); + assert( + await exists(join(locksDir, holderId)), + `the holder DO itself must be locked` + ); + const lockFile = join(locksDir, targetId); + assert( + !(await exists(lockFile)), + `storing/loading a stub must not create the lock file ${lockFile}` + ); + const nsEntries = await readdir(nsDir); + assert( + !nsEntries.some((name) => name.startsWith(targetId)), + `storing/loading a stub must not create actor storage; found: ${nsEntries.join(', ')}` + ); + + // Calling the loaded stub claims ownership and creates the lock file. + const called = await fetchJson( + nodes[1].httpPort, + '/call-stored-stub?name=holder' + ); + assert.strictEqual(called.status, 200, JSON.stringify(called.body)); + assert.strictEqual(called.body.target.id, targetId); + assert.strictEqual(called.body.target.count, 0); + assert( + await exists(lockFile), + `calling the stub must create the lock file ${lockFile}` + ); + }); +}); + +// Requests that would hang forever if a broken actor kept its ownership lock +// are bounded by this timeout so the failure is a clear error. +const HANG_TIMEOUT_MS = 5000; + +function lockFilePath(sharedPath, id) { + return join(sharedPath, 'counter-test-namespace', 'locks', id); +} + +test('cluster: a broken DO pinned by a stub does not keep its ownership lock', async () => { + await withCluster(2, async ({ nodes, sharedPath, nodeKeys }) => { + // Node 0 claims "pinned". DO "holder", served by node 1, then obtains a + // stub to it, uses it, and keeps it for as long as "holder" lives. Through + // node 0's bootstrap for that stub, node 0's container for "pinned" stays + // referenced for the rest of the test. + const first = await fetchJson(nodes[0].httpPort, '/increment?name=pinned'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + assert.strictEqual(first.body.count, 1); + const owner = first.body.nodeId; + const lockFile = lockFilePath(sharedPath, first.body.id); + assert.strictEqual(await readLockOwner(lockFile), nodeKeys.get(owner)); + + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=pinned' + ); + assert.strictEqual(held.status, 200, JSON.stringify(held.body)); + assert.strictEqual(held.body.target.nodeId, owner); + + // Break the DO. Its container leaves the owner's map but lives on, pinned + // by the parked stub. + const broken = await fetchJson(nodes[1].httpPort, '/break?name=pinned'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + // The lock must have been released along with the map entry. Otherwise a + // request from the non-owner routes to the owner, which finds no container + // and routes to itself, forever. + const other = nodes.find((n) => n.nodeId !== owner); + const after = await fetchJson(other.httpPort, '/increment?name=pinned', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual(after.body.count, 2, 'state must survive the break'); + assert.strictEqual( + await readLockOwner(lockFile), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + }); +}); + +test('cluster: a DO whose constructor fails releases its ownership lock', async () => { + await withCluster(2, async ({ nodes, sharedPath, nodeKeys }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=ctor'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const lockFile = lockFilePath(sharedPath, first.body.id); + + // Arm the one-shot constructor failure and abort the running instance so + // the next request has to construct a new one. + const armed = await fetchJson( + nodes[1].httpPort, + '/arm-constructor-failure?name=ctor' + ); + assert.strictEqual(armed.status, 500, JSON.stringify(armed.body)); + + // Re-instantiate through a stub kept by DO "holder". The constructor + // throws, which breaks the new instance; the kept stub pins its hollowed + // container on whichever node claimed the DO. + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=ctor', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(held.status, 500, JSON.stringify(held.body)); + assert.match(held.body.error, /constructor failed on purpose/); + + // A fresh request from the other node must not hang. The failed constructor + // committed the cleared flag before throwing, so this instantiation + // succeeds and sees the earlier increment. + const after = await fetchJson(nodes[0].httpPort, '/increment?name=ctor', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual(after.body.count, 2); + assert.strictEqual( + await readLockOwner(lockFile), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + }); +}); + +test('cluster: aborting a DO with a request in flight lets it be re-instantiated immediately', async () => { + await withCluster(2, async ({ nodes, sharedPath, nodeKeys }) => { + const first = await fetchJson( + nodes[0].httpPort, + '/increment?name=inflight' + ); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const owner = first.body.nodeId; + const lockFile = lockFilePath(sharedPath, first.body.id); + + // Start a request the DO holds open, give it time to reach the DO, then + // abort the DO underneath it. The in-flight request keeps a reference to + // the old Worker::Actor until it unwinds, but the storage must be closed + // and the lock released as soon as the actor breaks, not when that + // reference goes away. + const holding = fetchJson(nodes[0].httpPort, '/hold?name=inflight', { + timeoutMs: HANG_TIMEOUT_MS, + }); + await sleep(250); + const broken = await fetchJson(nodes[0].httpPort, '/break?name=inflight'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + const other = nodes.find((n) => n.nodeId !== owner); + const after = await fetchJson(other.httpPort, '/increment?name=inflight', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual( + after.body.count, + 2, + 'the new instance must see the state committed before the abort' + ); + assert.strictEqual( + await readLockOwner(lockFile), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + + const held = await holding; + assert.strictEqual(held.status, 500, JSON.stringify(held.body)); + }); +}); + +// Sets up the cross-node persistent-stub scenario shared by the tests below: +// DO "root" is claimed by node 0, then DO "holder", served through node 1 (and +// therefore claimed by node 1), asks "root" to vend a persistent stub of the +// given kind and stores it. `liveRestores` is how many times vending itself is +// expected to run root's [restore](). Returns the two DOs' node IDs and root's +// id. +async function vendAcrossNodes(nodes, kind, liveRestores = 1) { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=root'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const rootOwner = first.body.nodeId; + + const vended = await fetchJson( + nodes[1].httpPort, + `/vend-stub?name=holder&kind=${kind}&target=root`, + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(vended.status, 200, JSON.stringify(vended.body)); + assert.strictEqual(vended.body.hasStub, true); + const holderOwner = vended.body.nodeId; + assert.notStrictEqual( + holderOwner, + rootOwner, + 'the test requires root and holder to live on different nodes' + ); + + // Vending ran [restore]() live. Storing and loading the stub must not have + // replayed it. + const state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.status, 200, JSON.stringify(state.body)); + assert.strictEqual(state.body.restores, liveRestores); + + return { rootOwner, holderOwner, rootId: first.body.id }; +} + +test('cluster: a restored facet stub is redeemed on the root DO owner', async () => { + await withCluster(2, async ({ nodes, sharedPath, nodeKeys }) => { + const { rootOwner, rootId } = await vendAcrossNodes(nodes, 'facet'); + + // Redeeming from holder (node 1) forwards the whole token to root's owner, + // which replays [restore]() and runs the facet there. + const used = await fetchJson( + nodes[1].httpPort, + '/use-stub?name=holder&kind=facet', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(used.status, 200, JSON.stringify(used.body)); + assert.strictEqual(used.body.result.count, 1); + assert.strictEqual(used.body.result.nodeId, rootOwner); + let state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.body.restores, 2); + + // Break root so the next redemption has to re-instantiate it (on whichever + // node claims it) and replay [restore]() from scratch. The facet's storage + // lives under root's, so its count carries over. + const broken = await fetchJson(nodes[1].httpPort, '/break?name=root'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + const again = await fetchJson( + nodes[1].httpPort, + '/use-stub?name=holder&kind=facet', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(again.status, 200, JSON.stringify(again.body)); + assert.strictEqual(again.body.result.count, 2); + assert.strictEqual( + await readLockOwner(lockFilePath(sharedPath, rootId)), + nodeKeys.get(again.body.result.nodeId), + 'the facet must run on the node that now owns root' + ); + state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.body.restores, 3); + }); +}); + +test('cluster: a restored RpcTarget stub is redeemed on the root DO owner', async () => { + await withCluster(2, async ({ nodes }) => { + const { rootOwner } = await vendAcrossNodes(nodes, 'rpc'); + + // The RpcTarget mutates root's own storage, so it must run in root's + // context on root's owner. + const used = await fetchJson( + nodes[1].httpPort, + '/use-stub?name=holder&kind=rpc&amount=10', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(used.status, 200, JSON.stringify(used.body)); + assert.deepStrictEqual(used.body.result, { count: 11, nodeId: rootOwner }); + + const state = await fetchJson(nodes[0].httpPort, '/get?name=root'); + assert.strictEqual(state.body.count, 11); + assert.strictEqual(state.body.restores, 2); + }); +}); + +test('cluster: a two-level restore chain is redeemed on the root DO owner', async () => { + await withCluster(2, async ({ nodes }) => { + // root -> facet -> sub-facet. The facet's own [restore]() must run with a + // working ctx.restore(), which only holds when the entire chain is replayed + // in-process on root's owner. Vending runs root's [restore]() twice: once + // live for the facet stub, and once more when holder calls vendSub() on + // that stub, since a stub received over RPC is replayed on first use. + const { rootOwner } = await vendAcrossNodes(nodes, 'chained', 2); + + const used = await fetchJson( + nodes[1].httpPort, + '/use-stub?name=holder&kind=chained', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(used.status, 200, JSON.stringify(used.body)); + assert.strictEqual(used.body.result.count, 1); + assert.strictEqual(used.body.result.nodeId, rootOwner); + + const state = await fetchJson(nodes[0].httpPort, '/get?name=root'); + assert.strictEqual(state.body.restores, 3); + }); +}); + +test('cluster: a stub resolved to a remote DO keeps failing after that DO breaks', async () => { + await withCluster(2, async ({ nodes }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=sticky'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const owner = first.body.nodeId; + + // DO "holder" on node 1 obtains a stub to "sticky", uses it once (resolving + // it to node 0's instance), and keeps it. + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=sticky' + ); + assert.strictEqual(held.status, 200, JSON.stringify(held.body)); + assert.strictEqual(held.body.target.nodeId, owner); + assert.notStrictEqual(held.body.nodeId, owner); + + const broken = await fetchJson(nodes[1].httpPort, '/break?name=sticky'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + // The held stub points at the broken instance and must keep failing, even + // after a fresh stub has re-instantiated the DO. + const heldAfter = await fetchJson( + nodes[1].httpPort, + '/call-held-stub?name=holder&target=sticky', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(heldAfter.status, 500, JSON.stringify(heldAfter.body)); + assert.match(heldAfter.body.error, /broken on purpose/); + + const fresh = await fetchJson(nodes[1].httpPort, '/increment?name=sticky', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(fresh.status, 200, JSON.stringify(fresh.body)); + assert.strictEqual(fresh.body.count, 2); + + const heldAgain = await fetchJson( + nodes[1].httpPort, + '/call-held-stub?name=holder&target=sticky', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(heldAgain.status, 500, JSON.stringify(heldAgain.body)); + assert.match(heldAgain.body.error, /broken on purpose/); + }); +}); + +test('cluster: a freshly minted RpcTarget stub passed via props follows its DO to a new owner', async () => { + await withCluster(2, async ({ nodes }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=root'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const originalOwner = first.body.nodeId; + + // Root mints an RpcTarget stub and hands it to a Consumer entrypoint on the + // same node via props. The Consumer holds the stub's channel only -- the + // very channel object ctx.restore() created, not a decoded token -- and + // waits for the go signal before using it. + const vended = await fetchJson( + nodes[0].httpPort, + '/vend-to-consumer?name=root' + ); + assert.strictEqual(vended.status, 200, JSON.stringify(vended.body)); + + // Move root to the other node: break it, then re-instantiate it from there. + const broken = await fetchJson(nodes[1].httpPort, '/break?name=root'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + const moved = await fetchJson(nodes[1].httpPort, '/increment?name=root', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(moved.status, 200, JSON.stringify(moved.body)); + assert.strictEqual(moved.body.count, 2); + assert.notStrictEqual(moved.body.nodeId, originalOwner); + + // Now let the Consumer use its stub. The restore must be forwarded to the + // new owner as a whole token, not replayed over a remote vendor channel. + const go = await fetchJson(nodes[0].httpPort, '/set-go?name=sink'); + assert.strictEqual(go.status, 200, JSON.stringify(go.body)); + + const deadline = Date.now() + HANG_TIMEOUT_MS; + let recorded; + while (recorded === undefined) { + assert(Date.now() < deadline, 'Consumer never recorded a result'); + await sleep(50); + const sink = await fetchJson(nodes[0].httpPort, '/get?name=sink'); + recorded = sink.body.recorded; + } + assert.deepStrictEqual(recorded, { count: 3, nodeId: moved.body.nodeId }); + + const state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.body.restores, 2); + }); +}); diff --git a/src/workerd/server/tests/cluster/cluster-test.wd-test b/src/workerd/server/tests/cluster/cluster-test.wd-test new file mode 100644 index 00000000000..41bd3dea774 --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-test.wd-test @@ -0,0 +1,60 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +# Config for the cluster integration test. Designed to be reused across multiple +# workerd instances pointing at the same shared directory: +# - The `shared` disk service has no path; the test driver overrides it via +# `--directory-path shared=` so all instances see the same directory. +# - The `http` socket has address `*:0`; the test driver overrides this and +# reads the actual assigned port from the control fd. +# - The cluster `network` field defaults to "unix"; the test driver overrides +# it to "127.0.0.0/8" when testing the CIDR/IP variant by passing a different +# config (see cluster-test-cidr.wd-test). +# - The NODE_ID env binding lets each instance be identified per request, so +# the test can verify which workerd owns/serves a given DO. + +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + # The shared filesystem used by the whole cluster. Path is supplied via + # `--directory-path shared=` on the command line so all instances + # share the same directory. + ( name = "shared", + disk = ( + writable = true, + allowDotfiles = true, + ) + ), + + ( name = "main", + worker = ( + modules = [ + ( name = "cluster-worker.js", esModule = embed "cluster-worker.js" ), + ], + compatibilityDate = "2026-01-01", + compatibilityFlags = ["nodejs_compat", "allow_irrevocable_stub_storage"], + bindings = [ + ( name = "COUNTER", durableObjectNamespace = "Counter" ), + ( name = "NODE_ID", fromEnvironment = "NODE_ID" ), + ], + durableObjectNamespaces = [ + ( className = "Counter", uniqueKey = "counter-test-namespace" ), + ], + durableObjectStorage = ( localDisk = "shared" ), + ) + ), + ], + + sockets = [ + ( name = "http", address = "*:0", http = (), service = "main" ), + ], + + cluster = ( + sharedDirectory = "shared", + registrySubdir = "workerd-registry", + network = "unix", + key = "test-cluster-shared-secret", + ), +); diff --git a/src/workerd/server/tests/cluster/cluster-worker.js b/src/workerd/server/tests/cluster/cluster-worker.js new file mode 100644 index 00000000000..184f62883fa --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-worker.js @@ -0,0 +1,383 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Test worker for the cluster integration test. Exposes a Counter Durable Object +// and a top-level fetch() that routes requests to the DO by name. Requests that +// fail are reported as JSON `{ error }` with status 500 rather than letting the +// exception propagate, so the test can assert on the message. +// +// The DO supports the following request paths: +// /increment Increments and returns the current counter value. +// /get Returns the current counter value without incrementing it, +// along with `restores`, the number of times this DO's +// [restore]() method has run. +// /set-alarm Tries to schedule a DO alarm. In cluster mode this should +// fail with a "not yet supported" error. +// /identity Returns information identifying this instance (NODE_ID env) +// along with the DO id, so the test can verify which instance +// served the request. +// /store-stub Stores a stub to the DO named by the `target` query param in +// this DO's storage, then reads it back without calling it. +// Returns the target's id. +// /call-stored-stub +// Loads the stub stored by /store-stub and calls /get on it. +// Returns the target's response. +// /hold-stub Obtains a stub to the DO named by the `target` query param, +// calls /identity on it, and keeps the stub on this instance +// for as long as this instance lives. (A stub is owned by the +// request context that created it, so a module-level variable +// would not keep it alive past the request; an instance field +// of a long-lived DO does.) Returns the target's response. +// /call-held-stub Calls /identity on the stub kept by /hold-stub for the DO +// named by the `target` query param. Returns the target's +// response. +// /hold Waits one second before responding, so that the test can +// break the DO while this request is still in flight. +// /break Aborts the DO via ctx.abort(). The request fails. +// /arm-constructor-failure +// Sets a storage flag that makes the *next* instantiation of +// this DO throw from its constructor (the flag is cleared and +// committed before throwing, so the instantiation after that +// succeeds), then aborts the DO so that the next request +// re-instantiates it. +// +// Persistent-stub paths. Each `kind` below names a restorable object vended by +// the DO given by the `target` query param via its [restore]() method: +// facet a CounterFacet facet of the target, named by the `facet` param +// rpc a CounterAdder RpcTarget that mutates the target's counter +// chained a CounterFacet facet's own [restore]() result: a sub-facet, so the +// stored token has two restore layers +// /vend-stub?kind=K&target=T[&facet=F] +// Asks the target (over RPC) to vend a persistent stub of the +// given kind, stores it in this DO's storage under `kind`, and +// reads it back without calling it. +// /use-stub?kind=K[&amount=N] +// Loads the stub stored by /vend-stub and calls it: facets +// respond to increment(), the adder to add(amount). Returns +// the result. +// /vend-to-consumer +// Vends an `rpc` stub of *this* DO and hands it, via props, +// to a Consumer entrypoint in the same process, which keeps +// only the stub's channel (props carry no live capability). +// The Consumer waits until DO "sink" has its `go` flag set, +// then calls the stub and records the result in "sink". +// /set-go Sets this DO's `go` flag (read by Consumer via /get). +// /record?value=V Stores V (JSON) as this DO's `recorded` value. +// +// The top-level fetch() additionally honors a `delay` query parameter (ms): it +// waits that long *before* calling the DO. This lets a test put a request in +// flight on a node and SIGTERM that node while the request has not yet routed +// to the DO's owner. + +import { + DurableObject, + RpcTarget, + WorkerEntrypoint, + restore, +} from 'cloudflare:workers'; + +export default { + async fetch(request, env) { + const url = new URL(request.url); + + // The DO name to act on is given via the `name` query parameter (defaults + // to "default"). All instances using the same name will route to the same + // DO id. + const name = url.searchParams.get('name') ?? 'default'; + + // Synthesize a stub URL for the DO. + const id = env.COUNTER.idFromName(name); + const stub = env.COUNTER.get(id); + + const delay = url.searchParams.get('delay'); + if (delay !== null) { + await scheduler.wait(Number(delay)); + } + + // Forward to the DO with the same path and query. + const forwardUrl = new URL(url.pathname + url.search, 'http://do/'); + try { + return await stub.fetch(forwardUrl.toString(), { + method: request.method, + headers: request.headers, + }); + } catch (err) { + return Response.json({ error: String(err) }, { status: 500 }); + } + }, +}; + +// RpcTarget vended by Counter's [restore]() that mutates the counter's storage. +// It runs in the Counter's context on whichever node owns the Counter. +class CounterAdder extends RpcTarget { + constructor(counter) { + super(); + this.counter = counter; + } + + async add(amount) { + const count = + ((await this.counter.state.storage.get('count')) ?? 0) + amount; + await this.counter.state.storage.put('count', count); + return { count, nodeId: this.counter.env.NODE_ID ?? '' }; + } +} + +// See /vend-to-consumer. The stub in `props` was minted by ctx.restore() in this +// process and arrives here without its live capability, so the first call on it +// restores through the channel object the minting DO created, not through a +// decoded token. +export class Consumer extends WorkerEntrypoint { + useLater() { + const adder = this.ctx.props.adder; + const sink = this.env.COUNTER.get(this.env.COUNTER.idFromName('sink')); + this.ctx.waitUntil( + (async () => { + for (;;) { + const state = await (await sink.fetch('http://do/get')).json(); + if (state.go) break; + await scheduler.wait(50); + } + let result; + try { + result = await adder.add(1); + } catch (err) { + result = { error: String(err) }; + } + await sink.fetch( + 'http://do/record?value=' + encodeURIComponent(JSON.stringify(result)) + ); + })() + ); + } +} + +// Facet class used by Counter's [restore](). +export class CounterFacet extends DurableObject { + async increment() { + const count = ((await this.ctx.storage.get('count')) ?? 0) + 1; + await this.ctx.storage.put('count', count); + return { count, nodeId: this.env.NODE_ID ?? '' }; + } + + // Vend a persistent stub to a sub-facet. Only works when this facet was itself + // reached through a restored stub, which is what makes the two-level token. + vendSub() { + return this.ctx.restore({ sub: true }); + } + + [restore](params) { + if (params.sub) { + return this.ctx.facets.get('sub', () => ({ + class: this.ctx.exports.CounterFacet, + })); + } + throw new Error('unexpected facet restore params'); + } +} + +const BREAK_ON_CONSTRUCT = 'break-on-construct'; + +export class Counter extends DurableObject { + constructor(state, env) { + super(state, env); + this.state = state; + this.env = env; + // See /hold-stub. Maps target DO name -> stub. + this.heldStubs = new Map(); + + state.blockConcurrencyWhile(async () => { + if (await state.storage.get(BREAK_ON_CONSTRUCT)) { + // One-shot: make sure the cleared flag is durable before breaking, so + // the next instantiation succeeds. + await state.storage.delete(BREAK_ON_CONSTRUCT); + await state.storage.sync(); + throw new Error('constructor failed on purpose'); + } + }); + } + + async fetch(request) { + const url = new URL(request.url); + const nodeId = this.env.NODE_ID ?? ''; + const idHex = this.state.id.toString(); + + if (url.pathname === '/increment') { + let count = (await this.state.storage.get('count')) ?? 0; + count += 1; + await this.state.storage.put('count', count); + return Response.json({ + count, + nodeId, + id: idHex, + }); + } else if (url.pathname === '/get') { + const count = (await this.state.storage.get('count')) ?? 0; + const restores = (await this.state.storage.get('restores')) ?? 0; + const go = (await this.state.storage.get('go')) ?? false; + const recorded = await this.state.storage.get('recorded'); + return Response.json({ + count, + restores, + go, + recorded, + nodeId, + id: idHex, + }); + } else if (url.pathname === '/set-alarm') { + // Schedule an alarm `ms` milliseconds from now (default 60s). Across + // machines (the CIDR/NFS variant) this should throw a clear error. + try { + const ms = Number(url.searchParams.get('ms') ?? 60_000); + await this.state.storage.setAlarm(Date.now() + ms); + return Response.json({ ok: true, nodeId, id: idHex }); + } catch (err) { + return Response.json( + { ok: false, error: String(err), nodeId, id: idHex }, + { status: 500 } + ); + } + } else if (url.pathname === '/alarm-state') { + return Response.json({ + nodeId, + id: idHex, + alarm: await this.state.storage.getAlarm(), + fired: (await this.state.storage.get('alarm-fired-on')) ?? [], + }); + } else if (url.pathname === '/identity') { + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/store-stub') { + // Only stubs from the ctx.exports self-binding are storable. + const target = url.searchParams.get('target'); + const ns = this.state.exports.Counter; + const targetId = ns.idFromName(target); + await this.state.storage.put('stub', ns.get(targetId)); + // Read it back so that the stub is also deserialized, but do not call it. + const stub = await this.state.storage.get('stub'); + return Response.json({ + nodeId, + id: idHex, + targetId: targetId.toString(), + hasStub: typeof stub?.fetch === 'function', + }); + } else if (url.pathname === '/call-stored-stub') { + const stub = await this.state.storage.get('stub'); + const response = await stub.fetch('http://do/get'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); + } else if (url.pathname === '/hold-stub') { + const target = url.searchParams.get('target'); + const stub = this.env.COUNTER.get(this.env.COUNTER.idFromName(target)); + this.heldStubs.set(target, stub); + const response = await stub.fetch('http://do/identity'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); + } else if (url.pathname === '/call-held-stub') { + const target = url.searchParams.get('target'); + const stub = this.heldStubs.get(target); + if (stub === undefined) { + return Response.json({ error: 'no held stub' }, { status: 400 }); + } + const response = await stub.fetch('http://do/identity'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); + } else if (url.pathname === '/vend-stub') { + const kind = url.searchParams.get('kind'); + const target = url.searchParams.get('target'); + const facet = url.searchParams.get('facet') ?? 'default'; + const ns = this.state.exports.Counter; + const targetStub = ns.get(ns.idFromName(target)); + let stub; + if (kind === 'chained') { + // The facet's own ctx.restore() only works when the facet is reached + // through a restored stub, which vendFacet() returns. + const facetStub = await targetStub.vendFacet(facet); + stub = await facetStub.vendSub(); + } else { + stub = await targetStub.vend(kind); + } + await this.state.storage.put(kind, stub); + // Read it back so that the stub is also deserialized, but do not call it. + const loaded = await this.state.storage.get(kind); + return Response.json({ + nodeId, + id: idHex, + hasStub: loaded !== undefined, + }); + } else if (url.pathname === '/use-stub') { + const kind = url.searchParams.get('kind'); + const stub = await this.state.storage.get(kind); + const result = + kind === 'rpc' + ? await stub.add(Number(url.searchParams.get('amount') ?? '1')) + : await stub.increment(); + return Response.json({ nodeId, id: idHex, result }); + } else if (url.pathname === '/vend-to-consumer') { + const adder = await this.state.restore({ kind: 'rpc' }); + const consumer = this.state.exports.Consumer({ props: { adder } }); + await consumer.useLater(); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/set-go') { + await this.state.storage.put('go', true); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/record') { + await this.state.storage.put( + 'recorded', + JSON.parse(url.searchParams.get('value')) + ); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/hold') { + await scheduler.wait(1000); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/break') { + this.state.abort('broken on purpose'); + } else if (url.pathname === '/arm-constructor-failure') { + await this.state.storage.put(BREAK_ON_CONSTRUCT, true); + await this.state.storage.sync(); + this.state.abort('armed constructor failure'); + } + + return new Response('Not found', { status: 404 }); + } + + async alarm() { + // Record which instance ran it (the unix variant checks it ran exactly once). + const fired = (await this.state.storage.get('alarm-fired-on')) ?? []; + fired.push(this.env.NODE_ID ?? null); + await this.state.storage.put('alarm-fired-on', fired); + await this.state.storage.put('alarm-fired', true); + } + + // RPC methods used by /vend-stub on the *target* DO. + vend(kind) { + return this.state.restore({ kind }); + } + + vendFacet(facet) { + return this.state.restore({ kind: 'facet', facet }); + } + + async [restore](params) { + const restores = ((await this.state.storage.get('restores')) ?? 0) + 1; + await this.state.storage.put('restores', restores); + + if (params.kind === 'facet') { + return this.state.facets.get(params.facet, () => ({ + class: this.state.exports.CounterFacet, + })); + } else if (params.kind === 'rpc') { + return new CounterAdder(this); + } + throw new Error(`unexpected restore params: ${JSON.stringify(params)}`); + } +} diff --git a/src/workerd/server/workerd.capnp b/src/workerd/server/workerd.capnp index 310b60c3165..aa48df8c34d 100644 --- a/src/workerd/server/workerd.capnp +++ b/src/workerd/server/workerd.capnp @@ -94,6 +94,9 @@ struct Config { logging @6 : LoggingOptions; # Console and Stdio logging configuration options. + + cluster @7 :ClusterConfig; + # Set when the workerd instance should join a cluster. } struct LoggingOptions { @@ -111,6 +114,72 @@ struct LoggingOptions { # Set a custom prefix for process.stderr. Defaults to "stderr: ". } +struct ClusterConfig { + # workerd instances can work together in a cluster, in order to distribute work and Durable + # Objects among them. + # + # All instances must share a single filesystem where Durable Object data is stored. The + # filesystem can be NFSv4 or any filesystem that supports similar locking semantics with proper + # leases. It can also just be a local filesystem, when running multiple workerd instances on the + # same machine to utilize multiple cores. + # + # All instances in the cluster MUST have identical configuration. The idea is that all + # instances behave exactly the same and any instance could forward any request it receives to + # any other instance and it should produce the same result. For the purpose of rolling upgrades, + # instances may temporarily differ in ways that are backwards-compatible. + # + # Cluster mode is currently implemented only on Linux. + + sharedDirectory @0 :Text; + # Name of a `DiskDirectory` service which represents storage shared by all instances in the + # cluster. If instances are on different machines (not just different processes on the same + # machine), this should point to a network filesystem that implements proper locking and leasing, + # like NFSv4. + # + # All `durableObjectStorage.localDisk` configurations in this config file must point at this same + # directory. (Each Durable Object namespace will create a subdirectory.) + + registrySubdir @1 :Text = "workerd-registry"; + # Subdirectory of `sharedDirectory` to put the actual registry files into. (This can be customized + # in case the default name conflicts with one of the Durable Object namespace `uniqueKey`s, which + # determine the directory names under which they are stored.) + # + # The registry is used to discover what workerd instances are available. Each workerd instance + # registers itself in the registry such that other instances can find it and send traffic to it. + # + # Specifically, each instance will create a file in this directory named after the instance's + # public key, which is randomly-generated when the instance starts up. The file contains the + # instance's network address, so that other instances can connect to it. + + network @3 :Text; + # Each workerd instance in the cluster listens for connections from other instances over the + # network. The `network` field indicates which network to listen on. + # + # This can be either: + # * An IPv4 or IPv6 CIDR, which must match one of the machine's IP addresses. + # * The word "unix", to mean that the cluster is actually local to the machine and should + # communicate via Unix-domain sockets. + # + # In the CIDR case, workerd enumerates all IP addresses of all network interfaces available to + # it, selects the first one that matches the CIDR, and opens an ephemeral port on that address. + # Note that this means the port workerd uses is not predictable. That's OK because workerd will + # then write a file to the registry containing its own address. + # + # In the "unix" case, the registry will be populated not by files, but by Unix-domain sockets. + + key @2 :Text; + # Some secret value known only to members of the cluster. This is used to encrypt channel tokens, + # which are used when passing service stubs or Durable Object stubs over RPC. + # + # The secrecy of this key matters if you allow untrusted clients outside of the cluster to speak + # the low-level worker-interface.capnp protocol directly with workerd instances in the cluster. + # At present, this is only exposed via the `capnpConnectHost` and `workerdDebugPort` config + # settings. If you don't use either of those in production, then the secrecy of this key isn't + # very important. + # + # TODO(someday): Support some way to gradually roll out a rotated key. +} + # ======================================================================================== # Sockets @@ -729,8 +798,6 @@ struct Worker { # This mode is intended for local testing purposes. localDisk @12 :Text; - # ** EXPERIMENTAL; SUBJECT TO BACKWARDS-INCOMPATIBLE CHANGE ** - # # Durable Object data will be stored in a directory on local disk. This field is the name of # a service, which must be a DiskDirectory service. For each Durable Object class, a # subdirectory will be created using `uniqueKey` as the name. Within the directory, one or @@ -738,11 +805,14 @@ struct Worker { # a number of different extensions depending on the storage mode. (Currently, the main storage # is a file with the extension `.sqlite`, and in certain situations extra files with the # extensions `.sqlite-wal`, and `.sqlite-shm` may also be present.) + # + # Note that if workerd has been given a `ClusterConfig` at the top level, then it is expected + # this disk directory is shared by the whole cluster (e.g. via NFSv4), and workerd will + # automatically ensure that each Durable Object is scheduled on a single machine. When using + # NFSv4, all Durable Object directories MUST be on the SAME VOLUME as the registry directory + # defined in `ClusterConfig`, so that lease loss affects all files and locks simultaneously. } - # TODO(someday): Support distributing objects across a cluster. At present, objects are always - # local to one instance of the runtime. - moduleFallback @13 :Text; tails @14 :List(ServiceDesignator); @@ -923,6 +993,12 @@ struct DiskDirectory { # is no acceptable format for these, regardless of what the client says it accepts). # # `HEAD` requests are properly optimized to perform a stat() without actually opening the file. + # + # If workerd has been given a `ClusterConfig` at the top level, it is important that any + # `DiskDirectory` is set up in such a way that a request that uses it can be served correctly + # from any instance in the cluster. For read-only directories, this just means they all have to + # contain the same data. For writable directories, you may want to use a shared filesystem like + # NFSv4. path @0 :Text; # The filesystem path of the directory. If not specified, then it must be specified on the diff --git a/src/workerd/util/BUILD.bazel b/src/workerd/util/BUILD.bazel index 8d8589ce5ea..67332fedeea 100644 --- a/src/workerd/util/BUILD.bazel +++ b/src/workerd/util/BUILD.bazel @@ -510,3 +510,16 @@ kj_test( src = "state-machine-test.c++", deps = [":state-machine"], ) + +wd_cc_library( + name = "ofd-lock", + srcs = ["ofd-lock.c++"], + hdrs = ["ofd-lock.h"], + visibility = ["//visibility:public"], + deps = ["@capnp-cpp//src/kj"], +) + +kj_test( + src = "ofd-lock-test.c++", + deps = [":ofd-lock"], +) diff --git a/src/workerd/util/ofd-lock-test.c++ b/src/workerd/util/ofd-lock-test.c++ new file mode 100644 index 00000000000..1f120a7574e --- /dev/null +++ b/src/workerd/util/ofd-lock-test.c++ @@ -0,0 +1,182 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "ofd-lock.h" + +#include +#include +#include + +#if !_WIN32 +#include +#include +#include +#endif + +namespace workerd { +namespace { + +#ifdef __linux__ + +kj::OwnFd openTempFile() { + char path[] = "/tmp/ofd-lock-test-XXXXXX"; + int fd = mkstemp(path); + KJ_SYSCALL(fd); + unlink(path); + return kj::OwnFd(fd); +} + +KJ_TEST("exclusive lock conflicts with exclusive lock") { + auto fd1 = openTempFile(); + + // Open a second fd to the same file via /proc/self/fd. + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + + // Second exclusive lock on a different fd to the same file should fail. + KJ_EXPECT(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE) == kj::none); +} + +KJ_TEST("shared lock conflicts with exclusive lock") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + + // Shared lock should also fail when exclusive is held. + KJ_EXPECT(OfdLock::tryLock(fd2, OfdLock::SHARED) == kj::none); +} + +KJ_TEST("shared locks do not conflict with each other") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::SHARED)); + auto lock2 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd2, OfdLock::SHARED)); + // Both shared locks held simultaneously — no conflict. +} + +KJ_TEST("exclusive lock fails while shared lock is held") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::SHARED)); + + // Exclusive lock should fail when shared lock is held by a different fd. + KJ_EXPECT(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE) == kj::none); +} + +KJ_TEST("releasing lock allows re-acquisition") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + { + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + // lock1 goes out of scope here, releasing the lock. + } + + // Now fd2 should be able to acquire an exclusive lock. + auto lock2 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE)); +} + +KJ_TEST("move constructor transfers lock") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + + // Move the lock. + auto lock2 = kj::mv(lock1); + + // The lock should still be held (fd2 can't get it). + KJ_EXPECT(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE) == kj::none); +} + +KJ_TEST("move constructor releases on destruction of target") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + { + auto lock2 = kj::mv(lock1); + // lock2 goes out of scope, releasing the lock. + } + + // fd2 should now succeed. + auto lock3 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE)); +} + +KJ_TEST("move assignment transfers lock") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + + // Create a second lock on fd2... wait, we can't because fd1 already holds exclusive. + // Instead, test move assignment by creating a dummy that gets replaced. + auto fd3 = openTempFile(); + auto lock3 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd3, OfdLock::EXCLUSIVE)); + + lock3 = kj::mv(lock1); // Releases lock on fd3, takes lock from fd1. + + // fd2 still can't get the lock (lock on fd1 was transferred to lock3). + KJ_EXPECT(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE) == kj::none); +} + +KJ_TEST("same fd can re-lock (lock is per open file description)") { + auto fd1 = openTempFile(); + + // On the same fd, acquiring the same lock type again is a no-op re-lock, not a conflict. + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + auto lock2 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + // Both succeed because OFD locks are per open file description, and this is the same fd. +} + +KJ_TEST("verifyHeld succeeds when lock is held") { + auto fd1 = openTempFile(); + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + // Should not throw. + lock1.verifyHeld(); +} + +#else // #if __linux__ + +KJ_TEST("dummy test: platform not supported") {} + +#endif // #if __linux__, #else + +} // namespace +} // namespace workerd diff --git a/src/workerd/util/ofd-lock.c++ b/src/workerd/util/ofd-lock.c++ new file mode 100644 index 00000000000..96ad38aee9b --- /dev/null +++ b/src/workerd/util/ofd-lock.c++ @@ -0,0 +1,111 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "ofd-lock.h" + +#include + +#ifdef __linux__ +#include +#include +#endif + +namespace workerd { + +#ifdef __linux__ + +namespace { + +struct flock makeFlock(short type) { + struct flock fl = {}; + fl.l_type = type; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; // Lock the entire file. + return fl; +} + +} // namespace + +kj::Maybe OfdLock::tryLock(int fd, Type type) { + auto fl = makeFlock(type == SHARED ? F_RDLCK : F_WRLCK); + if (fcntl(fd, F_OFD_SETLK, &fl) == -1) { + int err = errno; + if (err == EAGAIN || err == EACCES) { + // Lock is held by another open file description. + return kj::none; + } + KJ_FAIL_SYSCALL("fcntl(F_OFD_SETLK)", err); + } + return OfdLock(fd, type); +} + +void OfdLock::verifyHeld() { + KJ_REQUIRE(fd >= 0, "OfdLock has been moved away"); + // Re-lock with the same type. If the underlying NFSv4 lock stateid has been invalidated + // (lease lost), the Linux NFS client surfaces this as EIO. If the stateid is still valid, + // the kernel treats the re-lock as a no-op. + auto fl = makeFlock(type == SHARED ? F_RDLCK : F_WRLCK); + KJ_SYSCALL(fcntl(fd, F_OFD_SETLK, &fl), "NFS lease may have been lost"); +} + +OfdLock::OfdLock(int fd, Type type): fd(fd), type(type) {} + +OfdLock::OfdLock(OfdLock&& other): fd(other.fd), type(other.type) { + other.fd = -1; +} + +OfdLock& OfdLock::operator=(OfdLock&& other) { + if (this != &other) { + // Release existing lock, if any. + if (fd >= 0) { + auto fl = makeFlock(F_UNLCK); + fcntl(fd, F_OFD_SETLK, &fl); + } + fd = other.fd; + type = other.type; + other.fd = -1; + } + return *this; +} + +OfdLock::~OfdLock() { + if (fd >= 0) { + auto fl = makeFlock(F_UNLCK); + fcntl(fd, F_OFD_SETLK, &fl); + } +} + +#else // !__linux__ + +kj::Maybe OfdLock::tryLock(int fd, Type type) { + KJ_FAIL_REQUIRE("OFD locks are only supported on Linux. " + "Cluster mode is currently implemented only on Linux."); +} + +void OfdLock::verifyHeld() { + KJ_FAIL_REQUIRE("OFD locks are only supported on Linux. " + "Cluster mode is currently implemented only on Linux."); +} + +OfdLock::OfdLock(int fd, Type type): fd(fd), type(type) {} + +OfdLock::OfdLock(OfdLock&& other): fd(other.fd), type(other.type) { + other.fd = -1; +} + +OfdLock& OfdLock::operator=(OfdLock&& other) { + if (this != &other) { + fd = other.fd; + type = other.type; + other.fd = -1; + } + return *this; +} + +OfdLock::~OfdLock() {} + +#endif // __linux__ + +} // namespace workerd diff --git a/src/workerd/util/ofd-lock.h b/src/workerd/util/ofd-lock.h new file mode 100644 index 00000000000..aebe84b4c8f --- /dev/null +++ b/src/workerd/util/ofd-lock.h @@ -0,0 +1,48 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once + +#include +#include + +namespace workerd { + +class OfdLock { + // RAII holder for a lock on an open file description. Releasing this object releases the lock. + // + // Uses Linux OFD (Open File Description) locks via fcntl(F_OFD_SETLK). OFD locks are per-fd + // rather than per-process, which is essential since a single workerd process may manage many + // Durable Objects concurrently. + public: + enum Type { SHARED, EXCLUSIVE }; + + // Try to acquire a lock (non-blocking). Returns kj::none if the lock is held in a conflicting + // mode by another open file description. + // + // On non-Linux platforms, fails at runtime with KJ_FAIL_REQUIRE rather than a compile error, + // so non-Linux builds still compile when cluster mode is unused. + static kj::Maybe tryLock(int fd, Type type); + + // Verify that a lock that this open file description was previously granted is still held. + // Specifically, this is meant to check for NFSv4 lease loss. Implemented as a no-op + // F_OFD_SETLK of the same lock type on the same fd: if the underlying NFSv4 lock stateid + // has been invalidated (lease lost), the Linux NFS client surfaces this as EIO and this + // function throws; if the stateid is still valid, the kernel treats the re-lock as a + // no-op and we return cleanly. + void verifyHeld(); + + OfdLock(OfdLock&& other); + OfdLock& operator=(OfdLock&& other); + KJ_DISALLOW_COPY(OfdLock); + + ~OfdLock(); // releases via F_OFD_SETLK with F_UNLCK + + private: + int fd; // -1 after move + Type type; + explicit OfdLock(int fd, Type type); +}; + +} // namespace workerd diff --git a/src/workerd/util/sqlite-kv-test.c++ b/src/workerd/util/sqlite-kv-test.c++ index 0d60038868a..9e29a5aa07a 100644 --- a/src/workerd/util/sqlite-kv-test.c++ +++ b/src/workerd/util/sqlite-kv-test.c++ @@ -338,5 +338,25 @@ KJ_TEST("SQLite-KV multi-put with allowUnconfirmed") { KJ_EXPECT(called); } +KJ_TEST("SQLite-KV reads throw after close()") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + SqliteKv kv(db); + + kv.put("foo", "abc"_kj.asBytes()); + kv.putExternals("foo", kj::arr(kj::heapArray("tok"_kj.asBytes()))); + + db.close(); + + // Unlike reset(), close() must not make the data appear deleted: the file still has it. + KJ_EXPECT_THROW_MESSAGE("database has been closed", + kv.get("foo", [](kj::ArrayPtr) { KJ_FAIL_EXPECT("should not be called"); })); + KJ_EXPECT_THROW_MESSAGE( + "database has been closed", kv.list(nullptr, kj::none, kj::none, SqliteKv::FORWARD)->next()); + KJ_EXPECT_THROW_MESSAGE("database has been closed", kv.getExternals("foo")); + KJ_EXPECT_THROW_MESSAGE("database has been closed", kv.put("bar", "def"_kj.asBytes())); +} + } // namespace } // namespace workerd diff --git a/src/workerd/util/sqlite-kv.c++ b/src/workerd/util/sqlite-kv.c++ index 7644878de9d..f7b631edb18 100644 --- a/src/workerd/util/sqlite-kv.c++ +++ b/src/workerd/util/sqlite-kv.c++ @@ -258,10 +258,16 @@ void SqliteKv::putExternals(kj::StringPtr key, kj::Array> tokens } } -void SqliteKv::beforeSqliteReset() { - // We'll need to recreate the tables on the next operation. - tableCreated = false; - externalsTableCreated = false; +void SqliteKv::beforeSqliteClose(SqliteDatabase::CloseReason reason) { + if (reason == SqliteDatabase::CloseReason::RESET) { + // The tables are deleted along with the database and will need to be recreated on the next + // operation. + tableCreated = false; + externalsTableCreated = false; + } + // On close(), leave the flags alone: the tables still exist in the file. Clearing them would + // make get()/list()/getExternals() report an empty database without consulting the (closed) + // connection; with the flags intact those reads go through a Statement and throw instead. } void SqliteKv::rollbackMultiPut(Initialized& stmts, WriteOptions options) { diff --git a/src/workerd/util/sqlite-kv.h b/src/workerd/util/sqlite-kv.h index 5292c990c77..bfaf3db63f6 100644 --- a/src/workerd/util/sqlite-kv.h +++ b/src/workerd/util/sqlite-kv.h @@ -222,7 +222,7 @@ class SqliteKv: private SqliteDatabase::ResetListener { // with `allowUnconfirmed = true` since the paired KV write decides confirmation semantics. void clearExternalsIfPresent(KeyPtr key); - void beforeSqliteReset() override; + void beforeSqliteClose(SqliteDatabase::CloseReason reason) override; // Helper function that rolls back a multi-put statement and swallows any exceptions that may // occur during the rollback. diff --git a/src/workerd/util/sqlite-metadata-test.c++ b/src/workerd/util/sqlite-metadata-test.c++ index b153a44e781..f899f8c545f 100644 --- a/src/workerd/util/sqlite-metadata-test.c++ +++ b/src/workerd/util/sqlite-metadata-test.c++ @@ -65,5 +65,23 @@ KJ_TEST("SQLite-METADATA") { KJ_EXPECT(metadata.getAlarm() == anAlarmTime2); } +KJ_TEST("SQLite-METADATA reads throw after close()") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + SqliteMetadata metadata(db); + + constexpr kj::Date anAlarmTime = kj::UNIX_EPOCH + 1734099316 * kj::SECONDS; + metadata.setAlarm(anAlarmTime, /*allowUnconfirmed=*/false); + KJ_EXPECT(metadata.getAlarm() == anAlarmTime); + + db.close(); + + // Unlike reset(), close() must not make the alarm appear unset: the file still has it. + KJ_EXPECT_THROW_MESSAGE("database has been closed", metadata.getAlarm()); + KJ_EXPECT_THROW_MESSAGE( + "database has been closed", metadata.setAlarm(kj::none, /*allowUnconfirmed=*/false)); +} + } // namespace } // namespace workerd diff --git a/src/workerd/util/sqlite-metadata.c++ b/src/workerd/util/sqlite-metadata.c++ index 6acdbdb2528..7cc4d5625a0 100644 --- a/src/workerd/util/sqlite-metadata.c++ +++ b/src/workerd/util/sqlite-metadata.c++ @@ -100,10 +100,18 @@ SqliteMetadata::Initialized& SqliteMetadata::ensureInitialized(bool allowUnconfi KJ_UNREACHABLE; } -void SqliteMetadata::beforeSqliteReset() { - // We'll need to recreate the table on the next operation. - tableCreated = false; +void SqliteMetadata::beforeSqliteClose(SqliteDatabase::CloseReason reason) { + // Drop the alarm cache in both cases. After a reset() the alarm is gone; after a close() the + // next read must reach the closed connection and throw rather than report a stale value. cacheState = kj::none; + + if (reason == SqliteDatabase::CloseReason::RESET) { + // The table is deleted along with the database and will need to be recreated on the next + // operation. Leave `tableCreated` alone on close(): the table still exists in the file, and + // clearing the flag would make getAlarmUncached() report "no alarm" without consulting the + // (closed) database. + tableCreated = false; + } } } // namespace workerd diff --git a/src/workerd/util/sqlite-metadata.h b/src/workerd/util/sqlite-metadata.h index c365166a213..9df50dbca6d 100644 --- a/src/workerd/util/sqlite-metadata.h +++ b/src/workerd/util/sqlite-metadata.h @@ -75,7 +75,7 @@ class SqliteMetadata final: private SqliteDatabase::ResetListener { // first write. // ResetListener interface: - void beforeSqliteReset() override; + void beforeSqliteClose(SqliteDatabase::CloseReason reason) override; }; } // namespace workerd diff --git a/src/workerd/util/sqlite.c++ b/src/workerd/util/sqlite.c++ index 9a05922dc03..940673d8175 100644 --- a/src/workerd/util/sqlite.c++ +++ b/src/workerd/util/sqlite.c++ @@ -674,7 +674,11 @@ SqliteDatabase::~SqliteDatabase() noexcept(false) { } SqliteDatabase::operator sqlite3*() { - return &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + KJ_IF_SOME(db, maybeDb) { + return &db; + } + KJ_REQUIRE(!closed, "database has been closed"); + KJ_FAIL_ASSERT("previous reset() failed"); } SqliteMemoryScope SqliteDatabase::enterMemoryScope() { @@ -700,7 +704,7 @@ void SqliteDatabase::handleCriticalError(kj::Maybe errorCode, if (code == SQLITE_FULL || code == SQLITE_IOERR || code == SQLITE_NOMEM || code == SQLITE_INTERRUPT) { - sqlite3* db = &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + sqlite3* db = *this; // We are in a transaction if (inTransaction || !savepoints.empty()) { // The transaction was auto-rolledback, re-enabling the auto commit mode, so we should fail @@ -815,7 +819,7 @@ SqliteDatabase::StatementAndEffect SqliteDatabase::prepareSql(StaticRegulator re uint prepFlags, Multi multi, kj::Maybe&> prelude) { - sqlite3* db = &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + sqlite3* db = *this; ParseContext parseContext; KJ_ASSERT(currentParseContext == kj::none, "recursive prepareSql()?"); @@ -1011,6 +1015,7 @@ void SqliteDatabase::executeWithRegulator( void SqliteDatabase::reset() { KJ_REQUIRE(!readOnly, "can't reset() read-only database"); + KJ_REQUIRE(!closed, "can't reset() a database that has been closed"); // If transactions are open during reset(), whatever had the transaction open is going to get // confused at best, or lose data at worst. Let's just not allow this. @@ -1024,7 +1029,7 @@ void SqliteDatabase::reset() { KJ_IF_SOME(db, maybeDb) { for (auto& listener: resetListeners) { - listener.beforeSqliteReset(); + listener.beforeSqliteClose(CloseReason::RESET); } auto err = sqlite3_close(&db); @@ -1043,6 +1048,44 @@ void SqliteDatabase::reset() { } } +void SqliteDatabase::close() noexcept { + if (closed) return; + + // If we're currently executing a SQLite statement, there's no safe way for us to close the DB. + // But, callers of close() rely on the fact that the DB will in fact be closed, and they often + // sit on error-handling paths that cannot tolerate further exceptions. These asserts, if they + // fail, will abort the process, due to close() being noexcept. As of this writing it is believed + // that no real code path is likely to hit these, so these are just safety checks / internal + // invariants. + KJ_ASSERT(currentStatement == kj::none, "close() called while a query is executing"); + KJ_ASSERT(currentRegulator == kj::none, "close() called while a query is compiling"); + + auto memoryScope = enterMemoryScope(); + + KJ_IF_SOME(db, maybeDb) { + for (auto& listener: resetListeners) { + listener.beforeSqliteClose(CloseReason::CLOSE); + } + + // Every prepared statement was finalized by the listener walk above, so SQLITE_BUSY here + // indicates a statement that isn't tracked by a ResetListener. Do not fall back to + // sqlite3_close_v2(): a lazily-closed handle keeps the file open, which is exactly what this + // method exists to prevent. + auto err = sqlite3_close(&db); + KJ_ASSERT( + err == SQLITE_OK, "close() found dependent objects that still exist", sqlite3_errstr(err)); + + maybeDb = kj::none; + } + + // sqlite3_close() rolled back any open transaction. Nothing may read the database again, so the + // rollback callbacks (which exist to keep in-memory caches in sync with it) are simply dropped. + inTransaction = false; + savepoints.clear(); + rollbackCallbacks.clear(); + closed = true; +} + bool SqliteDatabase::isAuthorized(int actionCode, kj::Maybe param1, kj::Maybe param2, @@ -1490,10 +1533,22 @@ SqliteDatabase::StatementAndEffect& SqliteDatabase::Statement::prepareForExecuti return KJ_ASSERT_NONNULL(stmt.tryGet()); } -void SqliteDatabase::Statement::beforeSqliteReset() { +void SqliteDatabase::Statement::beforeSqliteClose(CloseReason reason) { + // Regardless of `reason`, the prepared statement must be finalized (by destroying the + // StatementAndEffect) so that the connection can be closed. KJ_IF_SOME(prepared, stmt.tryGet()) { - // Pull the original SQL code out of the statement and store it. - stmt = kj::str(sqlite3_sql(prepared.statement)); + switch (reason) { + case CloseReason::RESET: + // Keep the SQL text so that the statement can be reprepared against the new database. + stmt = kj::str(sqlite3_sql(prepared.statement)); + break; + case CloseReason::CLOSE: + // The statement can never be reprepared, so don't bother copying the SQL. An empty string + // (no allocation) suffices: prepareForExecution() will attempt to prepare it and fail + // with "database has been closed" before ever looking at the text. + stmt = kj::String(); + break; + } } } @@ -1794,15 +1849,20 @@ bool SqliteDatabase::Query::isNull(uint column) { SqliteDatabase::StatementAndEffect& SqliteDatabase::Query::getStatementAndEffect() { return KJ_UNWRAP_OR(maybeStatement, { - regulator->onError(kj::none, "SQLite query was canceled because the database was deleted."); - KJ_FAIL_REQUIRE("query canceled because reset() was called on the database"); + if (db.closed) { + regulator->onError(kj::none, "SQLite query was canceled because the database was closed."); + KJ_FAIL_REQUIRE("query canceled because the database has been closed"); + } else { + regulator->onError(kj::none, "SQLite query was canceled because the database was deleted."); + KJ_FAIL_REQUIRE("query canceled because reset() was called on the database"); + } }); } -void SqliteDatabase::Query::beforeSqliteReset() { +void SqliteDatabase::Query::beforeSqliteClose(CloseReason reason) { // Note that if we don't own the statement, then `maybeStatement` is probably already dangling // here. Luckily, we don't need to reset it or anything because the statement will be destroyed - // by Statement::beforeSqliteReset(). + // by Statement::beforeSqliteClose(). maybeStatement = kj::none; ownStatement = {}; } @@ -2020,7 +2080,7 @@ sqlite3_vfs SqliteDatabase::Vfs::makeWrappedNativeVfs() { file->pMethods = nullptr; // Set up currentVfsRoot. - auto& self = *reinterpret_cast(vfs->pAppData); + auto& self = *reinterpret_cast(vfs->pAppData); KJ_ASSERT(currentVfsRoot == AT_FDCWD); currentVfsRoot = self.rootFd; KJ_DEFER(currentVfsRoot = AT_FDCWD); @@ -2032,6 +2092,16 @@ sqlite3_vfs SqliteDatabase::Vfs::makeWrappedNativeVfs() { if (file->pMethods == nullptr) { wrapper->pMethods = nullptr; } else { + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + try { + afterOpen(); + } catch (kj::Exception& e) { + reportVfsErrorCaught(kj::mv(e)); + file->pMethods->xClose(file); + wrapper->pMethods = nullptr; + return SQLITE_CANTOPEN; + } + } wrapper->pMethods = &WrappedNativeFileImpl::METHOD_TABLE; wrapper->vfs = &self; wrapper->rootFd = self.rootFd; @@ -2336,7 +2406,7 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { .pAppData = this, #define WRAP_METHOD(errorCode, block) \ - auto& self KJ_UNUSED = *static_cast(vfs->pAppData); \ + auto& self KJ_UNUSED = *static_cast(vfs->pAppData); \ try block catch (kj::Exception& e) { \ KJ_LOG(ERROR, "SQLite VFS I/O error", e); \ return errorCode; \ @@ -2353,6 +2423,11 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { auto path = kj::Path::parse(zName); auto kjFile = KJ_UNWRAP_OR(self.directory.tryOpenFile(path), { return SQLITE_CANTOPEN; }); + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + afterOpen(); + } else { + // squelch spurious "dangling else" warning from clang + } kj::Maybe> lock; if (flags & SQLITE_OPEN_MAIN_DB) { lock = self.lockManager.lock(path, *kjFile); @@ -2368,6 +2443,11 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { KJ_ASSERT(flags & SQLITE_OPEN_DELETEONCLOSE); KJ_ASSERT(!(flags & SQLITE_OPEN_MAIN_DB), "main DB can't be a temporary file"); kjFile = self.directory.createTemporary(); + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + afterOpen(); + } else { + // squelch spurious "dangling else" warning from clang + } } else { kj::WriteMode mode; if (flags & SQLITE_OPEN_CREATE) { @@ -2383,6 +2463,11 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { auto path = kj::Path::parse(zName); kjFile = KJ_UNWRAP_OR(self.directory.tryOpenFile(path, mode), { return SQLITE_CANTOPEN; }); + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + afterOpen(); + } else { + // squelch spurious "dangling else" warning from clang + } if (flags & SQLITE_OPEN_MAIN_DB) { lock = self.lockManager.lock(path, *kjFile); } diff --git a/src/workerd/util/sqlite.h b/src/workerd/util/sqlite.h index b007a54f160..cfdbc4f25a6 100644 --- a/src/workerd/util/sqlite.h +++ b/src/workerd/util/sqlite.h @@ -289,7 +289,31 @@ class SqliteDatabase { // may throw if they depend on tables that haven't been recreated yet). void reset(); - // Objects that need to be notified when reset() is called may inherit `ResetListener`. + // Closes the underlying SQLite connection without reopening it. Every subsequent attempt to + // use the database, including through existing Statements and Queries, throws. Any open + // transaction is rolled back (never committed). Idempotent. + // + // This exists so that an actor's storage can be severed at the moment the actor is shut down, + // even if references to the SqliteDatabase live on: closing the last connection checkpoints and + // deletes the WAL and releases any file locks, which the next opener of the same file needs. + // + // This is declared noexcept because it is expected to be used in shutdown paths which are not + // prepared to catch further errors and strongly depend on the database actually becoming closed + // (so that they can safely release file locks). If close() is unable to complete (e.g. because + // a sqlite statement is currently being executed) then we have no choice but to abort the + // process. + void close() noexcept; + + // Why a connection is being closed. Passed to `ResetListener::beforeSqliteClose()`. + enum class CloseReason { + // reset(): the database file is deleted and a fresh connection opened. + RESET, + // close(): the connection is gone for good. + CLOSE, + }; + + // Objects that need to be notified when reset() or close() is called may inherit + // `ResetListener`. class ResetListener { public: ResetListener(SqliteDatabase& db): db(db) { @@ -303,9 +327,16 @@ class SqliteDatabase { db.resetListeners.add(*this); } - // When the database's `reset()` method is called, all listeners' `beforeSqliteReset()` will be - // called before actually resetting the database. - virtual void beforeSqliteReset() = 0; + // When the database's `reset()` or `close()` method is called, all listeners' + // `beforeSqliteClose()` will be called before actually closing the connection. Any prepared + // statements must be finalized here so that `sqlite3_close()` can succeed. + // + // The `reason` matters for listeners that cache facts about the database's contents: after + // RESET the database is empty and will be reopened, so such caches must be cleared; after + // CLOSE nothing may read the database again, so such caches should be left alone so that + // attempted reads reach the closed connection and throw rather than reporting an empty + // database. + virtual void beforeSqliteClose(CloseReason reason) = 0; protected: // so that subclasses don't have to store their own copy of the `db` reference SqliteDatabase& db; @@ -339,6 +370,11 @@ class SqliteDatabase { // When a rollback occurs, callbacks are invoked in the reverse of the order in which they were // registered. The database content is rolled back first, before invoking any callbacks. // Callbacks may read from the database, but must not write to it. + // + // NOTE: close() technically rolls back all outstanding transactions, but does NOT call rollback + // callbacks. Instead, callers who need to handle the close() case should register a + // ResetListener. In practices, in-memory caches are generally moot after close() since the + // actor has shut down anyway. void onRollback(kj::Function callback) { if (inTransaction || !savepoints.empty()) { rollbackCallbacks.add(kj::mv(callback)); @@ -366,9 +402,12 @@ class SqliteDatabase { kj::Maybe actorAccountLimits; - // This pointer can be left null if a call to reset() failed to re-open the database. + // Null after a call to reset() failed to re-open the database, or after close(). kj::Maybe maybeDb; + // Set by close(). Distinguishes the closed state from a failed reset() in error messages. + bool closed = false; + // Set while a query is compiling. kj::Maybe currentRegulator; @@ -531,7 +570,7 @@ class SqliteDatabase::Statement final: private ResetListener { regulator(regulator), stmt(kj::mv(sqlCode)) {} - void beforeSqliteReset() override; + void beforeSqliteClose(CloseReason reason) override; // Get the underlying StatementAndEffect, which the caller will then execute. If `prelude` is // non-empty, prepareForExecution() actually executes the prelude. @@ -793,7 +832,7 @@ class SqliteDatabase::Query final: private ResetListener { return getStatementAndEffect().statement; } - void beforeSqliteReset() override; + void beforeSqliteClose(CloseReason reason) override; void nextRow(bool first); }; @@ -817,6 +856,11 @@ struct SqliteDatabase::VfsOptions { // will fall back to the native VFS implementation. In that case, the options you set here will // be ORed with the ones set by the underlying VFS. int deviceCharacteristics = 0x00001000; // = SQLITE_FCNTL_POWERSAFE_OVERWRITE + + // Called immediately after SQLite opens any file through this VFS. This is used by clustered + // Durable Object storage to verify that the original NFS lease is still valid and therefore + // covers the newly-opened file. + kj::Maybe> afterOpen; }; // Implements a SQLite VFS based on a KJ directory.