From 0526592a718e907b57d3e6c27c2ad34229d7c39d Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Thu, 21 May 2026 14:22:16 -0500 Subject: [PATCH 01/33] Update config capnp and worker-interface.capnp for clustering. This defines how the new feature is configured and exposed on the network. --- src/workerd/io/worker-interface.capnp | 12 ++++ src/workerd/server/workerd.capnp | 86 +++++++++++++++++++++++++-- 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/src/workerd/io/worker-interface.capnp b/src/workerd/io/worker-interface.capnp index 93e52542e9d..597ad67e1de 100644 --- a/src/workerd/io/worker-interface.capnp +++ b/src/workerd/io/worker-interface.capnp @@ -994,6 +994,18 @@ interface WorkerdDebugPort { # purposes. # # This interface is subject to change. It is intended for use by miniflare. + # + # This is also the same interface that workerd exports on its cluster port, which other workerd + # instances in the same cluster may connect to. See ClusterConfig in workerd.capnp for how to + # configure clustering. The cluster port is only intended to be used by other instances in the + # cluster, and does not implement a regular two-party RPC protocol. `ClusterRegistry` implements + # the `VatNetwork` intended to be used for this. Nobody else should try to connect to it. + # + # TODO(clustering): Once channel tokens have been extended to support actors, consider changing + # the cluster interface with one that takes a channel token. This seems a bit cleaner and safer + # than letting the client specify props and such directly. + # TODO(clustering): Automatically use encryption on non-localhost networks. We have an X25519 + # keypair for each node, no need for certs! getEntrypoint @0 (service :Text, entrypoint :Text, props :Frankenvalue) -> (entrypoint :WorkerdBootstrap); diff --git a/src/workerd/server/workerd.capnp b/src/workerd/server/workerd.capnp index 310b60c3165..aa48df8c34d 100644 --- a/src/workerd/server/workerd.capnp +++ b/src/workerd/server/workerd.capnp @@ -94,6 +94,9 @@ struct Config { logging @6 : LoggingOptions; # Console and Stdio logging configuration options. + + cluster @7 :ClusterConfig; + # Set when the workerd instance should join a cluster. } struct LoggingOptions { @@ -111,6 +114,72 @@ struct LoggingOptions { # Set a custom prefix for process.stderr. Defaults to "stderr: ". } +struct ClusterConfig { + # workerd instances can work together in a cluster, in order to distribute work and Durable + # Objects among them. + # + # All instances must share a single filesystem where Durable Object data is stored. The + # filesystem can be NFSv4 or any filesystem that supports similar locking semantics with proper + # leases. It can also just be a local filesystem, when running multiple workerd instances on the + # same machine to utilize multiple cores. + # + # All instances in the cluster MUST have identical configuration. The idea is that all + # instances behave exactly the same and any instance could forward any request it receives to + # any other instance and it should produce the same result. For the purpose of rolling upgrades, + # instances may temporarily differ in ways that are backwards-compatible. + # + # Cluster mode is currently implemented only on Linux. + + sharedDirectory @0 :Text; + # Name of a `DiskDirectory` service which represents storage shared by all instances in the + # cluster. If instances are on different machines (not just different processes on the same + # machine), this should point to a network filesystem that implements proper locking and leasing, + # like NFSv4. + # + # All `durableObjectStorage.localDisk` configurations in this config file must point at this same + # directory. (Each Durable Object namespace will create a subdirectory.) + + registrySubdir @1 :Text = "workerd-registry"; + # Subdirectory of `sharedDirectory` to put the actual registry files into. (This can be customized + # in case the default name conflicts with one of the Durable Object namespace `uniqueKey`s, which + # determine the directory names under which they are stored.) + # + # The registry is used to discover what workerd instances are available. Each workerd instance + # registers itself in the registry such that other instances can find it and send traffic to it. + # + # Specifically, each instance will create a file in this directory named after the instance's + # public key, which is randomly-generated when the instance starts up. The file contains the + # instance's network address, so that other instances can connect to it. + + network @3 :Text; + # Each workerd instance in the cluster listens for connections from other instances over the + # network. The `network` field indicates which network to listen on. + # + # This can be either: + # * An IPv4 or IPv6 CIDR, which must match one of the machine's IP addresses. + # * The word "unix", to mean that the cluster is actually local to the machine and should + # communicate via Unix-domain sockets. + # + # In the CIDR case, workerd enumerates all IP addresses of all network interfaces available to + # it, selects the first one that matches the CIDR, and opens an ephemeral port on that address. + # Note that this means the port workerd uses is not predictable. That's OK because workerd will + # then write a file to the registry containing its own address. + # + # In the "unix" case, the registry will be populated not by files, but by Unix-domain sockets. + + key @2 :Text; + # Some secret value known only to members of the cluster. This is used to encrypt channel tokens, + # which are used when passing service stubs or Durable Object stubs over RPC. + # + # The secrecy of this key matters if you allow untrusted clients outside of the cluster to speak + # the low-level worker-interface.capnp protocol directly with workerd instances in the cluster. + # At present, this is only exposed via the `capnpConnectHost` and `workerdDebugPort` config + # settings. If you don't use either of those in production, then the secrecy of this key isn't + # very important. + # + # TODO(someday): Support some way to gradually roll out a rotated key. +} + # ======================================================================================== # Sockets @@ -729,8 +798,6 @@ struct Worker { # This mode is intended for local testing purposes. localDisk @12 :Text; - # ** EXPERIMENTAL; SUBJECT TO BACKWARDS-INCOMPATIBLE CHANGE ** - # # Durable Object data will be stored in a directory on local disk. This field is the name of # a service, which must be a DiskDirectory service. For each Durable Object class, a # subdirectory will be created using `uniqueKey` as the name. Within the directory, one or @@ -738,11 +805,14 @@ struct Worker { # a number of different extensions depending on the storage mode. (Currently, the main storage # is a file with the extension `.sqlite`, and in certain situations extra files with the # extensions `.sqlite-wal`, and `.sqlite-shm` may also be present.) + # + # Note that if workerd has been given a `ClusterConfig` at the top level, then it is expected + # this disk directory is shared by the whole cluster (e.g. via NFSv4), and workerd will + # automatically ensure that each Durable Object is scheduled on a single machine. When using + # NFSv4, all Durable Object directories MUST be on the SAME VOLUME as the registry directory + # defined in `ClusterConfig`, so that lease loss affects all files and locks simultaneously. } - # TODO(someday): Support distributing objects across a cluster. At present, objects are always - # local to one instance of the runtime. - moduleFallback @13 :Text; tails @14 :List(ServiceDesignator); @@ -923,6 +993,12 @@ struct DiskDirectory { # is no acceptable format for these, regardless of what the client says it accepts). # # `HEAD` requests are properly optimized to perform a stat() without actually opening the file. + # + # If workerd has been given a `ClusterConfig` at the top level, it is important that any + # `DiskDirectory` is set up in such a way that a request that uses it can be served correctly + # from any instance in the cluster. For read-only directories, this just means they all have to + # contain the same data. For writable directories, you may want to use a shared filesystem like + # NFSv4. path @0 :Text; # The filesystem path of the directory. If not specified, then it must be specified on the From af5273660c826db4daec16c781c793f805eee51a Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Thu, 21 May 2026 15:05:02 -0500 Subject: [PATCH 02/33] Add OFD file lock primitive RAII wrapper around Linux OFD locks (fcntl F_OFD_SETLK) for per-fd file locking needed by the cluster DO ownership protocol. OFD locks are per open-file-description rather than per-process, which is essential since a single workerd process manages many DOs concurrently. Includes tryLock() (non-blocking), verifyHeld() (for NFSv4 lease verification), and move semantics. Non-Linux platforms fail at runtime rather than compile time so builds still succeed when cluster mode is unused. This code (and commit message) was entirely written by Opus 4.6. --- src/workerd/util/BUILD.bazel | 13 +++ src/workerd/util/ofd-lock-test.c++ | 182 +++++++++++++++++++++++++++++ src/workerd/util/ofd-lock.c++ | 111 ++++++++++++++++++ src/workerd/util/ofd-lock.h | 48 ++++++++ 4 files changed, 354 insertions(+) create mode 100644 src/workerd/util/ofd-lock-test.c++ create mode 100644 src/workerd/util/ofd-lock.c++ create mode 100644 src/workerd/util/ofd-lock.h diff --git a/src/workerd/util/BUILD.bazel b/src/workerd/util/BUILD.bazel index 8d8589ce5ea..67332fedeea 100644 --- a/src/workerd/util/BUILD.bazel +++ b/src/workerd/util/BUILD.bazel @@ -510,3 +510,16 @@ kj_test( src = "state-machine-test.c++", deps = [":state-machine"], ) + +wd_cc_library( + name = "ofd-lock", + srcs = ["ofd-lock.c++"], + hdrs = ["ofd-lock.h"], + visibility = ["//visibility:public"], + deps = ["@capnp-cpp//src/kj"], +) + +kj_test( + src = "ofd-lock-test.c++", + deps = [":ofd-lock"], +) diff --git a/src/workerd/util/ofd-lock-test.c++ b/src/workerd/util/ofd-lock-test.c++ new file mode 100644 index 00000000000..aa827c444b2 --- /dev/null +++ b/src/workerd/util/ofd-lock-test.c++ @@ -0,0 +1,182 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "ofd-lock.h" + +#include +#include +#include + +#if !_WIN32 +#include +#include +#include +#endif + +namespace workerd { +namespace { + +#ifdef __linux__ + +kj::OwnFd openTempFile() { + char path[] = "/tmp/ofd-lock-test-XXXXXX"; + int fd = mkstemp(path); + KJ_SYSCALL(fd); + unlink(path); + return kj::OwnFd(fd); +} + +KJ_TEST("exclusive lock conflicts with exclusive lock") { + auto fd1 = openTempFile(); + + // Open a second fd to the same file via /proc/self/fd. + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + + // Second exclusive lock on a different fd to the same file should fail. + KJ_EXPECT(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE) == kj::none); +} + +KJ_TEST("shared lock conflicts with exclusive lock") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + + // Shared lock should also fail when exclusive is held. + KJ_EXPECT(OfdLock::tryLock(fd2, OfdLock::SHARED) == kj::none); +} + +KJ_TEST("shared locks do not conflict with each other") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::SHARED)); + auto lock2 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd2, OfdLock::SHARED)); + // Both shared locks held simultaneously — no conflict. +} + +KJ_TEST("exclusive lock blocks while shared lock is held") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::SHARED)); + + // Exclusive lock should fail when shared lock is held by a different fd. + KJ_EXPECT(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE) == kj::none); +} + +KJ_TEST("releasing lock allows re-acquisition") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + { + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + // lock1 goes out of scope here, releasing the lock. + } + + // Now fd2 should be able to acquire an exclusive lock. + auto lock2 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE)); +} + +KJ_TEST("move constructor transfers lock") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + + // Move the lock. + auto lock2 = kj::mv(lock1); + + // The lock should still be held (fd2 can't get it). + KJ_EXPECT(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE) == kj::none); +} + +KJ_TEST("move constructor releases on destruction of target") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + { + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + auto lock2 = kj::mv(lock1); + // lock2 goes out of scope, releasing the lock. + } + + // fd2 should now succeed. + auto lock3 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE)); +} + +KJ_TEST("move assignment transfers lock") { + auto fd1 = openTempFile(); + + auto procPath = kj::str("/proc/self/fd/", fd1.get()); + int rawFd2; + KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); + auto fd2 = kj::OwnFd(rawFd2); + + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + + // Create a second lock on fd2... wait, we can't because fd1 already holds exclusive. + // Instead, test move assignment by creating a dummy that gets replaced. + auto fd3 = openTempFile(); + auto lock3 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd3, OfdLock::EXCLUSIVE)); + + lock3 = kj::mv(lock1); // Releases lock on fd3, takes lock from fd1. + + // fd2 still can't get the lock (lock on fd1 was transferred to lock3). + KJ_EXPECT(OfdLock::tryLock(fd2, OfdLock::EXCLUSIVE) == kj::none); +} + +KJ_TEST("same fd can re-lock (lock is per open file description)") { + auto fd1 = openTempFile(); + + // On the same fd, acquiring the same lock type again is a no-op re-lock, not a conflict. + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + auto lock2 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + // Both succeed because OFD locks are per open file description, and this is the same fd. +} + +KJ_TEST("verifyHeld succeeds when lock is held") { + auto fd1 = openTempFile(); + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); + // Should not throw. + lock1.verifyHeld(); +} + +#else // #if __linux__ + +KJ_TEST("dummy test: platform not supported") {} + +#endif // #if __linux__, #else + +} // namespace +} // namespace workerd diff --git a/src/workerd/util/ofd-lock.c++ b/src/workerd/util/ofd-lock.c++ new file mode 100644 index 00000000000..96ad38aee9b --- /dev/null +++ b/src/workerd/util/ofd-lock.c++ @@ -0,0 +1,111 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "ofd-lock.h" + +#include + +#ifdef __linux__ +#include +#include +#endif + +namespace workerd { + +#ifdef __linux__ + +namespace { + +struct flock makeFlock(short type) { + struct flock fl = {}; + fl.l_type = type; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; // Lock the entire file. + return fl; +} + +} // namespace + +kj::Maybe OfdLock::tryLock(int fd, Type type) { + auto fl = makeFlock(type == SHARED ? F_RDLCK : F_WRLCK); + if (fcntl(fd, F_OFD_SETLK, &fl) == -1) { + int err = errno; + if (err == EAGAIN || err == EACCES) { + // Lock is held by another open file description. + return kj::none; + } + KJ_FAIL_SYSCALL("fcntl(F_OFD_SETLK)", err); + } + return OfdLock(fd, type); +} + +void OfdLock::verifyHeld() { + KJ_REQUIRE(fd >= 0, "OfdLock has been moved away"); + // Re-lock with the same type. If the underlying NFSv4 lock stateid has been invalidated + // (lease lost), the Linux NFS client surfaces this as EIO. If the stateid is still valid, + // the kernel treats the re-lock as a no-op. + auto fl = makeFlock(type == SHARED ? F_RDLCK : F_WRLCK); + KJ_SYSCALL(fcntl(fd, F_OFD_SETLK, &fl), "NFS lease may have been lost"); +} + +OfdLock::OfdLock(int fd, Type type): fd(fd), type(type) {} + +OfdLock::OfdLock(OfdLock&& other): fd(other.fd), type(other.type) { + other.fd = -1; +} + +OfdLock& OfdLock::operator=(OfdLock&& other) { + if (this != &other) { + // Release existing lock, if any. + if (fd >= 0) { + auto fl = makeFlock(F_UNLCK); + fcntl(fd, F_OFD_SETLK, &fl); + } + fd = other.fd; + type = other.type; + other.fd = -1; + } + return *this; +} + +OfdLock::~OfdLock() { + if (fd >= 0) { + auto fl = makeFlock(F_UNLCK); + fcntl(fd, F_OFD_SETLK, &fl); + } +} + +#else // !__linux__ + +kj::Maybe OfdLock::tryLock(int fd, Type type) { + KJ_FAIL_REQUIRE("OFD locks are only supported on Linux. " + "Cluster mode is currently implemented only on Linux."); +} + +void OfdLock::verifyHeld() { + KJ_FAIL_REQUIRE("OFD locks are only supported on Linux. " + "Cluster mode is currently implemented only on Linux."); +} + +OfdLock::OfdLock(int fd, Type type): fd(fd), type(type) {} + +OfdLock::OfdLock(OfdLock&& other): fd(other.fd), type(other.type) { + other.fd = -1; +} + +OfdLock& OfdLock::operator=(OfdLock&& other) { + if (this != &other) { + fd = other.fd; + type = other.type; + other.fd = -1; + } + return *this; +} + +OfdLock::~OfdLock() {} + +#endif // __linux__ + +} // namespace workerd diff --git a/src/workerd/util/ofd-lock.h b/src/workerd/util/ofd-lock.h new file mode 100644 index 00000000000..aebe84b4c8f --- /dev/null +++ b/src/workerd/util/ofd-lock.h @@ -0,0 +1,48 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once + +#include +#include + +namespace workerd { + +class OfdLock { + // RAII holder for a lock on an open file description. Releasing this object releases the lock. + // + // Uses Linux OFD (Open File Description) locks via fcntl(F_OFD_SETLK). OFD locks are per-fd + // rather than per-process, which is essential since a single workerd process may manage many + // Durable Objects concurrently. + public: + enum Type { SHARED, EXCLUSIVE }; + + // Try to acquire a lock (non-blocking). Returns kj::none if the lock is held in a conflicting + // mode by another open file description. + // + // On non-Linux platforms, fails at runtime with KJ_FAIL_REQUIRE rather than a compile error, + // so non-Linux builds still compile when cluster mode is unused. + static kj::Maybe tryLock(int fd, Type type); + + // Verify that a lock that this open file description was previously granted is still held. + // Specifically, this is meant to check for NFSv4 lease loss. Implemented as a no-op + // F_OFD_SETLK of the same lock type on the same fd: if the underlying NFSv4 lock stateid + // has been invalidated (lease lost), the Linux NFS client surfaces this as EIO and this + // function throws; if the stateid is still valid, the kernel treats the re-lock as a + // no-op and we return cleanly. + void verifyHeld(); + + OfdLock(OfdLock&& other); + OfdLock& operator=(OfdLock&& other); + KJ_DISALLOW_COPY(OfdLock); + + ~OfdLock(); // releases via F_OFD_SETLK with F_UNLCK + + private: + int fd; // -1 after move + Type type; + explicit OfdLock(int fd, Type type); +}; + +} // namespace workerd From bbf2b1a7969cea5d901331af32b3533fadeb82d7 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Thu, 21 May 2026 15:56:34 -0500 Subject: [PATCH 03/33] Add cluster registry as a custom VatNetwork Implements ClusterRegistry, a capnp::VatNetwork for multi-node cluster RPC. Each node generates an X25519 keypair as its identity, binds a listener (Unix socket or ephemeral TCP port based on CIDR), and registers itself in a shared directory. Includes: - cluster.capnp: VatId and stub Level 3/4 types for VatNetwork params - X25519PublicKey: small wrapper with hex encoding and HashMap support - ConnectionImpl: per-peer Connection with MessageStream, idle timeout, and refcounted outbound caching - Handshake protocol: raw 32-byte public key exchange before RPC starts - Peer discovery: lazy directory scanning with mtime-based liveness - CIDR/IP mode: getifaddrs-based IP matching, OFD-locked registry file with periodic mtime heartbeat - Unix mode: socket-file-as-registry-entry, connect-based liveness The end-to-end RPC tests are present but currently failing; the non-network tests (keypair generation, self-bootstrap, peer discovery, self-connect) all pass. This code (and commit message) was originally written by Opus 4.6 but the code sucked and was heavily refactored by my human hand. --- src/workerd/server/BUILD.bazel | 27 + src/workerd/server/cluster-registry-test.c++ | 304 +++++++ src/workerd/server/cluster-registry.c++ | 825 +++++++++++++++++++ src/workerd/server/cluster-registry.h | 171 ++++ src/workerd/server/cluster.capnp | 25 + 5 files changed, 1352 insertions(+) create mode 100644 src/workerd/server/cluster-registry-test.c++ create mode 100644 src/workerd/server/cluster-registry.c++ create mode 100644 src/workerd/server/cluster-registry.h create mode 100644 src/workerd/server/cluster.capnp diff --git a/src/workerd/server/BUILD.bazel b/src/workerd/server/BUILD.bazel index 22284af7725..a0042482eb2 100644 --- a/src/workerd/server/BUILD.bazel +++ b/src/workerd/server/BUILD.bazel @@ -249,6 +249,23 @@ wd_cc_library( ], ) +wd_capnp_library(src = "cluster.capnp") + +wd_cc_library( + name = "cluster-registry", + srcs = ["cluster-registry.c++"], + hdrs = ["cluster-registry.h"], + visibility = ["//visibility:public"], + deps = [ + ":cluster_capnp", + "//src/workerd/util:ofd-lock", + "@capnp-cpp//src/capnp:capnp-rpc", + "@capnp-cpp//src/kj", + "@capnp-cpp//src/kj:kj-async", + "@ssl", + ], +) + wd_capnp_library(src = "docker-api.capnp") wd_capnp_library(src = "log-schema.capnp") @@ -422,6 +439,16 @@ kj_test( ], ) +kj_test( + src = "cluster-registry-test.c++", + deps = [ + ":cluster-registry", + "@capnp-cpp//src/capnp:capnp-rpc", + "@capnp-cpp//src/kj", + "@capnp-cpp//src/kj:kj-async", + ], +) + kj_test( src = "json-logger-test.c++", deps = [ diff --git a/src/workerd/server/cluster-registry-test.c++ b/src/workerd/server/cluster-registry-test.c++ new file mode 100644 index 00000000000..e7b274922d4 --- /dev/null +++ b/src/workerd/server/cluster-registry-test.c++ @@ -0,0 +1,304 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "cluster-registry.h" + +#include + +#if !_WIN32 +#include +#endif + +#include +#include +#include +#include +#include +#include + +namespace workerd { +namespace { + +#if __linux__ + +// A trivial capability server that counts calls. +class CallCountServer final: public capnp::Capability::Server { + public: + int callCount = 0; + + DispatchCallResult dispatchCall(uint64_t interfaceId, + uint16_t methodId, + capnp::CallContext context) override { + ++callCount; + return {kj::READY_NOW, false, true}; + } +}; + +// Helper to create a temp directory on disk. +class TempDir { + public: + TempDir() { + char tmpl[] = "/tmp/cluster-registry-test-XXXXXX"; + KJ_ASSERT(mkdtemp(tmpl) != nullptr); + pathStr = kj::str(tmpl); + disk = kj::newDiskFilesystem(); + dir = disk->getRoot().openSubdir(kj::Path::parse(pathStr.slice(1)), // remove leading / + kj::WriteMode::MODIFY); + } + + ~TempDir() noexcept(false) { + auto p = kj::Path::parse(pathStr.slice(1)); + disk->getRoot().remove(p); + } + + kj::Own get() { + return dir->clone(); + } + + private: + kj::String pathStr; + kj::Own disk; + kj::Own dir; +}; + +// Helper to make a simple RPC call on a capability client. +void makeCall(capnp::Capability::Client& client, kj::WaitScope& ws) { + auto req = client.typelessRequest( + 0, 0, capnp::MessageSize{4, 0}, capnp::Capability::Client::CallHints()); + req.send().wait(ws); +} + +KJ_TEST("X25519PublicKey: hex roundtrip") { + X25519PublicKey key; + memset(key.bytes, 0xAB, sizeof(key.bytes)); + auto hex = key.toHex(); + KJ_EXPECT(hex.size() == 64); + auto decoded = X25519PublicKey::fromHex(hex); + KJ_EXPECT(decoded == key); +} + +KJ_TEST("X25519PublicKey: equality") { + X25519PublicKey a, b; + memset(a.bytes, 1, sizeof(a.bytes)); + memset(b.bytes, 1, sizeof(b.bytes)); + KJ_EXPECT(a == b); + + b.bytes[0] = 2; + KJ_EXPECT(!(a == b)); +} + +KJ_TEST("ClusterRegistry: self is never dead") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + KJ_EXPECT(!reg.isPeerDead(reg.getPublicKey())); +} + +KJ_TEST("ClusterRegistry: self-connect returns none") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(kj::arrayPtr(reg.getPublicKey().bytes, 32)); + + KJ_EXPECT(reg.connect(vatId.asReader()) == kj::none); +} + +KJ_TEST("ClusterRegistry: self-bootstrap returns local capability") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto server = kj::heap(); + auto& serverRef = *server; + auto bootstrap = capnp::Capability::Client(kj::mv(server)); + auto rpc = capnp::makeRpcServer(reg, kj::mv(bootstrap)); + + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(kj::arrayPtr(reg.getPublicKey().bytes, 32)); + + auto client = rpc.bootstrap(vatId); + makeCall(client, io.waitScope); + KJ_EXPECT(serverRef.callCount == 1); +} + +KJ_TEST("ClusterRegistry: peer discovery via directory scan") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg1(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + ClusterRegistry reg2(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + // In unix mode, socket files in the directory are registry entries. + KJ_EXPECT(!reg1.isPeerDead(reg2.getPublicKey())); + KJ_EXPECT(!reg2.isPeerDead(reg1.getPublicKey())); +} + +KJ_TEST("ClusterRegistry: isPeerDead caches live peers briefly") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg1(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + ClusterRegistry reg2(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto peerPath = kj::Path({reg2.getPublicKey().toHex()}); + KJ_EXPECT(!reg1.isPeerDead(reg2.getPublicKey())); + + tmpDir.get()->tryRemove(peerPath); + KJ_EXPECT(!reg1.isPeerDead(reg2.getPublicKey())); +} + +KJ_TEST("ClusterRegistry: isPeerDead caches dead peers permanently") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + X25519PublicKey peerKey; + memset(peerKey.bytes, 0xCA, sizeof(peerKey.bytes)); + auto peerPath = kj::Path({peerKey.toHex()}); + + KJ_EXPECT(reg.isPeerDead(peerKey)); + tmpDir.get()->openFile(peerPath, kj::WriteMode::CREATE)->writeAll("placeholder"); + KJ_EXPECT(reg.isPeerDead(peerKey)); +} + +KJ_TEST("ClusterRegistry: end-to-end RPC between two registries") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg1(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + ClusterRegistry reg2(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto maint1 = reg1.runMaintenance(); + auto maint2 = reg2.runMaintenance(); + + auto server1 = kj::heap(); + auto& server1Ref = *server1; + auto server2 = kj::heap(); + auto& server2Ref = *server2; + + auto rpc1 = capnp::makeRpcServer(reg1, capnp::Capability::Client(kj::mv(server1))); + auto rpc2 = capnp::makeRpcServer(reg2, capnp::Capability::Client(kj::mv(server2))); + + auto run1 = rpc1.run(); + auto run2 = rpc2.run(); + + // reg1 -> reg2 + { + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(kj::arrayPtr(reg2.getPublicKey().bytes, 32)); + + auto client = rpc1.bootstrap(vatId); + makeCall(client, io.waitScope); + KJ_EXPECT(server2Ref.callCount == 1); + } + + // reg2 -> reg1 + { + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(kj::arrayPtr(reg1.getPublicKey().bytes, 32)); + + auto client = rpc2.bootstrap(vatId); + makeCall(client, io.waitScope); + KJ_EXPECT(server1Ref.callCount == 1); + } +} + +KJ_TEST("ClusterRegistry: end-to-end RPC with binary IP registry entries") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg1( + tmpDir.get(), "127.0.0.0/8"_kj, io.provider->getNetwork(), io.provider->getTimer()); + ClusterRegistry reg2( + tmpDir.get(), "127.0.0.0/8"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto maint1 = reg1.runMaintenance(); + auto maint2 = reg2.runMaintenance(); + + auto server2 = kj::heap(); + auto& server2Ref = *server2; + + auto rpc1 = capnp::makeRpcServer(reg1, capnp::Capability::Client(kj::heap())); + auto rpc2 = capnp::makeRpcServer(reg2, capnp::Capability::Client(kj::mv(server2))); + + auto run1 = rpc1.run(); + auto run2 = rpc2.run(); + + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(kj::arrayPtr(reg2.getPublicKey().bytes, 32)); + + auto client = rpc1.bootstrap(vatId); + makeCall(client, io.waitScope); + KJ_EXPECT(server2Ref.callCount == 1); +} + +KJ_TEST("ClusterRegistry: idle timeout closes connections") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + auto shortTimeout = 100 * kj::MILLISECONDS; + + ClusterRegistry reg1( + tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer(), shortTimeout); + ClusterRegistry reg2( + tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer(), shortTimeout); + + auto server2 = kj::heap(); + auto& server2Ref = *server2; + + auto rpc1 = capnp::makeRpcServer(reg1, capnp::Capability::Client(kj::heap())); + auto rpc2 = capnp::makeRpcServer(reg2, capnp::Capability::Client(kj::mv(server2))); + + auto run1 = rpc1.run(); + auto run2 = rpc2.run(); + + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(kj::arrayPtr(reg2.getPublicKey().bytes, 32)); + + { + auto client = rpc1.bootstrap(vatId); + makeCall(client, io.waitScope); + KJ_EXPECT(server2Ref.callCount == 1); + } + // Client dropped — connection should become idle. + + // Wait for the idle timeout to fire. + io.provider->getTimer().afterDelay(shortTimeout + 200 * kj::MILLISECONDS).wait(io.waitScope); + + // After idle close, a new bootstrap should still work (new connection). + auto client2 = rpc1.bootstrap(vatId); + makeCall(client2, io.waitScope); + KJ_EXPECT(server2Ref.callCount == 2); +} + +KJ_TEST("ClusterRegistry: verifyNfsLease() succeeds when lock is held") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + reg.verifyNfsLease(); +} + +#else // #if __linux__ + +KJ_TEST("dummy test: platform not supported") {} + +#endif // #if __linux__, #else + +} // namespace +} // namespace workerd diff --git a/src/workerd/server/cluster-registry.c++ b/src/workerd/server/cluster-registry.c++ new file mode 100644 index 00000000000..a430d059e86 --- /dev/null +++ b/src/workerd/server/cluster-registry.c++ @@ -0,0 +1,825 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "cluster-registry.h" + +#if !_WIN32 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#include +#include +#include +#include +#include + +namespace workerd { + +#if __linux__ + +namespace { + +constexpr kj::Duration REGISTRY_MAINTENANCE_INTERVAL = 60 * kj::SECONDS; +constexpr kj::Duration DIRECTORY_SCAN_INTERVAL = 30 * kj::SECONDS; +constexpr kj::Duration NEW_REGISTRY_ENTRY_GRACE_PERIOD = 15 * kj::SECONDS; +constexpr kj::Duration FAILED_PROBE_THROTTLE = 5 * kj::SECONDS; +constexpr kj::Duration SUSPECT_PEER_AVOIDANCE_PERIOD = 15 * kj::SECONDS; +constexpr kj::Duration PEER_LIVENESS_CACHE_TTL = 15 * kj::SECONDS; + +} // namespace + +// ======================================================================================= +// X25519PublicKey + +kj::String X25519PublicKey::toHex() const { + return kj::encodeHex(kj::arrayPtr(bytes, sizeof(bytes))); +} + +X25519PublicKey X25519PublicKey::fromHex(kj::StringPtr hex) { + KJ_REQUIRE(hex.size() == 64, "X25519 public key hex must be 64 characters", hex.size()); + auto decoded = kj::decodeHex(hex.asArray()); + KJ_REQUIRE(!decoded.hadErrors, "invalid hex in X25519 public key"); + KJ_ASSERT(decoded.size() == 32); + X25519PublicKey result; + memcpy(result.bytes, decoded.begin(), 32); + return result; +} + +// ======================================================================================= +// ClusterRegistry::NativeAddress + +void ClusterRegistry::NativeAddress::setPort(uint port) { + KJ_REQUIRE(port <= static_cast(kj::maxValue), "invalid port", port); + + switch (storage.ss_family) { + case AF_INET: { + KJ_REQUIRE(length == sizeof(sockaddr_in), "invalid IPv4 sockaddr length", length); + auto& sin = *reinterpret_cast(&storage); + sin.sin_port = htons(port); + break; + } + case AF_INET6: { + KJ_REQUIRE(length == sizeof(sockaddr_in6), "invalid IPv6 sockaddr length", length); + auto& sin6 = *reinterpret_cast(&storage); + sin6.sin6_port = htons(port); + break; + } + default: + KJ_FAIL_REQUIRE("address does not have a TCP port", storage.ss_family); + } +} + +ClusterRegistry::NativeAddress ClusterRegistry::NativeAddress::fromSockaddr(const sockaddr* addr) { + KJ_REQUIRE(addr != nullptr, "missing sockaddr"); + + NativeAddress result; + switch (addr->sa_family) { + case AF_INET: + result.length = sizeof(sockaddr_in); + break; + case AF_INET6: + result.length = sizeof(sockaddr_in6); + break; + default: + KJ_FAIL_REQUIRE("sockaddr is not an IP address", addr->sa_family); + } + + memcpy(&result.storage, addr, result.length); + return result; +} + +ClusterRegistry::NativeAddress ClusterRegistry::NativeAddress::parse( + kj::ArrayPtr bytes) { + KJ_REQUIRE( + bytes.size() >= sizeof(sa_family_t), "peer registry address is too short", bytes.size()); + KJ_REQUIRE( + bytes.size() <= sizeof(sockaddr_storage), "peer registry address is too long", bytes.size()); + + NativeAddress result; + result.length = bytes.size(); + memcpy(&result.storage, bytes.begin(), bytes.size()); + + switch (result.storage.ss_family) { + case AF_INET: + KJ_REQUIRE( + result.length == sizeof(sockaddr_in), "invalid IPv4 sockaddr length", result.length); + break; + case AF_INET6: + KJ_REQUIRE( + result.length == sizeof(sockaddr_in6), "invalid IPv6 sockaddr length", result.length); + break; + default: + KJ_FAIL_REQUIRE("peer registry address is not an IP sockaddr", result.storage.ss_family); + } + + return result; +} + +// ======================================================================================= +// ClusterRegistry::ConnectionImpl + +class ClusterRegistry::ConnectionImpl final: public ClusterVatNetworkBase::Connection, + public kj::Refcounted { + public: + ConnectionImpl(ClusterRegistry& registry, + kj::Own stream, + kj::Maybe peerKey, + kj::Maybe> wasRefused) + : registry(registry), + stream(kj::mv(stream)), + msgStream(*this->stream, capnp::IncomingRpcMessage::getShortLivedCallback()), + peerKey(peerKey), + wasRefused(kj::mv(wasRefused)), + outbound(peerKey != kj::none) { + // We always need to pump immediately to send the handshake. + ensurePumping(); + } + + ~ConnectionImpl() noexcept(false) { + unregister(); + } + + cluster::VatId::Reader getPeerVatId() override { + // We can safely expected the RPC system won't call this method until there is actually some + // activity on the stream that causes it to be needed. If this is an inbound stream, there + // is necessary no activity until we receive some messages, and we must have received the + // peer key before that. + // + // This is convenient because otherwise we'd have to figure out what to do with this call + // if we haven't received the handshake yet, and therefore don't actually know who the peer + // is. + auto pk = KJ_ASSERT_NONNULL(peerKey, "getPeerVatId() called too early"); + + // Build `peerVatId` on first use. + KJ_IF_SOME(p, peerVatId) { + return p.getRoot().asReader(); + } else { + auto builder = peerVatId.emplace(peerVatIdScratch).getRoot(); + builder.setPublicKey(kj::arrayPtr(pk.bytes, sizeof(pk.bytes))); + return builder.asReader(); + } + } + + kj::Own newOutgoingMessage(uint firstSegmentWordSize) override; + kj::Promise>> receiveIncomingMessage() override; + kj::Promise shutdown() override; + void setIdle(bool newIdle) override; + + private: + class OutgoingMessageImpl; + class IncomingMessageImpl; + + ClusterRegistry& registry; + kj::Own stream; + capnp::BufferedMessageStream msgStream; + kj::Maybe peerKey; + kj::Maybe> wasRefused; + bool outbound; // is this an outbound or inbound connection? + + capnp::word peerVatIdScratch[8]{}; + kj::Maybe peerVatId; + + kj::Vector> writeQueue; + kj::Maybe> pumpTask; + kj::Maybe brokenReason; + + bool pumping = false; // is pump() running? + bool isShutdown = false; // has shutdown() been called? + + bool idle = false; // tracks last value passed to setIdle() + bool idleTimedOut = false; + + bool sentHandshake = false; + bool receivedHandshake = false; + + kj::Maybe> idleTimer; + kj::Maybe>>>> + interruptReceiveFulfiller; + + void ensurePumping(); + kj::Promise pump(); + + void requireOpen(); + void unregister(); + + friend class ClusterRegistry; +}; + +class ClusterRegistry::ConnectionImpl::OutgoingMessageImpl final: public capnp::OutgoingRpcMessage, + public kj::Refcounted { + public: + OutgoingMessageImpl(ConnectionImpl& conn, uint firstSegmentWordSize) + : conn(conn), + message(firstSegmentWordSize == 0 ? capnp::SUGGESTED_FIRST_SEGMENT_WORDS + : firstSegmentWordSize) {} + + capnp::AnyPointer::Builder getBody() override { + return message.getRoot(); + } + + void send() override { + conn.requireOpen(); + KJ_REQUIRE(!conn.idle, "bug in RpcSystem: trying to send a message while idle"); + KJ_REQUIRE(!conn.idleTimedOut, "send() after idle period timed out"); + + conn.writeQueue.add(kj::addRef(*this)); + conn.ensurePumping(); + } + + size_t sizeInWords() override { + return message.sizeInWords(); + } + + private: + ConnectionImpl& conn; + capnp::MallocMessageBuilder message; + + friend class ConnectionImpl; +}; + +class ClusterRegistry::ConnectionImpl::IncomingMessageImpl final: public capnp::IncomingRpcMessage { + public: + IncomingMessageImpl(kj::Own message): message(kj::mv(message)) {} + + capnp::AnyPointer::Reader getBody() override { + return message->getRoot(); + } + + size_t sizeInWords() override { + return message->sizeInWords(); + } + + private: + kj::Own message; +}; + +kj::Own ClusterRegistry::ConnectionImpl::newOutgoingMessage( + uint firstSegmentWordSize) { + KJ_REQUIRE(!idle, "bug in RpcSystem: trying to send a message while idle"); + return kj::refcounted(*this, firstSegmentWordSize); +} + +kj::Promise>> ClusterRegistry::ConnectionImpl:: + receiveIncomingMessage() { + KJ_IF_SOME(b, brokenReason) { + kj::throwFatalException(b.clone()); + } + if (idleTimedOut) { + co_return kj::none; + } + + if (!receivedHandshake) { + X25519PublicKey receivedKey; + auto readHandshake = stream->read(receivedKey.bytes); + if (outbound && registry.nfsMode()) { + co_await registry.timer.timeoutAfter(registry.connectTimeout, kj::mv(readHandshake)); + } else { + co_await readHandshake; + } + KJ_IF_SOME(expectedKey, peerKey) { + KJ_REQUIRE(receivedKey == expectedKey, "peer did not send expected handshake bytes"); + } else { + peerKey = receivedKey; + } + receivedHandshake = true; + if (outbound) { + KJ_IF_SOME(entry, registry.peers.find(KJ_ASSERT_NONNULL(peerKey))) { + entry.suspect = false; + entry.suspectSince = kj::none; + } + } + } + + auto paf = kj::newPromiseAndFulfiller>>(); + interruptReceiveFulfiller = kj::mv(paf.fulfiller); + + auto message = co_await msgStream.tryReadMessage().exclusiveJoin(kj::mv(paf.promise)); + + KJ_IF_SOME(m, message) { + co_return kj::heap(kj::mv(m)); + } else { + co_return kj::none; + } +} + +kj::Promise ClusterRegistry::ConnectionImpl::shutdown() { + requireOpen(); + isShutdown = true; + ensurePumping(); + return KJ_ASSERT_NONNULL(kj::mv(pumpTask)); +} + +void ClusterRegistry::ConnectionImpl::setIdle(bool newIdle) { + if (newIdle == idle) return; // Already in the desired state. + idle = newIdle; + + // Only outbound connections should potentially close themselves when idle. Inbound should + // be closed at the other end. + if (outbound) { + if (idle) { + // Wait for the idle timeout. + idleTimer = registry.timer.afterDelay(registry.idleTimeout) + .then([this]() { + // Wait until the event queue is empty, just in case we're concurrently receiving a + // message that causes us to become non-idle. (But that shouldn't happen since this is + // an outbound connection. Once idle, it should only be revived from our end. But just to + // be safe.) + return kj::evalLast([this]() { + // We officially timed out. Make sure all future calls to receiveIncomingMessage() return + // kj::none. + idleTimedOut = true; + + // Cancel the current receiveIncomingMessage() by fulfilling the promise that is + // joined with the `tryReadMessage()` promise. + KJ_IF_SOME(i, interruptReceiveFulfiller) { + i->fulfill(kj::none); + } + + // Make sure a new call to connect() can't return this again. + unregister(); + }); + }).eagerlyEvaluate(nullptr); + } else { + // Cancel the last idle wait. + idleTimer = kj::none; + } + } +} + +void ClusterRegistry::ConnectionImpl::ensurePumping() { + if (!pumping) { + pumping = true; + pumpTask = pump(); + } +} + +kj::Promise ClusterRegistry::ConnectionImpl::pump() { + KJ_TRY { + if (!sentHandshake) { + co_await stream->write(registry.publicKey.bytes); + sentHandshake = true; + } + + // Give a chance for messages to accumulate to be sent all at once. + co_await kj::yieldUntilQueueEmpty(); + + while (!writeQueue.empty()) { + // Take ownership of all messages currently pending. `writeQueue` is left empty, but may + // be repopulated concurrently. + auto ownMessages = kj::mv(writeQueue); + + // Write them all at once. + auto messageSegments = + KJ_MAP(msg, ownMessages) { return msg->message.getSegmentsForOutput(); }; + co_await msgStream.writeMessages(messageSegments); + } + + if (isShutdown) { + co_await msgStream.end(); + } + + pumping = false; + } + KJ_CATCH(exception) { + brokenReason = exception.clone(); + KJ_IF_SOME(i, interruptReceiveFulfiller) { + i->reject(exception.clone()); + } + kj::throwFatalException(kj::mv(exception)); + } +} + +void ClusterRegistry::ConnectionImpl::requireOpen() { + KJ_IF_SOME(e, brokenReason) { + kj::throwFatalException(e.clone()); + } + KJ_REQUIRE(!isShutdown, "bug in RpcSystem: can't send() after shutdown()"); +} + +void ClusterRegistry::ConnectionImpl::unregister() { + if (outbound) { + auto key = KJ_ASSERT_NONNULL(peerKey); // peerKey is always set for outbounds + + KJ_TRY { + KJ_IF_SOME(entry, registry.peers.findEntry(key)) { + KJ_IF_SOME(conn, entry.value.outboundConnection) { + if (&conn == this) { + entry.value.outboundConnection = kj::none; + } + } + + // If we failed to receive the peer handshake, mark the peer as suspect and run the + // dead-peer cleanup path. We may have failed to connect entirely, or we may have + // connected to a server that wasn't the peer we expected. + if (!receivedHandshake) { + bool wasRefusedUnix = false; + KJ_IF_SOME(w, wasRefused) { + wasRefusedUnix = *w; + } + registry.cleanupFailedPeer(entry, wasRefusedUnix); + } + } + } + KJ_CATCH(exception) { + KJ_LOG(WARNING, "failed to clean up cluster peer after connection failure", key.toHex(), + exception); + } + } +} + +// ======================================================================================= +// ClusterRegistry constructor + +ClusterRegistry::ClusterRegistry(kj::Own registryDirParam, + kj::StringPtr networkConfig, + kj::Network& kjNetwork, + kj::Timer& timer, + kj::Duration idleTimeout, + kj::Duration connectTimeout) + : registryDir(kj::mv(registryDirParam)), + dirFd(KJ_REQUIRE_NONNULL( + registryDir->getFd(), "cluster registry directory must be a disk-backed directory")), + timer(timer), + network(kjNetwork), + idleTimeout(idleTimeout), + connectTimeout(connectTimeout) { + // Generate X25519 keypair. + X25519_keypair(publicKey.bytes, privateKey); + publicKeyHex = publicKey.toHex(); + + if (networkConfig == "unix") { + // Unix mode: bind a Unix domain socket at /. + // Use /proc/self/fd// as the socket path. + auto socketPath = kj::str("/proc/self/fd/", dirFd, "/", publicKeyHex); + sockaddr_un addr{}; + KJ_REQUIRE( + socketPath.size() < sizeof(addr.sun_path), "Unix socket path is too long", socketPath); + addr.sun_family = AF_UNIX; + memcpy(addr.sun_path, socketPath.begin(), socketPath.size()); + listener = kjNetwork.getSockaddr(&addr, sizeof(addr))->listen(); + } else { + // CIDR/IP/NFS mode: find a matching local IP and bind an ephemeral port. + auto addr = findMatchingIp(networkConfig); + listener = kjNetwork.getSockaddr(addr.asSockaddr(), addr.length)->listen(); + auto port = listener->getPort(); + addr.setPort(port); + boundAddress = addr; + } +} + +ClusterRegistry::~ClusterRegistry() noexcept(false) { + registryDir->tryRemove(kj::Path({publicKeyHex})); +} + +// Search all IP addresses for all interfaces available in the network namespace to find one that +// matches the given CIDR. +ClusterRegistry::NativeAddress ClusterRegistry::findMatchingIp(kj::StringPtr cidr) { + kj::CidrRange range(cidr); + + struct ifaddrs* ifap = nullptr; + KJ_SYSCALL(getifaddrs(&ifap)); + KJ_DEFER(freeifaddrs(ifap)); + + for (auto* ifa = ifap; ifa != nullptr; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == nullptr) continue; + if (range.matches(ifa->ifa_addr)) { + auto result = NativeAddress::fromSockaddr(ifa->ifa_addr); + result.setPort(0); + return result; + } + } + + KJ_FAIL_REQUIRE("no local interface matches CIDR", cidr); +} + +// ======================================================================================= +// Registry maintenance + +kj::Promise ClusterRegistry::runMaintenance() { + KJ_IF_SOME(addr, boundAddress) { + // CIDR/IP mode: create the registry file, lock it, write our address, then periodically + // verify that our registry entry and NFS lock lease are still valid. + + auto registryFile = registryDir->openFile(kj::Path({publicKeyHex}), kj::WriteMode::CREATE); + + // Acquire an exclusive OFD lock. + auto lock = KJ_ASSERT_NONNULL( + OfdLock::tryLock(KJ_ASSERT_NONNULL(registryFile->getFd()), OfdLock::EXCLUSIVE), + "our registry entry is already locked?"); + + // Write our address. + registryFile->write(0, addr.asBytes()); + registryFile->sync(); + + return maintenanceLoopCanceler.wrap(maintenanceLoop(registration.emplace(Registration{ + .file = kj::mv(registryFile), + .lock = kj::mv(lock), + }))); + } else { + // Unix socket mode: the listener socket's lifetime IS the registry entry. Nothing to maintain. + return kj::NEVER_DONE; + } +} + +kj::Promise ClusterRegistry::maintenanceLoop(Registration& registration) { + for (;;) { + co_await timer.afterDelay(REGISTRY_MAINTENANCE_INTERVAL); + + // Verify nlink > 0 (not unlinked by another node). + KJ_REQUIRE(registration.file->stat().linkCount > 0, + "registry file was unlinked by another node; this instance should shut down"); + + // Verify the OFD lock is still held (NFSv4 lease check). + registration.lock.verifyHeld(); + } +} + +void ClusterRegistry::verifyNfsLease() { + KJ_IF_SOME(r, registration) { + KJ_TRY { + r.lock.verifyHeld(); + } + KJ_CATCH(exception) { + // Might as well cancel the maintenance loop immediately. + maintenanceLoopCanceler.cancel(exception); + kj::throwFatalException(kj::mv(exception)); + } + } +} + +// ======================================================================================= +// Peer discovery + +void ClusterRegistry::scanDirectory() { + // List the registry to discover new peers and add them to the `peers` map, so that they can + // be considered by `pickRandomPeer()`. Keep in mind that on NFS, directory listings tend to be + // cached, and therefore we can's assume the existence or absence of a file really tells us + // anything about whether the peer is currently alive or dead. The best we can do is populate the + // map entries so that we know that peers exist at all, and then we'll need to probe them for + // liveness as usual later on. + + auto names = registryDir->listNames(); + + for (auto& name: names) { + if (name.size() != 64) continue; + if (name == publicKeyHex) continue; + + auto decoded = kj::decodeHex(name.asArray()); + if (decoded.hadErrors || decoded.size() != 32) continue; + + X25519PublicKey key; + memcpy(key.bytes, decoded.begin(), 32); + + peers.findOrCreate(key, [&]() -> decltype(peers)::Entry { return {.key = key}; }); + } +} + +void ClusterRegistry::scanDirectoryIfStale(kj::TimePoint now) { + KJ_IF_SOME(lastScan, lastDirectoryScan) { + if (now - lastScan < DIRECTORY_SCAN_INTERVAL) return; + } + + scanDirectory(); + lastDirectoryScan = now; +} + +bool ClusterRegistry::isPeerDead(const X25519PublicKey& key) { + if (key == publicKey) return false; + + auto& entry = + peers.findOrCreateEntry(key, [&]() -> decltype(peers)::Entry { return {.key = key}; }); + + if (entry.value.knownDead) return true; + + auto now = timer.now(); + KJ_IF_SOME(lastLive, entry.value.lastConfirmedLiveTime) { + if (now - lastLive < PEER_LIVENESS_CACHE_TTL) return false; + } + + if (registryDir->exists(kj::Path({key.toHex()}))) { + entry.value.lastConfirmedLiveTime = now; + return false; + } + + entry.value.knownDead = true; + return true; +} + +kj::Maybe ClusterRegistry::pickRandomPeer() { + kj::Vector candidates; + auto now = timer.now(); + scanDirectoryIfStale(now); + + for (auto& peer: peers) { + if (peer.key == publicKey) continue; + if (peer.value.knownDead) continue; + if (peer.value.suspect) { + KJ_IF_SOME(since, peer.value.suspectSince) { + if (now - since < SUSPECT_PEER_AVOIDANCE_PERIOD) continue; + } + } + candidates.add(peer.key); + } + + if (candidates.empty()) return kj::none; + + // Simple pseudo-random selection using the monotonic clock as a cheap source. (Don't use + // timer.now() since it only updates when the event loop waits for I/O.) + auto nanos = + (kj::systemPreciseMonotonicClock().now() - kj::origin()) / kj::NANOSECONDS; + auto idx = kj::hashCode(nanos) % candidates.size(); + return candidates[idx]; +} + +void ClusterRegistry::cleanupFailedPeer(decltype(peers)::Entry& entry, bool wasRefusedUnix) { + auto now = timer.now(); + + entry.value.suspect = true; + entry.value.suspectSince = now; + + auto path = kj::Path({entry.key.toHex()}); + + if (!nfsMode()) { + if (wasRefusedUnix) { + // We failed with ECONNREFUSED, which means the listener is gone (process exited or closed + // the listen socket), in which case we can certainly clean up the socket from disk. + registryDir->tryRemove(path); + entry.value.knownDead = true; + } else { + // We failed for some other reason. Check if the file has disappeared from disk. + if (registryDir->exists(path)) { + // File still exists. We may have failed with EAGAIN (the peer's listen queue is full) or + // some other error where the peer is not necessarily gone. We've already marked it + // suspect, but we should not actually erase it. + } else { + // File is gone from disk. This may be exactly why the connection failed, but regardless, + // we should now mark the peer dead in our cache. + entry.value.knownDead = true; + } + } + + return; + } + + KJ_IF_SOME(lastProbe, entry.value.lastFailedProbeTime) { + if (now - lastProbe < FAILED_PROBE_THROTTLE) return; + } + entry.value.lastFailedProbeTime = now; + + KJ_IF_SOME(file, registryDir->tryOpenFile(path)) { + auto meta = file->stat(); + auto calendarNow = kj::systemCoarseCalendarClock().now(); + if (calendarNow - meta.lastModified < NEW_REGISTRY_ENTRY_GRACE_PERIOD) return; + + if (OfdLock::tryLock(KJ_ASSERT_NONNULL(file->getFd()), OfdLock::SHARED) != kj::none) { + registryDir->tryRemove(path); + entry.value.knownDead = true; + } + } else { + entry.value.knownDead = true; + } +} + +// ======================================================================================= +// Peer address lookup + +ClusterRegistry::NativeAddress ClusterRegistry::readPeerAddress(const X25519PublicKey& key) { + auto path = kj::Path({key.toHex()}); + auto file = + KJ_REQUIRE_NONNULL(registryDir->tryOpenFile(path), "peer not found in registry", key.toHex()); + auto content = file->readAllBytes(); + KJ_REQUIRE(content.size() > 0, "peer registry file is empty", key.toHex()); + return NativeAddress::parse(content); +} + +// ======================================================================================= +// VatNetwork: connect() + +kj::Maybe> ClusterRegistry::connect( + cluster::VatId::Reader peer) { + auto keyData = peer.getPublicKey(); + KJ_REQUIRE(keyData.size() == 32, "invalid VatId: publicKey must be 32 bytes"); + + X25519PublicKey peerKey; + memcpy(peerKey.bytes, keyData.begin(), 32); + + // Self-connect returns kj::none per VatNetwork contract. + if (peerKey == publicKey) { + return kj::none; + } + + // Check for cached outbound connection. + KJ_IF_SOME(entry, peers.find(peerKey)) { + KJ_IF_SOME(conn, entry.outboundConnection) { + return kj::addRef(conn); + } + } + + auto& entry = peers.findOrCreateEntry( + peerKey, [&]() -> decltype(peers)::Entry { return {.key = peerKey}; }); + + KJ_REQUIRE(!entry.value.knownDead, "peer is known to be dead", peerKey.toHex()); + + // Need to establish a new connection. Look up the peer's address. + kj::Own networkAddress; + if (nfsMode()) { + NativeAddress address; + try { + address = readPeerAddress(peerKey); + } catch (...) { + cleanupFailedPeer(entry, false); + throw; + } + networkAddress = network.getSockaddr(address.asSockaddr(), address.length); + } else { + auto path = kj::str("/proc/self/fd/", dirFd, "/", peerKey.toHex()); + sockaddr_un addr{}; + KJ_REQUIRE(path.size() < sizeof(addr.sun_path), "Unix socket path is too long", path); + addr.sun_family = AF_UNIX; + memcpy(addr.sun_path, path.begin(), path.size()); + networkAddress = network.getSockaddr(&addr, sizeof(addr)); + } + + auto connectPromise = networkAddress->connect(); + + kj::Maybe> wasRefused; + if (!nfsMode()) { + auto wrapper = kj::refcountedWrapper(false); + wasRefused = wrapper->addWrappedRef(); + connectPromise = connectPromise.catch_( + [wrapper = kj::mv(wrapper)]( + kj::Exception&& e) mutable -> kj::Promise> { + if (e.getType() == kj::Exception::Type::DISCONNECTED) { + // If connect() failed with DISCONNECTED on a unix socket, this likely means we got + // ECONNREFUSED. Record this. + wrapper->getWrapped() = true; + } + kj::throwFatalException(kj::mv(e)); + }); + } + + auto promisedStream = kj::newPromisedStream(connectPromise.attach(kj::mv(networkAddress))); + + auto conn = + kj::refcounted(*this, kj::mv(promisedStream), peerKey, kj::mv(wasRefused)); + + // Cache the outbound connection. + entry.value.outboundConnection = *conn; + + return kj::Own(kj::addRef(*conn)); +} + +// ======================================================================================= +// VatNetwork: accept() + +kj::Promise> ClusterRegistry::accept() { + auto stream = co_await listener->accept(); + co_return kj::refcounted(*this, kj::mv(stream), kj::none, kj::none); +} + +// ======================================================================================= + +#else // #if __linux__ + +ClusterRegistry::ClusterRegistry(kj::Own registryDir, + kj::StringPtr network, + kj::Network& kjNetwork, + kj::Timer& timer, + kj::Duration idleTimeout, + kj::Duration connectTimeout) { + KJ_UNIMPLEMENTED("cluster mode is only implemented on linux"); +} +ClusterRegistry::~ClusterRegistry() noexcept(false) {} + +kj::Promise ClusterRegistry::runMaintenance() { + KJ_UNREACHABLE; +} +void ClusterRegistry::verifyNfsLease() { + KJ_UNREACHABLE; +} +bool ClusterRegistry::isPeerDead(const X25519PublicKey& key) { + KJ_UNREACHABLE; +} +kj::Maybe ClusterRegistry::pickRandomPeer() { + KJ_UNREACHABLE; +} +kj::Maybe> ClusterRegistry::connect( + cluster::VatId::Reader peer) { + KJ_UNREACHABLE; +} +kj::Promise> ClusterRegistry::accept() { + KJ_UNREACHABLE; +} + +#endif // #if __linux__, #else + +} // namespace workerd diff --git a/src/workerd/server/cluster-registry.h b/src/workerd/server/cluster-registry.h new file mode 100644 index 00000000000..781e91c4504 --- /dev/null +++ b/src/workerd/server/cluster-registry.h @@ -0,0 +1,171 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once + +#include +#include + +#if __linux__ +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +namespace workerd { + +using kj::byte; +using kj::uint; + +struct X25519PublicKey { + kj::byte bytes[32]; + + kj::String toHex() const; + static X25519PublicKey fromHex(kj::StringPtr hex); + + bool operator==(const X25519PublicKey& other) const { + return memcmp(bytes, other.bytes, sizeof(bytes)) == 0; + } + kj::uint hashCode() const { + return kj::hashCode(kj::ArrayPtr(bytes)); + } +}; + +// Type alias for the VatNetwork base that ClusterRegistry implements. +using ClusterVatNetworkBase = capnp::VatNetwork; + +class ClusterRegistry final: public ClusterVatNetworkBase { + // Manages this node's cluster identity, registry presence, peer discovery, and the + // VatNetwork that the cluster's RpcSystem sits on top of. + public: + ClusterRegistry(kj::Own registryDir, + kj::StringPtr network, + kj::Network& kjNetwork, + kj::Timer& timer, + kj::Duration idleTimeout = 60 * kj::SECONDS, + kj::Duration connectTimeout = 1 * kj::SECONDS); + ~ClusterRegistry() noexcept(false); + + const X25519PublicKey& getPublicKey() { + return publicKey; + } + + // Run the registry maintenance loop. Returns a promise that must be held alive for + // the lifetime of the server. + kj::Promise runMaintenance(); + + // Verify that the NFS lease is still valid on the registration file, or throw an exception if + // not (no-op if not in NFS mode). + // + // More specifically: If you call this immediately after open()ing a file, and it returns + // successfully, then the newly-opened file is guaranteed to be on the same lease as the + // registration. (Technically, the lease could have been lost already, but if so, the + // newly-opened file will not be writable.) + void verifyNfsLease(); + + // Definitive proof-of-death query. Returns true only when the peer's registry entry is absent. + bool isPeerDead(const X25519PublicKey& key); + + // Pick a random peer's key from the cached peer list, biased away from recent failures. + kj::Maybe pickRandomPeer(); + + // Returns true if this registry uses IP sockets (not unix sockets) and is therefore in + // "NFS mode" where it tries to be NFS-friendly. + bool nfsMode() { + return boundAddress != kj::none; + } + + // implements VatNetwork ----------------------------------------------------- + kj::Maybe> connect(cluster::VatId::Reader peer) override; + kj::Promise> accept() override; + + private: +#if __linux__ + class ConnectionImpl; + + struct NativeAddress { + sockaddr_storage storage{}; + socklen_t length = 0; + + const sockaddr* asSockaddr() const { + return reinterpret_cast(&storage); + } + + kj::ArrayPtr asBytes() const { + return kj::arrayPtr(reinterpret_cast(&storage), length); + } + + void setPort(uint port); + + static NativeAddress fromSockaddr(const sockaddr* addr); + static NativeAddress parse(kj::ArrayPtr bytes); + }; + + kj::Own registryDir; + int dirFd; // raw fd of registryDir, for POSIX ops + kj::Timer& timer; + kj::Network& network; + kj::Duration idleTimeout; + kj::Duration connectTimeout; + + // Identity + kj::byte privateKey[32]; + X25519PublicKey publicKey; + kj::String publicKeyHex; + + // Address we wrote to the registry file (CIDR/IP/NFS mode). Null in Unix socket mode. + kj::Maybe boundAddress; + + struct Registration { + kj::Own file; + OfdLock lock; + }; + kj::Maybe registration; // only in NFS mode + + kj::Canceler maintenanceLoopCanceler; + + // Listener + kj::Own listener; + + // Peer cache + struct PeerInfo { + kj::Maybe outboundConnection; + kj::Maybe lastConfirmedLiveTime; + kj::Maybe lastFailedProbeTime; + bool suspect = false; + kj::Maybe suspectSince; + bool knownDead = false; + }; + kj::HashMap peers; + kj::Maybe lastDirectoryScan; + + void scanDirectory(); + void scanDirectoryIfStale(kj::TimePoint now); + kj::Promise maintenanceLoop(Registration& registration); + void cleanupFailedPeer(decltype(peers)::Entry& entry, bool wasRefusedUnix); + + NativeAddress findMatchingIp(kj::StringPtr cidr); + + // Read the address for a peer from its registry file. CIDR/IP mode only. + NativeAddress readPeerAddress(const X25519PublicKey& key); + +#else // #if __linux__ + // Just so inline methods compile + X25519PublicKey publicKey; + kj::Maybe boundAddress; + +#endif // #if __linux__, #else +}; + +} // namespace workerd diff --git a/src/workerd/server/cluster.capnp b/src/workerd/server/cluster.capnp new file mode 100644 index 00000000000..35532c83478 --- /dev/null +++ b/src/workerd/server/cluster.capnp @@ -0,0 +1,25 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +@0xbd0c09739255a698; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("workerd::cluster"); +$Cxx.allowCancellation; + +struct VatId { + publicKey @0 :Data; + # 32-byte raw X25519 public key. +} + +# The following are stubs for v1. They are not used because canIntroduceTo() returns +# false (the default), so the RPC system falls back to proxying for any cross-peer +# capability passing. Cap'n Proto itself has not yet fully implemented Level 3, so +# leaving these empty is fine. +struct ThirdPartyToContact {} +struct ThirdPartyToAwait {} +struct ThirdPartyCompletion {} + +# Level 4 (Join) is not supported. +struct JoinResult {} From 0c5857ed961b1876f055133c935a41761c386159 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Fri, 22 May 2026 16:40:36 -0500 Subject: [PATCH 04/33] Add cluster DO ownership lock file protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements ClusterLockManager, which manages the lock files that serialize ownership of Durable Objects across a workerd cluster. Each lock file is named after the actor ID and contains the owner's 32-byte X25519 public key (or is empty if unowned). acquireOrRoute() encapsulates the full protocol: - If the lock file is the expected 32-byte size, read the owner key and look it up in the ClusterRegistry. If the owner is live (or might be), return a WorkerdDebugPort capability via clusterRpc.bootstrap() — the RpcSystem short-circuits self-routing without going through the VatNetwork. - Otherwise (empty file or confirmed-dead owner), try to acquire an exclusive OFD lock. If granted, ftruncate, write our identity, fsync, and return an OwnedLock. If refused, retry with exponential backoff and jitter. No checksum is needed on the lock file payload: a single pwrite() of 32 bytes is atomic with respect to concurrent readers (both on local filesystems and over NFS, where it's a single RPC to the server). The writer's sequence (ftruncate → pwrite → fsync) means readers see either size 0 or the full 32-byte key, never a partial write. This is a deviation from the original spec, which called for a CRC32. OwnedLock is an RAII handle whose destructor truncates the file and releases the lock, returning the DO to the unowned pool. verifyStillOwned() is provided for the NFSv4 lease check that the SQLite VFS will call on every file open (Commit 4). This code (and commit message) was originally written by Opus 4.6 with a fair amount of manual cleanup. --- src/workerd/server/BUILD.bazel | 28 ++ src/workerd/server/cluster-lock-test.c++ | 323 +++++++++++++++++++++++ src/workerd/server/cluster-lock.c++ | 162 ++++++++++++ src/workerd/server/cluster-lock.h | 77 ++++++ 4 files changed, 590 insertions(+) create mode 100644 src/workerd/server/cluster-lock-test.c++ create mode 100644 src/workerd/server/cluster-lock.c++ create mode 100644 src/workerd/server/cluster-lock.h diff --git a/src/workerd/server/BUILD.bazel b/src/workerd/server/BUILD.bazel index a0042482eb2..5ad0c476f06 100644 --- a/src/workerd/server/BUILD.bazel +++ b/src/workerd/server/BUILD.bazel @@ -266,6 +266,22 @@ wd_cc_library( ], ) +wd_cc_library( + name = "cluster-lock", + srcs = ["cluster-lock.c++"], + hdrs = ["cluster-lock.h"], + visibility = ["//visibility:public"], + deps = [ + ":cluster-registry", + ":cluster_capnp", + "//src/workerd/io:worker-interface_capnp", + "//src/workerd/util:ofd-lock", + "@capnp-cpp//src/capnp:capnp-rpc", + "@capnp-cpp//src/kj", + "@capnp-cpp//src/kj:kj-async", + ], +) + wd_capnp_library(src = "docker-api.capnp") wd_capnp_library(src = "log-schema.capnp") @@ -449,6 +465,18 @@ kj_test( ], ) +kj_test( + src = "cluster-lock-test.c++", + deps = [ + ":cluster-lock", + ":cluster-registry", + "//src/workerd/io:worker-interface_capnp", + "@capnp-cpp//src/capnp:capnp-rpc", + "@capnp-cpp//src/kj", + "@capnp-cpp//src/kj:kj-async", + ], +) + kj_test( src = "json-logger-test.c++", deps = [ diff --git a/src/workerd/server/cluster-lock-test.c++ b/src/workerd/server/cluster-lock-test.c++ new file mode 100644 index 00000000000..980a76de145 --- /dev/null +++ b/src/workerd/server/cluster-lock-test.c++ @@ -0,0 +1,323 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "cluster-lock.h" +#include "cluster-registry.h" + +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace workerd { +namespace { + +#if __linux__ + +// Helper to create a temp directory on disk with a `registry/` and `locks/` subdir, mirroring +// the production layout (the cluster registry directory and the per-namespace locks directory +// are always separate). +class TempDir { + public: + TempDir() { + char tmpl[] = "/tmp/cluster-lock-test-XXXXXX"; + KJ_ASSERT(mkdtemp(tmpl) != nullptr); + pathStr = kj::str(tmpl); + disk = kj::newDiskFilesystem(); + root = disk->getRoot().openSubdir(kj::Path::parse(pathStr.slice(1)), // remove leading / + kj::WriteMode::MODIFY); + locksDir = root->openSubdir(kj::Path({"locks"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + } + + ~TempDir() noexcept(false) { + auto p = kj::Path::parse(pathStr.slice(1)); + disk->getRoot().remove(p); + } + + kj::Own registry() { + return root->openSubdir(kj::Path({"registry"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + } + + kj::Own locks() { + return locksDir->clone(); + } + + // Read the content of a lock file directly (for asserting that the file is in the expected state). + kj::Array readLockFile(kj::StringPtr name) { + auto file = KJ_ASSERT_NONNULL(locksDir->tryOpenFile(kj::Path({name}))); + return file->readAllBytes(); + } + + // Write raw bytes to a lock file (for setting up test scenarios). + void writeLockFile(kj::StringPtr name, kj::ArrayPtr data) { + auto file = locksDir->openFile(kj::Path({name}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + file->writeAll(data); + } + + private: + kj::String pathStr; + kj::Own disk; + kj::Own root; + kj::Own locksDir; +}; + +// Stub WorkerdDebugPort server used as a per-node bootstrap. We don't actually invoke methods on +// it from these tests; we just need a capability to hand to RpcSystem. +class StubDebugPort final: public rpc::WorkerdDebugPort::Server {}; + +KJ_TEST("ClusterLockManager: unowned (empty file) -> acquire succeeds") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + auto result = lockManager.acquireOrRoute("a1").wait(io.waitScope); + KJ_ASSERT(result.is()); + + // The lock file should now contain our key. + auto content = tmpDir.readLockFile("a1"); + KJ_ASSERT(content.size() == 32); + KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); +} + +KJ_TEST("ClusterLockManager: owned by self -> returns self-bootstrap, no wire traffic") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + // First acquire to populate the lock file with our own key. + auto ownedResult = lockManager.acquireOrRoute("a2").wait(io.waitScope); + KJ_ASSERT(ownedResult.is()); + + // Now simulate someone else also calling acquireOrRoute on the same actor (i.e. a forwarding + // path). With the OwnedLock still held, the lock file is non-empty and our key is the owner. + // acquireOrRoute should return a bootstrap client (routing to ourselves). + auto routeResult = lockManager.acquireOrRoute("a2").wait(io.waitScope); + KJ_ASSERT(routeResult.is()); +} + +KJ_TEST("ClusterLockManager: stale owner (peer dead) -> claim path runs") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + // Manually create a lock file naming a non-existent peer as owner. Since `deadKey`'s registry + // file doesn't exist in tmpDir, isPeerDead(deadKey) returns true and the claim path should run. + X25519PublicKey deadKey; + memset(deadKey.bytes, 0xAB, sizeof(deadKey.bytes)); + + tmpDir.writeLockFile("a3", kj::arrayPtr(deadKey.bytes, 32)); + + // Confirm isPeerDead returns true for the dead key. + KJ_EXPECT(reg.isPeerDead(deadKey)); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + // acquireOrRoute should run the claim path and succeed. + auto result = lockManager.acquireOrRoute("a3").wait(io.waitScope); + KJ_ASSERT(result.is()); + + // The lock file should now contain *our* key. + auto content = tmpDir.readLockFile("a3"); + KJ_ASSERT(content.size() == 32); + KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); +} + +KJ_TEST("ClusterLockManager: wrong-sized stale content + writer dead -> claim immediately") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + // Write a lock file of unexpected size (e.g. a crashed writer left a half-written file). + // Nobody holds the lock, so the claim path should succeed. + kj::byte garbage[10]; + memset(garbage, 0xFF, sizeof(garbage)); + tmpDir.writeLockFile("a4", kj::arrayPtr(garbage, sizeof(garbage))); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + auto result = lockManager.acquireOrRoute("a4").wait(io.waitScope); + KJ_ASSERT(result.is()); + + // The lock file should now contain our key. + auto content = tmpDir.readLockFile("a4"); + KJ_ASSERT(content.size() == 32); + KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); +} + +KJ_TEST("ClusterLockManager: OwnedLock destructor truncates and releases") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + { + auto result = lockManager.acquireOrRoute("a5").wait(io.waitScope); + KJ_ASSERT(result.is()); + // Lock file should have our key. + auto content = tmpDir.readLockFile("a5"); + KJ_ASSERT(content.size() == 32); + } + + // After OwnedLock destructor, the file should be truncated to 0 bytes (and the OFD lock + // released). + auto content = tmpDir.readLockFile("a5"); + KJ_EXPECT(content.size() == 0); + + // A second acquire should now succeed via the claim path. + auto result2 = lockManager.acquireOrRoute("a5").wait(io.waitScope); + KJ_ASSERT(result2.is()); +} + +KJ_TEST("ClusterLockManager: owned by live peer -> returns bootstrap client") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + // Stand up two registries in the same dir (this creates registry entries for both, so neither + // is "dead"). + ClusterRegistry reg1( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + ClusterRegistry reg2( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc1 = capnp::makeRpcServer(reg1, capnp::Capability::Client(kj::heap())); + + // reg1 owns a ClusterLockManager; we'll set up a lock file naming reg2 as the owner. + // reg2 is alive (its registry file exists in tmpDir), so isPeerDead(reg2) returns false, + // and acquireOrRoute should return a bootstrap client without falling through to claim. + // + // Note: actor IDs aren't hex of 64 chars, so they won't collide with registry filenames. + tmpDir.writeLockFile("a7", kj::arrayPtr(reg2.getPublicKey().bytes, 32)); + + KJ_EXPECT(!reg1.isPeerDead(reg2.getPublicKey())); + + ClusterLockManager lockManager(tmpDir.locks(), reg1, rpc1, io.provider->getTimer()); + + auto result = lockManager.acquireOrRoute("a7").wait(io.waitScope); + KJ_ASSERT(result.is()); + + // The lock file should NOT have been modified (we did not claim). + auto content = tmpDir.readLockFile("a7"); + KJ_ASSERT(content.size() == 32); + KJ_EXPECT(memcmp(content.begin(), reg2.getPublicKey().bytes, 32) == 0); +} + +KJ_TEST("ClusterLockManager: writer alive -> retries until writer releases") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + // Open the lock file from outside the ClusterLockManager and take an exclusive lock. This + // simulates a writer that holds the lock but hasn't yet flushed its key. The file is left + // empty (matching the freshly-truncated state during the writer's claim path). + auto externalFile = + tmpDir.locks()->openFile(kj::Path({"a8"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + int externalFd = KJ_REQUIRE_NONNULL(externalFile->getFd()); + + auto externalLock = KJ_ASSERT_NONNULL(OfdLock::tryLock(externalFd, OfdLock::EXCLUSIVE)); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + // acquireOrRoute should see an empty file, attempt to lock, fail, then retry with backoff. + auto promise = lockManager.acquireOrRoute("a8"); + + bool completed = false; + auto trackedPromise = promise.then( + [&](kj::OneOf&& r) { + completed = true; + return kj::mv(r); + }); + + // Pump the event loop briefly; the implementation should be stuck in its backoff loop. + io.provider->getTimer().afterDelay(50 * kj::MILLISECONDS).wait(io.waitScope); + KJ_EXPECT(!completed, "acquireOrRoute should still be waiting for the external lock"); + + // Now the "external writer" writes its key (our own, for simplicity) and releases the lock. + // acquireOrRoute, on its next iteration, should see the valid key, recognize the owner as us, + // and return a bootstrap client. + externalFile->write(0, kj::arrayPtr(reg.getPublicKey().bytes, 32)); + { auto _ = kj::mv(externalLock); } + + auto result = trackedPromise.wait(io.waitScope); + KJ_EXPECT(completed); + KJ_ASSERT(result.is()); +} + +KJ_TEST("ClusterLockManager: concurrent acquisitions, exactly one wins") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + // Launch several concurrent calls. + constexpr uint N = 5; + kj::Vector>> + promises; + for (uint i = 0; i < N; ++i) { + promises.add(lockManager.acquireOrRoute("a9")); + } + + auto results = kj::joinPromises(promises.releaseAsArray()).wait(io.waitScope); + + uint ownedCount = 0; + uint routedCount = 0; + for (auto& r: results) { + if (r.is()) { + ++ownedCount; + } else { + ++routedCount; + } + } + // Exactly one should have won. + KJ_EXPECT(ownedCount == 1); + KJ_EXPECT(routedCount == N - 1); +} + +#else // #if __linux__ + +KJ_TEST("dummy test: platform not supported") {} + +#endif // #if __linux__, #else + +} // namespace +} // namespace workerd diff --git a/src/workerd/server/cluster-lock.c++ b/src/workerd/server/cluster-lock.c++ new file mode 100644 index 00000000000..c3480e8a95d --- /dev/null +++ b/src/workerd/server/cluster-lock.c++ @@ -0,0 +1,162 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "cluster-lock.h" + +#include +#include +#include + +namespace workerd { + +namespace { + +// The lock file contains exactly one X25519 public key (the owner). No checksum is needed: a +// single write of `sizeof(X25519PublicKey::bytes)` bytes is atomic with respect to concurrent +// readers on Linux (and likewise on NFS, where it's a single RPC to the server). The writer's +// protocol is truncate(0) → write(full key) → sync, so a reader interleaving sees either size +// 0 (treat as unowned, run claim path) or the full key. + +constexpr kj::Duration INITIAL_BACKOFF = 10 * kj::MILLISECONDS; +constexpr kj::Duration MAX_BACKOFF = 1 * kj::SECONDS; + +// Build a `cluster::VatId` Reader for a given public key, backed by an internal `MessageBuilder`. +class VatIdHolder { + public: + explicit VatIdHolder(const X25519PublicKey& key): msg(scratch) { + auto vatId = msg.initRoot(); + vatId.setPublicKey(kj::ArrayPtr(key.bytes)); + } + + cluster::VatId::Reader getReader() { + return msg.getRoot().asReader(); + } + + private: + capnp::word scratch[8]{}; + capnp::MallocMessageBuilder msg; +}; + +// Compute the next backoff delay with ±25% jitter. `attempt` starts at 0. +kj::Duration computeBackoff(uint attempt) { + // Exponential: 10ms, 20ms, 40ms, ..., capped at 1s. + kj::Duration base = INITIAL_BACKOFF; + for (uint i = 0; i < attempt && base < MAX_BACKOFF; ++i) { + base = base * 2; + } + if (base > MAX_BACKOFF) base = MAX_BACKOFF; + + // Jitter: multiply by a random value in [0.75, 1.25]. Use a cheap pseudo-random source. + auto nanos = + (kj::systemPreciseMonotonicClock().now() - kj::origin()) / kj::NANOSECONDS; + uint32_t r = kj::hashCode(nanos); + double jitter = 0.75 + (r / static_cast(0xFFFFFFFFu)) * 0.5; + return base * static_cast(jitter * 1000) / 1000; +} + +} // namespace + +// ======================================================================================= +// ClusterLockManager::OwnedLock + +ClusterLockManager::OwnedLock::~OwnedLock() noexcept(false) { + if (file.get() != nullptr) { + // Truncate the file back to zero so the next claimant sees an empty file. We then drop the + // OFD lock (via OfdLock's destructor) which makes the file available for other nodes to claim. + // + // Best-effort: don't throw from the destructor. + KJ_TRY { + file->truncate(0); + } + KJ_CATCH(e) { + KJ_LOG(WARNING, "truncate(0) on cluster lock file failed during release", e); + } + } +} + +// ======================================================================================= +// ClusterLockManager + +ClusterLockManager::ClusterLockManager(kj::Own dir, + ClusterRegistry& registry, + capnp::RpcSystem& clusterRpc, + kj::Timer& timer) + : dir(kj::mv(dir)), + registry(registry), + clusterRpc(clusterRpc), + timer(timer) {} + +kj::Promise> +ClusterLockManager::acquireOrRoute(kj::StringPtr actorId) { + // Validate that the actor ID is a hex string. This is the only form of actor ID we expect on + // this path (durable actors named by a 64-char hex SHA-256 hash, or by `idFromName` which also + // produces hex). Enforcing hex incidentally guarantees the ID is safe to use as a filename. + for (char c: actorId) { + KJ_REQUIRE((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'), + "actor ID is not a hex string", actorId); + } + KJ_REQUIRE(actorId.size() > 0, "actor ID is empty"); + + kj::Path path({actorId}); + + for (uint attempt = 0;; ++attempt) { + if (attempt > 0) { + co_await timer.afterDelay(computeBackoff(attempt - 1)); + } + + // Open or create the lock file. + auto file = dir->openFile(path, kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + + // Read the owner key directly. read() returns: + // 0 → file is empty (freshly created, or previous owner cleanly + // released) + // sizeof(ownerKey.bytes) → full key present + // anything else → unexpected partial state (e.g. writer crashed mid-protocol) + // + // Since the key has a fixed width, we can assume that if we read the entire key, it is the + // complete and valid key. If the key is not done being written, we could get a short read + // (unlikely, since you'd expect a 32-byte read to always be atomic, but it's not guaranteed). + X25519PublicKey ownerKey; + auto n = file->read(0, ownerKey.bytes); + + if (n == sizeof(ownerKey.bytes)) { + if (!registry.isPeerDead(ownerKey)) { + // Trust the lock file. Build a VatId and ask the RpcSystem to bootstrap. + // The owner might be us — clusterRpc.bootstrap() returns the local bootstrap directly + // without going through ClusterRegistry::connect() in that case. + // + // If the owner is actually unreachable, the bootstrap call will fail later. Internally, + // ClusterRegistry::ConnectionImpl::unregister() runs the dead-peer cleanup probe; + // if the peer's registry file is unlocked, the file is unlinked. Whoever retries + // acquireOrRoute() will then see isPeerDead(ownerKey) return true and fall through to + // the claim path. + VatIdHolder holder(ownerKey); + co_return clusterRpc.bootstrap(holder.getReader()).castAs(); + } + + // Owner is confirmed dead. Fall through to the claim path. + } + // else: n == 0 (empty file) or n is some unexpected partial size. In either case, the claim + // path is the right next step: if a live writer is currently producing the file, they + // hold the exclusive lock and our tryLock will fail; if no one holds the lock, we'll + // succeed and overwrite whatever was there. + + // Claim path. Try to acquire an exclusive OFD lock. + int fd = KJ_REQUIRE_NONNULL(file->getFd(), "lock directory must be disk-backed"); + KJ_IF_SOME(lock, OfdLock::tryLock(fd, OfdLock::EXCLUSIVE)) { + // We got the exclusive lock. Clear any stale content and write our identity. + file->truncate(0); + + const auto& myKey = registry.getPublicKey(); + file->write(0, myKey.bytes); + file->sync(); + + co_return OwnedLock(kj::mv(file), kj::mv(lock)); + } + + // tryLock failed — someone else owns it (or is racing us to claim it). Retry. + } +} + +} // namespace workerd diff --git a/src/workerd/server/cluster-lock.h b/src/workerd/server/cluster-lock.h new file mode 100644 index 00000000000..d259968a48e --- /dev/null +++ b/src/workerd/server/cluster-lock.h @@ -0,0 +1,77 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace workerd { + +// Manages a directory of lock files for DO ownership. Each lock file is named after the actor ID +// and contains the owning node's public key (or is empty if unowned). +// +// The ownership protocol: +// 1. Open/create the lock file. +// 2. Read it. If non-empty and CRC32 valid, parse the owner key. +// - If !registry.isPeerDead(ownerKey): route via clusterRpc.bootstrap(vatIdFor(ownerKey)). +// - If registry.isPeerDead(ownerKey): fall through to the claim path. +// 3. Claim path: tryLock(EXCLUSIVE). If granted, ftruncate, write our key, fsync, +// return OwnedLock. If refused, retry with backoff. +class ClusterLockManager { + public: + ClusterLockManager(kj::Own dir, + ClusterRegistry& registry, + capnp::RpcSystem& clusterRpc, + kj::Timer& timer); + + // RAII ownership handle. Holding this object means this node owns the DO. + // Destruction truncates the lock file and releases the exclusive OFD lock, + // making the DO available for other nodes to claim. + class OwnedLock { + public: + ~OwnedLock() noexcept(false); + OwnedLock(OwnedLock&&) = default; + OwnedLock& operator=(OwnedLock&&) = default; + KJ_DISALLOW_COPY(OwnedLock); + + private: + OwnedLock(kj::Own file, OfdLock lock): file(kj::mv(file)), lock(kj::mv(lock)) {} + + kj::Own file; + OfdLock lock; + friend class ClusterLockManager; + }; + + // The core routing operation. Either acquires local ownership or returns a capability for + // reaching the current owner. + // + // Encapsulates the full protocol including retries with backoff, registry liveness + // cross-referencing, and capability resolution (with retry on stale owners). + // + // Returns: + // OwnedLock — this node has newly acquired ownership (caller should start the DO). + // rpc::WorkerdDebugPort::Client — the DO has an existing owner. The caller calls + // getActor() on this to reach the actor. The owner might be this node, in which case + // the RpcSystem returns the local self-bootstrap directly (no wire traffic). The + // caller does not need to distinguish local vs remote. + kj::Promise> acquireOrRoute( + kj::StringPtr actorId); + + private: + kj::Own dir; + ClusterRegistry& registry; + capnp::RpcSystem& clusterRpc; + kj::Timer& timer; +}; + +} // namespace workerd From 43a3d303fb12c29ae57fc9aefc7e2a4474306c8f Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 23 May 2026 15:25:05 -0500 Subject: [PATCH 05/33] Add a post-open callback to SQLite. This callback is invoked immediately after `open()` opens a file. In NFS mode we'll use this hook to verify the new open was opened on the same NFS lease as the registry lock. --- src/workerd/util/sqlite.c++ | 29 +++++++++++++++++++++++++++-- src/workerd/util/sqlite.h | 5 +++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/workerd/util/sqlite.c++ b/src/workerd/util/sqlite.c++ index 9a05922dc03..96607b2f16a 100644 --- a/src/workerd/util/sqlite.c++ +++ b/src/workerd/util/sqlite.c++ @@ -2020,7 +2020,7 @@ sqlite3_vfs SqliteDatabase::Vfs::makeWrappedNativeVfs() { file->pMethods = nullptr; // Set up currentVfsRoot. - auto& self = *reinterpret_cast(vfs->pAppData); + auto& self = *reinterpret_cast(vfs->pAppData); KJ_ASSERT(currentVfsRoot == AT_FDCWD); currentVfsRoot = self.rootFd; KJ_DEFER(currentVfsRoot = AT_FDCWD); @@ -2032,6 +2032,16 @@ sqlite3_vfs SqliteDatabase::Vfs::makeWrappedNativeVfs() { if (file->pMethods == nullptr) { wrapper->pMethods = nullptr; } else { + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + try { + afterOpen(); + } catch (kj::Exception& e) { + reportVfsErrorCaught(kj::mv(e)); + file->pMethods->xClose(file); + wrapper->pMethods = nullptr; + return SQLITE_CANTOPEN; + } + } wrapper->pMethods = &WrappedNativeFileImpl::METHOD_TABLE; wrapper->vfs = &self; wrapper->rootFd = self.rootFd; @@ -2336,7 +2346,7 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { .pAppData = this, #define WRAP_METHOD(errorCode, block) \ - auto& self KJ_UNUSED = *static_cast(vfs->pAppData); \ + auto& self KJ_UNUSED = *static_cast(vfs->pAppData); \ try block catch (kj::Exception& e) { \ KJ_LOG(ERROR, "SQLite VFS I/O error", e); \ return errorCode; \ @@ -2353,6 +2363,11 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { auto path = kj::Path::parse(zName); auto kjFile = KJ_UNWRAP_OR(self.directory.tryOpenFile(path), { return SQLITE_CANTOPEN; }); + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + afterOpen(); + } else { + // sqelch spurious "dangling else" warning from clang + } kj::Maybe> lock; if (flags & SQLITE_OPEN_MAIN_DB) { lock = self.lockManager.lock(path, *kjFile); @@ -2368,6 +2383,11 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { KJ_ASSERT(flags & SQLITE_OPEN_DELETEONCLOSE); KJ_ASSERT(!(flags & SQLITE_OPEN_MAIN_DB), "main DB can't be a temporary file"); kjFile = self.directory.createTemporary(); + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + afterOpen(); + } else { + // sqelch spurious "dangling else" warning from clang + } } else { kj::WriteMode mode; if (flags & SQLITE_OPEN_CREATE) { @@ -2383,6 +2403,11 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { auto path = kj::Path::parse(zName); kjFile = KJ_UNWRAP_OR(self.directory.tryOpenFile(path, mode), { return SQLITE_CANTOPEN; }); + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + afterOpen(); + } else { + // sqelch spurious "dangling else" warning from clang + } if (flags & SQLITE_OPEN_MAIN_DB) { lock = self.lockManager.lock(path, *kjFile); } diff --git a/src/workerd/util/sqlite.h b/src/workerd/util/sqlite.h index b007a54f160..68535f5a8f0 100644 --- a/src/workerd/util/sqlite.h +++ b/src/workerd/util/sqlite.h @@ -817,6 +817,11 @@ struct SqliteDatabase::VfsOptions { // will fall back to the native VFS implementation. In that case, the options you set here will // be ORed with the ones set by the underlying VFS. int deviceCharacteristics = 0x00001000; // = SQLITE_FCNTL_POWERSAFE_OVERWRITE + + // Called immediately after SQLite opens any file through this VFS. This is used by clustered + // Durable Object storage to verify that the original NFS lease is still valid and therefore + // covers the newly-opened file. + kj::Maybe> afterOpen; }; // Implements a SQLite VFS based on a KJ directory. From aff1673eb087e0cb01e2df22ffcf248b7b7e26f5 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 23 May 2026 15:27:22 -0500 Subject: [PATCH 06/33] Extend ChannelTokenHandler to use shared keys in cluster mode. In cluster mode, all members of the cluster need to use the same key for channel tokens, so they can exchange them. --- src/workerd/server/channel-token.c++ | 27 +++++++++++++++++++++++++-- src/workerd/server/channel-token.h | 2 +- src/workerd/server/server.c++ | 16 ++++++++++++++-- src/workerd/server/server.h | 4 +++- 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/workerd/server/channel-token.c++ b/src/workerd/server/channel-token.c++ index c9b4c982caa..9686842ba62 100644 --- a/src/workerd/server/channel-token.c++ +++ b/src/workerd/server/channel-token.c++ @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -22,9 +23,31 @@ namespace workerd::server { -ChannelTokenHandler::ChannelTokenHandler(Resolver& resolver): resolver(resolver) { - getEntropy(tokenKey); +ChannelTokenHandler::ChannelTokenHandler(Resolver& resolver, kj::Maybe clusterKey) + : resolver(resolver) { + KJ_IF_SOME(key, clusterKey) { + // All nodes in the cluster must derive the same key, based on `clusterKey`. We HMAC the + // `clusterKey` with a randomly-chosen salt to derive the channel token key. + + // clang-format off + static constexpr byte SALT[] = { + 0x9a, 0x24, 0x71, 0x39, 0x1b, 0x85, 0xce, 0x97, + 0x6c, 0xf9, 0x1c, 0xdf, 0x93, 0xc0, 0xc6, 0x36, + 0xb9, 0xd2, 0x09, 0xfe, 0x01, 0xde, 0xb1, 0x9a, + 0xda, 0xd3, 0x8e, 0x76, 0xbb, 0xae, 0xeb, 0x89, + }; + // clang-format on + + uint outLen = 0; + KJ_ASSERT(HMAC(EVP_sha256(), SALT, sizeof(SALT), key.asBytes().begin(), key.size(), tokenKey, + &outLen) != nullptr); + KJ_ASSERT(outLen == sizeof(tokenKey)); + } else { + // Single-instance, generate a new key for every run. + getEntropy(tokenKey); + } + // Construct key ID by hashing the key. SHA256_CTX ctx{}; KJ_ASSERT(SHA256_Init(&ctx)); KJ_ASSERT(SHA256_Update(&ctx, tokenKey, sizeof(tokenKey))); diff --git a/src/workerd/server/channel-token.h b/src/workerd/server/channel-token.h index 06943c62e5d..39a2a4f875b 100644 --- a/src/workerd/server/channel-token.h +++ b/src/workerd/server/channel-token.h @@ -58,7 +58,7 @@ class ChannelTokenHandler { IoChannelFactory::ChannelTokenUsage usage) = 0; }; - explicit ChannelTokenHandler(Resolver& resolver); + explicit ChannelTokenHandler(Resolver& resolver, kj::Maybe clusterKey = kj::none); // Helpers to implement `IoChannelFactory::{SubrequestChannel,ActorClassChannel}::getToken()`. // diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index c7e137fde87..13bbfd79956 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -238,7 +238,6 @@ Server::Server(kj::Filesystem& fs, reportConfigWarning(kj::mv(reportConfigWarning)), loggingOptions(loggingOptions), memoryCacheProvider(kj::heap(timer)), - channelTokenHandler(*this), tasks(*this) {} struct Server::GlobalContext { @@ -6097,7 +6096,7 @@ kj::Promise> Server::makeWorkerImpl(kj::StringPtr kj::Maybe serviceName; if (!def.isDynamic) serviceName = name; - auto result = kj::refcounted(channelTokenHandler, serviceName, + auto result = kj::refcounted(KJ_ASSERT_NONNULL(channelTokenHandler), serviceName, globalContext->threadContext, monotonicClock, kj::mv(worker), kj::mv(errorReporter.defaultEntrypoint), kj::mv(errorReporter.namedEntrypoints), kj::mv(errorReporter.actorClasses), kj::mv(linkCallback), @@ -6877,6 +6876,8 @@ kj::Promise Server::run( loggingOptions.structuredLogging = StructuredLogging(config.getStructuredLogging()); } + setupChannelTokenHandler(config); + kj::HttpHeaderTable::Builder headerTableBuilder; globalContext = kj::heap(*this, v8System, headerTableBuilder); invalidConfigServiceSingleton = kj::refcounted(); @@ -6899,6 +6900,15 @@ kj::Promise Server::run( co_return co_await listenPromise.exclusiveJoin(kj::mv(fatalPromise)); } +void Server::setupChannelTokenHandler(config::Config::Reader config) { + ChannelTokenHandler::Resolver& tokenResolver = *this; + kj::Maybe channelTokenKey; + if (config.hasCluster()) { + channelTokenKey = config.getCluster().getKey(); + } + channelTokenHandler.emplace(tokenResolver, channelTokenKey); +} + // Configure and start the inspector socket, returning the port the socket started on. uint startInspector( kj::StringPtr inspectorAddress, Server::InspectorServiceIsolateRegistrar& registrar) { @@ -7304,6 +7314,8 @@ kj::Promise Server::test(jsg::V8System& v8System, loggingOptions.structuredLogging = StructuredLogging(config.getStructuredLogging()); } + setupChannelTokenHandler(config); + kj::HttpHeaderTable::Builder headerTableBuilder; globalContext = kj::heap(*this, v8System, headerTableBuilder); invalidConfigServiceSingleton = kj::refcounted(); diff --git a/src/workerd/server/server.h b/src/workerd/server/server.h index 83364059f2e..c3fe15c24e5 100644 --- a/src/workerd/server/server.h +++ b/src/workerd/server/server.h @@ -162,7 +162,7 @@ class Server final: private kj::TaskSet::ErrorHandler, private ChannelTokenHandl kj::Own memoryCacheProvider; - ChannelTokenHandler channelTokenHandler; + kj::Maybe channelTokenHandler; kj::HashMap>> socketOverrides; kj::HashMap directoryOverrides; @@ -358,6 +358,8 @@ class Server final: private kj::TaskSet::ErrorHandler, private ChannelTokenHandl void unlinkWorkerLoaders(); + void setupChannelTokenHandler(config::Config::Reader config); + kj::Promise preloadPython( kj::StringPtr workerName, const WorkerDef& workerDef, ErrorReporter& errorReporter); From 8d18d54377e8612509bc14d348e8e0dac1750bdd Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 23 May 2026 15:32:47 -0500 Subject: [PATCH 07/33] Extend WorkerdDebugPort to include actor name with ID. If the actor ID was derived from a name, it's important to pass the name along with the ID so that `ctx.id.name` works correctly. --- src/workerd/io/worker-interface.capnp | 5 ++++- src/workerd/server/server.c++ | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/workerd/io/worker-interface.capnp b/src/workerd/io/worker-interface.capnp index 597ad67e1de..adc17e184e8 100644 --- a/src/workerd/io/worker-interface.capnp +++ b/src/workerd/io/worker-interface.capnp @@ -1011,7 +1011,10 @@ interface WorkerdDebugPort { -> (entrypoint :WorkerdBootstrap); # Get direct access to a stateless entrypoint. - getActor @1 (service :Text, entrypoint :Text, actorId :Text) -> (actor :WorkerdBootstrap); + getActor @1 (service :Text, entrypoint :Text, actorId :Text, actorName :Text) + -> (actor :WorkerdBootstrap); # Get an actor (Durable Object) stub. # The actorId should be a hex string for Durable Objects or a plain string for ephemeral actors. + # `actorName` may optionally be specified for Durable Objects, if `actorId` is derived from the + # name. } diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 13bbfd79956..47c867f7df7 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -6790,8 +6790,12 @@ class Server::WorkerdDebugPortImpl final: public rpc::WorkerdDebugPort::Server { auto decoded = kj::decodeHex(actorIdStr); KJ_REQUIRE(decoded.size() == SHA256_DIGEST_LENGTH, "Invalid Durable Object ID: expected 64 hex characters (32 bytes)", decoded.size()); + kj::Maybe name; + if (params.hasActorName()) { + name = params.getActorName().clone(); + } kj::Own id = - kj::heap(decoded.begin(), kj::none); + kj::heap(decoded.begin(), kj::mv(name)); actorId = kj::mv(id); } KJ_CASE_ONEOF(c, Ephemeral) { From 60cbfd79204ab6e931172b1055ffbc3b499bc7f4 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 23 May 2026 16:58:22 -0500 Subject: [PATCH 08/33] Make self-hosted Durable Objects scalable across a cluster. (This is the commit that wires everything up, but this commit message describes the overall series.) workerd has always been intended to be something you can run in production, in order to self-host Workers outside of Cloudflare. For many use cases it does in fact work for this. But, there has been one missing piece: Durable Objects. workerd has always supported running Durable Objects for the purpose of local testing, but only within the scope of a single instance running on a single thread. Any Durable Objects created would run within that same workerd instance. But in production, you probably want to utilize more than one thread on one machine. For stateless Workers, this could be done just fine: just run multiple instances of workerd and load-balance across them. But doing this completely broke the model of Durable Objects, where each object is supposed to have a single instance globally, not one per workerd instance! This change fixes the problem, by introducing a new "cluster" mode. In cluster mode, you can run several workerd instances that are aware of each other and able to route requests to each other. All instances in a cluster run the same config. Mostly these workerd instances behave like they would normally, except when a request is made to a Durable Object, they coordinate to make sure only one workerd instance owns the DO, and others route to it. The design assumes a shared filesystem for underlying DO storage. All instances must be on the same filesystem. If all instances are on the same machine (useful for utilizing multiple cores), then this can be any local filesystem. Otherwise, it must be NFSv4, or some network filesystem that has exactly the same lock/lease semantics as NFSv4. The shared filesystem is also used for service discovery and locking. This is unconventional, but has a major advantage vs something like etcd or Consul: If a node loses its NFS lease, it simultaneously loses its locks *and* loses the ability to write to any open files. This provides "fencing": there's no way a node could continue writing after other nodes believe that it is dead. If locking were provided by a separate service, then it becomes extremely difficult to ensure that a node can't accidentally write after losing its lock due to a timeout. A complete design doc can be found here: https://gist.github.com/kentonv/cd8001237dc1181058193de0e7509972 The "Implementation Plan" in the design doc was largely written by AI (with several rounds of revision guided by me); the sections before that were largely written by me (with some AI review and minor changes). GPT 5.5, Opus 4.6, and Opus 4.7 were all employed. A blog post will come later. This particular commit was initially written by GPT 5.5 but heavily refactored by hand. --- src/workerd/server/BUILD.bazel | 5 + src/workerd/server/server.c++ | 315 ++++++++++++++++++++++++++++++--- src/workerd/server/server.h | 5 + 3 files changed, 302 insertions(+), 23 deletions(-) diff --git a/src/workerd/server/BUILD.bazel b/src/workerd/server/BUILD.bazel index 5ad0c476f06..57a66453830 100644 --- a/src/workerd/server/BUILD.bazel +++ b/src/workerd/server/BUILD.bazel @@ -212,6 +212,7 @@ wd_cc_library( ":channel-token_capnp", "//src/workerd/io", "//src/workerd/util:entropy", + "@ssl", ], ) @@ -228,6 +229,9 @@ wd_cc_library( ":alarm-scheduler", ":channel-token", ":channel-token_capnp", + ":cluster-lock", + ":cluster-registry", + ":cluster_capnp", ":container-client", ":facet-tree-index", ":fallback-service", @@ -244,6 +248,7 @@ wd_cc_library( "//src/workerd/jsg", "//src/workerd/util:perfetto", "//src/workerd/util:websocket-error-handler", + "@capnp-cpp//src/capnp:capnp-rpc", "@capnp-cpp//src/kj/compat:kj-gzip", "@capnp-cpp//src/kj/compat:kj-tls", ], diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 47c867f7df7..4d916a7502f 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -5,6 +5,7 @@ #include "server.h" #include "alarm-scheduler.h" +#include "cluster-lock.h" #include "container-client.h" #include "pyodide.h" #include "workerd-api.h" @@ -49,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -361,10 +363,13 @@ class Server::ActorNamespace final { friend class Server; ActorNamespace(kj::Own actorClass, + kj::StringPtr serviceName, + kj::StringPtr className, const ActorConfig& config, const kj::Clock& clock, kj::Timer& timer, capnp::ByteStreamFactory& byteStreamFactory, + capnp::HttpOverCapnpFactory& httpOverCapnpFactory, ChannelTokenHandler& channelTokenHandler, kj::Network& dockerNetwork, kj::Maybe dockerPath, @@ -372,10 +377,13 @@ class Server::ActorNamespace final { kj::TaskSet& waitUntilTasks, Persistent selfTokensArePersistent) : actorClass(kj::mv(actorClass)), + serviceName(serviceName), + className(className), config(config), clock(clock), timer(timer), byteStreamFactory(byteStreamFactory), + httpOverCapnpFactory(httpOverCapnpFactory), channelTokenHandler(channelTokenHandler), dockerNetwork(dockerNetwork), dockerPath(dockerPath), @@ -383,15 +391,35 @@ class Server::ActorNamespace final { waitUntilTasks(waitUntilTasks), selfTokensArePersistent(selfTokensArePersistent) {} - void link(kj::Maybe serviceActorStorage) { + class ClusterActorChannel; + + void link(kj::Maybe serviceActorStorage, + kj::Maybe clusterRegistry, + kj::Maybe&> clusterRpc) { KJ_IF_SOME(dir, serviceActorStorage) { KJ_IF_SOME(d, config.tryGet()) { this->actorStorage.emplace( - dir.openSubdir(kj::Path({d.uniqueKey}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY)); + dir.openSubdir(kj::Path({d.uniqueKey}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY), + *this); + KJ_IF_SOME(registry, clusterRegistry) { + auto& rpc = KJ_ASSERT_NONNULL(clusterRpc); + this->clusterRegistry = registry; + auto lockDirectory = KJ_ASSERT_NONNULL(this->actorStorage) + .directory->openSubdir(kj::Path({"locks"}), + kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + this->lockManager.emplace(kj::mv(lockDirectory), registry, rpc, timer); + } } } KJ_IF_SOME(d, config.tryGet()) { + if (lockManager != kj::none) { + // TODO(clustering): Support alarms in cluster mode. Until then, we must skip creating + // an AlarmScheduler altogether otherwise multiple instances will fight over the alarm + // database. + return; + } + auto idFactory = kj::heap(d.uniqueKey); AlarmScheduler::GetActorFn getActor = [this, idFactory = kj::mv(idFactory)]( @@ -438,6 +466,8 @@ class Server::ActorNamespace final { return result; } + // Get the ActorChannel for the given actor ID, routing to the appropriate workerd instance + // if needed. kj::Own getActorChannel( Worker::Actor::Id id, Persistent persistent = Persistent::NO) { KJ_IF_SOME(doId, id.tryGet>()) { @@ -451,7 +481,43 @@ class Server::ActorNamespace final { } } - return kj::refcounted(getActorContainer(kj::mv(id)), persistent); + KJ_IF_SOME(ld, lockManager) { + auto key = actorKey(id); + + // First check if the container exists locally. + KJ_IF_SOME(container, actors.find(key)) { + return kj::refcounted(container->addRef(), persistent); + } + + auto promise = ld.acquireOrRoute(key); + return newPromisedChannel( + promise.then([this, id = kj::mv(id), key = kj::mv(key), persistent]( + kj::OneOf + lockOrClient) mutable -> kj::Own { + KJ_SWITCH_ONEOF(lockOrClient) { + KJ_CASE_ONEOF(ownership, ClusterLockManager::OwnedLock) { + return kj::refcounted( + createActorContainer(kj::mv(key), kj::mv(id), kj::mv(ownership)), persistent); + } + KJ_CASE_ONEOF(debugPort, rpc::WorkerdDebugPort::Client) { + auto req = debugPort.getActorRequest(capnp::MessageSize{32, 0}); + req.setService(serviceName); + req.setEntrypoint(className); + req.setActorId(key); + KJ_IF_SOME(i, id.tryGet>()) { + KJ_IF_SOME(name, i->getName()) { + req.setActorName(name); + } + } + return kj::refcounted( + *this, kj::mv(id), persistent, req.send().getActor()); + } + } + KJ_UNREACHABLE; + })); + } else { + return kj::refcounted(getActorContainer(kj::mv(id)), persistent); + } } class ActorContainer; @@ -484,7 +550,8 @@ class Server::ActorNamespace final { ActorNamespace& ns, kj::Maybe parent, kj::OneOf> classAndIdParam, - kj::Timer& timer) + kj::Timer& timer, + kj::Maybe ownershipLock = kj::none) : key(kj::mv(key)), tracker(kj::refcounted(*this)), ns(ns), @@ -492,7 +559,8 @@ class Server::ActorNamespace final { .orDefault(*this)), parent(parent), timer(timer), - lastAccess(timer.now()) { + lastAccess(timer.now()), + ownershipLock(kj::mv(ownershipLock)) { KJ_SWITCH_ONEOF(classAndIdParam) { KJ_CASE_ONEOF(value, ClassAndId) { // `classAndId` is immediately available. @@ -839,6 +907,7 @@ class Server::ActorNamespace final { kj::Maybe> onBrokenTask; kj::Maybe brokenReason; kj::Vector>> inactiveFulfillers; + kj::Maybe ownershipLock; // in cluster mode // Reference to the ContainerClient (if container is enabled for this actor) kj::Maybe> containerClient; @@ -882,6 +951,7 @@ class Server::ActorNamespace final { ns.actorStorage, "can't call getFacetId() when there's no backing storage"); auto indexFile = as.directory->openFile( kj::Path({kj::str(key, ".facets")}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + ns.verifyRegistration(); return *facetTreeIndex.emplace(kj::heap(kj::mv(indexFile))); } } @@ -899,6 +969,7 @@ class Server::ActorNamespace final { auto indexFile = KJ_UNWRAP_OR( as.directory->tryOpenFile(kj::Path({kj::str(key, ".facets")}), kj::WriteMode::MODIFY), return kj::none); + ns.verifyRegistration(); return *facetTreeIndex.emplace(kj::heap(kj::mv(indexFile))); } } @@ -1244,7 +1315,9 @@ class Server::ActorNamespace final { KJ_IF_SOME(as, ns.actorStorage) { kj::Own sqliteHooks; if (parent == kj::none) { - KJ_IF_SOME(a, ns.alarmScheduler) { + if (ns.lockManager != kj::none) { + sqliteHooks = kj::heap(); + } else KJ_IF_SOME(a, ns.alarmScheduler) { // clone() copies the id and name into storage owned by the returned ActorKey, so // the temporary StringPtrs below only need to outlive this call. auto actorKey = ActorKey{.actorId = key, .name = actorName.map([](kj::String& n) { @@ -1267,9 +1340,15 @@ class Server::ActorNamespace final { // Before we do anything, make sure the database is in WAL mode. We also need to // do this after reset() is used, so register a callback for that. + if (ns.nfsMode()) { + db->run("PRAGMA locking_mode=EXCLUSIVE;"); + } db->run("PRAGMA journal_mode=WAL;"); db->afterReset([this, &dir = *as.directory, selfId](SqliteDatabase& db) { + if (ns.nfsMode()) { + db.run("PRAGMA locking_mode=EXCLUSIVE;"); + } db.run("PRAGMA journal_mode=WAL;"); // reset() is used when the app called deleteAll(), in which case we also want to @@ -1389,28 +1468,29 @@ class Server::ActorNamespace final { kj::Array getChannelTokenImpl(IoChannelFactory::ChannelTokenUsage usage, const Worker::Actor::Id& id, Persistent persistent) { - kj::StringPtr uniqueKey = KJ_ASSERT_NONNULL(ns.getConfig().tryGet()).uniqueKey; - auto& abstractId = *KJ_ASSERT_NONNULL(id.tryGet>()); - auto& idImpl = - KJ_ASSERT_NONNULL(kj::tryDowncast(abstractId)); - return ns.channelTokenHandler.encodeActorChannelToken( - usage, uniqueKey, idImpl.getRaw(), idImpl.getName(), persistent); + return ns.encodeActorChannelToken(usage, id, persistent); } }; - kj::Own getActorContainer(Worker::Actor::Id id) { - kj::String key; - + kj::String actorKey(Worker::Actor::Id& id) { KJ_SWITCH_ONEOF(id) { KJ_CASE_ONEOF(obj, kj::Own) { KJ_REQUIRE(config.is()); - key = obj->toString(); + return obj->toString(); } KJ_CASE_ONEOF(str, kj::String) { KJ_REQUIRE(config.is()); - key = kj::str(str); + return kj::str(str); } } + KJ_UNREACHABLE; + } + + // Get or create an ActorContainer. Use only when NOT in cluster mode. + kj::Own getActorContainer(Worker::Actor::Id id) { + KJ_REQUIRE(lockManager == kj::none); + + kj::String key = actorKey(id); return actors .findOrCreate(key, [&]() mutable { @@ -1422,6 +1502,30 @@ class Server::ActorNamespace final { })->addRef(); } + // Create an ActorContainer in cluster mode. + kj::Own createActorContainer( + kj::String key, Worker::Actor::Id id, ClusterLockManager::OwnedLock ownership) { + KJ_ASSERT(actors.find(key) == kj::none); + + auto container = kj::refcounted(kj::str(key), *this, kj::none, + ActorContainer::ClassAndId(kj::addRef(*actorClass), kj::mv(id)), timer, kj::mv(ownership)); + actors.insert(container->getKey(), container->addRef()); + return container; + } + + // Encode a channel token pointing at the given actor in this namespace. The caller is + // responsible for having checked transferrability. + kj::Array encodeActorChannelToken(IoChannelFactory::ChannelTokenUsage usage, + const Worker::Actor::Id& id, + Persistent persistent) { + kj::StringPtr uniqueKey = KJ_ASSERT_NONNULL(config.tryGet()).uniqueKey; + auto& abstractId = *KJ_ASSERT_NONNULL(id.tryGet>()); + auto& idImpl = + KJ_ASSERT_NONNULL(kj::tryDowncast(abstractId)); + return channelTokenHandler.encodeActorChannelToken( + usage, uniqueKey, idImpl.getRaw(), idImpl.getName(), persistent); + } + kj::Own getContainerClient( kj::StringPtr containerId, kj::StringPtr imageName, ContainerPrivileges privileges) { KJ_IF_SOME(existingClient, containerClients.find(containerId)) { @@ -1534,6 +1638,8 @@ class Server::ActorNamespace final { private: kj::Own actorClass; + kj::StringPtr serviceName; + kj::StringPtr className; const ActorConfig& config; const kj::Clock& clock; @@ -1541,9 +1647,12 @@ class Server::ActorNamespace final { kj::Own directory; SqliteDatabase::Vfs vfs; - ActorStorage(kj::Own directoryParam) + ActorStorage(kj::Own directoryParam, ActorNamespace& ns) : directory(kj::mv(directoryParam)), - vfs(*directory) {} + vfs(*directory, + SqliteDatabase::VfsOptions{ + .afterOpen = kj::Function(KJ_BIND_METHOD(ns, verifyRegistration)), + }) {} }; // Note: The Vfs, actorStorage, and ownAlarmScheduler must not be torn down until all actors @@ -1551,6 +1660,9 @@ class Server::ActorNamespace final { kj::Maybe actorStorage; kj::Maybe> ownAlarmScheduler; + kj::Maybe clusterRegistry; + kj::Maybe lockManager; + // Tracks the canceler and cleanup promise for a Docker container's lifecycle cleanup. // Useful to await on async calls of a ContainerClient destructor when the new // one appears before they've been resolved. @@ -1586,6 +1698,7 @@ class Server::ActorNamespace final { kj::Maybe> cleanupTask; kj::Timer& timer; capnp::ByteStreamFactory& byteStreamFactory; + capnp::HttpOverCapnpFactory& httpOverCapnpFactory; ChannelTokenHandler& channelTokenHandler; kj::Network& dockerNetwork; kj::Maybe dockerPath; @@ -1634,6 +1747,23 @@ class Server::ActorNamespace final { } } + bool nfsMode() { + KJ_IF_SOME(r, clusterRegistry) { + return r.nfsMode(); + } else { + return false; + } + } + + // Callback called immediately after any actor storage file is opened to verify that the + // registry lock is still held, and therefore the new open must be on the same NFS lease as + // the registry lock. + void verifyRegistration() { + KJ_IF_SOME(r, clusterRegistry) { + r.verifyNfsLease(); + } + } + class ActorChannelImpl final: public IoChannelFactory::ActorChannel { public: ActorChannelImpl(kj::Own actorContainer, Persistent persistent) @@ -1754,6 +1884,65 @@ class Server::ActorNamespace final { JSG_FAIL_REQUIRE(Error, "Facets currently cannot set alarms."); } }; + + class ClusterActorSqliteHooks final: public ActorSqlite::Hooks { + public: + kj::Promise scheduleRun( + kj::Maybe newAlarmTime, kj::Promise priorTask) override { + // TODO(clustering): Support alarms. + JSG_FAIL_REQUIRE(Error, "Durable Object alarms are not yet supported in cluster mode"); + } + }; + + // ActorChannel implementation wrapping a `WorkerdBootstrap` RPC stub pointing at an actor + // owned by another workerd instance in the cluster. + class RpcActorChannel final: public IoChannelFactory::ActorChannel { + public: + RpcActorChannel(ActorNamespace& ns, + Worker::Actor::Id id, + Persistent persistent, + rpc::WorkerdBootstrap::Client client) + : ns(ns), + id(kj::mv(id)), + persistent(persistent), + client(kj::mv(client)) {} + + kj::Own startRequest(IoChannelFactory::SubrequestMetadata metadata) override { + capnp::MessageSize sizeHint{4, 0}; + KJ_IF_SOME(cf, metadata.cfBlobJson) { + sizeHint.wordCount += cf.size() / sizeof(capnp::word) + 1; + } + + // The remote instance reconstructs SubrequestMetadata from these params. + auto req = client.startEventRequest(sizeHint); + KJ_IF_SOME(cf, metadata.cfBlobJson) { + req.setCfBlobJson(cf); + } + req.setFromPersistentStub(bool(metadata.fromPersistentStub || persistent)); + + auto dispatcher = req.sendForPipeline().getDispatcher(); + // NOTE: We don't support restore() over cluster RPC so we can use + // getUnsupportedFrankenvalueHandler() here for now. + return kj::heap(ns.httpOverCapnpFactory, ns.byteStreamFactory, + getUnsupportedFrankenvalueHandler(), kj::mv(dispatcher)); + } + + void requireAllowsTransfer() override { + // Cluster mode only routes root actors of durable namespaces, which are always + // transferrable via channel tokens. + } + + kj::OneOf, kj::Promise>> getTokenMaybeSync( + IoChannelFactory::ChannelTokenUsage usage) override { + return ns.encodeActorChannelToken(usage, id, persistent); + } + + private: + ActorNamespace& ns; + Worker::Actor::Id id; + Persistent persistent; + rpc::WorkerdBootstrap::Client client; + }; }; // ======================================================================================= @@ -3431,6 +3620,8 @@ class Server::WorkerService final: public Service, kj::Array> workerLoaders; kj::Maybe workerdDebugPortNetwork; kj::Maybe workerdDebugPortServer; + kj::Maybe clusterRegistry; + kj::Maybe&> clusterRpc; }; using LinkCallback = kj::Function; @@ -3490,9 +3681,11 @@ class Server::WorkerService final: public Service, } auto actorClass = kj::refcounted(*this, entry.key, Frankenvalue()); - auto ns = kj::heap(kj::mv(actorClass), entry.value, - kj::systemPreciseCalendarClock(), threadContext.getUnsafeTimer(), - threadContext.getByteStreamFactory(), channelTokenHandler, network, dockerPath, + auto ns = kj::heap(kj::mv(actorClass), + // Dynamic workers can't have actor namespaces, so `serviceName` must be available here. + KJ_ASSERT_NONNULL(serviceName), entry.key, entry.value, kj::systemPreciseCalendarClock(), + threadContext.getUnsafeTimer(), threadContext.getByteStreamFactory(), + threadContext.getHttpOverCapnpFactory(), channelTokenHandler, network, dockerPath, containerEgressInterceptorImage, waitUntilTasks, selfTokensArePersistent()); KJ_IF_SOME(d, entry.value.tryGet()) { actorNamespacesByUniqueKey.insert(d.uniqueKey, ns.get()); @@ -3626,7 +3819,7 @@ class Server::WorkerService final: public Service, auto linked = callback(*this, errorReporter); for (auto& ns: actorNamespaces) { - ns.value->link(linked.actorStorage); + ns.value->link(linked.actorStorage, linked.clusterRegistry, linked.clusterRpc); } ioChannels = kj::mv(linked); @@ -5919,6 +6112,10 @@ kj::Promise> Server::makeWorkerImpl(kj::StringPtr auto linkCallback = [this, def = kj::mv(def), totalActorChannels](WorkerService& workerService, Worker::ValidationErrorReporter& errorReporter) mutable { WorkerService::LinkedIoChannels result; + KJ_IF_SOME(registry, this->clusterRegistry) { + result.clusterRegistry = *registry; + result.clusterRpc = KJ_ASSERT_NONNULL(this->clusterRpc); + } auto entrypointNames = workerService.getEntrypointNames(); auto actorClassNames = workerService.getActorClassNames(); @@ -6827,6 +7024,15 @@ class Server::DebugPortListener { co_return co_await server.listen(*listener); } + // Expose WorkerdDebugPortImpl for use by the cluster RPC system. + // + // TODO(cleanup): Maybe move WorkerDebugPortImpl up into Server. However, note the clustering + // code might also shift to a different interface. + static rpc::WorkerdDebugPort::Client makeBootstrap( + Server& owner, capnp::HttpOverCapnpFactory& httpOverCapnpFactory) { + return kj::heap(owner, httpOverCapnpFactory); + } + private: Server& owner; kj::Own listener; @@ -6860,6 +7066,12 @@ kj::Promise Server::handleDrain(kj::Promise drainWhen) { // doc comment, we instead add the promise to `tasks` to be safe. tasks.add(httpServer.httpServer.drain()); } + + // TODO(clustering): This is a harsh ending for actors on this machine. We should really evict + // each one at a time that is convenient, and wait some time for RPC requests to drain. + // Three-party handoff would help with straggler requests that we forward to the new owner. + clusterRpc = kj::none; + abortAllActors(kj::none); } kj::Promise Server::run( @@ -7038,7 +7250,22 @@ kj::Promise Server::startServices(jsg::V8System& v8System, } goto validDurableObjectStorage; case config::Worker::DurableObjectStorage::IN_MEMORY: + if (config.hasCluster() && hadDurable) { + reportConfigError(kj::str("Worker service \"", name, + "\" uses in-memory Durable Object storage, which is not supported in cluster " + "mode.")); + } + goto validDurableObjectStorage; case config::Worker::DurableObjectStorage::LOCAL_DISK: + if (config.hasCluster() && hadDurable && + workerConf.getDurableObjectStorage().getLocalDisk() != + config.getCluster().getSharedDirectory()) { + reportConfigError( + kj::str("Worker service \"", name, "\" uses Durable Object storage disk service \"", + workerConf.getDurableObjectStorage().getLocalDisk(), + "\", but cluster.sharedDirectory is \"", + config.getCluster().getSharedDirectory(), "\".")); + } goto validDurableObjectStorage; } reportConfigError(kj::str("Encountered unknown durableObjectStorage type in service \"", name, @@ -7101,6 +7328,48 @@ kj::Promise Server::startServices(jsg::V8System& v8System, return decltype(services)::Entry{kj::str("internet"_kj), kj::mv(service)}; }); + if (config.hasCluster()) { + if (!experimental) { + reportConfigError( + "Cluster mode is experimental and subject to change. You must run workerd with " + "`--experimental` to use it."_kj.clone()); + } +#ifndef __linux__ + reportConfigError("Cluster mode is currently implemented only on Linux."_kj.clone()); +#endif + + auto cluster = config.getCluster(); + KJ_IF_SOME(sharedDirService, services.find(cluster.getSharedDirectory())) { + auto diskSvc = dynamic_cast(sharedDirService.get()); + if (diskSvc == nullptr) { + reportConfigError(kj::str("cluster.sharedDirectory refers to service \"", + cluster.getSharedDirectory(), "\", but that service is not a disk directory.")); + } else KJ_IF_SOME(dir, diskSvc->getWritable()) { + auto registryDir = dir.openSubdir( + kj::Path({cluster.getRegistrySubdir()}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + auto registry = + kj::heap(kj::mv(registryDir), cluster.getNetwork(), network, timer); + tasks.add(registry->runMaintenance().exclusiveJoin(forkedDrainWhen.addBranch())); + auto& reg = *clusterRegistry.emplace(kj::mv(registry)); + clusterRpc.emplace(capnp::makeRpcServer( + reg, DebugPortListener::makeBootstrap(*this, globalContext->httpOverCapnpFactory))); + + // TODO(clustering): We really shouldn't start accepting connections until the server + // is totally up, but the RPC system actually starts accepting immediately on + // construction, even if you don't call run(). + KJ_IF_SOME(rpc, clusterRpc) { + tasks.add(rpc.run().exclusiveJoin(forkedDrainWhen.addBranch())); + } + } else { + reportConfigError(kj::str("cluster.sharedDirectory refers to disk service \"", + cluster.getSharedDirectory(), "\", but that service is defined read-only.")); + } + } else { + reportConfigError(kj::str("cluster.sharedDirectory refers to service \"", + cluster.getSharedDirectory(), "\", but no such service is defined.")); + } + } + // Third pass: Cross-link services. for (auto& service: services) { ConfigErrorReporter errorReporter(*this, service.key); diff --git a/src/workerd/server/server.h b/src/workerd/server/server.h index c3fe15c24e5..7005340485e 100644 --- a/src/workerd/server/server.h +++ b/src/workerd/server/server.h @@ -9,8 +9,10 @@ #include #include #include +#include #include +#include #include #include #include @@ -164,6 +166,9 @@ class Server final: private kj::TaskSet::ErrorHandler, private ChannelTokenHandl kj::Maybe channelTokenHandler; + kj::Maybe> clusterRegistry; + kj::Maybe> clusterRpc; + kj::HashMap>> socketOverrides; kj::HashMap directoryOverrides; From 4ff98b4dd50b306c359899cdde5adcf107f9c95f Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 23 May 2026 18:10:41 -0500 Subject: [PATCH 09/33] Add integration tests for clustering mode. This test suite was written entirely by Claude Opus 4.7. --- src/workerd/server/tests/cluster/BUILD.bazel | 45 ++ .../tests/cluster/cluster-test-cidr.wd-test | 49 ++ .../server/tests/cluster/cluster-test.mjs | 465 ++++++++++++++++++ .../server/tests/cluster/cluster-test.wd-test | 60 +++ .../server/tests/cluster/cluster-worker.js | 86 ++++ 5 files changed, 705 insertions(+) create mode 100644 src/workerd/server/tests/cluster/BUILD.bazel create mode 100644 src/workerd/server/tests/cluster/cluster-test-cidr.wd-test create mode 100644 src/workerd/server/tests/cluster/cluster-test.mjs create mode 100644 src/workerd/server/tests/cluster/cluster-test.wd-test create mode 100644 src/workerd/server/tests/cluster/cluster-worker.js diff --git a/src/workerd/server/tests/cluster/BUILD.bazel b/src/workerd/server/tests/cluster/BUILD.bazel new file mode 100644 index 00000000000..ae7ad386f7f --- /dev/null +++ b/src/workerd/server/tests/cluster/BUILD.bazel @@ -0,0 +1,45 @@ +load("@aspect_rules_js//js:defs.bzl", "js_test") + +js_test( + name = "cluster-test-unix", + size = "medium", + data = [ + ":cluster-test.wd-test", + ":cluster-worker.js", + "//src/workerd/server:workerd", + ], + entry_point = "cluster-test.mjs", + env = { + "WORKERD_BINARY": "$(rootpath //src/workerd/server:workerd)", + "WD_TEST_CONFIG": "$(rootpath :cluster-test.wd-test)", + }, + tags = ["js-test"], + target_compatible_with = select({ + "@platforms//os:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) + +js_test( + name = "cluster-test-cidr", + # The CIDR variant exercises the dead-peer detection path, which has to + # wait out the new-entry grace period before unlinking the dead peer's + # registry file. That makes the takeover test ~15s on its own, so the + # combined run needs more than the default "medium" budget. + size = "large", + data = [ + ":cluster-test-cidr.wd-test", + ":cluster-worker.js", + "//src/workerd/server:workerd", + ], + entry_point = "cluster-test.mjs", + env = { + "WORKERD_BINARY": "$(rootpath //src/workerd/server:workerd)", + "WD_TEST_CONFIG": "$(rootpath :cluster-test-cidr.wd-test)", + }, + tags = ["js-test"], + target_compatible_with = select({ + "@platforms//os:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) diff --git a/src/workerd/server/tests/cluster/cluster-test-cidr.wd-test b/src/workerd/server/tests/cluster/cluster-test-cidr.wd-test new file mode 100644 index 00000000000..3bebcbb6634 --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-test-cidr.wd-test @@ -0,0 +1,49 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +# CIDR/IP variant of cluster-test.wd-test. Uses 127.0.0.0/8 (the loopback) so +# the registry-file locking, TCP peer connections, new-entry grace period, and +# dead-peer lock probes are exercised independently of the unix-socket variant. + +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + ( name = "shared", + disk = ( + writable = true, + allowDotfiles = true, + ) + ), + + ( name = "main", + worker = ( + modules = [ + ( name = "cluster-worker.js", esModule = embed "cluster-worker.js" ), + ], + compatibilityDate = "2024-09-01", + compatibilityFlags = ["nodejs_compat"], + bindings = [ + ( name = "COUNTER", durableObjectNamespace = "Counter" ), + ( name = "NODE_ID", fromEnvironment = "NODE_ID" ), + ], + durableObjectNamespaces = [ + ( className = "Counter", uniqueKey = "counter-test-namespace" ), + ], + durableObjectStorage = ( localDisk = "shared" ), + ) + ), + ], + + sockets = [ + ( name = "http", address = "*:0", http = (), service = "main" ), + ], + + cluster = ( + sharedDirectory = "shared", + registrySubdir = "workerd-registry", + network = "127.0.0.0/8", + key = "test-cluster-shared-secret", + ), +); diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs new file mode 100644 index 00000000000..3203dc34812 --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -0,0 +1,465 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +// +// Integration test for the cluster mode described in scalable-durable-objects.md. +// +// The test spawns several workerd instances pointing at the same shared +// directory, then exercises: +// 1. Consistent DO state across instances (single-writer correctness). +// 2. Transparent forwarding when a request arrives at the "wrong" instance. +// 3. Takeover after a node is killed. +// 4. Alarms are rejected with a clear error in cluster mode. +// 5. No `metadata.sqlite` is ever created (no AlarmScheduler in cluster mode). +// +// Two variants are run: the unix-socket cluster network mode (default) and a +// localhost CIDR (`127.0.0.0/8`) IP-socket mode that exercises the registry +// file locking and TCP peer connections separately. The variant is selected +// by the WD_TEST_CONFIG environment variable supplied by the BUILD target. + +import { spawn } from 'node:child_process'; +import { mkdtemp, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { env } from 'node:process'; +import { test } from 'node:test'; +import assert from 'node:assert'; +import { setTimeout as sleep } from 'node:timers/promises'; + +assert( + env.WORKERD_BINARY !== undefined, + 'You must set the WORKERD_BINARY environment variable.' +); +assert( + env.WD_TEST_CONFIG !== undefined, + 'You must set the WD_TEST_CONFIG environment variable.' +); + +const CONTROL_FD = 3; + +// A minimal harness specialized for the cluster test. Unlike server-harness.mjs +// it: +// - allows overriding bindings via process environment (NODE_ID), +// - permits multiple instances to share a directory, +// - supports SIGKILL in addition to SIGTERM for the takeover test. +class ClusterNode { + #binary; + #config; + #sharedPath; + #nodeId; + #child = null; + #httpPort = null; + #closed = null; + #stderrBuffer = ''; + + constructor({ binary, config, sharedPath, nodeId }) { + this.#binary = binary; + this.#config = config; + this.#sharedPath = sharedPath; + this.#nodeId = nodeId; + } + + get nodeId() { + return this.#nodeId; + } + + get httpPort() { + assert(this.#httpPort !== null, 'node not started'); + return this.#httpPort; + } + + // Returns the contents of the node's stderr observed so far. Useful for + // verifying that internal error messages (e.g. the cluster alarm rejection) + // were emitted. + get stderr() { + return this.#stderrBuffer; + } + + async start() { + assert.strictEqual(this.#child, null); + + const args = [ + 'serve', + this.#config, + '--experimental', + '--verbose', + `--control-fd=${CONTROL_FD}`, + `--directory-path=shared=${this.#sharedPath}`, + `--socket-addr=http=127.0.0.1:0`, + ]; + + const child = spawn(this.#binary, args, { + stdio: ['ignore', 'inherit', 'pipe', 'pipe'], + env: { ...env, NODE_ID: this.#nodeId }, + }); + this.#child = child; + + child.stderr.on('data', (data) => { + const chunk = data.toString('utf8'); + this.#stderrBuffer += chunk; + // Mirror to our own stderr so test output remains useful. + process.stderr.write(`[${this.#nodeId}] ${chunk}`); + }); + + // Watch the control fd for the assigned http port. + const portPromise = new Promise((resolve, reject) => { + let buffer = ''; + const onData = (data) => { + buffer += data.toString('utf8'); + let nl; + while ((nl = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, nl).trim(); + buffer = buffer.slice(nl + 1); + if (!line) continue; + try { + const parsed = JSON.parse(line); + if (parsed.event === 'listen' && parsed.socket === 'http') { + child.stdio[CONTROL_FD].off('data', onData); + resolve(parsed.port); + return; + } + } catch (err) { + reject( + new Error( + `Failed to parse control message from node ${this.#nodeId}: ${line}` + ) + ); + return; + } + } + }; + child.stdio[CONTROL_FD].on('data', onData); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (this.#httpPort === null) { + reject( + new Error( + `Node ${this.#nodeId} exited before listening (code=${code}, signal=${signal})` + ) + ); + } + }); + }); + + this.#closed = new Promise((resolve) => { + child.once('exit', (code, signal) => resolve({ code, signal })); + }); + + await new Promise((resolve, reject) => { + child.once('spawn', resolve).once('error', reject); + }); + + this.#httpPort = await portPromise; + } + + async stop({ signal = 'SIGTERM', timeoutMs = 10_000 } = {}) { + if (this.#child === null) return null; + const child = this.#child; + this.#child = null; + child.kill(signal); + + const killTimer = setTimeout(() => { + try { + child.kill('SIGKILL'); + } catch (_) { + // process may already be gone. + } + }, timeoutMs); + + const result = await this.#closed; + clearTimeout(killTimer); + this.#httpPort = null; + return result; + } +} + +async function fetchJson(port, path, init) { + const url = `http://127.0.0.1:${port}${path}`; + const res = await fetch(url, init); + const text = await res.text(); + let body; + try { + body = JSON.parse(text); + } catch (err) { + throw new Error( + `Non-JSON response from ${url} (status ${res.status}): ${text}` + ); + } + return { status: res.status, body }; +} + +async function withCluster(numNodes, fn) { + // Each test run gets its own shared directory, so the registry/lock-file + // state doesn't leak between tests. + const sharedPath = await mkdtemp(join(tmpdir(), 'workerd-cluster-')); + const nodes = []; + try { + for (let i = 0; i < numNodes; i++) { + const node = new ClusterNode({ + binary: env.WORKERD_BINARY, + config: env.WD_TEST_CONFIG, + sharedPath, + nodeId: `node${i}`, + }); + await node.start(); + nodes.push(node); + } + await fn({ nodes, sharedPath }); + } finally { + // Stop all nodes (ignore errors -- some may already be stopped by the + // test itself). + for (const node of nodes) { + try { + await node.stop({ timeoutMs: 5000 }); + } catch (_) { + // ignore + } + } + await rm(sharedPath, { recursive: true, force: true }); + } +} + +// --------------------------------------------------------------------------- +// Tests + +test('cluster: DO state is consistent across instances (single writer)', async () => { + await withCluster(3, async ({ nodes }) => { + // Send a series of /increment requests to a round-robin of nodes, all for + // the same DO name. Track the maximum returned count -- it must be a + // strictly-increasing sequence (1, 2, 3, ...) because only one instance is + // the writer at any given time and the DO state is persistent. + const totalRequests = 12; + const expected = []; + for (let i = 1; i <= totalRequests; i++) expected.push(i); + + const observed = []; + const ownerIds = new Set(); + for (let i = 0; i < totalRequests; i++) { + const node = nodes[i % nodes.length]; + const { status, body } = await fetchJson( + node.httpPort, + '/increment?name=consistency' + ); + assert.strictEqual( + status, + 200, + `request to ${node.nodeId} failed: ${JSON.stringify(body)}` + ); + observed.push(body.count); + ownerIds.add(body.nodeId); + } + observed.sort((a, b) => a - b); + assert.deepStrictEqual( + observed, + expected, + `each /increment must be a unique strictly-increasing count` + ); + + // The DO is owned by exactly one node at a time. Even though we directed + // requests at all 3 nodes, every observed response should report the same + // nodeId -- the actual owner -- because incoming requests are forwarded. + assert.strictEqual( + ownerIds.size, + 1, + `expected exactly one DO owner across ${totalRequests} requests, got: ${[...ownerIds].join(', ')}` + ); + }); +}); + +test('cluster: requests to non-owner are transparently forwarded', async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO by sending an increment to node 0. That node should claim + // ownership and respond. Then send a /get to node 1. Node 1 should forward + // to node 0 (the owner) and return the same state, and node 0 should be + // identified as the responder. + const first = await fetchJson( + nodes[0].httpPort, + '/increment?name=forwarding' + ); + assert.strictEqual(first.status, 200); + assert.strictEqual(first.body.count, 1); + const owner = first.body.nodeId; + + const second = await fetchJson(nodes[1].httpPort, '/get?name=forwarding'); + assert.strictEqual(second.status, 200); + assert.strictEqual( + second.body.count, + 1, + `forwarded /get must see prior increment` + ); + assert.strictEqual( + second.body.nodeId, + owner, + `forwarded request must be served by the original owner (${owner}), not the forwarding node (${nodes[1].nodeId})` + ); + + // Same id must be reported for both calls (sanity check for idFromName + // determinism across instances). + assert.strictEqual(first.body.id, second.body.id); + }); +}); + +test('cluster: killing the owner allows another node to take over', async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO on whichever node ends up owning it. + const initial = await fetchJson( + nodes[0].httpPort, + '/increment?name=takeover' + ); + assert.strictEqual(initial.status, 200); + assert.strictEqual(initial.body.count, 1); + const originalOwner = initial.body.nodeId; + const ownerIndex = nodes.findIndex((n) => n.nodeId === originalOwner); + assert.notStrictEqual(ownerIndex, -1); + const survivorIndex = ownerIndex === 0 ? 1 : 0; + const survivor = nodes[survivorIndex]; + + // Hard-kill the owner. SIGKILL leaves the registry file in place, so the + // survivor has to detect death via a failed RPC + dead-peer cleanup probe. + await nodes[ownerIndex].stop({ signal: 'SIGKILL', timeoutMs: 2000 }); + + // Hit the survivor repeatedly. The first request may fail or take time + // while the survivor's RPC attempt to the dead owner is in flight; once + // the dead-peer cleanup completes, subsequent requests succeed and the + // survivor takes ownership. The persisted count must be preserved. + let response = null; + let lastErr = null; + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + response = await fetchJson( + survivor.httpPort, + '/increment?name=takeover' + ); + if (response.status === 200) break; + } catch (err) { + lastErr = err; + } + await sleep(200); + } + assert( + response !== null && response.status === 200, + `survivor never succeeded; last error: ${lastErr}` + ); + assert.strictEqual( + response.body.nodeId, + survivor.nodeId, + `survivor (${survivor.nodeId}) must own the DO after takeover, got ${response.body.nodeId}` + ); + assert.strictEqual( + response.body.count, + 2, + `persisted state must be preserved across takeover` + ); + + // Further requests to the survivor continue to work and increment. + const followup = await fetchJson( + survivor.httpPort, + '/increment?name=takeover' + ); + assert.strictEqual(followup.status, 200); + assert.strictEqual(followup.body.count, 3); + assert.strictEqual(followup.body.nodeId, survivor.nodeId); + }); +}); + +test('cluster: alarms are rejected with a clear error', async () => { + await withCluster(1, async ({ nodes, sharedPath }) => { + // Fire the request. In cluster mode `setAlarm()` does not throw + // synchronously to the JS code -- the rejection happens during output-gate + // flush, after the worker's fetch handler has already returned. The + // observable behaviour is therefore a non-2xx HTTP status, while the + // canonical error message is logged to stderr where the test can detect + // it. (The spec mandates the message "Durable Object alarms are not yet + // supported in cluster mode".) + let response; + try { + response = await fetch( + `http://127.0.0.1:${nodes[0].httpPort}/set-alarm?name=alarm-test` + ); + } catch (err) { + // A network-level failure is acceptable too: it confirms that the alarm + // path did not silently succeed. + response = null; + } + if (response !== null) { + // Drain the body to allow logging to flush. + try { + await response.text(); + } catch (_) { + // ignore + } + assert.notStrictEqual( + Math.floor(response.status / 100), + 2, + `expected non-2xx for set-alarm in cluster mode, got ${response.status}` + ); + } + + // Wait briefly for stderr to flush the error message. + const deadline = Date.now() + 5000; + const expectedMessage = + 'Durable Object alarms are not yet supported in cluster mode'; + while ( + Date.now() < deadline && + !nodes[0].stderr.includes(expectedMessage) + ) { + await sleep(50); + } + assert( + nodes[0].stderr.includes(expectedMessage), + `expected stderr to contain the cluster alarm rejection message ` + + `("${expectedMessage}"); stderr so far:\n${nodes[0].stderr}` + ); + + // No per-namespace metadata.sqlite should exist anywhere under the + // shared directory -- in cluster mode the AlarmScheduler is never + // constructed. + const nsDir = join(sharedPath, 'counter-test-namespace'); + let entries = []; + try { + entries = await readdir(nsDir); + } catch (err) { + // Namespace dir might not exist if the DO was never created on disk + // because the request failed before storage was opened. That is a + // valid (and even stronger) signal that no metadata.sqlite was made. + } + assert( + !entries.some((name) => name.startsWith('metadata.sqlite')), + `cluster mode must not create metadata.sqlite; found: ${entries.join(', ')}` + ); + }); +}); + +test('cluster: registry directory is populated for each running instance', async () => { + await withCluster(2, async ({ nodes, sharedPath }) => { + // Give the instances a brief moment to settle. Both should have written + // their registry entries before they reported their listening sockets, + // but allow a small grace period for filesystem visibility. + const registryDir = join(sharedPath, 'workerd-registry'); + let entries = []; + const deadline = Date.now() + 5000; + while (Date.now() < deadline) { + try { + entries = await readdir(registryDir); + } catch (_) { + entries = []; + } + if (entries.length >= nodes.length) break; + await sleep(100); + } + assert.strictEqual( + entries.length, + nodes.length, + `expected ${nodes.length} registry entries, got ${entries.length}: [${entries.join(', ')}]` + ); + // Each entry should be a 64-char hex public key. + for (const name of entries) { + assert.match( + name, + /^[0-9a-f]{64}$/, + `registry entry name should be 64-char hex, got: ${name}` + ); + } + }); +}); diff --git a/src/workerd/server/tests/cluster/cluster-test.wd-test b/src/workerd/server/tests/cluster/cluster-test.wd-test new file mode 100644 index 00000000000..6c6d53b2d7e --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-test.wd-test @@ -0,0 +1,60 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +# Config for the cluster integration test. Designed to be reused across multiple +# workerd instances pointing at the same shared directory: +# - The `shared` disk service has no path; the test driver overrides it via +# `--directory-path shared=` so all instances see the same directory. +# - The `http` socket has address `*:0`; the test driver overrides this and +# reads the actual assigned port from the control fd. +# - The cluster `network` field defaults to "unix"; the test driver overrides +# it to "127.0.0.0/8" when testing the CIDR/IP variant by passing a different +# config (see cluster-test-cidr.wd-test). +# - The NODE_ID env binding lets each instance be identified per request, so +# the test can verify which workerd owns/serves a given DO. + +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + # The shared filesystem used by the whole cluster. Path is supplied via + # `--directory-path shared=` on the command line so all instances + # share the same directory. + ( name = "shared", + disk = ( + writable = true, + allowDotfiles = true, + ) + ), + + ( name = "main", + worker = ( + modules = [ + ( name = "cluster-worker.js", esModule = embed "cluster-worker.js" ), + ], + compatibilityDate = "2024-09-01", + compatibilityFlags = ["nodejs_compat"], + bindings = [ + ( name = "COUNTER", durableObjectNamespace = "Counter" ), + ( name = "NODE_ID", fromEnvironment = "NODE_ID" ), + ], + durableObjectNamespaces = [ + ( className = "Counter", uniqueKey = "counter-test-namespace" ), + ], + durableObjectStorage = ( localDisk = "shared" ), + ) + ), + ], + + sockets = [ + ( name = "http", address = "*:0", http = (), service = "main" ), + ], + + cluster = ( + sharedDirectory = "shared", + registrySubdir = "workerd-registry", + network = "unix", + key = "test-cluster-shared-secret", + ), +); diff --git a/src/workerd/server/tests/cluster/cluster-worker.js b/src/workerd/server/tests/cluster/cluster-worker.js new file mode 100644 index 00000000000..dbd107fee3f --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-worker.js @@ -0,0 +1,86 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Test worker for the cluster integration test. Exposes a Counter Durable Object +// and a top-level fetch() that routes requests to the DO by name. +// +// The DO supports the following request paths: +// /increment Increments and returns the current counter value. +// /get Returns the current counter value without incrementing it. +// /set-alarm Tries to schedule a DO alarm. In cluster mode this should +// fail with a "not yet supported" error. +// /identity Returns information identifying this instance (NODE_ID env) +// along with the DO id, so the test can verify which instance +// served the request. + +export default { + async fetch(request, env) { + const url = new URL(request.url); + + // The DO name to act on is given via the `name` query parameter (defaults + // to "default"). All instances using the same name will route to the same + // DO id. + const name = url.searchParams.get('name') ?? 'default'; + + // Synthesize a stub URL for the DO. + const id = env.COUNTER.idFromName(name); + const stub = env.COUNTER.get(id); + + // Forward to the DO with the same pathname. + const forwardUrl = new URL(url.pathname, 'http://do/'); + return await stub.fetch(forwardUrl.toString(), { + method: request.method, + headers: request.headers, + }); + }, +}; + +export class Counter { + constructor(state, env) { + this.state = state; + this.env = env; + } + + async fetch(request) { + const url = new URL(request.url); + const nodeId = this.env.NODE_ID ?? ''; + const idHex = this.state.id.toString(); + + if (url.pathname === '/increment') { + let count = (await this.state.storage.get('count')) ?? 0; + count += 1; + await this.state.storage.put('count', count); + return Response.json({ + count, + nodeId, + id: idHex, + }); + } else if (url.pathname === '/get') { + const count = (await this.state.storage.get('count')) ?? 0; + return Response.json({ count, nodeId, id: idHex }); + } else if (url.pathname === '/set-alarm') { + // Try to schedule an alarm 60s in the future. In cluster mode this should + // throw a clear error. + try { + await this.state.storage.setAlarm(Date.now() + 60_000); + return Response.json({ ok: true, nodeId, id: idHex }); + } catch (err) { + return Response.json( + { ok: false, error: String(err), nodeId, id: idHex }, + { status: 500 } + ); + } + } else if (url.pathname === '/identity') { + return Response.json({ nodeId, id: idHex }); + } + + return new Response('Not found', { status: 404 }); + } + + async alarm() { + // Should never be called in cluster mode; if it ever is, surface that via + // a stored marker so the test can observe the failure mode. + await this.state.storage.put('alarm-fired', true); + } +} From 9732a00c48aa411ebf927621e450859dc5105230 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 5 Sep 2026 18:04:50 -0500 Subject: [PATCH 10/33] Drive-by: Fix compile_flags.txt for kj async library split. --- compile_flags.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compile_flags.txt b/compile_flags.txt index 9ffc5c68c3e..fec26b8509d 100644 --- a/compile_flags.txt +++ b/compile_flags.txt @@ -21,7 +21,10 @@ -isystembazel-bin/external/+http+capnp-cpp/src/capnp/compat/_virtual_includes/http-over-capnp -isystembazel-bin/external/+http+capnp-cpp/src/capnp/compat/_virtual_includes/json -isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj --isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj-async +-isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj-async-core +-isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj-async-io +-isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj-async-os-hdrs +-isystembazel-bin/external/+http+capnp-cpp/src/kj/_virtual_includes/kj-test -isystembazel-bin/external/+http+capnp-cpp/src/kj/compat/_virtual_includes/kj-brotli -isystembazel-bin/external/+http+capnp-cpp/src/kj/compat/_virtual_includes/kj-gzip -isystembazel-bin/external/+http+capnp-cpp/src/kj/compat/_virtual_includes/kj-http From 9eb96c0ef7852c1a01b971ac291c8ca6e931707c Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 5 Sep 2026 22:27:58 -0500 Subject: [PATCH 11/33] Propogate the new `fromPersistentStub` flag correctly over RPC. This flag was introduced by the persistent stubs feature, which was written and landed while DO clustering was still in flight. This commit is the first commit added after rebasing. It's essentially fixing a hidden merge conflict. --- src/workerd/server/server-test.c++ | 49 ++++++++++++++++++++++++++++++ src/workerd/server/server.c++ | 9 ++++-- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/workerd/server/server-test.c++ b/src/workerd/server/server-test.c++ index 2be79f35d54..b02d529d824 100644 --- a/src/workerd/server/server-test.c++ +++ b/src/workerd/server/server-test.c++ @@ -7132,6 +7132,55 @@ KJ_TEST("Server: debug port RPC calls") { auto jsonResult = deserializeV8ToJson(resultData); KJ_EXPECT(jsonResult == "8", jsonResult, "Expected result to be 8"); } + + // Test 7: `startEvent(fromPersistentStub = true)` is propagated to the target for non-HTTP + // events. `rpc-service` lacks `allow_irrevocable_stub_storage`, so the target must reject the + // event; this is the only observable that proves the flag reached WorkerEntrypoint::construct(). + { + auto startSession = [&](bool fromPersistentStub) { + auto bootstrap = + getBootstrap("rpc-service", kj::none, [](auto& props) { props.setEmptyObject(); }); + auto eventReq = bootstrap.startEventRequest(); + eventReq.setFromPersistentStub(fromPersistentStub); + auto dispatcher = eventReq.send().wait(test.ws).getDispatcher(); + return dispatcher.jsRpcSessionRequest().send(); + }; + + auto callAdd = [&](rpc::JsRpcTarget::Client rpcTarget) { + auto v8SerializedArgs = serializeJsArguments({[](jsg::Lock& js) { + return jsg::JsValue(js.num(2)); + }, [](jsg::Lock& js) { return jsg::JsValue(js.num(3)); }}); + auto callReq = rpcTarget.callRequest(); + callReq.setMethodName("add"); + callReq.initOperation().initCallWithArgs().setV8Serialized(v8SerializedArgs); + return callReq.send(); + }; + + // Control: without the flag, the session works. + { + auto sessionPromise = startSession(false); + auto callResp = callAdd(sessionPromise.getTopLevel()).wait(test.ws); + KJ_EXPECT(deserializeV8ToJson(callResp.getResult().getV8Serialized()) == "5"); + } + + // With the flag, the target rejects the RPC session. + { + auto sessionPromise = startSession(true); + KJ_EXPECT_THROW_MESSAGE("no longer has the allow_irrevocable_stub_storage", + callAdd(sessionPromise.getTopLevel()).wait(test.ws)); + } + + // Control: the HTTP path rejects it too. + { + auto bootstrap = + getBootstrap("rpc-service", kj::none, [](auto& props) { props.setEmptyObject(); }); + auto eventReq = bootstrap.startEventRequest(); + eventReq.setFromPersistentStub(true); + auto dispatcher = eventReq.send().wait(test.ws).getDispatcher(); + KJ_EXPECT_THROW_MESSAGE("no longer has the allow_irrevocable_stub_storage", + makeHttpRequestFromDispatcher(kj::mv(dispatcher), "/")); + } + } } KJ_TEST("Server: workerdDebugPort binding current process test") { diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 4d916a7502f..e5c94285c24 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -6608,9 +6608,12 @@ class Server::WorkerdBootstrapImpl final: public rpc::WorkerdBootstrap::Server { } kj::Own getWorker() { - // For non-HTTP events (RPC, traces, etc.), create WorkerInterface with - // empty metadata since there's no HTTP request to extract cf from. - return getService()->startRequest({}); + // For non-HTTP events (RPC, traces, etc.), there's no HTTP request to extract cf from, so + // cfBlobJson is left unset. fromPersistentStub must still be propagated so that the target + // re-verifies allow_irrevocable_stub_storage. + IoChannelFactory::SubrequestMetadata metadata; + metadata.fromPersistentStub = fromPersistentStub; + return getService()->startRequest(kj::mv(metadata)); } [[noreturn]] void throwUnsupported() { From e686678f312fa5842b0c3c2f9c22d36aec64d86f Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sat, 5 Sep 2026 22:58:18 -0500 Subject: [PATCH 12/33] Make cluster-routed actor channels resolve the actor lazily. It's important that restoring a stub from storage does not immediately acquire the DO's lock or make network requests -- that should happen only if a request is made on the stub. Thus, we introduce a new `ClusterActorChannel` type (replacing `RpcActorChannel`) which does the lazily resolution. All stubs in cluster mode use this type, including when the actor ultimately ends up being local. --- src/workerd/server/server.c++ | 209 +++++++++++++----- .../tests/cluster/cluster-test-cidr.wd-test | 4 +- .../server/tests/cluster/cluster-test.mjs | 60 ++++- .../server/tests/cluster/cluster-test.wd-test | 4 +- .../server/tests/cluster/cluster-worker.js | 32 ++- 5 files changed, 241 insertions(+), 68 deletions(-) diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index e5c94285c24..16a9feefee9 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -391,8 +391,6 @@ class Server::ActorNamespace final { waitUntilTasks(waitUntilTasks), selfTokensArePersistent(selfTokensArePersistent) {} - class ClusterActorChannel; - void link(kj::Maybe serviceActorStorage, kj::Maybe clusterRegistry, kj::Maybe&> clusterRpc) { @@ -481,40 +479,10 @@ class Server::ActorNamespace final { } } - KJ_IF_SOME(ld, lockManager) { - auto key = actorKey(id); - - // First check if the container exists locally. - KJ_IF_SOME(container, actors.find(key)) { - return kj::refcounted(container->addRef(), persistent); - } - - auto promise = ld.acquireOrRoute(key); - return newPromisedChannel( - promise.then([this, id = kj::mv(id), key = kj::mv(key), persistent]( - kj::OneOf - lockOrClient) mutable -> kj::Own { - KJ_SWITCH_ONEOF(lockOrClient) { - KJ_CASE_ONEOF(ownership, ClusterLockManager::OwnedLock) { - return kj::refcounted( - createActorContainer(kj::mv(key), kj::mv(id), kj::mv(ownership)), persistent); - } - KJ_CASE_ONEOF(debugPort, rpc::WorkerdDebugPort::Client) { - auto req = debugPort.getActorRequest(capnp::MessageSize{32, 0}); - req.setService(serviceName); - req.setEntrypoint(className); - req.setActorId(key); - KJ_IF_SOME(i, id.tryGet>()) { - KJ_IF_SOME(name, i->getName()) { - req.setActorName(name); - } - } - return kj::refcounted( - *this, kj::mv(id), persistent, req.send().getActor()); - } - } - KJ_UNREACHABLE; - })); + if (lockManager != kj::none) { + // Cluster mode: resolution of the owning node is deferred until the channel is first used, + // so that merely obtaining a stub never takes cluster locks or wakes actors. + return kj::refcounted(*this, kj::mv(id), persistent); } else { return kj::refcounted(getActorContainer(kj::mv(id)), persistent); } @@ -1502,7 +1470,10 @@ class Server::ActorNamespace final { })->addRef(); } - // Create an ActorContainer in cluster mode. + // Create an ActorContainer in cluster mode. Must be called in the same synchronous continuation + // that received `ownership` from ClusterLockManager::acquireOrRoute(): the lock file naming this + // node while `actors` has no entry for the key would make requests for the actor loop through + // self-bootstrap until the entry appears. kj::Own createActorContainer( kj::String key, Worker::Actor::Id id, ClusterLockManager::OwnedLock ownership) { KJ_ASSERT(actors.find(key) == kj::none); @@ -1513,6 +1484,29 @@ class Server::ActorNamespace final { return container; } + // Which node currently owns the actor with the given key: a container on this node (created + // here if we just claimed the lock), or the owning node's debug port. Not cached; callers cache + // what they derive from it. Every cluster-routed channel type resolves through this method so + // that the lock protocol lives in exactly one place. `key` and `id` must remain valid until the + // returned promise settles. Cluster mode only. + using Node = kj::OneOf, rpc::WorkerdDebugPort::Client>; + kj::Promise resolveNode(kj::StringPtr key, Worker::Actor::Id& id) { + KJ_IF_SOME(container, actors.find(key)) { + co_return container->addRef(); + } + + auto lockOrClient = co_await KJ_ASSERT_NONNULL(lockManager).acquireOrRoute(key); + KJ_SWITCH_ONEOF(lockOrClient) { + KJ_CASE_ONEOF(ownership, ClusterLockManager::OwnedLock) { + co_return createActorContainer(kj::str(key), Worker::Actor::cloneId(id), kj::mv(ownership)); + } + KJ_CASE_ONEOF(debugPort, rpc::WorkerdDebugPort::Client) { + co_return kj::mv(debugPort); + } + } + KJ_UNREACHABLE; + } + // Encode a channel token pointing at the given actor in this namespace. The caller is // responsible for having checked transferrability. kj::Array encodeActorChannelToken(IoChannelFactory::ChannelTokenUsage usage, @@ -1894,37 +1888,51 @@ class Server::ActorNamespace final { } }; - // ActorChannel implementation wrapping a `WorkerdBootstrap` RPC stub pointing at an actor - // owned by another workerd instance in the cluster. - class RpcActorChannel final: public IoChannelFactory::ActorChannel { + // Channel to an actor in this namespace in cluster mode. Resolution of which node owns the actor + // is deferred until first use, so that obtaining or deserializing a stub does not take cluster + // locks or wake remote actors. + class ClusterActorChannel final: public IoChannelFactory::ActorChannel { public: - RpcActorChannel(ActorNamespace& ns, - Worker::Actor::Id id, - Persistent persistent, - rpc::WorkerdBootstrap::Client client) + ClusterActorChannel(ActorNamespace& ns, Worker::Actor::Id id, Persistent persistent) : ns(ns), id(kj::mv(id)), - persistent(persistent), - client(kj::mv(client)) {} + key(ns.actorKey(this->id)), + persistent(persistent) {} + ~ClusterActorChannel() noexcept(false) { + KJ_IF_SOME(t, target) { + KJ_IF_SOME(container, t.tryGet>()) { + container->updateAccessTime(); + } + } + } kj::Own startRequest(IoChannelFactory::SubrequestMetadata metadata) override { - capnp::MessageSize sizeHint{4, 0}; - KJ_IF_SOME(cf, metadata.cfBlobJson) { - sizeHint.wordCount += cf.size() / sizeof(capnp::word) + 1; - } + // If this channel was reconstructed from a persistent (stored) stub, signal the target so it + // can re-verify that it still allows persistent stubs. + metadata.fromPersistentStub = metadata.fromPersistentStub || persistent; + return newPromisedWorkerInterface(startRequestImpl(kj::addRef(*this), kj::mv(metadata))); + } - // The remote instance reconstructs SubrequestMetadata from these params. - auto req = client.startEventRequest(sizeHint); - KJ_IF_SOME(cf, metadata.cfBlobJson) { - req.setCfBlobJson(cf); + kj::Promise evictForTest(IoChannelFactory::EvictWebSocketMode webSocketMode) override { + KJ_IF_SOME(t, target) { + KJ_SWITCH_ONEOF(t) { + KJ_CASE_ONEOF(container, kj::Own) { + return container->evictForTest(webSocketMode).attach(kj::addRef(*this)); + } + KJ_CASE_ONEOF(_, rpc::WorkerdBootstrap::Client) { + JSG_FAIL_REQUIRE(Error, + "Cannot evict Durable Object: it is owned by another workerd instance in the " + "cluster."); + } + } } - req.setFromPersistentStub(bool(metadata.fromPersistentStub || persistent)); - auto dispatcher = req.sendForPipeline().getDispatcher(); - // NOTE: We don't support restore() over cluster RPC so we can use - // getUnsupportedFrankenvalueHandler() here for now. - return kj::heap(ns.httpOverCapnpFactory, ns.byteStreamFactory, - getUnsupportedFrankenvalueHandler(), kj::mv(dispatcher)); + // Not resolved yet. If the actor is running on this node, evict it without pinning it to + // this channel. + KJ_IF_SOME(container, ns.actors.find(key)) { + return container->evictForTest(webSocketMode).attach(container->addRef()); + } + JSG_FAIL_REQUIRE(Error, "Cannot evict Durable Object: it is not currently running."); } void requireAllowsTransfer() override { @@ -1940,8 +1948,87 @@ class Server::ActorNamespace final { private: ActorNamespace& ns; Worker::Actor::Id id; + kj::String key; Persistent persistent; - rpc::WorkerdBootstrap::Client client; + + // Where requests on this channel go, resolved on first startRequest() and retained for the + // life of the channel. Local: the container, pinned exactly as ActorChannelImpl pins it (and + // therefore the ownership lock). Remote: a WorkerdBootstrap for this actor on the owning node. + // Retaining the target gives remote stubs the same failure semantics as local ones: a local + // channel keeps throwing `brokenReason` once its container breaks; a retained bootstrap + // likewise keeps failing once the far end breaks, rather than silently reaching a freshly + // re-instantiated actor on the next call. + using Target = kj::OneOf, rpc::WorkerdBootstrap::Client>; + kj::Maybe target; + + // Set once the first resolution starts. Its branches are only ever held by coroutines that + // also hold a reference to this channel, so the underlying resolution never outlives `this`. + kj::Maybe> resolving; + + // Resolves `target`, caching the result. Safe to call concurrently; all callers share one + // ns.resolveNode() attempt. A failed resolution is retained too, so every later request on the + // channel fails with the same exception. + kj::Promise resolveTarget() { + if (target != kj::none) return kj::READY_NOW; + KJ_IF_SOME(r, resolving) return r.addBranch(); + return resolving.emplace(resolveTargetImpl().fork()).addBranch(); + } + + kj::Promise resolveTargetImpl() { + auto node = co_await ns.resolveNode(key, id); + KJ_SWITCH_ONEOF(node) { + KJ_CASE_ONEOF(container, kj::Own) { + target = kj::mv(container); + } + KJ_CASE_ONEOF(debugPort, rpc::WorkerdDebugPort::Client) { + // Convert the owner's debug port into a bootstrap for this actor. Nothing else on this + // channel needs the debug port, so only the (pipelined) bootstrap is retained: if + // getActor() fails, the bootstrap is broken and every later startEvent() fails with + // that exception. + auto req = debugPort.getActorRequest(capnp::MessageSize{32, 0}); + req.setService(ns.serviceName); + req.setEntrypoint(ns.className); + req.setActorId(key); + KJ_IF_SOME(i, id.tryGet>()) { + KJ_IF_SOME(name, i->getName()) { + req.setActorName(name); + } + } + target = req.send().getActor(); + } + } + } + + static kj::Promise> startRequestImpl( + kj::Own self, IoChannelFactory::SubrequestMetadata metadata) { + co_await self->resolveTarget(); + + KJ_SWITCH_ONEOF(KJ_ASSERT_NONNULL(self->target)) { + KJ_CASE_ONEOF(container, kj::Own) { + co_return co_await container->startRequest(kj::mv(metadata)); + } + KJ_CASE_ONEOF(bootstrap, rpc::WorkerdBootstrap::Client) { + capnp::MessageSize sizeHint{4, 0}; + KJ_IF_SOME(cf, metadata.cfBlobJson) { + sizeHint.wordCount += cf.size() / sizeof(capnp::word) + 1; + } + + // The remote instance reconstructs SubrequestMetadata from these params. + auto req = bootstrap.startEventRequest(sizeHint); + KJ_IF_SOME(cf, metadata.cfBlobJson) { + req.setCfBlobJson(cf); + } + req.setFromPersistentStub(metadata.fromPersistentStub.toBool()); + + auto dispatcher = req.sendForPipeline().getDispatcher(); + // NOTE: We don't support restore() over cluster RPC so we can use + // getUnsupportedFrankenvalueHandler() here for now. + co_return kj::heap(self->ns.httpOverCapnpFactory, + self->ns.byteStreamFactory, getUnsupportedFrankenvalueHandler(), kj::mv(dispatcher)); + } + } + KJ_UNREACHABLE; + } }; }; diff --git a/src/workerd/server/tests/cluster/cluster-test-cidr.wd-test b/src/workerd/server/tests/cluster/cluster-test-cidr.wd-test index 3bebcbb6634..837070346ae 100644 --- a/src/workerd/server/tests/cluster/cluster-test-cidr.wd-test +++ b/src/workerd/server/tests/cluster/cluster-test-cidr.wd-test @@ -22,8 +22,8 @@ const config :Workerd.Config = ( modules = [ ( name = "cluster-worker.js", esModule = embed "cluster-worker.js" ), ], - compatibilityDate = "2024-09-01", - compatibilityFlags = ["nodejs_compat"], + compatibilityDate = "2026-01-01", + compatibilityFlags = ["nodejs_compat", "allow_irrevocable_stub_storage"], bindings = [ ( name = "COUNTER", durableObjectNamespace = "Counter" ), ( name = "NODE_ID", fromEnvironment = "NODE_ID" ), diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index 3203dc34812..a42623b42a6 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -11,6 +11,7 @@ // 3. Takeover after a node is killed. // 4. Alarms are rejected with a clear error in cluster mode. // 5. No `metadata.sqlite` is ever created (no AlarmScheduler in cluster mode). +// 6. Storing/loading a DO stub without calling it takes no ownership lock. // // Two variants are run: the unix-socket cluster network mode (default) and a // localhost CIDR (`127.0.0.0/8`) IP-socket mode that exercises the registry @@ -18,7 +19,7 @@ // by the WD_TEST_CONFIG environment variable supplied by the BUILD target. import { spawn } from 'node:child_process'; -import { mkdtemp, readdir, rm } from 'node:fs/promises'; +import { mkdtemp, readdir, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { env } from 'node:process'; @@ -463,3 +464,60 @@ test('cluster: registry directory is populated for each running instance', async } }); }); + +async function exists(path) { + try { + await stat(path); + return true; + } catch (err) { + if (err.code === 'ENOENT') return false; + throw err; + } +} + +test('cluster: storing and loading a stub does not take the ownership lock', async () => { + await withCluster(2, async ({ nodes, sharedPath }) => { + const nsDir = join(sharedPath, 'counter-test-namespace'); + const locksDir = join(nsDir, 'locks'); + + // DO "holder" (on whichever node claims it) stores a stub to DO "lazy" and + // reads it back. Serializing and deserializing the stub must not claim + // "lazy": no lock file, and no storage for it. + const stored = await fetchJson( + nodes[1].httpPort, + '/store-stub?name=holder&target=lazy' + ); + assert.strictEqual(stored.status, 200, JSON.stringify(stored.body)); + assert.strictEqual(stored.body.hasStub, true); + const holderId = stored.body.id; + const targetId = stored.body.targetId; + assert.notStrictEqual(holderId, targetId); + assert( + await exists(join(locksDir, holderId)), + `the holder DO itself must be locked` + ); + const lockFile = join(locksDir, targetId); + assert( + !(await exists(lockFile)), + `storing/loading a stub must not create the lock file ${lockFile}` + ); + const nsEntries = await readdir(nsDir); + assert( + !nsEntries.some((name) => name.startsWith(targetId)), + `storing/loading a stub must not create actor storage; found: ${nsEntries.join(', ')}` + ); + + // Calling the loaded stub claims ownership and creates the lock file. + const called = await fetchJson( + nodes[1].httpPort, + '/call-stored-stub?name=holder' + ); + assert.strictEqual(called.status, 200, JSON.stringify(called.body)); + assert.strictEqual(called.body.target.id, targetId); + assert.strictEqual(called.body.target.count, 0); + assert( + await exists(lockFile), + `calling the stub must create the lock file ${lockFile}` + ); + }); +}); diff --git a/src/workerd/server/tests/cluster/cluster-test.wd-test b/src/workerd/server/tests/cluster/cluster-test.wd-test index 6c6d53b2d7e..41bd3dea774 100644 --- a/src/workerd/server/tests/cluster/cluster-test.wd-test +++ b/src/workerd/server/tests/cluster/cluster-test.wd-test @@ -33,8 +33,8 @@ const config :Workerd.Config = ( modules = [ ( name = "cluster-worker.js", esModule = embed "cluster-worker.js" ), ], - compatibilityDate = "2024-09-01", - compatibilityFlags = ["nodejs_compat"], + compatibilityDate = "2026-01-01", + compatibilityFlags = ["nodejs_compat", "allow_irrevocable_stub_storage"], bindings = [ ( name = "COUNTER", durableObjectNamespace = "Counter" ), ( name = "NODE_ID", fromEnvironment = "NODE_ID" ), diff --git a/src/workerd/server/tests/cluster/cluster-worker.js b/src/workerd/server/tests/cluster/cluster-worker.js index dbd107fee3f..f885c3db19a 100644 --- a/src/workerd/server/tests/cluster/cluster-worker.js +++ b/src/workerd/server/tests/cluster/cluster-worker.js @@ -13,6 +13,12 @@ // /identity Returns information identifying this instance (NODE_ID env) // along with the DO id, so the test can verify which instance // served the request. +// /store-stub Stores a stub to the DO named by the `target` query param in +// this DO's storage, then reads it back without calling it. +// Returns the target's id. +// /call-stored-stub +// Loads the stub stored by /store-stub and calls /get on it. +// Returns the target's response. export default { async fetch(request, env) { @@ -27,8 +33,8 @@ export default { const id = env.COUNTER.idFromName(name); const stub = env.COUNTER.get(id); - // Forward to the DO with the same pathname. - const forwardUrl = new URL(url.pathname, 'http://do/'); + // Forward to the DO with the same path and query. + const forwardUrl = new URL(url.pathname + url.search, 'http://do/'); return await stub.fetch(forwardUrl.toString(), { method: request.method, headers: request.headers, @@ -73,6 +79,28 @@ export class Counter { } } else if (url.pathname === '/identity') { return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/store-stub') { + // Only stubs from the ctx.exports self-binding are storable. + const target = url.searchParams.get('target'); + const ns = this.state.exports.Counter; + const targetId = ns.idFromName(target); + await this.state.storage.put('stub', ns.get(targetId)); + // Read it back so that the stub is also deserialized, but do not call it. + const stub = await this.state.storage.get('stub'); + return Response.json({ + nodeId, + id: idHex, + targetId: targetId.toString(), + hasStub: typeof stub?.fetch === 'function', + }); + } else if (url.pathname === '/call-stored-stub') { + const stub = await this.state.storage.get('stub'); + const response = await stub.fetch('http://do/get'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); } return new Response('Not found', { status: 404 }); From 127fe4cc8123bdc25f6a20a0288377eb4c3cfe68 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sun, 6 Sep 2026 12:30:14 -0500 Subject: [PATCH 13/33] Force closure of SQLite database before releasing DO lock. We need to drop the cluster lock on the DO: * No earlier than when we close the SQLite connection, since we need to stop all modifications after we've dropped the lock. (Practically speaking, SQLite has its own lock which prevents corruption, but if it is still held while the cluster lock is not, it'll cause confusion elsewhere.) * No later than when we remove the actor from the map, otherwise other nodes will keep routing to us and we might even get stuck in a loop of discovering we own the actor but it isn't in the map. However, historically, the actor could be removed from the map long before the sqlite connection was actually closed, since other references to the actor (e.g. stubs held by clients) could hold it open. The actor would block all storage operations in the meantime, but since it still held open the sqlite connection, the underlying files would still be locked. This was probably a bug even before the cluster changes. This commit changes things so that: * We forcefully close sqlite in `ActorSqlite::shutdown()`. The existing machinery for reset() (which closed and re-opened the database) makes this relatively easy to achieve. * We don't drop `ownershipLock` until after `shutdown()` has definitely been called. --- src/workerd/io/actor-sqlite-test.c++ | 126 ++++++++--- src/workerd/io/actor-sqlite.c++ | 11 + src/workerd/io/actor-sqlite.h | 1 + src/workerd/server/server.c++ | 59 ++++- .../server/tests/cluster/cluster-test.mjs | 206 +++++++++++++++++- .../server/tests/cluster/cluster-worker.js | 64 +++++- src/workerd/util/sqlite.c++ | 58 ++++- src/workerd/util/sqlite.h | 27 ++- 8 files changed, 494 insertions(+), 58 deletions(-) diff --git a/src/workerd/io/actor-sqlite-test.c++ b/src/workerd/io/actor-sqlite-test.c++ index 502e5df03c0..9903393fcf8 100644 --- a/src/workerd/io/actor-sqlite-test.c++ +++ b/src/workerd/io/actor-sqlite-test.c++ @@ -2569,42 +2569,36 @@ KJ_TEST("unconfirmed setAlarm failure still breaks output gate") { KJ_EXPECT_THROW_MESSAGE("alarm commit failed", promise.wait(test.ws)); } +// Drives `test` into the broken state through SqliteDatabase's critical-error path: a write that +// fails with SQLITE_NOMEM inside `txn` makes SQLite auto-rollback the transaction, which invokes +// ActorSqlite::onCriticalError(). The global heap limit is restored before returning so that +// connections opened later are unaffected. +void breakViaCriticalError(ActorSqliteTest& test, ActorCacheInterface::Transaction& txn) { + auto heapLimit = test.db.run("PRAGMA hard_heap_limit").getInt64(0); + KJ_DEFER(sqlite3_hard_heap_limit64(heapLimit)); + + test.db.run("PRAGMA hard_heap_limit=8192"); // 8KB limit + + // A value large enough to exceed the limit. Copied once more to convert Array to + // Array. + auto largeData = kj::heapArray(50000, 'X'); + try { + txn.put(kj::str("large_key"), kj::heapArray(largeData.asBytes()), {}, nullptr); + KJ_FAIL_ASSERT("Query should have failed with SQLITE_NOMEM"); + } catch (kj::Exception& e) { + KJ_ASSERT(e.getDescription().contains("SENTRY_DO"), e); + KJ_ASSERT(e.getDescription().contains("out of memory"), e); + } +} + KJ_TEST("sync() throws after critical error in explicit transaction") { ActorSqliteTest test({.monitorOutputGate = false}); - auto heapLimit = [&]() { - auto row = test.db.run("PRAGMA hard_heap_limit"); - return row.getInt(0); - }(); - KJ_DBG(heapLimit); - KJ_DEFER(sqlite3_hard_heap_limit64(heapLimit);); - // Start an explicit transaction + // Start an explicit transaction and do a write within it. auto txn = test.startTransaction(); - - // Do a write within the transaction txn->put(kj::str("foo"), kj::heapArray("bar"_kj.asBytes()), {}, nullptr); - // Trigger a critical error using SQLITE_NOMEM by setting a very low heap limit - // and then trying to insert a large value. - try { - // Set SQLite's memory limit very low to trigger SQLITE_NOMEM - test.db.run("PRAGMA hard_heap_limit=8192"); // 8KB limit - - // Create data that will exceed the memory limit - auto largeData = kj::heapArray(50000, 'X'); // 50KB - - // This should trigger SQLITE_NOMEM, causing SQLite to auto-rollback the transaction, - // which will trigger the critical error handler. - // - // We have to copy it again in order to convert to a Array from an Array. - txn->put(kj::str("large_key"), kj::heapArray(largeData.asBytes()), /*options=*/{}, - nullptr); - KJ_FAIL_ASSERT("Query should have failed with SQLITE_NOMEM"); - } catch (kj::Exception& e) { - // Expected: out of memory error. We catch and ignore this to continue the test. - KJ_ASSERT(e.getDescription().contains("SENTRY_DO")); - KJ_ASSERT(e.getDescription().contains("out of memory")); - } + breakViaCriticalError(test, *txn); // sync() should also throw an exception because the storage is now broken auto syncResult = test.sync(); @@ -2617,6 +2611,78 @@ KJ_TEST("sync() throws after critical error in explicit transaction") { KJ_EXPECT_THROW_MESSAGE("broken", txn->commit()); } +// Asserts that `test.db` has been closed. Direct use fails, and a second connection to the same +// file can take an exclusive lock (which the closed connection's open transaction would have +// blocked) and sees the committed write but not the uncommitted one. +void expectDatabaseClosed(ActorSqliteTest& test) { + KJ_EXPECT_THROW_MESSAGE("database has been closed", test.db.run("SELECT 1")); + + SqliteDatabase fresh(test.vfs, kj::Path({"foo"}), kj::WriteMode::MODIFY); + fresh.run("BEGIN EXCLUSIVE"); + KJ_EXPECT(fresh.run("SELECT count(*) FROM _cf_KV WHERE key = 'committed'").getInt(0) == 1); + KJ_EXPECT(fresh.run("SELECT count(*) FROM _cf_KV WHERE key = 'uncommitted'").getInt(0) == 0); + fresh.run("COMMIT"); +} + +KJ_TEST("shutdown() closes the database") { + ActorSqliteTest test({.monitorOutputGate = false}); + + test.put("committed", "yes"); + test.pollAndExpectCalls({"commit"})[0]->fulfill(); + + // Leave a second write open in an implicit transaction whose commit has not run yet. + test.put("uncommitted", "yes"); + + test.actor.shutdown(kj::none); + + KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.actor.getSqliteDatabase()); + expectDatabaseClosed(test); + + // The scheduled commit observes the shutdown and breaks the output gate with the shutdown + // exception rather than touching the closed database. + KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.gate.onBroken().wait(test.ws)); + test.pollAndExpectCalls({}); + + // A second shutdown() is a no-op. + test.actor.shutdown(kj::none); + KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.actor.getSqliteDatabase()); + expectDatabaseClosed(test); +} + +KJ_TEST("shutdown() closes the database when storage is already broken") { + ActorSqliteTest test({.monitorOutputGate = false}); + + test.put("committed", "yes"); + test.pollAndExpectCalls({"commit"})[0]->fulfill(); + + auto txn = test.startTransaction(); + txn->put(kj::str("uncommitted"), kj::heapArray("yes"_kj.asBytes()), {}, nullptr); + breakViaCriticalError(test, *txn); + + // Storage is broken but the connection is still open until shutdown() runs, as it does when the + // broken output gate aborts the IoContext. + KJ_EXPECT_THROW_MESSAGE("broken.outputGateBroken", test.actor.getSqliteDatabase()); + test.db.run("SELECT 1"); + + test.actor.shutdown(kj::none); + + auto expectOriginalBrokenException = [&]() { + KJ_IF_SOME(e, kj::runCatchingExceptions([&]() { test.actor.getSqliteDatabase(); })) { + // The first exception wins: the shutdown exception must not replace the storage error. + KJ_EXPECT(e.getDescription().contains("broken.outputGateBroken"), e); + KJ_EXPECT(!e.getDescription().contains(ActorCache::SHUTDOWN_ERROR_MESSAGE), e); + } else { + KJ_FAIL_EXPECT("getSqliteDatabase() should have thrown"); + } + }; + expectOriginalBrokenException(); + expectDatabaseClosed(test); + + test.actor.shutdown(kj::none); + expectOriginalBrokenException(); + expectDatabaseClosed(test); +} + KJ_TEST("allowUnconfirmed put in explicit transaction does not block output gate") { ActorSqliteTest test; diff --git a/src/workerd/io/actor-sqlite.c++ b/src/workerd/io/actor-sqlite.c++ index 2d016268911..9131001207f 100644 --- a/src/workerd/io/actor-sqlite.c++ +++ b/src/workerd/io/actor-sqlite.c++ @@ -997,6 +997,17 @@ void ActorSqlite::shutdown(kj::Maybe maybeException) { // We've already experienced a terminal exception either from shutdown or OOM, there should // already be a flush scheduled that will break the output gate. } + + // Sever the sqlite connection, so that the underlying database file is fully and synchcronously + // closed. Callers of shutdown() rely on this. Do it even if `broken` was already set because not + // all paths that set `broken` call `close()`. Note `close()` is idempotent. + // + // TODO(cleanup): Maybe the entire `broken` mechanism should be pushed down into SqliteDatabase? + // Currently `ActorSqlite` carefully checks `broken` in every path that touches the database, + // but `SqliteDatabase` itself checks if it is closed on every path. We could change `close()` + // to something like `abort(reason)` and then rely on that instead of having `broken` in + // `ActorSqlite`. Needs careful investigation of all code paths, though. + db->close(); } kj::OneOf ActorSqlite:: diff --git a/src/workerd/io/actor-sqlite.h b/src/workerd/io/actor-sqlite.h index 527353fe2ae..36ffdbffeef 100644 --- a/src/workerd/io/actor-sqlite.h +++ b/src/workerd/io/actor-sqlite.h @@ -88,6 +88,7 @@ class ActorSqlite final: public ActorCacheInterface, private kj::TaskSet::ErrorH void blockTransaction(kj::Promise promise) override; kj::Maybe getSqliteDatabase() override { + requireNotBroken(); return *db; } diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 16a9feefee9..6bfc666bb86 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -520,15 +520,15 @@ class Server::ActorNamespace final { kj::OneOf> classAndIdParam, kj::Timer& timer, kj::Maybe ownershipLock = kj::none) - : key(kj::mv(key)), + : ownershipLock(kj::mv(ownershipLock)), + key(kj::mv(key)), tracker(kj::refcounted(*this)), ns(ns), root(parent.map([](ActorContainer& p) -> ActorContainer& { return p.root; }) .orDefault(*this)), parent(parent), timer(timer), - lastAccess(timer.now()), - ownershipLock(kj::mv(ownershipLock)) { + lastAccess(timer.now()) { KJ_SWITCH_ONEOF(classAndIdParam) { KJ_CASE_ONEOF(value, ClassAndId) { // `classAndId` is immediately available. @@ -706,7 +706,9 @@ class Server::ActorNamespace final { // removed from any maps that would cause it to receive further traffic, since any further // requests will be expected to fail. abort() does NOT attempt to remove the ActorContainer // from the parent facet map since at most call sites it makes more sense to handle this - // directly. + // directly. Callers that remove the container from `ns.actors` must do so in the same + // synchronous continuation as this call: abort() releases the ownership lock (see + // `ownershipLock`). void abort(kj::Maybe reason) { if (brokenReason != kj::none) return; @@ -730,6 +732,12 @@ class Server::ActorNamespace final { actor = kj::none; containerClient = kj::none; + // The actor (and every facet) shut down above, which closed their databases, so nothing on + // this node holds a handle into the actor's storage anymore. Drop ours and let another node + // (or a fresh container on this one) claim the actor. + facetTreeIndex = kj::none; + ownershipLock = kj::none; + KJ_IF_SOME(r, reason) { brokenReason = r.clone(); } else { @@ -738,13 +746,15 @@ class Server::ActorNamespace final { } // Resets the actor's SQLite database while the connection is still open, - // avoiding file-locking issues on Windows. + // avoiding file-locking issues on Windows. Storage that is already broken is skipped. void resetStorage() { KJ_IF_SOME(a, actor) { KJ_IF_SOME(cache, a->getPersistent()) { - KJ_IF_SOME(db, cache.getSqliteDatabase()) { - kj::runCatchingExceptions([&]() { db.reset(); }); - } + kj::runCatchingExceptions([&]() { + KJ_IF_SOME(db, cache.getSqliteDatabase()) { + db.reset(); + } + }); } } } @@ -860,6 +870,21 @@ class Server::ActorNamespace final { } private: + // In cluster mode, the root container's claim on the actor. Declared first so that on plain + // destruction it is released only after everything below it -- the Worker::Actor, facets, + // hibernation manager, and facet tree index -- has been torn down. + // + // Invariant: the lock file names this node if and only if this container is in `ns.actors`. + // A request for an actor whose lock file names this node but which has no map entry loops + // through self-bootstrap (see ActorNamespace::resolveNode()), so the lock is released at the + // moment the container leaves the map (abort(), monitorOnBroken()), not when the last + // reference to the container is dropped. Releasing it there is only safe because + // ActorSqlite::shutdown() closes the database: by the time either path hollows the container, + // every SQLite handle under the actor and its facets is already closed, so no handle into the + // actor's storage outlives the claim even if an in-flight request still holds the + // Worker::Actor. + kj::Maybe ownershipLock; + // The actor is constructed after the ActorContainer so it starts off empty. kj::Maybe> actor; @@ -875,7 +900,6 @@ class Server::ActorNamespace final { kj::Maybe> onBrokenTask; kj::Maybe brokenReason; kj::Vector>> inactiveFulfillers; - kj::Maybe ownershipLock; // in cluster mode // Reference to the ContainerClient (if container is enabled for this actor) kj::Maybe> containerClient; @@ -1059,6 +1083,13 @@ class Server::ActorNamespace final { tracker->shutdown(); auto managerToDrop = kj::mv(manager); + // onBroken fired only after IoContext::abort() shut down the actor cache, and the facets + // were aborted above, so every database under this actor is closed. Release the claim now, + // in the same synchronous continuation as the erase() below (see `ownershipLock`). Any stub + // still pinning this container after the erase keeps only a hollow shell. + facetTreeIndex = kj::none; + ownershipLock = kj::none; + // Note that we remove the entire ActorContainer from the map -- this drops the // HibernationManager so any connected hibernatable websockets will be disconnected. KJ_IF_SOME(p, parent) { @@ -1473,9 +1504,13 @@ class Server::ActorNamespace final { // Create an ActorContainer in cluster mode. Must be called in the same synchronous continuation // that received `ownership` from ClusterLockManager::acquireOrRoute(): the lock file naming this // node while `actors` has no entry for the key would make requests for the actor loop through - // self-bootstrap until the entry appears. + // self-bootstrap until the entry appears (see resolveNode()). kj::Own createActorContainer( kj::String key, Worker::Actor::Id id, ClusterLockManager::OwnedLock ownership) { + // Invariant: the lock file names this node iff a container holding the OwnedLock is in + // `actors`. We hold the lock and are about to insert; nothing else may already be mapped. + // The other direction is upheld by ActorContainer::abort() and monitorOnBroken(), which + // release the lock as the container leaves the map. KJ_ASSERT(actors.find(key) == kj::none); auto container = kj::refcounted(kj::str(key), *this, kj::none, @@ -1491,6 +1526,10 @@ class Server::ActorNamespace final { // returned promise settles. Cluster mode only. using Node = kj::OneOf, rpc::WorkerdDebugPort::Client>; kj::Promise resolveNode(kj::StringPtr key, Worker::Actor::Id& id) { + // Local-first. This lookup is what terminates the routing loop: when the lock file names this + // node, acquireOrRoute() returns our own debug port with no backoff, and the resulting + // getActor() lands right back here. That is only correct because "lock file names this node" + // implies "container is in `actors`" -- see ActorContainer::ownershipLock. KJ_IF_SOME(container, actors.find(key)) { co_return container->addRef(); } diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index a42623b42a6..92f98546205 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -12,6 +12,8 @@ // 4. Alarms are rejected with a clear error in cluster mode. // 5. No `metadata.sqlite` is ever created (no AlarmScheduler in cluster mode). // 6. Storing/loading a DO stub without calling it takes no ownership lock. +// 7. A broken DO releases its ownership lock even while stubs still pin it, +// so it can be re-instantiated (on any node) instead of hanging requests. // // Two variants are run: the unix-socket cluster network mode (default) and a // localhost CIDR (`127.0.0.0/8`) IP-socket mode that exercises the registry @@ -19,7 +21,7 @@ // by the WD_TEST_CONFIG environment variable supplied by the BUILD target. import { spawn } from 'node:child_process'; -import { mkdtemp, readdir, rm, stat } from 'node:fs/promises'; +import { mkdtemp, readdir, readFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { env } from 'node:process'; @@ -174,9 +176,26 @@ class ClusterNode { } } -async function fetchJson(port, path, init) { +// Fetches `path` from the given node and parses the JSON response. When +// `timeoutMs` is given, a request that does not complete in time fails with a +// clear error instead of hanging the test (used where a bug would manifest as +// a request that never completes). +async function fetchJson(port, path, { timeoutMs } = {}) { const url = `http://127.0.0.1:${port}${path}`; - const res = await fetch(url, init); + let res; + try { + res = await fetch(url, { + signal: + timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs), + }); + } catch (err) { + if (err.name === 'TimeoutError') { + throw new Error( + `request to ${url} did not complete within ${timeoutMs}ms` + ); + } + throw err; + } const text = await res.text(); let body; try { @@ -189,11 +208,32 @@ async function fetchJson(port, path, init) { return { status: res.status, body }; } +// Returns the hex-encoded public key of the node that currently owns the lock +// file at `path` (the node's registry entry name), or null if the file is +// absent or empty. +async function readLockOwner(path) { + let content; + try { + content = await readFile(path); + } catch (err) { + if (err.code === 'ENOENT') return null; + throw err; + } + return content.length === 0 ? null : content.toString('hex'); +} + +// Starts `numNodes` workerd instances sharing one directory and runs `fn` with: +// nodes the ClusterNode instances, in start order +// sharedPath the shared directory +// nodeKeys Map from nodeId to the node's registry key (64-char hex), which +// is also what ownership lock files contain async function withCluster(numNodes, fn) { // Each test run gets its own shared directory, so the registry/lock-file // state doesn't leak between tests. const sharedPath = await mkdtemp(join(tmpdir(), 'workerd-cluster-')); + const registryDir = join(sharedPath, 'workerd-registry'); const nodes = []; + const nodeKeys = new Map(); try { for (let i = 0; i < numNodes; i++) { const node = new ClusterNode({ @@ -204,8 +244,32 @@ async function withCluster(numNodes, fn) { }); await node.start(); nodes.push(node); + + // Each node writes its registry entry before it reports its listening + // socket, but allow a small grace period for filesystem visibility. Since + // nodes start one at a time, the entry that is new is this node's. + const known = new Set(nodeKeys.values()); + const deadline = Date.now() + 5000; + let fresh = []; + while (Date.now() < deadline) { + let entries = []; + try { + entries = await readdir(registryDir); + } catch (_) { + // Not created yet. + } + fresh = entries.filter((name) => !known.has(name)); + if (fresh.length > 0) break; + await sleep(50); + } + assert.strictEqual( + fresh.length, + 1, + `expected exactly one new registry entry after starting ${node.nodeId}, got: [${fresh.join(', ')}]` + ); + nodeKeys.set(node.nodeId, fresh[0]); } - await fn({ nodes, sharedPath }); + await fn({ nodes, sharedPath, nodeKeys }); } finally { // Stop all nodes (ignore errors -- some may already be stopped by the // test itself). @@ -521,3 +585,137 @@ test('cluster: storing and loading a stub does not take the ownership lock', asy ); }); }); + +// Requests that would hang forever if a broken actor kept its ownership lock +// are bounded by this timeout so the failure is a clear error. +const HANG_TIMEOUT_MS = 5000; + +function lockFilePath(sharedPath, id) { + return join(sharedPath, 'counter-test-namespace', 'locks', id); +} + +test('cluster: a broken DO pinned by a stub does not keep its ownership lock', async () => { + await withCluster(2, async ({ nodes, sharedPath, nodeKeys }) => { + // Node 0 claims "pinned". DO "holder", served by node 1, then obtains a + // stub to it, uses it, and keeps it for as long as "holder" lives. Through + // node 0's bootstrap for that stub, node 0's container for "pinned" stays + // referenced for the rest of the test. + const first = await fetchJson(nodes[0].httpPort, '/increment?name=pinned'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + assert.strictEqual(first.body.count, 1); + const owner = first.body.nodeId; + const lockFile = lockFilePath(sharedPath, first.body.id); + assert.strictEqual(await readLockOwner(lockFile), nodeKeys.get(owner)); + + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=pinned' + ); + assert.strictEqual(held.status, 200, JSON.stringify(held.body)); + assert.strictEqual(held.body.target.nodeId, owner); + + // Break the DO. Its container leaves the owner's map but lives on, pinned + // by the parked stub. + const broken = await fetchJson(nodes[1].httpPort, '/break?name=pinned'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + // The lock must have been released along with the map entry. Otherwise a + // request from the non-owner routes to the owner, which finds no container + // and routes to itself, forever. + const other = nodes.find((n) => n.nodeId !== owner); + const after = await fetchJson(other.httpPort, '/increment?name=pinned', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual(after.body.count, 2, 'state must survive the break'); + assert.strictEqual( + await readLockOwner(lockFile), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + }); +}); + +test('cluster: a DO whose constructor fails releases its ownership lock', async () => { + await withCluster(2, async ({ nodes, sharedPath, nodeKeys }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=ctor'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const lockFile = lockFilePath(sharedPath, first.body.id); + + // Arm the one-shot constructor failure and abort the running instance so + // the next request has to construct a new one. + const armed = await fetchJson( + nodes[1].httpPort, + '/arm-constructor-failure?name=ctor' + ); + assert.strictEqual(armed.status, 500, JSON.stringify(armed.body)); + + // Re-instantiate through a stub kept by DO "holder". The constructor + // throws, which breaks the new instance; the kept stub pins its hollowed + // container on whichever node claimed the DO. + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=ctor', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(held.status, 500, JSON.stringify(held.body)); + assert.match(held.body.error, /constructor failed on purpose/); + + // A fresh request from the other node must not hang. The failed constructor + // committed the cleared flag before throwing, so this instantiation + // succeeds and sees the earlier increment. + const after = await fetchJson(nodes[0].httpPort, '/increment?name=ctor', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual(after.body.count, 2); + assert.strictEqual( + await readLockOwner(lockFile), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + }); +}); + +test('cluster: aborting a DO with a request in flight lets it be re-instantiated immediately', async () => { + await withCluster(2, async ({ nodes, sharedPath, nodeKeys }) => { + const first = await fetchJson( + nodes[0].httpPort, + '/increment?name=inflight' + ); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const owner = first.body.nodeId; + const lockFile = lockFilePath(sharedPath, first.body.id); + + // Start a request the DO holds open, give it time to reach the DO, then + // abort the DO underneath it. The in-flight request keeps a reference to + // the old Worker::Actor until it unwinds, but the storage must be closed + // and the lock released as soon as the actor breaks, not when that + // reference goes away. + const holding = fetchJson(nodes[0].httpPort, '/hold?name=inflight', { + timeoutMs: HANG_TIMEOUT_MS, + }); + await sleep(250); + const broken = await fetchJson(nodes[0].httpPort, '/break?name=inflight'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + const other = nodes.find((n) => n.nodeId !== owner); + const after = await fetchJson(other.httpPort, '/increment?name=inflight', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual( + after.body.count, + 2, + 'the new instance must see the state committed before the abort' + ); + assert.strictEqual( + await readLockOwner(lockFile), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + + const held = await holding; + assert.strictEqual(held.status, 500, JSON.stringify(held.body)); + }); +}); diff --git a/src/workerd/server/tests/cluster/cluster-worker.js b/src/workerd/server/tests/cluster/cluster-worker.js index f885c3db19a..4f8f1706df9 100644 --- a/src/workerd/server/tests/cluster/cluster-worker.js +++ b/src/workerd/server/tests/cluster/cluster-worker.js @@ -3,7 +3,9 @@ // https://opensource.org/licenses/Apache-2.0 // Test worker for the cluster integration test. Exposes a Counter Durable Object -// and a top-level fetch() that routes requests to the DO by name. +// and a top-level fetch() that routes requests to the DO by name. Requests that +// fail are reported as JSON `{ error }` with status 500 rather than letting the +// exception propagate, so the test can assert on the message. // // The DO supports the following request paths: // /increment Increments and returns the current counter value. @@ -19,6 +21,21 @@ // /call-stored-stub // Loads the stub stored by /store-stub and calls /get on it. // Returns the target's response. +// /hold-stub Obtains a stub to the DO named by the `target` query param, +// calls /identity on it, and keeps the stub on this instance +// for as long as this instance lives. (A stub is owned by the +// request context that created it, so a module-level variable +// would not keep it alive past the request; an instance field +// of a long-lived DO does.) Returns the target's response. +// /hold Waits one second before responding, so that the test can +// break the DO while this request is still in flight. +// /break Aborts the DO via ctx.abort(). The request fails. +// /arm-constructor-failure +// Sets a storage flag that makes the *next* instantiation of +// this DO throw from its constructor (the flag is cleared and +// committed before throwing, so the instantiation after that +// succeeds), then aborts the DO so that the next request +// re-instantiates it. export default { async fetch(request, env) { @@ -35,17 +52,35 @@ export default { // Forward to the DO with the same path and query. const forwardUrl = new URL(url.pathname + url.search, 'http://do/'); - return await stub.fetch(forwardUrl.toString(), { - method: request.method, - headers: request.headers, - }); + try { + return await stub.fetch(forwardUrl.toString(), { + method: request.method, + headers: request.headers, + }); + } catch (err) { + return Response.json({ error: String(err) }, { status: 500 }); + } }, }; +const BREAK_ON_CONSTRUCT = 'break-on-construct'; + export class Counter { constructor(state, env) { this.state = state; this.env = env; + // See /hold-stub. + this.heldStubs = []; + + state.blockConcurrencyWhile(async () => { + if (await state.storage.get(BREAK_ON_CONSTRUCT)) { + // One-shot: make sure the cleared flag is durable before breaking, so + // the next instantiation succeeds. + await state.storage.delete(BREAK_ON_CONSTRUCT); + await state.storage.sync(); + throw new Error('constructor failed on purpose'); + } + }); } async fetch(request) { @@ -101,6 +136,25 @@ export class Counter { id: idHex, target: await response.json(), }); + } else if (url.pathname === '/hold-stub') { + const target = url.searchParams.get('target'); + const stub = this.env.COUNTER.get(this.env.COUNTER.idFromName(target)); + this.heldStubs.push(stub); + const response = await stub.fetch('http://do/identity'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); + } else if (url.pathname === '/hold') { + await scheduler.wait(1000); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/break') { + this.state.abort('broken on purpose'); + } else if (url.pathname === '/arm-constructor-failure') { + await this.state.storage.put(BREAK_ON_CONSTRUCT, true); + await this.state.storage.sync(); + this.state.abort('armed constructor failure'); } return new Response('Not found', { status: 404 }); diff --git a/src/workerd/util/sqlite.c++ b/src/workerd/util/sqlite.c++ index 96607b2f16a..124b17a4db1 100644 --- a/src/workerd/util/sqlite.c++ +++ b/src/workerd/util/sqlite.c++ @@ -674,7 +674,11 @@ SqliteDatabase::~SqliteDatabase() noexcept(false) { } SqliteDatabase::operator sqlite3*() { - return &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + KJ_IF_SOME(db, maybeDb) { + return &db; + } + KJ_REQUIRE(!closed, "database has been closed"); + KJ_FAIL_ASSERT("previous reset() failed"); } SqliteMemoryScope SqliteDatabase::enterMemoryScope() { @@ -700,7 +704,7 @@ void SqliteDatabase::handleCriticalError(kj::Maybe errorCode, if (code == SQLITE_FULL || code == SQLITE_IOERR || code == SQLITE_NOMEM || code == SQLITE_INTERRUPT) { - sqlite3* db = &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + sqlite3* db = *this; // We are in a transaction if (inTransaction || !savepoints.empty()) { // The transaction was auto-rolledback, re-enabling the auto commit mode, so we should fail @@ -815,7 +819,7 @@ SqliteDatabase::StatementAndEffect SqliteDatabase::prepareSql(StaticRegulator re uint prepFlags, Multi multi, kj::Maybe&> prelude) { - sqlite3* db = &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + sqlite3* db = *this; ParseContext parseContext; KJ_ASSERT(currentParseContext == kj::none, "recursive prepareSql()?"); @@ -1011,6 +1015,7 @@ void SqliteDatabase::executeWithRegulator( void SqliteDatabase::reset() { KJ_REQUIRE(!readOnly, "can't reset() read-only database"); + KJ_REQUIRE(!closed, "can't reset() a database that has been closed"); // If transactions are open during reset(), whatever had the transaction open is going to get // confused at best, or lose data at worst. Let's just not allow this. @@ -1043,6 +1048,44 @@ void SqliteDatabase::reset() { } } +void SqliteDatabase::close() noexcept { + if (closed) return; + + // If we're currently executing a SQLite statement, there's no safe way for us to close the DB. + // But, callers of close() rely on the fact that the DB will in fact be closed, and they often + // sit on error-handling paths that cannot tolerate further exceptions. These asserts, if they + // fail, will abort the process, due to close() being noexcept. As of this writing it is believed + // that no real code path is likely to hit these, so these are just safety checks / internal + // invariants. + KJ_ASSERT(currentStatement == kj::none, "close() called while a query is executing"); + KJ_ASSERT(currentRegulator == kj::none, "close() called while a query is compiling"); + + auto memoryScope = enterMemoryScope(); + + KJ_IF_SOME(db, maybeDb) { + for (auto& listener: resetListeners) { + listener.beforeSqliteReset(); + } + + // Every prepared statement was finalized by the listener walk above, so SQLITE_BUSY here + // indicates a statement that isn't tracked by a ResetListener. Do not fall back to + // sqlite3_close_v2(): a lazily-closed handle keeps the file open, which is exactly what this + // method exists to prevent. + auto err = sqlite3_close(&db); + KJ_ASSERT( + err == SQLITE_OK, "close() found dependent objects that still exist", sqlite3_errstr(err)); + + maybeDb = kj::none; + } + + // sqlite3_close() rolled back any open transaction. Nothing may read the database again, so the + // rollback callbacks (which exist to keep in-memory caches in sync with it) are simply dropped. + inTransaction = false; + savepoints.clear(); + rollbackCallbacks.clear(); + closed = true; +} + bool SqliteDatabase::isAuthorized(int actionCode, kj::Maybe param1, kj::Maybe param2, @@ -1794,8 +1837,13 @@ bool SqliteDatabase::Query::isNull(uint column) { SqliteDatabase::StatementAndEffect& SqliteDatabase::Query::getStatementAndEffect() { return KJ_UNWRAP_OR(maybeStatement, { - regulator->onError(kj::none, "SQLite query was canceled because the database was deleted."); - KJ_FAIL_REQUIRE("query canceled because reset() was called on the database"); + if (db.closed) { + regulator->onError(kj::none, "SQLite query was canceled because the database was closed."); + KJ_FAIL_REQUIRE("query canceled because the database has been closed"); + } else { + regulator->onError(kj::none, "SQLite query was canceled because the database was deleted."); + KJ_FAIL_REQUIRE("query canceled because reset() was called on the database"); + } }); } diff --git a/src/workerd/util/sqlite.h b/src/workerd/util/sqlite.h index 68535f5a8f0..5a37fc6140d 100644 --- a/src/workerd/util/sqlite.h +++ b/src/workerd/util/sqlite.h @@ -289,7 +289,23 @@ class SqliteDatabase { // may throw if they depend on tables that haven't been recreated yet). void reset(); - // Objects that need to be notified when reset() is called may inherit `ResetListener`. + // Closes the underlying SQLite connection without reopening it. Every subsequent attempt to + // use the database, including through existing Statements and Queries, throws. Any open + // transaction is rolled back (never committed). Idempotent. + // + // This exists so that an actor's storage can be severed at the moment the actor is shut down, + // even if references to the SqliteDatabase live on: closing the last connection checkpoints and + // deletes the WAL and releases any file locks, which the next opener of the same file needs. + // + // This is declared noexcept because it is expected to be used in shutdown paths which are not + // prepared to catch further errors and strongly depend on the database actually becoming closed + // (so that they can safely release file locks). If close() is unable to complete (e.g. because + // a sqlite statement is currently being executed) then we have no choice but to abort the + // process. + void close() noexcept; + + // Objects that need to be notified when reset() or close() is called may inherit + // `ResetListener`. class ResetListener { public: ResetListener(SqliteDatabase& db): db(db) { @@ -303,8 +319,8 @@ class SqliteDatabase { db.resetListeners.add(*this); } - // When the database's `reset()` method is called, all listeners' `beforeSqliteReset()` will be - // called before actually resetting the database. + // When the database's `reset()` or `close()` method is called, all listeners' + // `beforeSqliteReset()` will be called before actually closing the connection. virtual void beforeSqliteReset() = 0; protected: // so that subclasses don't have to store their own copy of the `db` reference @@ -366,9 +382,12 @@ class SqliteDatabase { kj::Maybe actorAccountLimits; - // This pointer can be left null if a call to reset() failed to re-open the database. + // Null after a call to reset() failed to re-open the database, or after close(). kj::Maybe maybeDb; + // Set by close(). Distinguishes the closed state from a failed reset() in error messages. + bool closed = false; + // Set while a query is compiling. kj::Maybe currentRegulator; From b58b0de4e44ee4bf9d662bbd413fedf66af91243 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sun, 6 Sep 2026 17:08:58 -0500 Subject: [PATCH 14/33] Bugfix: Cease post-commit alarm updates when broken. Previously, after shutdown() was called, existing in-flight commits would continue running, potentially performing post-commit alarm updates. But at this point the actor could be dead and some other instance of the actor could have taken over, so this is incorrect. This commit makes us check for brokenness before these alarm updates. Probably we should really be cancelling the commit tasks, but that may have impact on SRS, so I have left it as a TODO for now. --- src/workerd/io/actor-sqlite-test.c++ | 30 ++++++++++++++++++++++++++++ src/workerd/io/actor-sqlite.c++ | 24 +++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/workerd/io/actor-sqlite-test.c++ b/src/workerd/io/actor-sqlite-test.c++ index 9903393fcf8..58618469383 100644 --- a/src/workerd/io/actor-sqlite-test.c++ +++ b/src/workerd/io/actor-sqlite-test.c++ @@ -2683,6 +2683,36 @@ KJ_TEST("shutdown() closes the database when storage is already broken") { expectDatabaseClosed(test); } +KJ_TEST("shutdown() during in-flight commit does not unschedule the persisted alarm") { + ActorSqliteTest test({.monitorOutputGate = false}); + + // Persist an alarm at 2ms, so that the next write moves the alarm earlier and thus starts a + // precommit scheduling request that the commit must wait on. + test.setAlarm(twoMs); + test.pollAndExpectCalls({"scheduleRun(2ms)"})[0]->fulfill(); + test.pollAndExpectCalls({"commit"})[0]->fulfill(); + + // Move the alarm earlier. The local transaction commits, then commitImpl() suspends waiting for + // the scheduler to confirm the 1ms request. + test.setAlarm(oneMs); + auto scheduleFulfiller = kj::mv(test.pollAndExpectCalls({"scheduleRun(1ms)"})[0]); + + // Shut down (and thereby close the database) while that commit is parked. + test.actor.shutdown(kj::none); + + // Resume the commit. It must not consult the closed database's alarm state and must not issue + // any further scheduler requests. In particular it must not issue `scheduleRun(none)`, which + // would cancel the alarm that the database still persists. + scheduleFulfiller->fulfill(); + test.pollAndExpectCalls({}); + KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.gate.onBroken().wait(test.ws)); + + // The database still holds the 1ms alarm for the next instance to pick up. + SqliteDatabase fresh(test.vfs, kj::Path({"foo"}), kj::WriteMode::MODIFY); + KJ_EXPECT(fresh.run("SELECT value FROM _cf_METADATA WHERE key = 1").getInt64(0) == + (oneMs - kj::UNIX_EPOCH) / kj::NANOSECONDS); +} + KJ_TEST("allowUnconfirmed put in explicit transaction does not block output gate") { ActorSqliteTest test; diff --git a/src/workerd/io/actor-sqlite.c++ b/src/workerd/io/actor-sqlite.c++ index 9131001207f..6fdf1610a0a 100644 --- a/src/workerd/io/actor-sqlite.c++ +++ b/src/workerd/io/actor-sqlite.c++ @@ -445,6 +445,9 @@ kj::Promise ActorSqlite::requestScheduledAlarm( } void ActorSqlite::scheduleLaterAlarm(kj::Maybe newAlarmTime, SpanParent parentSpan) { + // A queued update may resume after shutdown, when another instance owns the actor. + if (broken != kj::none) return; + if (alarmLaterIsInFlight) { // There's already a move-later request in-flight. Just store the desired time; the in-flight // request's completion handler will pick it up and start a new request. This overwrites any @@ -524,7 +527,7 @@ kj::Promise ActorSqlite::commitImpl( } commitSpan.setTag("merged_with_pending_commit"_kjc, true); co_await pending.addBranch(); - if (debugAlarmSync) { + if (debugAlarmSync && broken == kj::none) { auto alarmAfterMerge = metadata.getAlarm(); KJ_LOG(WARNING, "NOSENTRY DEBUG_ALARM: Commit merge resumed", logDate(alarmBeforeMerge), logDate(alarmAfterMerge), alarmVersion); @@ -548,6 +551,11 @@ kj::Promise ActorSqlite::commitImpl( auto alarmSpan = commitSpan.newChild("actor_sqlite_alarm_sync"_kjc); haveAlarmForDebug = true; co_await p; + + // We may have become broken (e.g. shutdown() closed the database) while suspended. Once + // broken, we no longer own the database, and there could be another instance of this actor + // running elsewhere, therefore we should not continue with alarm updates. + requireNotBroken(); } // While the local db state requires an earlier alarm than is known might be scheduled, issue an @@ -577,6 +585,7 @@ kj::Promise ActorSqlite::commitImpl( // future turn of the event loop. That means we're going to suspend, even if the promise is // ready, which means we'd take a performance hit. co_await requestScheduledAlarm(metadata.getAlarm(), kj::READY_NOW); + requireNotBroken(); // See above. syncIterations++; } if (debugAlarmSync && syncIterations > 0) { @@ -615,6 +624,13 @@ kj::Promise ActorSqlite::commitImpl( // Notify any merged commitImpl() requests that the db persistence completed. fulfiller->fulfill(); + // The commit itself is complete. If we became broken while it was in flight, skip the optional + // post-commit alarm sync rather than throwing: the merged waiters' writes did persist, and the + // alarm will self-correct when it fires (see armAlarmHandler()). + if (broken != kj::none) { + co_return; + } + if (debugAlarmSync) { KJ_LOG(WARNING, "NOSENTRY DEBUG_ALARM: Version check", alarmVersionBeforeAsync, alarmVersion, "match", (alarmVersion == alarmVersionBeforeAsync)); @@ -988,6 +1004,12 @@ void ActorSqlite::shutdown(kj::Maybe maybeException) { // `maybeTerminalException` has a value. Any in-flight flushes will continue to run in the // background. Remember that these in-flight flushes may or may not be awaited by the worker, // but they still hold the output lock as long as `allowUnconfirmed` wasn't used. + // + // TODO(cleanup): Instead of letting in-flight commits continue and having every resume point + // in commitImpl() re-check `broken`, consider cancelling them here outright. (Use a + // kj::Canceler for this to make sure all forks are canceled.) This changes the policy + // above (in-flight flushes no longer complete), which embedders with a real commitCallback + // (SRS / internal codebase) may care about -- need to study consequences there. broken.emplace(kj::mv(exception)); // We explicitly do not schedule a flush to break the output gate. This means that if a request From 19056df139960d2eb0dad7ce1d8125a9242ad70f Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sun, 6 Sep 2026 18:12:39 -0500 Subject: [PATCH 15/33] Bugfix: ResetListeners need to distinguish reset() vs. close(). The ResetListener machinery turned out to be less reusable than expected, since several implementations explicitly assume the reset is deleting all the DB content, not just closing. To fix this, we rename `beforeSqliteReset()` to `beforeSqliteClose()` and give it a parameter specifying if it's `close()` or `reset()`, and update all the implementations to do the right things. This is belt-and-suspenders (as Claude would say) with the previous commit. Before either commit, we could incorrectly cancel alarms after close() if an existing commit were in-flight, since after the close() attempts to read the metadata would incorrectly report that the alarm had been erased. --- src/workerd/util/sqlite-kv-test.c++ | 20 ++++++++++++++++ src/workerd/util/sqlite-kv.c++ | 14 ++++++++---- src/workerd/util/sqlite-kv.h | 2 +- src/workerd/util/sqlite-metadata-test.c++ | 18 +++++++++++++++ src/workerd/util/sqlite-metadata.c++ | 14 +++++++++--- src/workerd/util/sqlite-metadata.h | 2 +- src/workerd/util/sqlite.c++ | 26 +++++++++++++++------ src/workerd/util/sqlite.h | 28 +++++++++++++++++++---- 8 files changed, 104 insertions(+), 20 deletions(-) diff --git a/src/workerd/util/sqlite-kv-test.c++ b/src/workerd/util/sqlite-kv-test.c++ index 0d60038868a..9e29a5aa07a 100644 --- a/src/workerd/util/sqlite-kv-test.c++ +++ b/src/workerd/util/sqlite-kv-test.c++ @@ -338,5 +338,25 @@ KJ_TEST("SQLite-KV multi-put with allowUnconfirmed") { KJ_EXPECT(called); } +KJ_TEST("SQLite-KV reads throw after close()") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + SqliteKv kv(db); + + kv.put("foo", "abc"_kj.asBytes()); + kv.putExternals("foo", kj::arr(kj::heapArray("tok"_kj.asBytes()))); + + db.close(); + + // Unlike reset(), close() must not make the data appear deleted: the file still has it. + KJ_EXPECT_THROW_MESSAGE("database has been closed", + kv.get("foo", [](kj::ArrayPtr) { KJ_FAIL_EXPECT("should not be called"); })); + KJ_EXPECT_THROW_MESSAGE( + "database has been closed", kv.list(nullptr, kj::none, kj::none, SqliteKv::FORWARD)->next()); + KJ_EXPECT_THROW_MESSAGE("database has been closed", kv.getExternals("foo")); + KJ_EXPECT_THROW_MESSAGE("database has been closed", kv.put("bar", "def"_kj.asBytes())); +} + } // namespace } // namespace workerd diff --git a/src/workerd/util/sqlite-kv.c++ b/src/workerd/util/sqlite-kv.c++ index 7644878de9d..f7b631edb18 100644 --- a/src/workerd/util/sqlite-kv.c++ +++ b/src/workerd/util/sqlite-kv.c++ @@ -258,10 +258,16 @@ void SqliteKv::putExternals(kj::StringPtr key, kj::Array> tokens } } -void SqliteKv::beforeSqliteReset() { - // We'll need to recreate the tables on the next operation. - tableCreated = false; - externalsTableCreated = false; +void SqliteKv::beforeSqliteClose(SqliteDatabase::CloseReason reason) { + if (reason == SqliteDatabase::CloseReason::RESET) { + // The tables are deleted along with the database and will need to be recreated on the next + // operation. + tableCreated = false; + externalsTableCreated = false; + } + // On close(), leave the flags alone: the tables still exist in the file. Clearing them would + // make get()/list()/getExternals() report an empty database without consulting the (closed) + // connection; with the flags intact those reads go through a Statement and throw instead. } void SqliteKv::rollbackMultiPut(Initialized& stmts, WriteOptions options) { diff --git a/src/workerd/util/sqlite-kv.h b/src/workerd/util/sqlite-kv.h index 5292c990c77..bfaf3db63f6 100644 --- a/src/workerd/util/sqlite-kv.h +++ b/src/workerd/util/sqlite-kv.h @@ -222,7 +222,7 @@ class SqliteKv: private SqliteDatabase::ResetListener { // with `allowUnconfirmed = true` since the paired KV write decides confirmation semantics. void clearExternalsIfPresent(KeyPtr key); - void beforeSqliteReset() override; + void beforeSqliteClose(SqliteDatabase::CloseReason reason) override; // Helper function that rolls back a multi-put statement and swallows any exceptions that may // occur during the rollback. diff --git a/src/workerd/util/sqlite-metadata-test.c++ b/src/workerd/util/sqlite-metadata-test.c++ index b153a44e781..f899f8c545f 100644 --- a/src/workerd/util/sqlite-metadata-test.c++ +++ b/src/workerd/util/sqlite-metadata-test.c++ @@ -65,5 +65,23 @@ KJ_TEST("SQLite-METADATA") { KJ_EXPECT(metadata.getAlarm() == anAlarmTime2); } +KJ_TEST("SQLite-METADATA reads throw after close()") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + SqliteMetadata metadata(db); + + constexpr kj::Date anAlarmTime = kj::UNIX_EPOCH + 1734099316 * kj::SECONDS; + metadata.setAlarm(anAlarmTime, /*allowUnconfirmed=*/false); + KJ_EXPECT(metadata.getAlarm() == anAlarmTime); + + db.close(); + + // Unlike reset(), close() must not make the alarm appear unset: the file still has it. + KJ_EXPECT_THROW_MESSAGE("database has been closed", metadata.getAlarm()); + KJ_EXPECT_THROW_MESSAGE( + "database has been closed", metadata.setAlarm(kj::none, /*allowUnconfirmed=*/false)); +} + } // namespace } // namespace workerd diff --git a/src/workerd/util/sqlite-metadata.c++ b/src/workerd/util/sqlite-metadata.c++ index 6acdbdb2528..7cc4d5625a0 100644 --- a/src/workerd/util/sqlite-metadata.c++ +++ b/src/workerd/util/sqlite-metadata.c++ @@ -100,10 +100,18 @@ SqliteMetadata::Initialized& SqliteMetadata::ensureInitialized(bool allowUnconfi KJ_UNREACHABLE; } -void SqliteMetadata::beforeSqliteReset() { - // We'll need to recreate the table on the next operation. - tableCreated = false; +void SqliteMetadata::beforeSqliteClose(SqliteDatabase::CloseReason reason) { + // Drop the alarm cache in both cases. After a reset() the alarm is gone; after a close() the + // next read must reach the closed connection and throw rather than report a stale value. cacheState = kj::none; + + if (reason == SqliteDatabase::CloseReason::RESET) { + // The table is deleted along with the database and will need to be recreated on the next + // operation. Leave `tableCreated` alone on close(): the table still exists in the file, and + // clearing the flag would make getAlarmUncached() report "no alarm" without consulting the + // (closed) database. + tableCreated = false; + } } } // namespace workerd diff --git a/src/workerd/util/sqlite-metadata.h b/src/workerd/util/sqlite-metadata.h index c365166a213..9df50dbca6d 100644 --- a/src/workerd/util/sqlite-metadata.h +++ b/src/workerd/util/sqlite-metadata.h @@ -75,7 +75,7 @@ class SqliteMetadata final: private SqliteDatabase::ResetListener { // first write. // ResetListener interface: - void beforeSqliteReset() override; + void beforeSqliteClose(SqliteDatabase::CloseReason reason) override; }; } // namespace workerd diff --git a/src/workerd/util/sqlite.c++ b/src/workerd/util/sqlite.c++ index 124b17a4db1..ea101db72ce 100644 --- a/src/workerd/util/sqlite.c++ +++ b/src/workerd/util/sqlite.c++ @@ -1029,7 +1029,7 @@ void SqliteDatabase::reset() { KJ_IF_SOME(db, maybeDb) { for (auto& listener: resetListeners) { - listener.beforeSqliteReset(); + listener.beforeSqliteClose(CloseReason::RESET); } auto err = sqlite3_close(&db); @@ -1064,7 +1064,7 @@ void SqliteDatabase::close() noexcept { KJ_IF_SOME(db, maybeDb) { for (auto& listener: resetListeners) { - listener.beforeSqliteReset(); + listener.beforeSqliteClose(CloseReason::CLOSE); } // Every prepared statement was finalized by the listener walk above, so SQLITE_BUSY here @@ -1533,10 +1533,22 @@ SqliteDatabase::StatementAndEffect& SqliteDatabase::Statement::prepareForExecuti return KJ_ASSERT_NONNULL(stmt.tryGet()); } -void SqliteDatabase::Statement::beforeSqliteReset() { +void SqliteDatabase::Statement::beforeSqliteClose(CloseReason reason) { + // Regardless of `reason`, the prepared statement must be finalized (by destroying the + // StatementAndEffect) so that the connection can be closed. KJ_IF_SOME(prepared, stmt.tryGet()) { - // Pull the original SQL code out of the statement and store it. - stmt = kj::str(sqlite3_sql(prepared.statement)); + switch (reason) { + case CloseReason::RESET: + // Keep the SQL text so that the statement can be reprepared against the new database. + stmt = kj::str(sqlite3_sql(prepared.statement)); + break; + case CloseReason::CLOSE: + // The statement can never be reprepared, so don't bother copying the SQL. An empty string + // (no allocation) suffices: prepareForExecution() will attempt to prepare it and fail + // with "database has been closed" before ever looking at the text. + stmt = kj::String(); + break; + } } } @@ -1847,10 +1859,10 @@ SqliteDatabase::StatementAndEffect& SqliteDatabase::Query::getStatementAndEffect }); } -void SqliteDatabase::Query::beforeSqliteReset() { +void SqliteDatabase::Query::beforeSqliteClose(CloseReason reason) { // Note that if we don't own the statement, then `maybeStatement` is probably already dangling // here. Luckily, we don't need to reset it or anything because the statement will be destroyed - // by Statement::beforeSqliteReset(). + // by Statement::beforeSqliteClose(). maybeStatement = kj::none; ownStatement = {}; } diff --git a/src/workerd/util/sqlite.h b/src/workerd/util/sqlite.h index 5a37fc6140d..cfdbc4f25a6 100644 --- a/src/workerd/util/sqlite.h +++ b/src/workerd/util/sqlite.h @@ -304,6 +304,14 @@ class SqliteDatabase { // process. void close() noexcept; + // Why a connection is being closed. Passed to `ResetListener::beforeSqliteClose()`. + enum class CloseReason { + // reset(): the database file is deleted and a fresh connection opened. + RESET, + // close(): the connection is gone for good. + CLOSE, + }; + // Objects that need to be notified when reset() or close() is called may inherit // `ResetListener`. class ResetListener { @@ -320,8 +328,15 @@ class SqliteDatabase { } // When the database's `reset()` or `close()` method is called, all listeners' - // `beforeSqliteReset()` will be called before actually closing the connection. - virtual void beforeSqliteReset() = 0; + // `beforeSqliteClose()` will be called before actually closing the connection. Any prepared + // statements must be finalized here so that `sqlite3_close()` can succeed. + // + // The `reason` matters for listeners that cache facts about the database's contents: after + // RESET the database is empty and will be reopened, so such caches must be cleared; after + // CLOSE nothing may read the database again, so such caches should be left alone so that + // attempted reads reach the closed connection and throw rather than reporting an empty + // database. + virtual void beforeSqliteClose(CloseReason reason) = 0; protected: // so that subclasses don't have to store their own copy of the `db` reference SqliteDatabase& db; @@ -355,6 +370,11 @@ class SqliteDatabase { // When a rollback occurs, callbacks are invoked in the reverse of the order in which they were // registered. The database content is rolled back first, before invoking any callbacks. // Callbacks may read from the database, but must not write to it. + // + // NOTE: close() technically rolls back all outstanding transactions, but does NOT call rollback + // callbacks. Instead, callers who need to handle the close() case should register a + // ResetListener. In practices, in-memory caches are generally moot after close() since the + // actor has shut down anyway. void onRollback(kj::Function callback) { if (inTransaction || !savepoints.empty()) { rollbackCallbacks.add(kj::mv(callback)); @@ -550,7 +570,7 @@ class SqliteDatabase::Statement final: private ResetListener { regulator(regulator), stmt(kj::mv(sqlCode)) {} - void beforeSqliteReset() override; + void beforeSqliteClose(CloseReason reason) override; // Get the underlying StatementAndEffect, which the caller will then execute. If `prelude` is // non-empty, prepareForExecution() actually executes the prelude. @@ -812,7 +832,7 @@ class SqliteDatabase::Query final: private ResetListener { return getStatementAndEffect().statement; } - void beforeSqliteReset() override; + void beforeSqliteClose(CloseReason reason) override; void nextRow(bool first); }; From ce9a73340ed9bd67042e640b2edadcc910ef3e6d Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Sun, 6 Sep 2026 21:50:27 -0500 Subject: [PATCH 16/33] Make persistent stubs work in clustered mode. Specifically, this makes it so channel tokens that represent a directive to call `[restore]()` on some other token's target work when the first token is a remote actor in the cluster. The strategy we take is to figure out when the root of a restore chain turns out to be a remote actor and, in that case, send the full channel token (encoding the whole restore chain) to the remote machine, restoring it there, to handle any requests. This commit also changes the RPC interface exposed on the cluster port from WorkerDebugPort to a new WorkerClusterPort. The new interface deals strictly in (encrypted, MAC'd) channel token -- it does not accept arbitrary service props. This is a better fit for what we want and more secure to boot. --- src/workerd/api/restore.c++ | 82 ++-- src/workerd/api/restore.h | 11 + src/workerd/io/worker-interface.capnp | 38 +- src/workerd/io/worker.c++ | 2 +- src/workerd/io/worker.h | 2 +- src/workerd/server/channel-token.c++ | 79 ++-- src/workerd/server/channel-token.h | 14 + src/workerd/server/cluster-lock-test.c++ | 30 +- src/workerd/server/cluster-lock.c++ | 4 +- src/workerd/server/cluster-lock.h | 10 +- src/workerd/server/server.c++ | 415 +++++++++++++----- src/workerd/server/server.h | 4 + .../server/tests/cluster/cluster-test.mjs | 215 +++++++++ .../server/tests/cluster/cluster-worker.js | 207 ++++++++- 14 files changed, 907 insertions(+), 206 deletions(-) diff --git a/src/workerd/api/restore.c++ b/src/workerd/api/restore.c++ index 142623dcabe..33d404903f8 100644 --- a/src/workerd/api/restore.c++ +++ b/src/workerd/api/restore.c++ @@ -495,46 +495,78 @@ kj::Promise RestoreRpcStubCustomEvent::run } } -kj::Promise RestoreRpcStubCustomEvent::sendRpc( - capnp::HttpOverCapnpFactory& httpOverCapnpFactory, - capnp::ByteStreamFactory& byteStreamFactory, - FrankenvalueHandler& frankenvalueHandler, - rpc::EventDispatcher::Client dispatcher) { - // This contains a lot of similar code to JsRpcSessionCustomEvent::sendRpc() for setting - // up membranes, handling the returned JsRpcSession, etc. +namespace { - auto revokePaf = kj::newPromiseAndFulfiller(); +kj::Promise remoteRpcSessionTask(rpc::JsRpcSession::Client session, + kj::Promise allStubsDropped, + kj::Own> revoker) { + // Revoke all stubs in the session as soon as this task completes or is canceled. Normally the + // server side doesn't end the session while any stubs still exist, so this only makes a + // difference when some sort of error occurred. The stubs are probably already broken in that + // case, but revoking them ensures the underlying transport isn't held open waiting for the JS + // garbage collector to actually collect the JsRpcStub objects. KJ_DEFER({ - if (revokePaf.fulfiller->isWaiting()) { - revokePaf.fulfiller->reject(KJ_EXCEPTION(DISCONNECTED, "JS-RPC-restore session canceled")); + if (revoker->isWaiting()) { + revoker->reject(KJ_EXCEPTION(DISCONNECTED, "JS-RPC session canceled")); } }); - auto req = dispatcher.restoreRpcStubRequest(); - frankenvalueHandler.toCapnp(restoreParams, req.initParams()); - auto sent = req.send(); + try { + // Wait for `session` to resolve to a null capability, which the server does once the session + // is over. (If the server is using the "old approach" of not returning a `session` at all, the + // return itself represents the end of the session and `session` resolves to null just the + // same.) + co_await session.whenResolved().exclusiveJoin(kj::mv(allStubsDropped)); + } catch (...) { + auto e = kj::getCaughtExceptionAsKj(); + if (revoker->isWaiting()) { + revoker->reject(e.clone()); + } + kj::throwFatalException(kj::mv(e)); + } +} - rpc::JsRpcTarget::Client cap = sent.getTarget(); +} // namespace +IoChannelFactory::RpcChannel::Session wrapRemoteRpcSession( + rpc::JsRpcTarget::Client cap, rpc::JsRpcSession::Client session) { + auto revokePaf = kj::newPromiseAndFulfiller(); cap = capnp::membrane(kj::mv(cap), kj::refcounted(kj::mv(revokePaf.promise))); + // When no more stubs exist in the session, proactively end it. This matters in particular when + // the client drops the stub without making any calls at all, e.g. because serializing the + // arguments failed: `cap` may be a pipelined capability on a call that has not returned yet, in + // which case simply dropping it is not detected by the server. auto completionPaf = kj::newPromiseAndFulfiller(); cap = capnp::membrane( kj::mv(cap), kj::refcounted(kj::mv(completionPaf.fulfiller))); - capFulfiller->fulfill(kj::mv(cap)); + return { + .cap = kj::mv(cap), + .task = remoteRpcSessionTask( + kj::mv(session), kj::mv(completionPaf.promise), kj::mv(revokePaf.fulfiller)), + }; +} + +kj::Promise RestoreRpcStubCustomEvent::sendRpc( + capnp::HttpOverCapnpFactory& httpOverCapnpFactory, + capnp::ByteStreamFactory& byteStreamFactory, + FrankenvalueHandler& frankenvalueHandler, + rpc::EventDispatcher::Client dispatcher) { + auto req = dispatcher.restoreRpcStubRequest(); + frankenvalueHandler.toCapnp(restoreParams, req.initParams()); + auto sent = req.send(); + + auto session = wrapRemoteRpcSession(sent.getTarget(), sent.getSession()); - auto session = sent.getSession(); + // `session.task` already propagates any errors from the call, so the call promise itself can be + // dropped. (It would NOT work to use `req.sendForPipeline()` above, since pipelined capabilities + // cannot resolve until the call returns, but `sendForPipeline()` explicitly inhibits the return + // message.) { auto drop = kj::mv(sent); } - try { - co_await session.whenResolved().exclusiveJoin(kj::mv(completionPaf.promise)); - } catch (...) { - auto e = kj::getCaughtExceptionAsKj(); - if (revokePaf.fulfiller->isWaiting()) { - revokePaf.fulfiller->reject(e.clone()); - } - kj::throwFatalException(kj::mv(e)); - } + + capFulfiller->fulfill(kj::mv(session.cap)); + co_await session.task; co_return WorkerInterface::CustomEvent::Result{.outcome = EventOutcome::OK}; } diff --git a/src/workerd/api/restore.h b/src/workerd/api/restore.h index 9a899c90af8..81d9bf3f3fc 100644 --- a/src/workerd/api/restore.h +++ b/src/workerd/api/restore.h @@ -55,6 +55,17 @@ jsg::Promise restoreCurrentEntrypoint(jsg::Lock& js, const jsg::TypeHandler>& fetcherHandler, const jsg::TypeHandler>& rpcStubHandler); +// Wraps the `target` and `session` capabilities returned by an RPC that opened a JS RPC session on +// a remote worker (`EventDispatcher.restoreRpcStub()`, `WorkerdDebugPort.getRpcTargetFromToken()`) +// into an `RpcChannel::Session`. The returned `cap` is membraned so that dropping every stub +// derived from it ends the session promptly, without waiting for the server to notice, and so that +// completing or cancelling the returned `task` revokes any stubs still outstanding, so they cannot +// hold the underlying transport open. `task` completes when `session` resolves (to null, i.e. the +// server ended the session) or all local stubs are dropped, whichever comes first, and fails if the +// session fails. +IoChannelFactory::RpcChannel::Session wrapRemoteRpcSession( + rpc::JsRpcTarget::Client target, rpc::JsRpcSession::Client session); + // Custom event that calls the `[restore]()` method of the entrypoint and expects that the result // is a `Fetcher`. (When the result is expected to be an RpcStub instead, we use // `RestoreRpcStubCustomEvent`.) diff --git a/src/workerd/io/worker-interface.capnp b/src/workerd/io/worker-interface.capnp index adc17e184e8..ab630272a84 100644 --- a/src/workerd/io/worker-interface.capnp +++ b/src/workerd/io/worker-interface.capnp @@ -994,18 +994,6 @@ interface WorkerdDebugPort { # purposes. # # This interface is subject to change. It is intended for use by miniflare. - # - # This is also the same interface that workerd exports on its cluster port, which other workerd - # instances in the same cluster may connect to. See ClusterConfig in workerd.capnp for how to - # configure clustering. The cluster port is only intended to be used by other instances in the - # cluster, and does not implement a regular two-party RPC protocol. `ClusterRegistry` implements - # the `VatNetwork` intended to be used for this. Nobody else should try to connect to it. - # - # TODO(clustering): Once channel tokens have been extended to support actors, consider changing - # the cluster interface with one that takes a channel token. This seems a bit cleaner and safer - # than letting the client specify props and such directly. - # TODO(clustering): Automatically use encryption on non-localhost networks. We have an X25519 - # keypair for each node, no need for certs! getEntrypoint @0 (service :Text, entrypoint :Text, props :Frankenvalue) -> (entrypoint :WorkerdBootstrap); @@ -1018,3 +1006,29 @@ interface WorkerdDebugPort { # `actorName` may optionally be specified for Durable Objects, if `actorId` is derived from the # name. } + +interface WorkerdClusterPort { + # Bootstrap interface that workerd exports on its cluster port, which other workerd instances in + # the same cluster connect to in order to reach Durable Objects owned by this instance. See + # ClusterConfig in workerd.capnp for how to configure clustering. The cluster port does not + # implement a regular two-party RPC protocol; `ClusterRegistry` implements the `VatNetwork` used + # for it. Nobody else should try to connect to it. + # + # Every method takes a channel token authenticated with the cluster key, so a peer that does not + # hold the key can do nothing with this interface, and a peer that does gains no authority beyond + # what its tokens already grant. (Contrast WorkerdDebugPort, which lets the client name arbitrary + # services, props, and actors.) + # + # TODO(clustering): Automatically use encryption on non-localhost networks. We have an X25519 + # keypair for each node, no need for certs! + + getChannelFromToken @0 (token :Data) -> (bootstrap :WorkerdBootstrap); + # Resolve a `subrequest`-type channel token (in RPC-usage encoding) on this node and return a + # bootstrap for it. Used to route requests -- including restored stubs whose `[restore]()` chain + # must run on the node that owns the root actor -- to the node that owns the target. + + getRpcTargetFromToken @1 (token :Data) -> (target :JsRpcTarget, session :JsRpcSession); + # Like getChannelFromToken(), but for `rpc`-type tokens. Opens a fresh JS RPC session to the + # restored RpcTarget. `session` has the same semantics as the field of the same name in + # `EventDispatcher.restoreRpcStub()`. +} diff --git a/src/workerd/io/worker.c++ b/src/workerd/io/worker.c++ index 465ad1731c7..41172987d62 100644 --- a/src/workerd/io/worker.c++ +++ b/src/workerd/io/worker.c++ @@ -4166,7 +4166,7 @@ bool Worker::Actor::idsEqual(const Id& a, const Id& b) { KJ_UNREACHABLE; } -Worker::Actor::Id Worker::Actor::cloneId(Worker::Actor::Id& id) { +Worker::Actor::Id Worker::Actor::cloneId(const Worker::Actor::Id& id) { KJ_SWITCH_ONEOF(id) { KJ_CASE_ONEOF(coloLocalId, kj::String) { return kj::str(coloLocalId); diff --git a/src/workerd/io/worker.h b/src/workerd/io/worker.h index 9ce9f3f4e77..b9c2a1ac167 100644 --- a/src/workerd/io/worker.h +++ b/src/workerd/io/worker.h @@ -1022,7 +1022,7 @@ class Worker::Actor final: public kj::Refcounted { const Id& getId(); Id cloneId(); - static Id cloneId(Id& id); + static Id cloneId(const Id& id); kj::Maybe> getTransient(Worker::Lock& lock); kj::Maybe getPersistent(); kj::Own getLoopback(); diff --git a/src/workerd/server/channel-token.c++ b/src/workerd/server/channel-token.c++ index 9686842ba62..6d420cc477f 100644 --- a/src/workerd/server/channel-token.c++ +++ b/src/workerd/server/channel-token.c++ @@ -500,8 +500,10 @@ class ChannelTokenHandler::RestoredSubrequestChannel final class ChannelTokenHandler::RestoredRpcChannel final: public IoChannelFactory::RpcChannel { public: // Constructor for the "fresh" path: created by `ctx.restore()`. The vendor is the current - // entrypoint's `ServerSelfTokenFactory`, used only to construct the token. `restore()` is never - // called in this case, because the freshly-created stub is paired with a live capability. + // entrypoint's `ServerSelfTokenFactory`, used to construct the token. The freshly-created stub + // is paired with a live capability, so `restore()` is only reached if the stub is passed + // somewhere that keeps the channel but not the capability (`ctx.props`, a dynamic worker's + // `env`) and then used. RestoredRpcChannel(ChannelTokenHandler& handler, kj::Own vendor, Frankenvalue restoreArg, @@ -526,7 +528,38 @@ class ChannelTokenHandler::RestoredRpcChannel final: public IoChannelFactory::Rp // Unlike RestoredSubrequestChannel, we expect the caller of restore() already caches the // resulting stub, rather than calling again for every request. So, we don't need to cache // our result. - return restoreWith(getVendorChannel()); + KJ_SWITCH_ONEOF(vendor) { + KJ_CASE_ONEOF(channel, kj::Own) { + return restoreWith(kj::addRef(*channel)); + } + KJ_CASE_ONEOF(_, kj::Own) { + // Decode our own token rather than just the vendor's, so that the result is exactly what + // the stub would have been had it arrived as a token -- in particular, so that + // `Resolver::wrapRestoredChannel()` gets to route the whole token to whichever node owns + // the vendor, rather than us trying to replay `[restore]()` over a remote vendor channel. + auto& handlerRef = handler; + kj::Own decoded; + KJ_SWITCH_ONEOF(getTokenMaybeSync(IoChannelFactory::ChannelTokenUsage::RPC)) { + KJ_CASE_ONEOF(token, kj::Array) { + decoded = + handlerRef.decodeRpcChannelToken(IoChannelFactory::ChannelTokenUsage::RPC, token); + } + KJ_CASE_ONEOF(promise, kj::Promise>) { + decoded = newPromisedChannel( + promise.then([&handlerRef](kj::Array token) { + return handlerRef.decodeRpcChannelToken( + IoChannelFactory::ChannelTokenUsage::RPC, token); + })); + } + } + // The decoded channel's session task does not keep the channel itself alive; callers of + // restore() only keep *us* alive. + auto session = decoded->restore(); + session.task = session.task.attach(kj::mv(decoded)); + return session; + } + } + KJ_UNREACHABLE; } void requireAllowsTransfer() override {} @@ -552,34 +585,6 @@ class ChannelTokenHandler::RestoredRpcChannel final: public IoChannelFactory::Rp Frankenvalue restoreArg; Persistent persistent; - // Obtain a callable channel to the vendor, i.e. the entrypoint whose `[restore]()` method must - // be invoked to (re)create the live RPC target. - kj::Own getVendorChannel() { - KJ_SWITCH_ONEOF(vendor) { - KJ_CASE_ONEOF(channel, kj::Own) { - return kj::addRef(*channel); - } - KJ_CASE_ONEOF(factory, kj::Own) { - KJ_SWITCH_ONEOF(factory->getSelfToken(IoChannelFactory::ChannelTokenUsage::RPC)) { - KJ_CASE_ONEOF(token, kj::Array) { - return handler.decodeSubrequestChannelToken( - IoChannelFactory::ChannelTokenUsage::RPC, token); - } - KJ_CASE_ONEOF(promise, kj::Promise>) { - auto& handlerRef = handler; - return newPromisedChannel( - promise.then([&handlerRef](kj::Array token) { - return handlerRef.decodeSubrequestChannelToken( - IoChannelFactory::ChannelTokenUsage::RPC, token); - })); - } - } - KJ_UNREACHABLE; - } - } - KJ_UNREACHABLE; - } - Session restoreWith(kj::Own vendorChannel) { // Just like RestoredSubrequestChannel::ensureRestored(), we must verify that the worker on // which we call `[restore]()` allows persistence, if we are marked persistent. @@ -754,16 +759,24 @@ kj::Own ChannelTokenHandler::decodeChannelTokenImpl auto vendor = decodeSubrequestChannelToken(usage, restored.getVendor()); auto restoreArg = decodeFrankenvalue(usage, restored.getRestoreArg()); + // The restored channel takes ownership of `vendor`, so this reference stays valid for the + // `wrapRestoredChannel()` call below. + auto& vendorRef = *vendor; + kj::Own local; switch (type) { case ChannelToken::Type::SUBREQUEST: - return kj::refcounted( + local = kj::refcounted( *this, kj::mv(vendor), kj::mv(restoreArg), persistent); + break; case ChannelToken::Type::ACTOR_CLASS: KJ_FAIL_REQUIRE("actor class channel tokens cannot have a restore chain"); case ChannelToken::Type::RPC: - return kj::refcounted( + local = kj::refcounted( *this, kj::mv(vendor), kj::mv(restoreArg), persistent); + break; } + + return resolver.wrapRestoredChannel(type, vendorRef, kj::mv(local)); } } diff --git a/src/workerd/server/channel-token.h b/src/workerd/server/channel-token.h index 39a2a4f875b..34b005dc827 100644 --- a/src/workerd/server/channel-token.h +++ b/src/workerd/server/channel-token.h @@ -45,6 +45,20 @@ class ChannelTokenHandler { kj::ArrayPtr id, kj::Maybe name, Persistent persistent) = 0; + + // Called after decoding a `restored` token (a token that encodes calling [restore]() on the + // result of some other token). This gives the resolver an opportunity to replace the channel + // with something else. Simply returning `local` is valid when running in single-instance mode. + // When running in clustered mode (Durable Objects distributed over a cluster), the restorer + // checks whether `vendor` refers to a remote actor and, if so, returns a channel that makes + // an RPC to the remote actor's machine, sending the full original token, to be parsed there + // instead. + virtual kj::Own wrapRestoredChannel( + ChannelToken::Type type, + IoChannelFactory::SubrequestChannel& vendor, + kj::Own local) { + return kj::mv(local); + } }; // workerd's implementation of `IoChannelFactory::SelfTokenFactory`. Produces the encoded diff --git a/src/workerd/server/cluster-lock-test.c++ b/src/workerd/server/cluster-lock-test.c++ index 980a76de145..dbd9123cb81 100644 --- a/src/workerd/server/cluster-lock-test.c++ +++ b/src/workerd/server/cluster-lock-test.c++ @@ -68,9 +68,9 @@ class TempDir { kj::Own locksDir; }; -// Stub WorkerdDebugPort server used as a per-node bootstrap. We don't actually invoke methods on +// Stub WorkerdClusterPort server used as a per-node bootstrap. We don't actually invoke methods on // it from these tests; we just need a capability to hand to RpcSystem. -class StubDebugPort final: public rpc::WorkerdDebugPort::Server {}; +class StubClusterPort final: public rpc::WorkerdClusterPort::Server {}; KJ_TEST("ClusterLockManager: unowned (empty file) -> acquire succeeds") { auto io = kj::setupAsyncIo(); @@ -79,7 +79,7 @@ KJ_TEST("ClusterLockManager: unowned (empty file) -> acquire succeeds") { ClusterRegistry reg( tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); - auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); @@ -99,7 +99,7 @@ KJ_TEST("ClusterLockManager: owned by self -> returns self-bootstrap, no wire tr ClusterRegistry reg( tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); - auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); @@ -111,7 +111,7 @@ KJ_TEST("ClusterLockManager: owned by self -> returns self-bootstrap, no wire tr // path). With the OwnedLock still held, the lock file is non-empty and our key is the owner. // acquireOrRoute should return a bootstrap client (routing to ourselves). auto routeResult = lockManager.acquireOrRoute("a2").wait(io.waitScope); - KJ_ASSERT(routeResult.is()); + KJ_ASSERT(routeResult.is()); } KJ_TEST("ClusterLockManager: stale owner (peer dead) -> claim path runs") { @@ -121,7 +121,7 @@ KJ_TEST("ClusterLockManager: stale owner (peer dead) -> claim path runs") { ClusterRegistry reg( tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); - auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); // Manually create a lock file naming a non-existent peer as owner. Since `deadKey`'s registry // file doesn't exist in tmpDir, isPeerDead(deadKey) returns true and the claim path should run. @@ -152,7 +152,7 @@ KJ_TEST("ClusterLockManager: wrong-sized stale content + writer dead -> claim im ClusterRegistry reg( tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); - auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); // Write a lock file of unexpected size (e.g. a crashed writer left a half-written file). // Nobody holds the lock, so the claim path should succeed. @@ -178,7 +178,7 @@ KJ_TEST("ClusterLockManager: OwnedLock destructor truncates and releases") { ClusterRegistry reg( tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); - auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); @@ -211,7 +211,7 @@ KJ_TEST("ClusterLockManager: owned by live peer -> returns bootstrap client") { ClusterRegistry reg2( tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); - auto rpc1 = capnp::makeRpcServer(reg1, capnp::Capability::Client(kj::heap())); + auto rpc1 = capnp::makeRpcServer(reg1, capnp::Capability::Client(kj::heap())); // reg1 owns a ClusterLockManager; we'll set up a lock file naming reg2 as the owner. // reg2 is alive (its registry file exists in tmpDir), so isPeerDead(reg2) returns false, @@ -225,7 +225,7 @@ KJ_TEST("ClusterLockManager: owned by live peer -> returns bootstrap client") { ClusterLockManager lockManager(tmpDir.locks(), reg1, rpc1, io.provider->getTimer()); auto result = lockManager.acquireOrRoute("a7").wait(io.waitScope); - KJ_ASSERT(result.is()); + KJ_ASSERT(result.is()); // The lock file should NOT have been modified (we did not claim). auto content = tmpDir.readLockFile("a7"); @@ -240,7 +240,7 @@ KJ_TEST("ClusterLockManager: writer alive -> retries until writer releases") { ClusterRegistry reg( tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); - auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); // Open the lock file from outside the ClusterLockManager and take an exclusive lock. This // simulates a writer that holds the lock but hasn't yet flushed its key. The file is left @@ -258,7 +258,7 @@ KJ_TEST("ClusterLockManager: writer alive -> retries until writer releases") { bool completed = false; auto trackedPromise = promise.then( - [&](kj::OneOf&& r) { + [&](kj::OneOf&& r) { completed = true; return kj::mv(r); }); @@ -275,7 +275,7 @@ KJ_TEST("ClusterLockManager: writer alive -> retries until writer releases") { auto result = trackedPromise.wait(io.waitScope); KJ_EXPECT(completed); - KJ_ASSERT(result.is()); + KJ_ASSERT(result.is()); } KJ_TEST("ClusterLockManager: concurrent acquisitions, exactly one wins") { @@ -285,13 +285,13 @@ KJ_TEST("ClusterLockManager: concurrent acquisitions, exactly one wins") { ClusterRegistry reg( tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); - auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); // Launch several concurrent calls. constexpr uint N = 5; - kj::Vector>> + kj::Vector>> promises; for (uint i = 0; i < N; ++i) { promises.add(lockManager.acquireOrRoute("a9")); diff --git a/src/workerd/server/cluster-lock.c++ b/src/workerd/server/cluster-lock.c++ index c3480e8a95d..c77e8d71437 100644 --- a/src/workerd/server/cluster-lock.c++ +++ b/src/workerd/server/cluster-lock.c++ @@ -87,7 +87,7 @@ ClusterLockManager::ClusterLockManager(kj::Own dir, clusterRpc(clusterRpc), timer(timer) {} -kj::Promise> +kj::Promise> ClusterLockManager::acquireOrRoute(kj::StringPtr actorId) { // Validate that the actor ID is a hex string. This is the only form of actor ID we expect on // this path (durable actors named by a 64-char hex SHA-256 hash, or by `idFromName` which also @@ -132,7 +132,7 @@ ClusterLockManager::acquireOrRoute(kj::StringPtr actorId) { // acquireOrRoute() will then see isPeerDead(ownerKey) return true and fall through to // the claim path. VatIdHolder holder(ownerKey); - co_return clusterRpc.bootstrap(holder.getReader()).castAs(); + co_return clusterRpc.bootstrap(holder.getReader()).castAs(); } // Owner is confirmed dead. Fall through to the claim path. diff --git a/src/workerd/server/cluster-lock.h b/src/workerd/server/cluster-lock.h index d259968a48e..d473d31d20f 100644 --- a/src/workerd/server/cluster-lock.h +++ b/src/workerd/server/cluster-lock.h @@ -60,11 +60,11 @@ class ClusterLockManager { // // Returns: // OwnedLock — this node has newly acquired ownership (caller should start the DO). - // rpc::WorkerdDebugPort::Client — the DO has an existing owner. The caller calls - // getActor() on this to reach the actor. The owner might be this node, in which case - // the RpcSystem returns the local self-bootstrap directly (no wire traffic). The - // caller does not need to distinguish local vs remote. - kj::Promise> acquireOrRoute( + // rpc::WorkerdClusterPort::Client — the DO has an existing owner. The caller sends the + // actor's channel token to this to reach the actor. The owner might be this node, in + // which case the RpcSystem returns the local self-bootstrap directly (no wire traffic). + // The caller does not need to distinguish local vs remote. + kj::Promise> acquireOrRoute( kj::StringPtr actorId); private: diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 6bfc666bb86..87759b01cb4 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -363,8 +364,6 @@ class Server::ActorNamespace final { friend class Server; ActorNamespace(kj::Own actorClass, - kj::StringPtr serviceName, - kj::StringPtr className, const ActorConfig& config, const kj::Clock& clock, kj::Timer& timer, @@ -377,8 +376,6 @@ class Server::ActorNamespace final { kj::TaskSet& waitUntilTasks, Persistent selfTokensArePersistent) : actorClass(kj::mv(actorClass)), - serviceName(serviceName), - className(className), config(config), clock(clock), timer(timer), @@ -1471,7 +1468,7 @@ class Server::ActorNamespace final { } }; - kj::String actorKey(Worker::Actor::Id& id) { + kj::String actorKey(const Worker::Actor::Id& id) { KJ_SWITCH_ONEOF(id) { KJ_CASE_ONEOF(obj, kj::Own) { KJ_REQUIRE(config.is()); @@ -1520,16 +1517,17 @@ class Server::ActorNamespace final { } // Which node currently owns the actor with the given key: a container on this node (created - // here if we just claimed the lock), or the owning node's debug port. Not cached; callers cache - // what they derive from it. Every cluster-routed channel type resolves through this method so - // that the lock protocol lives in exactly one place. `key` and `id` must remain valid until the - // returned promise settles. Cluster mode only. - using Node = kj::OneOf, rpc::WorkerdDebugPort::Client>; - kj::Promise resolveNode(kj::StringPtr key, Worker::Actor::Id& id) { + // here if we just claimed the lock), or the owning node's cluster port. Not cached; callers + // cache what they derive from it. Every cluster-routed channel type resolves through this method + // so that the lock protocol lives in exactly one place. `key` and `id` must remain valid until + // the returned promise settles. Cluster mode only. + using Node = kj::OneOf, rpc::WorkerdClusterPort::Client>; + kj::Promise resolveNode(kj::StringPtr key, const Worker::Actor::Id& id) { // Local-first. This lookup is what terminates the routing loop: when the lock file names this - // node, acquireOrRoute() returns our own debug port with no backoff, and the resulting - // getActor() lands right back here. That is only correct because "lock file names this node" - // implies "container is in `actors`" -- see ActorContainer::ownershipLock. + // node, acquireOrRoute() returns our own cluster port with no backoff, and the resulting + // getChannelFromToken() decodes the token into a channel that lands right back here. That is + // only correct because "lock file names this node" implies "container is in `actors`" -- see + // ActorContainer::ownershipLock. KJ_IF_SOME(container, actors.find(key)) { co_return container->addRef(); } @@ -1539,8 +1537,8 @@ class Server::ActorNamespace final { KJ_CASE_ONEOF(ownership, ClusterLockManager::OwnedLock) { co_return createActorContainer(kj::str(key), Worker::Actor::cloneId(id), kj::mv(ownership)); } - KJ_CASE_ONEOF(debugPort, rpc::WorkerdDebugPort::Client) { - co_return kj::mv(debugPort); + KJ_CASE_ONEOF(clusterPort, rpc::WorkerdClusterPort::Client) { + co_return kj::mv(clusterPort); } } KJ_UNREACHABLE; @@ -1671,8 +1669,6 @@ class Server::ActorNamespace final { private: kj::Own actorClass; - kj::StringPtr serviceName; - kj::StringPtr className; const ActorConfig& config; const kj::Clock& clock; @@ -1927,17 +1923,22 @@ class Server::ActorNamespace final { } }; - // Channel to an actor in this namespace in cluster mode. Resolution of which node owns the actor - // is deferred until first use, so that obtaining or deserializing a stub does not take cluster - // locks or wake remote actors. - class ClusterActorChannel final: public IoChannelFactory::ActorChannel { + // Base class for SubrequestChannels in cluster mode whose requests are routed to whichever node + // currently owns some root actor in this namespace: plain actor stubs (ClusterActorChannel) and + // restored stubs whose vendor chain bottoms out at such an actor + // (ClusterRestoredSubrequestChannel). Resolution of the owning node is deferred until first use, + // so that obtaining or deserializing a stub does not take cluster locks or wake remote actors. + // + // Server::wrapRestoredChannel() recognizes a cluster-routed vendor by this type and copies the + // root actor's identity out of it so that the wrapper can resolve that node for itself. No + // resolution state is shared between channels; each caches its own `target`. + class ClusterRoutedChannel: public IoChannelFactory::SubrequestChannel { public: - ClusterActorChannel(ActorNamespace& ns, Worker::Actor::Id id, Persistent persistent) + ClusterRoutedChannel(ActorNamespace& ns, Worker::Actor::Id id) : ns(ns), id(kj::mv(id)), - key(ns.actorKey(this->id)), - persistent(persistent) {} - ~ClusterActorChannel() noexcept(false) { + key(ns.actorKey(this->id)) {} + ~ClusterRoutedChannel() noexcept(false) { KJ_IF_SOME(t, target) { KJ_IF_SOME(container, t.tryGet>()) { container->updateAccessTime(); @@ -1945,65 +1946,33 @@ class Server::ActorNamespace final { } } - kj::Own startRequest(IoChannelFactory::SubrequestMetadata metadata) override { - // If this channel was reconstructed from a persistent (stored) stub, signal the target so it - // can re-verify that it still allows persistent stubs. - metadata.fromPersistentStub = metadata.fromPersistentStub || persistent; - return newPromisedWorkerInterface(startRequestImpl(kj::addRef(*this), kj::mv(metadata))); - } - - kj::Promise evictForTest(IoChannelFactory::EvictWebSocketMode webSocketMode) override { - KJ_IF_SOME(t, target) { - KJ_SWITCH_ONEOF(t) { - KJ_CASE_ONEOF(container, kj::Own) { - return container->evictForTest(webSocketMode).attach(kj::addRef(*this)); - } - KJ_CASE_ONEOF(_, rpc::WorkerdBootstrap::Client) { - JSG_FAIL_REQUIRE(Error, - "Cannot evict Durable Object: it is owned by another workerd instance in the " - "cluster."); - } - } - } - - // Not resolved yet. If the actor is running on this node, evict it without pinning it to - // this channel. - KJ_IF_SOME(container, ns.actors.find(key)) { - return container->evictForTest(webSocketMode).attach(container->addRef()); - } - JSG_FAIL_REQUIRE(Error, "Cannot evict Durable Object: it is not currently running."); + ActorNamespace& getRootNamespace() { + return ns; } - - void requireAllowsTransfer() override { - // Cluster mode only routes root actors of durable namespaces, which are always - // transferrable via channel tokens. + const Worker::Actor::Id& getRootId() { + return id; } - kj::OneOf, kj::Promise>> getTokenMaybeSync( - IoChannelFactory::ChannelTokenUsage usage) override { - return ns.encodeActorChannelToken(usage, id, persistent); + kj::Own startRequest(IoChannelFactory::SubrequestMetadata metadata) override { + return newPromisedWorkerInterface(startRequestImpl(kj::addRef(*this), kj::mv(metadata))); } - private: + protected: ActorNamespace& ns; Worker::Actor::Id id; - kj::String key; - Persistent persistent; + kj::String key; // actorKey(id), computed once // Where requests on this channel go, resolved on first startRequest() and retained for the // life of the channel. Local: the container, pinned exactly as ActorChannelImpl pins it (and - // therefore the ownership lock). Remote: a WorkerdBootstrap for this actor on the owning node. - // Retaining the target gives remote stubs the same failure semantics as local ones: a local - // channel keeps throwing `brokenReason` once its container breaks; a retained bootstrap - // likewise keeps failing once the far end breaks, rather than silently reaching a freshly - // re-instantiated actor on the next call. + // therefore the ownership lock). Remote: a WorkerdBootstrap for this channel's target on the + // owning node, obtained by sending the channel's own token to that node's + // WorkerdClusterPort.getChannelFromToken(). Retaining the target gives remote stubs the same + // failure semantics as local ones: a local channel keeps throwing `brokenReason` once its + // container breaks; a retained bootstrap likewise keeps failing once the far end breaks, rather + // than silently reaching a freshly re-instantiated actor on the next call. using Target = kj::OneOf, rpc::WorkerdBootstrap::Client>; kj::Maybe target; - // Set once the first resolution starts. Its branches are only ever held by coroutines that - // also hold a reference to this channel, so the underlying resolution never outlives `this`. - kj::Maybe> resolving; - // Resolves `target`, caching the result. Safe to call concurrently; all callers share one // ns.resolveNode() attempt. A failed resolution is retained too, so every later request on the // channel fails with the same exception. @@ -2013,38 +1982,58 @@ class Server::ActorNamespace final { return resolving.emplace(resolveTargetImpl().fork()).addBranch(); } + // Start a request when `target` resolved to a container on this node. + virtual kj::Promise> startLocalRequest( + ActorContainer& container, IoChannelFactory::SubrequestMetadata metadata) = 0; + + private: + // Set once the first resolution starts. Its branches are only ever held by coroutines that + // also hold a reference to this channel, so the underlying resolution never outlives `this`. + kj::Maybe> resolving; + + static rpc::WorkerdBootstrap::Client bootstrapFromToken( + rpc::WorkerdClusterPort::Client& clusterPort, kj::ArrayPtr token) { + auto req = clusterPort.getChannelFromTokenRequest( + capnp::MessageSize{token.size() / sizeof(capnp::word) + 4, 0}); + req.setToken(token); + return req.send().getBootstrap(); + } + kj::Promise resolveTargetImpl() { auto node = co_await ns.resolveNode(key, id); KJ_SWITCH_ONEOF(node) { KJ_CASE_ONEOF(container, kj::Own) { target = kj::mv(container); } - KJ_CASE_ONEOF(debugPort, rpc::WorkerdDebugPort::Client) { - // Convert the owner's debug port into a bootstrap for this actor. Nothing else on this - // channel needs the debug port, so only the (pipelined) bootstrap is retained: if - // getActor() fails, the bootstrap is broken and every later startEvent() fails with - // that exception. - auto req = debugPort.getActorRequest(capnp::MessageSize{32, 0}); - req.setService(ns.serviceName); - req.setEntrypoint(ns.className); - req.setActorId(key); - KJ_IF_SOME(i, id.tryGet>()) { - KJ_IF_SOME(name, i->getName()) { - req.setActorName(name); + KJ_CASE_ONEOF(clusterPort, rpc::WorkerdClusterPort::Client) { + // Convert the owner's cluster port into a bootstrap for this channel's target by sending + // it our own token. Nothing else on this channel needs the cluster port, so only the + // (pipelined) bootstrap is retained: if getChannelFromToken() fails, the bootstrap is + // broken and every later startEvent() fails with that exception. A restored token may + // be asynchronous (when its restoreArg holds channels with asynchronous tokens); a + // promise capability covers that case with the same semantics. + KJ_SWITCH_ONEOF(getTokenMaybeSync(IoChannelFactory::ChannelTokenUsage::RPC)) { + KJ_CASE_ONEOF(token, kj::Array) { + target = bootstrapFromToken(clusterPort, token); + } + KJ_CASE_ONEOF(promise, kj::Promise>) { + target = rpc::WorkerdBootstrap::Client( + promise.then([clusterPort = kj::mv(clusterPort)](kj::Array token) mutable { + return bootstrapFromToken(clusterPort, token); + })); } } - target = req.send().getActor(); } } } static kj::Promise> startRequestImpl( - kj::Own self, IoChannelFactory::SubrequestMetadata metadata) { + kj::Own self, IoChannelFactory::SubrequestMetadata metadata) { co_await self->resolveTarget(); KJ_SWITCH_ONEOF(KJ_ASSERT_NONNULL(self->target)) { KJ_CASE_ONEOF(container, kj::Own) { - co_return co_await container->startRequest(kj::mv(metadata)); + co_return co_await self->startLocalRequest(*container, kj::mv(metadata)); } KJ_CASE_ONEOF(bootstrap, rpc::WorkerdBootstrap::Client) { capnp::MessageSize sizeHint{4, 0}; @@ -2060,8 +2049,9 @@ class Server::ActorNamespace final { req.setFromPersistentStub(metadata.fromPersistentStub.toBool()); auto dispatcher = req.sendForPipeline().getDispatcher(); - // NOTE: We don't support restore() over cluster RPC so we can use - // getUnsupportedFrankenvalueHandler() here for now. + // Restore events never cross the cluster -- restored stubs are forwarded as whole + // tokens instead (see ClusterRestoredSubrequestChannel) -- so no FrankenvalueHandler is + // needed to marshal restore params here. co_return kj::heap(self->ns.httpOverCapnpFactory, self->ns.byteStreamFactory, getUnsupportedFrankenvalueHandler(), kj::mv(dispatcher)); } @@ -2069,6 +2059,164 @@ class Server::ActorNamespace final { KJ_UNREACHABLE; } }; + + // Channel to a root actor in this namespace in cluster mode. + class ClusterActorChannel final: public ClusterRoutedChannel { + public: + ClusterActorChannel(ActorNamespace& ns, Worker::Actor::Id id, Persistent persistent) + : ClusterRoutedChannel(ns, kj::mv(id)), + persistent(persistent) {} + + kj::Own startRequest(IoChannelFactory::SubrequestMetadata metadata) override { + // If this channel was reconstructed from a persistent (stored) stub, signal the target so it + // can re-verify that it still allows persistent stubs. + metadata.fromPersistentStub = metadata.fromPersistentStub || persistent; + return ClusterRoutedChannel::startRequest(kj::mv(metadata)); + } + + kj::Promise evictForTest(IoChannelFactory::EvictWebSocketMode webSocketMode) override { + KJ_IF_SOME(t, target) { + KJ_SWITCH_ONEOF(t) { + KJ_CASE_ONEOF(container, kj::Own) { + return container->evictForTest(webSocketMode).attach(kj::addRef(*this)); + } + KJ_CASE_ONEOF(_, rpc::WorkerdBootstrap::Client) { + JSG_FAIL_REQUIRE(Error, + "Cannot evict Durable Object: it is owned by another workerd instance in the " + "cluster."); + } + } + } + + // Not resolved yet. If the actor is running on this node, evict it without pinning it to + // this channel. + KJ_IF_SOME(container, ns.actors.find(key)) { + return container->evictForTest(webSocketMode).attach(container->addRef()); + } + JSG_FAIL_REQUIRE(Error, "Cannot evict Durable Object: it is not currently running."); + } + + void requireAllowsTransfer() override { + // Cluster mode only routes root actors of durable namespaces, which are always + // transferrable via channel tokens. + } + + kj::OneOf, kj::Promise>> getTokenMaybeSync( + IoChannelFactory::ChannelTokenUsage usage) override { + return ns.encodeActorChannelToken(usage, id, persistent); + } + + private: + Persistent persistent; + + kj::Promise> startLocalRequest( + ActorContainer& container, IoChannelFactory::SubrequestMetadata metadata) override { + return container.startRequest(kj::mv(metadata)).attach(container.addRef()); + } + }; + + // Wrapper around a decoded `restored` SubrequestChannel whose vendor chain bottoms out at a + // cluster-routed actor. Rather than replaying `[restore]()` hop-by-hop across the network (which + // could not carry `restoredSelfTokenFactory`, so a remotely-restored facet could not itself call + // `ctx.restore()`), the whole token is either run locally (this node owns the root actor) or + // shipped to the owning node, which decodes it and runs the entire restore chain in-process. + // + // The wrapped `local` tree already knows how to re-encode itself for the wire, including + // converting STORAGE-usage tokens to RPC-usage at every nesting level. For a multiply-nested + // token each level is wrapped; only the outermost wrapper is ever exercised on the remote path, + // and on the local path each level's resolution finds the container the outer one created or + // found (see ActorNamespace::resolveNode()). + class ClusterRestoredSubrequestChannel final: public ClusterRoutedChannel { + public: + ClusterRestoredSubrequestChannel( + ClusterRoutedChannel& vendor, kj::Own local) + : ClusterRoutedChannel( + vendor.getRootNamespace(), Worker::Actor::cloneId(vendor.getRootId())), + local(kj::mv(local)) {} + + kj::Promise evictForTest(IoChannelFactory::EvictWebSocketMode webSocketMode) override { + return local->evictForTest(webSocketMode); + } + + void requireAllowsTransfer() override { + local->requireAllowsTransfer(); + } + + kj::OneOf, kj::Promise>> getTokenMaybeSync( + IoChannelFactory::ChannelTokenUsage usage) override { + return local->getTokenMaybeSync(usage); + } + + private: + kj::Own local; + + kj::Promise> startLocalRequest( + ActorContainer& container, IoChannelFactory::SubrequestMetadata metadata) override { + // `local` caches its restored channel across requests, as it does outside cluster mode. The + // root ClusterActorChannel inside it resolves to the container we just found or created. + return local->startRequest(kj::mv(metadata)); + } + }; + + // RpcChannel counterpart of ClusterRestoredSubrequestChannel. `restore()` is a fresh-session + // operation by contract, so nothing is retained across calls here: each call resolves the owning + // node anew and either restores locally or opens a session on the owner via + // WorkerdClusterPort.getRpcTargetFromToken(). + class ClusterRestoredRpcChannel final: public IoChannelFactory::RpcChannel { + public: + ClusterRestoredRpcChannel( + ClusterRoutedChannel& vendor, kj::Own local) + : ns(vendor.getRootNamespace()), + id(Worker::Actor::cloneId(vendor.getRootId())), + key(ns.actorKey(id)), + local(kj::mv(local)) {} + + Session restore() override { + auto split = restoreImpl(kj::addRef(*this)).split(); + return { + .cap = kj::mv(kj::get<0>(split)), + .task = kj::mv(kj::get<1>(split)), + }; + } + + void requireAllowsTransfer() override { + local->requireAllowsTransfer(); + } + + kj::OneOf, kj::Promise>> getTokenMaybeSync( + IoChannelFactory::ChannelTokenUsage usage) override { + return local->getTokenMaybeSync(usage); + } + + private: + ActorNamespace& ns; + Worker::Actor::Id id; + kj::String key; + kj::Own local; + + static kj::Promise>> restoreImpl( + kj::Own self) { + auto node = co_await self->ns.resolveNode(self->key, self->id); + KJ_SWITCH_ONEOF(node) { + KJ_CASE_ONEOF(container, kj::Own) { + // The root ClusterActorChannel inside `local` resolves to the container we just found or + // created and pins it for the session. + auto session = self->local->restore(); + co_return kj::tuple(kj::mv(session.cap), session.task.attach(kj::mv(self))); + } + KJ_CASE_ONEOF(clusterPort, rpc::WorkerdClusterPort::Client) { + auto token = co_await self->local->getToken(IoChannelFactory::ChannelTokenUsage::RPC); + auto req = clusterPort.getRpcTargetFromTokenRequest( + capnp::MessageSize{token.size() / sizeof(capnp::word) + 4, 0}); + req.setToken(token); + auto sent = req.send(); + auto session = api::wrapRemoteRpcSession(sent.getTarget(), sent.getSession()); + co_return kj::tuple(kj::mv(session.cap), session.task.attach(kj::mv(self))); + } + } + KJ_UNREACHABLE; + } + }; }; // ======================================================================================= @@ -3807,12 +3955,11 @@ class Server::WorkerService final: public Service, } auto actorClass = kj::refcounted(*this, entry.key, Frankenvalue()); - auto ns = kj::heap(kj::mv(actorClass), - // Dynamic workers can't have actor namespaces, so `serviceName` must be available here. - KJ_ASSERT_NONNULL(serviceName), entry.key, entry.value, kj::systemPreciseCalendarClock(), - threadContext.getUnsafeTimer(), threadContext.getByteStreamFactory(), - threadContext.getHttpOverCapnpFactory(), channelTokenHandler, network, dockerPath, - containerEgressInterceptorImage, waitUntilTasks, selfTokensArePersistent()); + auto ns = kj::heap(kj::mv(actorClass), entry.value, + kj::systemPreciseCalendarClock(), threadContext.getUnsafeTimer(), + threadContext.getByteStreamFactory(), threadContext.getHttpOverCapnpFactory(), + channelTokenHandler, network, dockerPath, containerEgressInterceptorImage, waitUntilTasks, + selfTokensArePersistent()); KJ_IF_SOME(d, entry.value.tryGet()) { actorNamespacesByUniqueKey.insert(d.uniqueKey, ns.get()); } @@ -6624,6 +6771,28 @@ kj::Own Server::resolveActor(kj::StringPtr names return ns.getActorChannel(kj::mv(idObj), persistent); } +kj::Own Server::wrapRestoredChannel(ChannelToken::Type type, + IoChannelFactory::SubrequestChannel& vendor, + kj::Own local) { + KJ_IF_SOME(routed, kj::tryDowncast(vendor)) { + switch (type) { + case ChannelToken::Type::SUBREQUEST: + return kj::refcounted( + routed, kj::mv(local).downcast()); + case ChannelToken::Type::RPC: + return kj::refcounted( + routed, kj::mv(local).downcast()); + case ChannelToken::Type::ACTOR_CLASS: + KJ_UNREACHABLE; // decodeChannelTokenImpl() already rejected this + } + KJ_UNREACHABLE; + } + + // The vendor is a static service, or we are not in cluster mode (resolveActor() then returns a + // plain ActorChannelImpl). Nothing to route. + return kj::mv(local); +} + // ======================================================================================= class Server::WorkerdBootstrapImpl final: public rpc::WorkerdBootstrap::Server { @@ -7142,6 +7311,48 @@ class Server::WorkerdDebugPortImpl final: public rpc::WorkerdDebugPort::Server { capnp::HttpOverCapnpFactory& httpOverCapnpFactory; }; +// Bootstrap served to cluster peers. Every method takes a channel token, so this grants nothing +// beyond what the caller's tokens already do; see WorkerdClusterPort in worker-interface.capnp. +class Server::WorkerdClusterPortImpl final: public rpc::WorkerdClusterPort::Server { + public: + WorkerdClusterPortImpl( + workerd::server::Server& srv, capnp::HttpOverCapnpFactory& httpOverCapnpFactory) + : srv(srv), + httpOverCapnpFactory(httpOverCapnpFactory) {} + + kj::Promise getChannelFromToken(GetChannelFromTokenContext context) override { + // Decoding on this node yields the same lazy channel tree the sender has. If we own the root + // actor, its ClusterActorChannel resolves locally and any restore chain runs in-process with + // correct self-tokens; if we have just lost ownership, it re-claims or forwards to the new + // owner. Either way this is bounded by the invariant described at ActorContainer::ownershipLock. + auto channel = KJ_ASSERT_NONNULL(srv.channelTokenHandler) + .decodeSubrequestChannelToken(IoChannelFactory::ChannelTokenUsage::RPC, + context.getParams().getToken()); + context.initResults(capnp::MessageSize{4, 1}) + .setBootstrap(kj::heap(kj::mv(channel), httpOverCapnpFactory)); + return kj::READY_NOW; + } + + kj::Promise getRpcTargetFromToken(GetRpcTargetFromTokenContext context) override { + auto channel = KJ_ASSERT_NONNULL(srv.channelTokenHandler) + .decodeRpcChannelToken(IoChannelFactory::ChannelTokenUsage::RPC, + context.getParams().getToken()); + auto session = channel->restore(); + + auto results = context.getResults(capnp::MessageSize{4, 2}); + results.setTarget(kj::mv(session.cap)); + // The returned session capability keeps the session alive; dropping it cancels the task. Same + // shape as RestoreRpcStubCustomEvent::receiveRpc(). + results.setSession(session.task.then( + [channel = kj::mv(channel)]() { return rpc::JsRpcSession::Client(nullptr); })); + return kj::READY_NOW; + } + + private: + workerd::server::Server& srv; + capnp::HttpOverCapnpFactory& httpOverCapnpFactory; +}; + class Server::DebugPortListener { public: DebugPortListener(Server& owner, kj::Own listener) @@ -7153,15 +7364,6 @@ class Server::DebugPortListener { co_return co_await server.listen(*listener); } - // Expose WorkerdDebugPortImpl for use by the cluster RPC system. - // - // TODO(cleanup): Maybe move WorkerDebugPortImpl up into Server. However, note the clustering - // code might also shift to a different interface. - static rpc::WorkerdDebugPort::Client makeBootstrap( - Server& owner, capnp::HttpOverCapnpFactory& httpOverCapnpFactory) { - return kj::heap(owner, httpOverCapnpFactory); - } - private: Server& owner; kj::Own listener; @@ -7480,8 +7682,9 @@ kj::Promise Server::startServices(jsg::V8System& v8System, kj::heap(kj::mv(registryDir), cluster.getNetwork(), network, timer); tasks.add(registry->runMaintenance().exclusiveJoin(forkedDrainWhen.addBranch())); auto& reg = *clusterRegistry.emplace(kj::mv(registry)); - clusterRpc.emplace(capnp::makeRpcServer( - reg, DebugPortListener::makeBootstrap(*this, globalContext->httpOverCapnpFactory))); + clusterRpc.emplace(capnp::makeRpcServer(reg, + rpc::WorkerdClusterPort::Client( + kj::heap(*this, globalContext->httpOverCapnpFactory)))); // TODO(clustering): We really shouldn't start accepting connections until the server // is totally up, but the RPC system actually starts accepting immediately on diff --git a/src/workerd/server/server.h b/src/workerd/server/server.h index 7005340485e..ba4982e8919 100644 --- a/src/workerd/server/server.h +++ b/src/workerd/server/server.h @@ -295,6 +295,9 @@ class Server final: private kj::TaskSet::ErrorHandler, private ChannelTokenHandl kj::ArrayPtr id, kj::Maybe name, Persistent persistent) override; + kj::Own wrapRestoredChannel(ChannelToken::Type type, + IoChannelFactory::SubrequestChannel& vendor, + kj::Own local) override; kj::Array encodeChannelToken(IoChannelFactory::ChannelTokenUsage usage, kj::StringPtr serviceName, @@ -331,6 +334,7 @@ class Server final: private kj::TaskSet::ErrorHandler, private ChannelTokenHandl class TcpListener; class DebugPortListener; class WorkerdDebugPortImpl; + class WorkerdClusterPortImpl; struct ErrorReporter; struct ConfigErrorReporter; diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index 92f98546205..424044e20f9 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -14,6 +14,13 @@ // 6. Storing/loading a DO stub without calling it takes no ownership lock. // 7. A broken DO releases its ownership lock even while stubs still pin it, // so it can be re-instantiated (on any node) instead of hanging requests. +// 8. Persistent stubs to restored targets (facets, RpcTargets, and chains of +// them) rooted at a DO on one node can be stored and redeemed from a DO on +// another node; the restore chain runs on the root DO's owner. This holds +// whether the stub arrives as a token or was minted in-process and passed +// through props. +// 9. A stub that has resolved to a remote DO keeps failing once that DO +// breaks, while a fresh stub reaches the re-instantiated DO. // // Two variants are run: the unix-socket cluster network mode (default) and a // localhost CIDR (`127.0.0.0/8`) IP-socket mode that exercises the registry @@ -719,3 +726,211 @@ test('cluster: aborting a DO with a request in flight lets it be re-instantiated assert.strictEqual(held.status, 500, JSON.stringify(held.body)); }); }); + +// Sets up the cross-node persistent-stub scenario shared by the tests below: +// DO "root" is claimed by node 0, then DO "holder", served through node 1 (and +// therefore claimed by node 1), asks "root" to vend a persistent stub of the +// given kind and stores it. `liveRestores` is how many times vending itself is +// expected to run root's [restore](). Returns the two DOs' node IDs and root's +// id. +async function vendAcrossNodes(nodes, kind, liveRestores = 1) { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=root'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const rootOwner = first.body.nodeId; + + const vended = await fetchJson( + nodes[1].httpPort, + `/vend-stub?name=holder&kind=${kind}&target=root`, + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(vended.status, 200, JSON.stringify(vended.body)); + assert.strictEqual(vended.body.hasStub, true); + const holderOwner = vended.body.nodeId; + assert.notStrictEqual( + holderOwner, + rootOwner, + 'the test requires root and holder to live on different nodes' + ); + + // Vending ran [restore]() live. Storing and loading the stub must not have + // replayed it. + const state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.status, 200, JSON.stringify(state.body)); + assert.strictEqual(state.body.restores, liveRestores); + + return { rootOwner, holderOwner, rootId: first.body.id }; +} + +test('cluster: a restored facet stub is redeemed on the root DO owner', async () => { + await withCluster(2, async ({ nodes, sharedPath, nodeKeys }) => { + const { rootOwner, rootId } = await vendAcrossNodes(nodes, 'facet'); + + // Redeeming from holder (node 1) forwards the whole token to root's owner, + // which replays [restore]() and runs the facet there. + const used = await fetchJson( + nodes[1].httpPort, + '/use-stub?name=holder&kind=facet', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(used.status, 200, JSON.stringify(used.body)); + assert.strictEqual(used.body.result.count, 1); + assert.strictEqual(used.body.result.nodeId, rootOwner); + let state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.body.restores, 2); + + // Break root so the next redemption has to re-instantiate it (on whichever + // node claims it) and replay [restore]() from scratch. The facet's storage + // lives under root's, so its count carries over. + const broken = await fetchJson(nodes[1].httpPort, '/break?name=root'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + const again = await fetchJson( + nodes[1].httpPort, + '/use-stub?name=holder&kind=facet', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(again.status, 200, JSON.stringify(again.body)); + assert.strictEqual(again.body.result.count, 2); + assert.strictEqual( + await readLockOwner(lockFilePath(sharedPath, rootId)), + nodeKeys.get(again.body.result.nodeId), + 'the facet must run on the node that now owns root' + ); + state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.body.restores, 3); + }); +}); + +test('cluster: a restored RpcTarget stub is redeemed on the root DO owner', async () => { + await withCluster(2, async ({ nodes }) => { + const { rootOwner } = await vendAcrossNodes(nodes, 'rpc'); + + // The RpcTarget mutates root's own storage, so it must run in root's + // context on root's owner. + const used = await fetchJson( + nodes[1].httpPort, + '/use-stub?name=holder&kind=rpc&amount=10', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(used.status, 200, JSON.stringify(used.body)); + assert.deepStrictEqual(used.body.result, { count: 11, nodeId: rootOwner }); + + const state = await fetchJson(nodes[0].httpPort, '/get?name=root'); + assert.strictEqual(state.body.count, 11); + assert.strictEqual(state.body.restores, 2); + }); +}); + +test('cluster: a two-level restore chain is redeemed on the root DO owner', async () => { + await withCluster(2, async ({ nodes }) => { + // root -> facet -> sub-facet. The facet's own [restore]() must run with a + // working ctx.restore(), which only holds when the entire chain is replayed + // in-process on root's owner. Vending runs root's [restore]() twice: once + // live for the facet stub, and once more when holder calls vendSub() on + // that stub, since a stub received over RPC is replayed on first use. + const { rootOwner } = await vendAcrossNodes(nodes, 'chained', 2); + + const used = await fetchJson( + nodes[1].httpPort, + '/use-stub?name=holder&kind=chained', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(used.status, 200, JSON.stringify(used.body)); + assert.strictEqual(used.body.result.count, 1); + assert.strictEqual(used.body.result.nodeId, rootOwner); + + const state = await fetchJson(nodes[0].httpPort, '/get?name=root'); + assert.strictEqual(state.body.restores, 3); + }); +}); + +test('cluster: a stub resolved to a remote DO keeps failing after that DO breaks', async () => { + await withCluster(2, async ({ nodes }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=sticky'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const owner = first.body.nodeId; + + // DO "holder" on node 1 obtains a stub to "sticky", uses it once (resolving + // it to node 0's instance), and keeps it. + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=sticky' + ); + assert.strictEqual(held.status, 200, JSON.stringify(held.body)); + assert.strictEqual(held.body.target.nodeId, owner); + assert.notStrictEqual(held.body.nodeId, owner); + + const broken = await fetchJson(nodes[1].httpPort, '/break?name=sticky'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + // The held stub points at the broken instance and must keep failing, even + // after a fresh stub has re-instantiated the DO. + const heldAfter = await fetchJson( + nodes[1].httpPort, + '/call-held-stub?name=holder&target=sticky', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(heldAfter.status, 500, JSON.stringify(heldAfter.body)); + assert.match(heldAfter.body.error, /broken on purpose/); + + const fresh = await fetchJson(nodes[1].httpPort, '/increment?name=sticky', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(fresh.status, 200, JSON.stringify(fresh.body)); + assert.strictEqual(fresh.body.count, 2); + + const heldAgain = await fetchJson( + nodes[1].httpPort, + '/call-held-stub?name=holder&target=sticky', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(heldAgain.status, 500, JSON.stringify(heldAgain.body)); + assert.match(heldAgain.body.error, /broken on purpose/); + }); +}); + +test('cluster: a freshly minted RpcTarget stub passed via props follows its DO to a new owner', async () => { + await withCluster(2, async ({ nodes }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=root'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const originalOwner = first.body.nodeId; + + // Root mints an RpcTarget stub and hands it to a Consumer entrypoint on the + // same node via props. The Consumer holds the stub's channel only -- the + // very channel object ctx.restore() created, not a decoded token -- and + // waits for the go signal before using it. + const vended = await fetchJson( + nodes[0].httpPort, + '/vend-to-consumer?name=root' + ); + assert.strictEqual(vended.status, 200, JSON.stringify(vended.body)); + + // Move root to the other node: break it, then re-instantiate it from there. + const broken = await fetchJson(nodes[1].httpPort, '/break?name=root'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + const moved = await fetchJson(nodes[1].httpPort, '/increment?name=root', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(moved.status, 200, JSON.stringify(moved.body)); + assert.strictEqual(moved.body.count, 2); + assert.notStrictEqual(moved.body.nodeId, originalOwner); + + // Now let the Consumer use its stub. The restore must be forwarded to the + // new owner as a whole token, not replayed over a remote vendor channel. + const go = await fetchJson(nodes[0].httpPort, '/set-go?name=sink'); + assert.strictEqual(go.status, 200, JSON.stringify(go.body)); + + const deadline = Date.now() + HANG_TIMEOUT_MS; + let recorded; + while (recorded === undefined) { + assert(Date.now() < deadline, 'Consumer never recorded a result'); + await sleep(50); + const sink = await fetchJson(nodes[0].httpPort, '/get?name=sink'); + recorded = sink.body.recorded; + } + assert.deepStrictEqual(recorded, { count: 3, nodeId: moved.body.nodeId }); + + const state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.body.restores, 2); + }); +}); diff --git a/src/workerd/server/tests/cluster/cluster-worker.js b/src/workerd/server/tests/cluster/cluster-worker.js index 4f8f1706df9..ebe38e16214 100644 --- a/src/workerd/server/tests/cluster/cluster-worker.js +++ b/src/workerd/server/tests/cluster/cluster-worker.js @@ -9,7 +9,9 @@ // // The DO supports the following request paths: // /increment Increments and returns the current counter value. -// /get Returns the current counter value without incrementing it. +// /get Returns the current counter value without incrementing it, +// along with `restores`, the number of times this DO's +// [restore]() method has run. // /set-alarm Tries to schedule a DO alarm. In cluster mode this should // fail with a "not yet supported" error. // /identity Returns information identifying this instance (NODE_ID env) @@ -27,6 +29,9 @@ // request context that created it, so a module-level variable // would not keep it alive past the request; an instance field // of a long-lived DO does.) Returns the target's response. +// /call-held-stub Calls /identity on the stub kept by /hold-stub for the DO +// named by the `target` query param. Returns the target's +// response. // /hold Waits one second before responding, so that the test can // break the DO while this request is still in flight. // /break Aborts the DO via ctx.abort(). The request fails. @@ -36,6 +41,36 @@ // committed before throwing, so the instantiation after that // succeeds), then aborts the DO so that the next request // re-instantiates it. +// +// Persistent-stub paths. Each `kind` below names a restorable object vended by +// the DO given by the `target` query param via its [restore]() method: +// facet a CounterFacet facet of the target, named by the `facet` param +// rpc a CounterAdder RpcTarget that mutates the target's counter +// chained a CounterFacet facet's own [restore]() result: a sub-facet, so the +// stored token has two restore layers +// /vend-stub?kind=K&target=T[&facet=F] +// Asks the target (over RPC) to vend a persistent stub of the +// given kind, stores it in this DO's storage under `kind`, and +// reads it back without calling it. +// /use-stub?kind=K[&amount=N] +// Loads the stub stored by /vend-stub and calls it: facets +// respond to increment(), the adder to add(amount). Returns +// the result. +// /vend-to-consumer +// Vends an `rpc` stub of *this* DO and hands it, via props, +// to a Consumer entrypoint in the same process, which keeps +// only the stub's channel (props carry no live capability). +// The Consumer waits until DO "sink" has its `go` flag set, +// then calls the stub and records the result in "sink". +// /set-go Sets this DO's `go` flag (read by Consumer via /get). +// /record?value=V Stores V (JSON) as this DO's `recorded` value. + +import { + DurableObject, + RpcTarget, + WorkerEntrypoint, + restore, +} from 'cloudflare:workers'; export default { async fetch(request, env) { @@ -63,14 +98,84 @@ export default { }, }; +// RpcTarget vended by Counter's [restore]() that mutates the counter's storage. +// It runs in the Counter's context on whichever node owns the Counter. +class CounterAdder extends RpcTarget { + constructor(counter) { + super(); + this.counter = counter; + } + + async add(amount) { + const count = + ((await this.counter.state.storage.get('count')) ?? 0) + amount; + await this.counter.state.storage.put('count', count); + return { count, nodeId: this.counter.env.NODE_ID ?? '' }; + } +} + +// See /vend-to-consumer. The stub in `props` was minted by ctx.restore() in this +// process and arrives here without its live capability, so the first call on it +// restores through the channel object the minting DO created, not through a +// decoded token. +export class Consumer extends WorkerEntrypoint { + useLater() { + const adder = this.ctx.props.adder; + const sink = this.env.COUNTER.get(this.env.COUNTER.idFromName('sink')); + this.ctx.waitUntil( + (async () => { + for (;;) { + const state = await (await sink.fetch('http://do/get')).json(); + if (state.go) break; + await scheduler.wait(50); + } + let result; + try { + result = await adder.add(1); + } catch (err) { + result = { error: String(err) }; + } + await sink.fetch( + 'http://do/record?value=' + encodeURIComponent(JSON.stringify(result)) + ); + })() + ); + } +} + +// Facet class used by Counter's [restore](). +export class CounterFacet extends DurableObject { + async increment() { + const count = ((await this.ctx.storage.get('count')) ?? 0) + 1; + await this.ctx.storage.put('count', count); + return { count, nodeId: this.env.NODE_ID ?? '' }; + } + + // Vend a persistent stub to a sub-facet. Only works when this facet was itself + // reached through a restored stub, which is what makes the two-level token. + vendSub() { + return this.ctx.restore({ sub: true }); + } + + [restore](params) { + if (params.sub) { + return this.ctx.facets.get('sub', () => ({ + class: this.ctx.exports.CounterFacet, + })); + } + throw new Error('unexpected facet restore params'); + } +} + const BREAK_ON_CONSTRUCT = 'break-on-construct'; -export class Counter { +export class Counter extends DurableObject { constructor(state, env) { + super(state, env); this.state = state; this.env = env; - // See /hold-stub. - this.heldStubs = []; + // See /hold-stub. Maps target DO name -> stub. + this.heldStubs = new Map(); state.blockConcurrencyWhile(async () => { if (await state.storage.get(BREAK_ON_CONSTRUCT)) { @@ -99,7 +204,17 @@ export class Counter { }); } else if (url.pathname === '/get') { const count = (await this.state.storage.get('count')) ?? 0; - return Response.json({ count, nodeId, id: idHex }); + const restores = (await this.state.storage.get('restores')) ?? 0; + const go = (await this.state.storage.get('go')) ?? false; + const recorded = await this.state.storage.get('recorded'); + return Response.json({ + count, + restores, + go, + recorded, + nodeId, + id: idHex, + }); } else if (url.pathname === '/set-alarm') { // Try to schedule an alarm 60s in the future. In cluster mode this should // throw a clear error. @@ -139,13 +254,70 @@ export class Counter { } else if (url.pathname === '/hold-stub') { const target = url.searchParams.get('target'); const stub = this.env.COUNTER.get(this.env.COUNTER.idFromName(target)); - this.heldStubs.push(stub); + this.heldStubs.set(target, stub); const response = await stub.fetch('http://do/identity'); return Response.json({ nodeId, id: idHex, target: await response.json(), }); + } else if (url.pathname === '/call-held-stub') { + const target = url.searchParams.get('target'); + const stub = this.heldStubs.get(target); + if (stub === undefined) { + return Response.json({ error: 'no held stub' }, { status: 400 }); + } + const response = await stub.fetch('http://do/identity'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); + } else if (url.pathname === '/vend-stub') { + const kind = url.searchParams.get('kind'); + const target = url.searchParams.get('target'); + const facet = url.searchParams.get('facet') ?? 'default'; + const ns = this.state.exports.Counter; + const targetStub = ns.get(ns.idFromName(target)); + let stub; + if (kind === 'chained') { + // The facet's own ctx.restore() only works when the facet is reached + // through a restored stub, which vendFacet() returns. + const facetStub = await targetStub.vendFacet(facet); + stub = await facetStub.vendSub(); + } else { + stub = await targetStub.vend(kind); + } + await this.state.storage.put(kind, stub); + // Read it back so that the stub is also deserialized, but do not call it. + const loaded = await this.state.storage.get(kind); + return Response.json({ + nodeId, + id: idHex, + hasStub: loaded !== undefined, + }); + } else if (url.pathname === '/use-stub') { + const kind = url.searchParams.get('kind'); + const stub = await this.state.storage.get(kind); + const result = + kind === 'rpc' + ? await stub.add(Number(url.searchParams.get('amount') ?? '1')) + : await stub.increment(); + return Response.json({ nodeId, id: idHex, result }); + } else if (url.pathname === '/vend-to-consumer') { + const adder = await this.state.restore({ kind: 'rpc' }); + const consumer = this.state.exports.Consumer({ props: { adder } }); + await consumer.useLater(); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/set-go') { + await this.state.storage.put('go', true); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/record') { + await this.state.storage.put( + 'recorded', + JSON.parse(url.searchParams.get('value')) + ); + return Response.json({ nodeId, id: idHex }); } else if (url.pathname === '/hold') { await scheduler.wait(1000); return Response.json({ nodeId, id: idHex }); @@ -165,4 +337,27 @@ export class Counter { // a stored marker so the test can observe the failure mode. await this.state.storage.put('alarm-fired', true); } + + // RPC methods used by /vend-stub on the *target* DO. + vend(kind) { + return this.state.restore({ kind }); + } + + vendFacet(facet) { + return this.state.restore({ kind: 'facet', facet }); + } + + async [restore](params) { + const restores = ((await this.state.storage.get('restores')) ?? 0) + 1; + await this.state.storage.put('restores', restores); + + if (params.kind === 'facet') { + return this.state.facets.get(params.facet, () => ({ + class: this.state.exports.CounterFacet, + })); + } else if (params.kind === 'rpc') { + return new CounterAdder(this); + } + throw new Error(`unexpected restore params: ${JSON.stringify(params)}`); + } } From f02308f3a3450f7d8f78c89a2f4430c0138e9d8c Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Mon, 7 Sep 2026 12:38:01 -0500 Subject: [PATCH 17/33] Fix UAF of cluster RPC system. The RpcSystem needs to remain live all the way until process exit, but we were destroying it at drain time. We should only stop accepting new connections then. (Caught by Astra.) --- src/workerd/server/server.c++ | 18 ++++++++- .../server/tests/cluster/cluster-test.mjs | 37 +++++++++++++++++++ .../server/tests/cluster/cluster-worker.js | 10 +++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 87759b01cb4..e103a48cb43 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -342,6 +342,14 @@ Server::~Server() noexcept { abortAllActors(KJ_EXCEPTION(DISCONNECTED, "Server shutting down.")); tasks.clear(); + // Tear down the cluster RPC system now that nothing can issue requests through it. This must + // happen after `tasks.clear()` (the accept loop task references the RpcSystem) and before the + // services are unlinked: open cluster connections export capabilities that hold references into + // services, and dropping those connections is what lets the unlink below eliminate all + // references. The ClusterLockManagers that reference `clusterRpc` live in ActorNamespaces, which + // are destroyed by WorkerService::unlink(). + clusterRpc = kj::none; + // Unlink all the services, which should remove all refcount cycles. unlinkWorkerLoaders(); for (auto& service: services) { @@ -7398,10 +7406,18 @@ kj::Promise Server::handleDrain(kj::Promise drainWhen) { tasks.add(httpServer.httpServer.drain()); } + // Note that `clusterRpc` is intentionally NOT torn down here. Its accept loop is a task joined + // with `forkedDrainWhen` (see startServices()), so no new cluster connections are accepted once + // draining begins. But the RpcSystem itself must stay alive: every ClusterLockManager holds a + // reference to it, and in-flight HTTP requests -- which are permitted to run to completion -- + // may still call a remotely owned actor, which routes through clusterRpc.bootstrap(). Dynamic + // workers linked during the drain window likewise need it (see the link callback's + // KJ_ASSERT_NONNULL(this->clusterRpc)). The RpcSystem is destroyed in ~Server(), after all + // requests have finished and before services are unlinked. + // // TODO(clustering): This is a harsh ending for actors on this machine. We should really evict // each one at a time that is convenient, and wait some time for RPC requests to drain. // Three-party handoff would help with straggler requests that we forward to the new owner. - clusterRpc = kj::none; abortAllActors(kj::none); } diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index 424044e20f9..153c1e211e5 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -435,6 +435,43 @@ test('cluster: killing the owner allows another node to take over', async () => }); }); +test('cluster: draining node still routes in-flight requests to a remote owner', async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO so that exactly one node owns it. + const initial = await fetchJson(nodes[0].httpPort, '/increment?name=drain'); + assert.strictEqual(initial.status, 200); + assert.strictEqual(initial.body.count, 1); + const owner = nodes.find((n) => n.nodeId === initial.body.nodeId); + const nonOwner = nodes.find((n) => n.nodeId !== initial.body.nodeId); + assert(owner !== undefined && nonOwner !== undefined); + + // Put a request in flight on the non-owner whose stateless handler waits + // before calling the DO, then SIGTERM the non-owner while it is waiting. + // Draining lets in-flight requests finish, so this request must still be + // able to route to the owner over cluster RPC after draining began. + const requestPromise = fetchJson( + nonOwner.httpPort, + '/increment?name=drain&delay=1000', + { timeoutMs: 10_000 } + ); + await sleep(250); + const exitPromise = nonOwner.stop({ signal: 'SIGTERM', timeoutMs: 10_000 }); + + const response = await requestPromise; + assert.strictEqual( + response.status, + 200, + `in-flight request on draining node failed: ${JSON.stringify(response.body)}` + ); + assert.strictEqual(response.body.nodeId, owner.nodeId); + assert.strictEqual(response.body.count, 2); + + // The draining node must have exited cleanly once the request finished. + const exit = await exitPromise; + assert.deepStrictEqual(exit, { code: 0, signal: null }); + }); +}); + test('cluster: alarms are rejected with a clear error', async () => { await withCluster(1, async ({ nodes, sharedPath }) => { // Fire the request. In cluster mode `setAlarm()` does not throw diff --git a/src/workerd/server/tests/cluster/cluster-worker.js b/src/workerd/server/tests/cluster/cluster-worker.js index ebe38e16214..4a6d9dcbed2 100644 --- a/src/workerd/server/tests/cluster/cluster-worker.js +++ b/src/workerd/server/tests/cluster/cluster-worker.js @@ -64,6 +64,11 @@ // then calls the stub and records the result in "sink". // /set-go Sets this DO's `go` flag (read by Consumer via /get). // /record?value=V Stores V (JSON) as this DO's `recorded` value. +// +// The top-level fetch() additionally honors a `delay` query parameter (ms): it +// waits that long *before* calling the DO. This lets a test put a request in +// flight on a node and SIGTERM that node while the request has not yet routed +// to the DO's owner. import { DurableObject, @@ -85,6 +90,11 @@ export default { const id = env.COUNTER.idFromName(name); const stub = env.COUNTER.get(id); + const delay = url.searchParams.get('delay'); + if (delay !== null) { + await scheduler.wait(Number(delay)); + } + // Forward to the DO with the same path and query. const forwardUrl = new URL(url.pathname + url.search, 'http://do/'); try { From 74919d2078f29b2bc556922d20c7639e5cedce37 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Mon, 7 Sep 2026 14:07:06 -0500 Subject: [PATCH 18/33] Bugfix: A stale lockfile referencing this node could cause an infinite routing loop. If a node opened a lockfile on disk and found it referenced itself, it would route to itself. But if no actor was actually loaded, the self-route would go load the lock again, and route to self again, in an infinite loop. This commit fixes this by adding a rule that when we encounter a self-reference, we don't optimistically route without checking whether the lock is held first. If the self-reference is stale, then it'll have no lock on it, so we'll end up creating a new lock and thus a new actor instance, avoiding the loop. Normally, skipping the lock is an important optimization, but in this case it doesn't matter because we always check for local presence in the actual in-memory actor table first. So, the only way we can end up finding a self-reference on disk is if it's actually stale or in certain narrow race conditions -- both unlikely. --- src/workerd/server/cluster-lock-test.c++ | 37 +++++++++++++++++-- src/workerd/server/cluster-lock.c++ | 41 +++++++++++++++------ src/workerd/server/cluster-lock.h | 16 +++++--- src/workerd/server/server.c++ | 47 +++++++++++++----------- 4 files changed, 98 insertions(+), 43 deletions(-) diff --git a/src/workerd/server/cluster-lock-test.c++ b/src/workerd/server/cluster-lock-test.c++ index dbd9123cb81..5db08ab4116 100644 --- a/src/workerd/server/cluster-lock-test.c++ +++ b/src/workerd/server/cluster-lock-test.c++ @@ -200,6 +200,34 @@ KJ_TEST("ClusterLockManager: OwnedLock destructor truncates and releases") { KJ_ASSERT(result2.is()); } +KJ_TEST("ClusterLockManager: stale file naming self with no lock held -> reclaims") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + ClusterRegistry reg( + tmpDir.registry(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + + ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); + + // Simulate a claim or release that published our key but failed before clearing it: the file + // names this node, but no OwnedLock (and hence no OFD lock) exists. Routing to ourselves here + // would loop forever; we must reclaim instead. + tmpDir.writeLockFile("a6", kj::arrayPtr(reg.getPublicKey().bytes, 32)); + + auto result = lockManager.acquireOrRoute("a6").wait(io.waitScope); + KJ_ASSERT(result.is()); + + auto content = tmpDir.readLockFile("a6"); + KJ_ASSERT(content.size() == 32); + KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); + + // With the OwnedLock now held, a further lookup routes to ourselves rather than reclaiming. + auto routeResult = lockManager.acquireOrRoute("a6").wait(io.waitScope); + KJ_ASSERT(routeResult.is()); +} + KJ_TEST("ClusterLockManager: owned by live peer -> returns bootstrap client") { auto io = kj::setupAsyncIo(); TempDir tmpDir; @@ -267,11 +295,12 @@ KJ_TEST("ClusterLockManager: writer alive -> retries until writer releases") { io.provider->getTimer().afterDelay(50 * kj::MILLISECONDS).wait(io.waitScope); KJ_EXPECT(!completed, "acquireOrRoute should still be waiting for the external lock"); - // Now the "external writer" writes its key (our own, for simplicity) and releases the lock. - // acquireOrRoute, on its next iteration, should see the valid key, recognize the owner as us, - // and return a bootstrap client. + // Now the "external writer" finishes publishing its key (our own, for simplicity) while still + // holding the lock, as a real owner does. acquireOrRoute, on its next iteration, should see the + // key, find the OFD lock held, and return a bootstrap client routing to the owner. (Releasing + // the lock instead would leave a self-naming file with no holder, which acquireOrRoute reclaims; + // see the "stale file naming self" test.) externalFile->write(0, kj::arrayPtr(reg.getPublicKey().bytes, 32)); - { auto _ = kj::mv(externalLock); } auto result = trackedPromise.wait(io.waitScope); KJ_EXPECT(completed); diff --git a/src/workerd/server/cluster-lock.c++ b/src/workerd/server/cluster-lock.c++ index c77e8d71437..7edea152849 100644 --- a/src/workerd/server/cluster-lock.c++ +++ b/src/workerd/server/cluster-lock.c++ @@ -65,7 +65,9 @@ ClusterLockManager::OwnedLock::~OwnedLock() noexcept(false) { // Truncate the file back to zero so the next claimant sees an empty file. We then drop the // OFD lock (via OfdLock's destructor) which makes the file available for other nodes to claim. // - // Best-effort: don't throw from the destructor. + // Best-effort: don't throw from the destructor. If this fails, the file keeps naming this node + // with no lock held. That's benign: the next acquireOrRoute() on this node sees its own key, + // takes the OFD lock, and reclaims (remote nodes route here, so they land on that path too). KJ_TRY { file->truncate(0); } @@ -120,11 +122,17 @@ ClusterLockManager::acquireOrRoute(kj::StringPtr actorId) { X25519PublicKey ownerKey; auto n = file->read(0, ownerKey.bytes); + const auto& myKey = registry.getPublicKey(); + bool namesSelf = n == sizeof(ownerKey.bytes) && ownerKey == myKey; + if (n == sizeof(ownerKey.bytes)) { - if (!registry.isPeerDead(ownerKey)) { + if (namesSelf) { + // The file names this node. That is authoritative only if an OwnedLock on this node still + // holds the OFD lock; otherwise a claim or release failed partway and left our key behind. + // Routing to ourselves in the latter case would loop forever (resolveNode() finds no + // container and lands back here), so the OFD lock decides: fall through to tryLock(). + } else if (!registry.isPeerDead(ownerKey)) { // Trust the lock file. Build a VatId and ask the RpcSystem to bootstrap. - // The owner might be us — clusterRpc.bootstrap() returns the local bootstrap directly - // without going through ClusterRegistry::connect() in that case. // // If the owner is actually unreachable, the bootstrap call will fail later. Internally, // ClusterRegistry::ConnectionImpl::unregister() runs the dead-peer cleanup probe; @@ -135,7 +143,7 @@ ClusterLockManager::acquireOrRoute(kj::StringPtr actorId) { co_return clusterRpc.bootstrap(holder.getReader()).castAs(); } - // Owner is confirmed dead. Fall through to the claim path. + // else: owner is confirmed dead. Fall through to the claim path. } // else: n == 0 (empty file) or n is some unexpected partial size. In either case, the claim // path is the right next step: if a live writer is currently producing the file, they @@ -145,14 +153,23 @@ ClusterLockManager::acquireOrRoute(kj::StringPtr actorId) { // Claim path. Try to acquire an exclusive OFD lock. int fd = KJ_REQUIRE_NONNULL(file->getFd(), "lock directory must be disk-backed"); KJ_IF_SOME(lock, OfdLock::tryLock(fd, OfdLock::EXCLUSIVE)) { - // We got the exclusive lock. Clear any stale content and write our identity. - file->truncate(0); - - const auto& myKey = registry.getPublicKey(); - file->write(0, myKey.bytes); - file->sync(); + // We got the exclusive lock. Clear any stale content and write our identity. Construct the + // OwnedLock first so that if any of these operations throws, unwinding runs ~OwnedLock, + // which clears the file (best-effort) before the OFD lock is dropped. + OwnedLock owned(kj::mv(file), kj::mv(lock)); + owned.file->truncate(0); + owned.file->write(0, myKey.bytes); + owned.file->sync(); + + co_return kj::mv(owned); + } - co_return OwnedLock(kj::mv(file), kj::mv(lock)); + if (namesSelf) { + // The file names this node and an OwnedLock on this node holds the OFD lock, so the actor's + // container exists here (see ActorContainer::ownershipLock). Route to ourselves; the + // RpcSystem returns the local bootstrap directly. + VatIdHolder holder(ownerKey); + co_return clusterRpc.bootstrap(holder.getReader()).castAs(); } // tryLock failed — someone else owns it (or is racing us to claim it). Retry. diff --git a/src/workerd/server/cluster-lock.h b/src/workerd/server/cluster-lock.h index d473d31d20f..90c5c4e5ac0 100644 --- a/src/workerd/server/cluster-lock.h +++ b/src/workerd/server/cluster-lock.h @@ -22,9 +22,13 @@ namespace workerd { // // The ownership protocol: // 1. Open/create the lock file. -// 2. Read it. If non-empty and CRC32 valid, parse the owner key. -// - If !registry.isPeerDead(ownerKey): route via clusterRpc.bootstrap(vatIdFor(ownerKey)). -// - If registry.isPeerDead(ownerKey): fall through to the claim path. +// 2. Read it. If it holds a full key, that's the owner. +// - If it's another node and !registry.isPeerDead(ownerKey): route via +// clusterRpc.bootstrap(vatIdFor(ownerKey)). +// - If it's another node and registry.isPeerDead(ownerKey): fall through to the claim path. +// - If it's this node: the file alone is not authoritative (a failed claim or release can +// leave our key behind). Fall through to the claim path; if tryLock() is refused, an +// OwnedLock on this node holds the actor and we route to ourselves. // 3. Claim path: tryLock(EXCLUSIVE). If granted, ftruncate, write our key, fsync, // return OwnedLock. If refused, retry with backoff. class ClusterLockManager { @@ -34,9 +38,9 @@ class ClusterLockManager { capnp::RpcSystem& clusterRpc, kj::Timer& timer); - // RAII ownership handle. Holding this object means this node owns the DO. - // Destruction truncates the lock file and releases the exclusive OFD lock, - // making the DO available for other nodes to claim. + // RAII ownership handle. Holding this object means this node owns the DO: it holds the + // exclusive OFD lock on the lock file. Destruction truncates the lock file (best-effort) and + // releases the OFD lock, making the DO available for other nodes to claim. class OwnedLock { public: ~OwnedLock() noexcept(false); diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index e103a48cb43..82360a2e83e 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -879,15 +879,20 @@ class Server::ActorNamespace final { // destruction it is released only after everything below it -- the Worker::Actor, facets, // hibernation manager, and facet tree index -- has been torn down. // - // Invariant: the lock file names this node if and only if this container is in `ns.actors`. - // A request for an actor whose lock file names this node but which has no map entry loops - // through self-bootstrap (see ActorNamespace::resolveNode()), so the lock is released at the - // moment the container leaves the map (abort(), monitorOnBroken()), not when the last - // reference to the container is dropped. Releasing it there is only safe because - // ActorSqlite::shutdown() closes the database: by the time either path hollows the container, - // every SQLite handle under the actor and its facets is already closed, so no handle into the - // actor's storage outlives the claim even if an in-flight request still holds the - // Worker::Actor. + // Invariant: an OwnedLock for the actor is alive on this node if and only if its container is + // in `ns.actors`. ClusterLockManager::acquireOrRoute() routes to this node whenever the lock + // file names it and the OFD lock is held, so an OwnedLock with no map entry would make every + // request for the actor loop through self-bootstrap (see ActorNamespace::resolveNode()). The + // lock is therefore released at the moment the container leaves the map (abort(), + // monitorOnBroken()), not when the last reference to the container is dropped. Releasing it + // there is only safe because ActorSqlite::shutdown() closes the database: by the time either + // path hollows the container, every SQLite handle under the actor and its facets is already + // closed, so no handle into the actor's storage outlives the claim even if an in-flight + // request still holds the Worker::Actor. + // + // The lock file's contents alone carry no such guarantee: a failed claim or release can leave + // this node's key behind with no OwnedLock held. acquireOrRoute() detects that (the OFD lock + // is free) and reclaims, so it needs no handling here. kj::Maybe ownershipLock; // The actor is constructed after the ActorContainer so it starts off empty. @@ -1507,15 +1512,15 @@ class Server::ActorNamespace final { } // Create an ActorContainer in cluster mode. Must be called in the same synchronous continuation - // that received `ownership` from ClusterLockManager::acquireOrRoute(): the lock file naming this - // node while `actors` has no entry for the key would make requests for the actor loop through - // self-bootstrap until the entry appears (see resolveNode()). + // that received `ownership` from ClusterLockManager::acquireOrRoute(): an OwnedLock alive on + // this node while `actors` has no entry for the key would make requests for the actor loop + // through self-bootstrap until the entry appears (see resolveNode()). kj::Own createActorContainer( kj::String key, Worker::Actor::Id id, ClusterLockManager::OwnedLock ownership) { - // Invariant: the lock file names this node iff a container holding the OwnedLock is in - // `actors`. We hold the lock and are about to insert; nothing else may already be mapped. - // The other direction is upheld by ActorContainer::abort() and monitorOnBroken(), which - // release the lock as the container leaves the map. + // Invariant: an OwnedLock is alive on this node iff a container holding it is in `actors`. We + // hold the lock and are about to insert; nothing else may already be mapped. The other + // direction is upheld by ActorContainer::abort() and monitorOnBroken(), which release the + // lock as the container leaves the map. KJ_ASSERT(actors.find(key) == kj::none); auto container = kj::refcounted(kj::str(key), *this, kj::none, @@ -1531,11 +1536,11 @@ class Server::ActorNamespace final { // the returned promise settles. Cluster mode only. using Node = kj::OneOf, rpc::WorkerdClusterPort::Client>; kj::Promise resolveNode(kj::StringPtr key, const Worker::Actor::Id& id) { - // Local-first. This lookup is what terminates the routing loop: when the lock file names this - // node, acquireOrRoute() returns our own cluster port with no backoff, and the resulting - // getChannelFromToken() decodes the token into a channel that lands right back here. That is - // only correct because "lock file names this node" implies "container is in `actors`" -- see - // ActorContainer::ownershipLock. + // Local-first. This lookup is what terminates the routing loop: when an OwnedLock on this + // node holds the actor, acquireOrRoute() returns our own cluster port with no backoff, and the + // resulting getChannelFromToken() decodes the token into a channel that lands right back here. + // That is only correct because "OwnedLock alive on this node" implies "container is in + // `actors`" -- see ActorContainer::ownershipLock. KJ_IF_SOME(container, actors.find(key)) { co_return container->addRef(); } From d548e0022719cacc57442a7f507cfb7b5acd8f7f Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Mon, 7 Sep 2026 14:34:15 -0500 Subject: [PATCH 19/33] Bugfix: Enforce timeout on network connections. We were already enforcing a timeout on the handshake after the connection was formed, but obviously the connect() itself can also time out. --- src/workerd/server/cluster-registry-test.c++ | 97 ++++++++++++++++++++ src/workerd/server/cluster-registry.c++ | 5 + 2 files changed, 102 insertions(+) diff --git a/src/workerd/server/cluster-registry-test.c++ b/src/workerd/server/cluster-registry-test.c++ index e7b274922d4..0b7e97be563 100644 --- a/src/workerd/server/cluster-registry-test.c++ +++ b/src/workerd/server/cluster-registry-test.c++ @@ -7,6 +7,7 @@ #include #if !_WIN32 +#include #include #endif @@ -69,6 +70,54 @@ void makeCall(capnp::Capability::Client& client, kj::WaitScope& ws) { req.send().wait(ws); } +// A kj::Network wrapper whose outbound connects never complete, simulating a peer address that +// blackholes our SYNs. Listening is delegated unchanged. +class BlackholeNetwork final: public kj::Network { + public: + BlackholeNetwork(kj::Network& inner): inner(inner) {} + + kj::Promise> parseAddress( + kj::StringPtr addr, uint portHint) override { + return inner.parseAddress(addr, portHint) + .then([](kj::Own a) -> kj::Own { + return kj::heap
(kj::mv(a)); + }); + } + + kj::Own getSockaddr(const void* sockaddr, uint len) override { + return kj::heap
(inner.getSockaddr(sockaddr, len)); + } + + kj::Own restrictPeers( + kj::ArrayPtr allow, kj::ArrayPtr deny) override { + KJ_UNIMPLEMENTED("not needed in test"); + } + + private: + class Address final: public kj::NetworkAddress { + public: + Address(kj::Own inner): inner(kj::mv(inner)) {} + + kj::Promise> connect() override { + return kj::NEVER_DONE; + } + kj::Own listen() override { + return inner->listen(); + } + kj::Own clone() override { + return kj::heap
(inner->clone()); + } + kj::String toString() override { + return inner->toString(); + } + + private: + kj::Own inner; + }; + + kj::Network& inner; +}; + KJ_TEST("X25519PublicKey: hex roundtrip") { X25519PublicKey key; memset(key.bytes, 0xAB, sizeof(key.bytes)); @@ -285,6 +334,54 @@ KJ_TEST("ClusterRegistry: idle timeout closes connections") { KJ_EXPECT(server2Ref.callCount == 2); } +KJ_TEST("ClusterRegistry: hung connect is torn down and dead peer cleaned up within timeout") { + auto io = kj::setupAsyncIo(); + TempDir tmpDir; + + auto connectTimeout = 100 * kj::MILLISECONDS; + BlackholeNetwork network(io.provider->getNetwork()); + + ClusterRegistry reg(tmpDir.get(), "127.0.0.0/8"_kj, network, io.provider->getTimer(), + 60 * kj::SECONDS, connectTimeout); + auto maint = reg.runMaintenance(); + + // Forge a registry entry for a peer that is gone: a valid address (copied from our own entry), + // not locked by anyone, and old enough to be past the new-entry grace period. + X25519PublicKey peerKey; + memset(peerKey.bytes, 0xBE, sizeof(peerKey.bytes)); + auto peerPath = kj::Path({peerKey.toHex()}); + { + auto dir = tmpDir.get(); + auto ownEntry = dir->openFile(kj::Path({reg.getPublicKey().toHex()}))->readAllBytes(); + auto peerFile = dir->openFile(peerPath, kj::WriteMode::CREATE); + peerFile->writeAll(ownEntry); + timespec old[2] = {{.tv_sec = 1, .tv_nsec = 0}, {.tv_sec = 1, .tv_nsec = 0}}; + KJ_SYSCALL(futimens(KJ_ASSERT_NONNULL(peerFile->getFd()), old)); + } + + auto rpc = capnp::makeRpcServer(reg, capnp::Capability::Client(kj::heap())); + auto run = rpc.run(); + + capnp::MallocMessageBuilder msg; + auto vatId = msg.initRoot(); + vatId.setPublicKey(kj::arrayPtr(peerKey.bytes, 32)); + + auto& timer = io.provider->getTimer(); + auto start = timer.now(); + { + auto client = rpc.bootstrap(vatId); + KJ_EXPECT_THROW_MESSAGE("timed out", makeCall(client, io.waitScope)); + } + + // Give the RPC system a chance to finish dropping the connection. + timer.afterDelay(connectTimeout).wait(io.waitScope); + KJ_EXPECT(timer.now() - start < 10 * connectTimeout); + + // The dead-peer probe must have run: the unlocked, stale entry is removed and the peer is dead. + KJ_EXPECT(!tmpDir.get()->exists(peerPath)); + KJ_EXPECT(reg.isPeerDead(peerKey)); +} + KJ_TEST("ClusterRegistry: verifyNfsLease() succeeds when lock is held") { auto io = kj::setupAsyncIo(); TempDir tmpDir; diff --git a/src/workerd/server/cluster-registry.c++ b/src/workerd/server/cluster-registry.c++ index a430d059e86..bf4cf767815 100644 --- a/src/workerd/server/cluster-registry.c++ +++ b/src/workerd/server/cluster-registry.c++ @@ -750,6 +750,11 @@ kj::Maybe> ClusterRegistry::connect( } auto connectPromise = networkAddress->connect(); + if (nfsMode()) { + // Enforce timeout when connecting over the network. (When not in nfsMode(), we're connecting + // to a local unix socket, in which case no timeout is needed.) + connectPromise = timer.timeoutAfter(connectTimeout, kj::mv(connectPromise)); + } kj::Maybe> wasRefused; if (!nfsMode()) { From 8ce56a11a2c205569d5df5165a89c569ae03e9eb Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Mon, 7 Sep 2026 15:05:07 -0500 Subject: [PATCH 20/33] Bugfix: Only abort actors at drain time in cluster mode. In non-cluster mode, adding `abortAllActors()` to `handleDrain()` was arguably a regression, so make it apply in cluster mode only. (It is necessary in cluster mode since once we stop accepting new requests, any actor hosted locally would be unreachable by the cluster.) Additionally, make sure that we do not acquire a clustered actor after we've started draining. --- src/workerd/server/cluster-lock.c++ | 12 +++++ src/workerd/server/cluster-registry.h | 14 ++++++ src/workerd/server/server.c++ | 35 +++++++++------ .../server/tests/cluster/cluster-test.mjs | 44 +++++++++++++++++++ 4 files changed, 92 insertions(+), 13 deletions(-) diff --git a/src/workerd/server/cluster-lock.c++ b/src/workerd/server/cluster-lock.c++ index 7edea152849..d5a9ad1335f 100644 --- a/src/workerd/server/cluster-lock.c++ +++ b/src/workerd/server/cluster-lock.c++ @@ -157,6 +157,18 @@ ClusterLockManager::acquireOrRoute(kj::StringPtr actorId) { // OwnedLock first so that if any of these operations throws, unwinding runs ~OwnedLock, // which clears the file (best-effort) before the OFD lock is dropped. OwnedLock owned(kj::mv(file), kj::mv(lock)); + + // Holding the lock proves the actor is unowned (a file naming this node was stale). A + // draining node must not become its owner: its cluster listener no longer accepts + // connections, so no other node could reach the actor until this process exits. The check + // comes after tryLock() because the lock is what distinguishes "unowned" from "owned here", + // and the latter must still route to ourselves below. Unwinding runs ~OwnedLock, which + // leaves the file empty and unlocked for a live node to claim. + if (registry.isDraining()) { + kj::throwFatalException( + KJ_EXCEPTION(DISCONNECTED, "cannot claim Durable Object: this node is shutting down")); + } + owned.file->truncate(0); owned.file->write(0, myKey.bytes); owned.file->sync(); diff --git a/src/workerd/server/cluster-registry.h b/src/workerd/server/cluster-registry.h index 781e91c4504..55a249950d9 100644 --- a/src/workerd/server/cluster-registry.h +++ b/src/workerd/server/cluster-registry.h @@ -80,6 +80,18 @@ class ClusterRegistry final: public ClusterVatNetworkBase { // Pick a random peer's key from the cached peer list, biased away from recent failures. kj::Maybe pickRandomPeer(); + // Marks this node as leaving the cluster. Once the server begins draining, its cluster listener + // stops accepting connections, so an actor claimed here would be unreachable from every other + // node until this process exits. ClusterLockManager::acquireOrRoute() consults isDraining() and + // refuses to claim unowned actors; routing to an existing owner is unaffected, since outbound + // connections keep working for in-flight requests. + void startDraining() { + draining = true; + } + bool isDraining() { + return draining; + } + // Returns true if this registry uses IP sockets (not unix sockets) and is therefore in // "NFS mode" where it tries to be NFS-friendly. bool nfsMode() { @@ -91,6 +103,8 @@ class ClusterRegistry final: public ClusterVatNetworkBase { kj::Promise> accept() override; private: + bool draining = false; + #if __linux__ class ConnectionImpl; diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 82360a2e83e..00ac37fa93b 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -7411,19 +7411,28 @@ kj::Promise Server::handleDrain(kj::Promise drainWhen) { tasks.add(httpServer.httpServer.drain()); } - // Note that `clusterRpc` is intentionally NOT torn down here. Its accept loop is a task joined - // with `forkedDrainWhen` (see startServices()), so no new cluster connections are accepted once - // draining begins. But the RpcSystem itself must stay alive: every ClusterLockManager holds a - // reference to it, and in-flight HTTP requests -- which are permitted to run to completion -- - // may still call a remotely owned actor, which routes through clusterRpc.bootstrap(). Dynamic - // workers linked during the drain window likewise need it (see the link callback's - // KJ_ASSERT_NONNULL(this->clusterRpc)). The RpcSystem is destroyed in ~Server(), after all - // requests have finished and before services are unlinked. - // - // TODO(clustering): This is a harsh ending for actors on this machine. We should really evict - // each one at a time that is convenient, and wait some time for RPC requests to drain. - // Three-party handoff would help with straggler requests that we forward to the new owner. - abortAllActors(kj::none); + KJ_IF_SOME(registry, clusterRegistry) { + // When draining begins, we stop accepting cluster RPC connections (`clusterRpc`'s accept + // loop is joined with `forkedDrainWhen` -- see `startServices()`). As such, as soon as we + // start draining, all actors hosted here become unreachable from the rest of the cluster. We + // MUST, therefore, abort them, and refuse to claim any more: an in-flight local request that + // reaches an unowned actor during the drain would otherwise make this node its owner (see + // ClusterLockManager::acquireOrRoute()). + // + // TODO(clustering): This is a harsh ending for actors on this machine. We should really evict + // each one at a time that is convenient, and wait some time for RPC requests to drain. + // As of this writing, though, we don't really have any machinery for determining what time + // is "convenient" nor for doing any sort of clean hand-off between instances of an actor. + // The Cloudflare production behavior is widely considered not great: draining instances lose + // access to storage but are allowed to finish in-flight requests. The DO team has plans to + // replace this behavior with something better in the not-so-distant future (maybe), so for + // now we don't try to recreate it here -- we just hard-abort actors when drain starts. Note + // that it is important that we abort at drain start rather than continue accepting cluster + // connections until fully drained since we want actor locks to be released cleanly; a stale + // lock takes more effort to clean up. + registry->startDraining(); + abortAllActors(kj::none); + } } kj::Promise Server::run( diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index 153c1e211e5..cf1c72499d9 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -472,6 +472,50 @@ test('cluster: draining node still routes in-flight requests to a remote owner', }); }); +test('cluster: draining node refuses to claim an unowned DO', async () => { + await withCluster(2, async ({ nodes, sharedPath, nodeKeys }) => { + // Put a request in flight on node 0 for a DO that nobody owns yet, then + // SIGTERM node 0 while its stateless handler is still waiting. By the time + // the handler calls the DO, node 0 is draining: its cluster listener no + // longer accepts connections, so claiming the DO there would leave it + // unreachable from the rest of the cluster until node 0 exits. The claim + // must be refused instead. + const requestPromise = fetchJson( + nodes[0].httpPort, + '/increment?name=lame-duck&delay=1000', + { timeoutMs: 10_000 } + ); + await sleep(250); + const exitPromise = nodes[0].stop({ signal: 'SIGTERM', timeoutMs: 10_000 }); + + const response = await requestPromise; + assert.strictEqual( + response.status, + 500, + `draining node must not claim the DO: ${JSON.stringify(response.body)}` + ); + // The refusal is a DISCONNECTED exception, which JS sees as a generic + // connection-loss error. + assert.match(response.body.error, /Network connection lost/); + + const exit = await exitPromise; + assert.deepStrictEqual(exit, { code: 0, signal: null }); + + // The DO was never instantiated anywhere: the survivor claims it fresh. + const survivor = await fetchJson( + nodes[1].httpPort, + '/increment?name=lame-duck' + ); + assert.strictEqual(survivor.status, 200, JSON.stringify(survivor.body)); + assert.strictEqual(survivor.body.count, 1); + assert.strictEqual(survivor.body.nodeId, nodes[1].nodeId); + assert.strictEqual( + await readLockOwner(lockFilePath(sharedPath, survivor.body.id)), + nodeKeys.get(nodes[1].nodeId) + ); + }); +}); + test('cluster: alarms are rejected with a clear error', async () => { await withCluster(1, async ({ nodes, sharedPath }) => { // Fire the request. In cluster mode `setAlarm()` does not throw From 4df457a47337a39935a8ccd10cbdb9c7f4288b1b Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Mon, 7 Sep 2026 15:43:09 -0500 Subject: [PATCH 21/33] Mark cluster-test-unix "large". This test takes 10s locally, and hit the 15s timeout in CI. --- src/workerd/server/tests/cluster/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workerd/server/tests/cluster/BUILD.bazel b/src/workerd/server/tests/cluster/BUILD.bazel index ae7ad386f7f..250d88507f0 100644 --- a/src/workerd/server/tests/cluster/BUILD.bazel +++ b/src/workerd/server/tests/cluster/BUILD.bazel @@ -2,7 +2,7 @@ load("@aspect_rules_js//js:defs.bzl", "js_test") js_test( name = "cluster-test-unix", - size = "medium", + size = "large", data = [ ":cluster-test.wd-test", ":cluster-worker.js", From 500541dd038cb5b432f968bb77d744a4d4556f07 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Mon, 7 Sep 2026 16:33:26 -0500 Subject: [PATCH 22/33] Bugfix: Pull db.close() out of ActorSqlite::shutdown(). It turns out the production runtime sometimes calls ActorSqlite::shutdown() synchronously inside a VFS callback from sqlite (e.g. when hitting the storage size limit). close() there would abort the process. The edge runtime doesn't really need the `close()`, though -- it has its own way of doing things. So pull the close call up into server.c++, in all the places where we call shutdown(). --- src/workerd/io/actor-sqlite-test.c++ | 36 +++++++++++--------- src/workerd/io/actor-sqlite.c++ | 17 ++-------- src/workerd/server/server.c++ | 51 ++++++++++++++++++++++------ 3 files changed, 64 insertions(+), 40 deletions(-) diff --git a/src/workerd/io/actor-sqlite-test.c++ b/src/workerd/io/actor-sqlite-test.c++ index 58618469383..f2c4b7e5dbb 100644 --- a/src/workerd/io/actor-sqlite-test.c++ +++ b/src/workerd/io/actor-sqlite-test.c++ @@ -2624,7 +2624,7 @@ void expectDatabaseClosed(ActorSqliteTest& test) { fresh.run("COMMIT"); } -KJ_TEST("shutdown() closes the database") { +KJ_TEST("shutdown() leaves the connection open; the owner closes it afterwards") { ActorSqliteTest test({.monitorOutputGate = false}); test.put("committed", "yes"); @@ -2635,21 +2635,27 @@ KJ_TEST("shutdown() closes the database") { test.actor.shutdown(kj::none); + // Storage is broken, but shutdown() must not close the connection itself: embedders may call it + // from inside a SQLite callback, where closing is unsafe. The owner closes it later. KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.actor.getSqliteDatabase()); - expectDatabaseClosed(test); + test.db.run("SELECT 1"); // The scheduled commit observes the shutdown and breaks the output gate with the shutdown - // exception rather than touching the closed database. + // exception rather than committing. KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.gate.onBroken().wait(test.ws)); test.pollAndExpectCalls({}); - // A second shutdown() is a no-op. + // Closing with the implicit transaction still open rolls it back and releases the file. + test.db.close(); + expectDatabaseClosed(test); + + // A second shutdown() on a closed database is a no-op. test.actor.shutdown(kj::none); KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.actor.getSqliteDatabase()); expectDatabaseClosed(test); } -KJ_TEST("shutdown() closes the database when storage is already broken") { +KJ_TEST("shutdown() after a critical error keeps the original exception") { ActorSqliteTest test({.monitorOutputGate = false}); test.put("committed", "yes"); @@ -2658,13 +2664,7 @@ KJ_TEST("shutdown() closes the database when storage is already broken") { auto txn = test.startTransaction(); txn->put(kj::str("uncommitted"), kj::heapArray("yes"_kj.asBytes()), {}, nullptr); breakViaCriticalError(test, *txn); - - // Storage is broken but the connection is still open until shutdown() runs, as it does when the - // broken output gate aborts the IoContext. KJ_EXPECT_THROW_MESSAGE("broken.outputGateBroken", test.actor.getSqliteDatabase()); - test.db.run("SELECT 1"); - - test.actor.shutdown(kj::none); auto expectOriginalBrokenException = [&]() { KJ_IF_SOME(e, kj::runCatchingExceptions([&]() { test.actor.getSqliteDatabase(); })) { @@ -2675,12 +2675,16 @@ KJ_TEST("shutdown() closes the database when storage is already broken") { KJ_FAIL_EXPECT("getSqliteDatabase() should have thrown"); } }; + + // shutdown() runs after the break (the broken output gate aborts the IoContext), then the owner + // closes the connection. + test.actor.shutdown(kj::none); expectOriginalBrokenException(); + test.db.close(); expectDatabaseClosed(test); test.actor.shutdown(kj::none); expectOriginalBrokenException(); - expectDatabaseClosed(test); } KJ_TEST("shutdown() during in-flight commit does not unschedule the persisted alarm") { @@ -2697,12 +2701,12 @@ KJ_TEST("shutdown() during in-flight commit does not unschedule the persisted al test.setAlarm(oneMs); auto scheduleFulfiller = kj::mv(test.pollAndExpectCalls({"scheduleRun(1ms)"})[0]); - // Shut down (and thereby close the database) while that commit is parked. + // Shut down while that commit is parked. test.actor.shutdown(kj::none); - // Resume the commit. It must not consult the closed database's alarm state and must not issue - // any further scheduler requests. In particular it must not issue `scheduleRun(none)`, which - // would cancel the alarm that the database still persists. + // Resume the commit. Now that storage is broken, it must not consult the database's alarm state + // and must not issue any further scheduler requests. In particular it must not issue + // `scheduleRun(none)`, which would cancel the alarm that the database still persists. scheduleFulfiller->fulfill(); test.pollAndExpectCalls({}); KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.gate.onBroken().wait(test.ws)); diff --git a/src/workerd/io/actor-sqlite.c++ b/src/workerd/io/actor-sqlite.c++ index 6fdf1610a0a..e0de5c78d38 100644 --- a/src/workerd/io/actor-sqlite.c++ +++ b/src/workerd/io/actor-sqlite.c++ @@ -552,9 +552,9 @@ kj::Promise ActorSqlite::commitImpl( haveAlarmForDebug = true; co_await p; - // We may have become broken (e.g. shutdown() closed the database) while suspended. Once - // broken, we no longer own the database, and there could be another instance of this actor - // running elsewhere, therefore we should not continue with alarm updates. + // We may have become broken (e.g. shutdown() was called) while suspended. Once broken, we no + // longer own the database, and there could be another instance of this actor running + // elsewhere, therefore we should not continue with alarm updates. requireNotBroken(); } @@ -1019,17 +1019,6 @@ void ActorSqlite::shutdown(kj::Maybe maybeException) { // We've already experienced a terminal exception either from shutdown or OOM, there should // already be a flush scheduled that will break the output gate. } - - // Sever the sqlite connection, so that the underlying database file is fully and synchcronously - // closed. Callers of shutdown() rely on this. Do it even if `broken` was already set because not - // all paths that set `broken` call `close()`. Note `close()` is idempotent. - // - // TODO(cleanup): Maybe the entire `broken` mechanism should be pushed down into SqliteDatabase? - // Currently `ActorSqlite` carefully checks `broken` in every path that touches the database, - // but `SqliteDatabase` itself checks if it is closed on every path. We could change `close()` - // to something like `abort(reason)` and then rely on that instead of having `broken` in - // `ActorSqlite`. Needs careful investigation of all code paths, though. - db->close(); } kj::OneOf ActorSqlite:: diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 00ac37fa93b..07a2dd0dc3b 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -572,6 +572,7 @@ class Server::ActorNamespace final { auto reason = 0; a->shutdown(reason); } + closeStorage(); // Drop the container client reference // If setInactivityTimeout() was called, there's still a timer holding a reference @@ -734,12 +735,13 @@ class Server::ActorNamespace final { shutdownTask = kj::none; manager = kj::none; tracker->shutdown(); + closeStorage(); actor = kj::none; containerClient = kj::none; - // The actor (and every facet) shut down above, which closed their databases, so nothing on - // this node holds a handle into the actor's storage anymore. Drop ours and let another node - // (or a fresh container on this one) claim the actor. + // The actor (and every facet) shut down above and closeStorage() closed the database, so + // nothing on this node holds a handle into the actor's storage anymore. Drop ours and let + // another node (or a fresh container on this one) claim the actor. facetTreeIndex = kj::none; ownershipLock = kj::none; @@ -764,6 +766,17 @@ class Server::ActorNamespace final { } } + // Closes the actor's SQLite connection so the database file is fully and synchronously + // released (WAL checkpointed and removed; in NFS mode the EXCLUSIVE lock dropped). Must run + // after the actor has been shut down (so ActorSqlite is `broken` and will not touch the + // connection again) and before `actor` is dropped (see `sqliteDb`). Idempotent. + void closeStorage() { + KJ_IF_SOME(db, sqliteDb) { + db.close(); + } + sqliteDb = kj::none; + } + kj::Own getFacetContainer( kj::String childKey, kj::Function()> getStartInfo) { auto makeContainer = [&]() { @@ -885,10 +898,10 @@ class Server::ActorNamespace final { // request for the actor loop through self-bootstrap (see ActorNamespace::resolveNode()). The // lock is therefore released at the moment the container leaves the map (abort(), // monitorOnBroken()), not when the last reference to the container is dropped. Releasing it - // there is only safe because ActorSqlite::shutdown() closes the database: by the time either - // path hollows the container, every SQLite handle under the actor and its facets is already - // closed, so no handle into the actor's storage outlives the claim even if an in-flight - // request still holds the Worker::Actor. + // there is only safe because closeStorage() runs first: by the time either path hollows the + // container, every SQLite handle under the actor and its facets is already closed, so no handle + // into the actor's storage outlives the claim even if an in-flight request still holds the + // Worker::Actor. // // The lock file's contents alone carry no such guarantee: a failed claim or release can leave // this node's key behind with no OwnedLock held. acquireOrRoute() detects that (the OFD lock @@ -898,6 +911,15 @@ class Server::ActorNamespace final { // The actor is constructed after the ActorContainer so it starts off empty. kj::Maybe> actor; + // The actor's SQLite connection, owned by the ActorSqlite inside `actor`. Set by start() (the + // actor constructs its cache synchronously) and cleared by closeStorage(), which every path + // that drops `actor` calls first, so this never outlives the connection it refers to. + // + // The connection is closed here rather than in ActorSqlite::shutdown() because shutdown() can + // run synchronously inside a SQLite callback (e.g. a VFS write that hits a storage limit), + // where closing the handle is unsafe. The container only drops the actor from the event loop. + kj::Maybe sqliteDb; + kj::String key; kj::Own tracker; ActorNamespace& ns; @@ -1094,9 +1116,11 @@ class Server::ActorNamespace final { auto managerToDrop = kj::mv(manager); // onBroken fired only after IoContext::abort() shut down the actor cache, and the facets - // were aborted above, so every database under this actor is closed. Release the claim now, - // in the same synchronous continuation as the erase() below (see `ownershipLock`). Any stub - // still pinning this container after the erase keeps only a hollow shell. + // were aborted above, so closing the database now is safe and leaves every database under + // this actor closed. Release the claim in the same synchronous continuation as the erase() + // below (see `ownershipLock`). Any stub still pinning this container after the erase keeps + // only a hollow shell. + closeStorage(); facetTreeIndex = kj::none; ownershipLock = kj::none; @@ -1151,6 +1175,7 @@ class Server::ActorNamespace final { a->shutdown(0, KJ_EXCEPTION(DISCONNECTED, "broken.dropped; Actor freed due to inactivity")); } // Destroy the last strong Worker::Actor reference. + closeStorage(); actor = kj::none; // Drop our reference to the ContainerClient @@ -1219,6 +1244,7 @@ class Server::ActorNamespace final { onBrokenTask = kj::none; // Destroy the last strong Worker::Actor reference. + closeStorage(); actor = kj::none; if (webSocketMode == IoChannelFactory::EvictWebSocketMode::CLOSE) { @@ -1371,6 +1397,8 @@ class Server::ActorNamespace final { deleteDescendantStorage(dir, selfId); }); + KJ_ASSERT(sqliteDb == kj::none); + sqliteDb = *db; return kj::heap(kj::mv(db), outputGate, [](SpanParent) -> kj::Promise { return kj::READY_NOW; }, *sqliteHooks) .attach(kj::mv(sqliteHooks)); @@ -1458,6 +1486,9 @@ class Server::ActorNamespace final { kj::mv(privileges)); } + // If actor construction throws after makeActorCache ran, the database it recorded is + // destroyed along with the half-built actor; don't leave `sqliteDb` pointing at it. + KJ_ON_SCOPE_FAILURE(sqliteDb = kj::none); auto actor = actorClass->newActor(getTracker(), Worker::Actor::cloneId(id), kj::mv(makeActorCache), kj::mv(makeStorage), kj::mv(loopback), tryGetManagerRef(), kj::mv(container), kj::mv(containerImages), *this); From ef97acb3e0a766c57c63cddfd8afa29587e4b770 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Mon, 7 Sep 2026 17:49:36 -0500 Subject: [PATCH 23/33] Fix Justin's review nits. --- src/workerd/server/cluster-lock-test.c++ | 58 ++++++++++++++---------- src/workerd/server/cluster-lock.c++ | 15 +++--- src/workerd/server/cluster-registry.c++ | 4 +- src/workerd/server/cluster-registry.h | 2 +- src/workerd/util/ofd-lock-test.c++ | 4 +- src/workerd/util/sqlite.c++ | 6 +-- 6 files changed, 49 insertions(+), 40 deletions(-) diff --git a/src/workerd/server/cluster-lock-test.c++ b/src/workerd/server/cluster-lock-test.c++ index 5db08ab4116..858c9f3cb77 100644 --- a/src/workerd/server/cluster-lock-test.c++ +++ b/src/workerd/server/cluster-lock-test.c++ @@ -72,6 +72,14 @@ class TempDir { // it from these tests; we just need a capability to hand to RpcSystem. class StubClusterPort final: public rpc::WorkerdClusterPort::Server {}; +// Make a valid actor ID (64 digits) given some shorter base. +kj::StringPtr testId(kj::StringPtr base) { + static kj::HashMap cache; + return cache.findOrCreate(base, [&]() -> decltype(cache)::Entry { + return {base, kj::str(base, kj::repeat('0', 64 - base.size()))}; + }); +} + KJ_TEST("ClusterLockManager: unowned (empty file) -> acquire succeeds") { auto io = kj::setupAsyncIo(); TempDir tmpDir; @@ -83,11 +91,11 @@ KJ_TEST("ClusterLockManager: unowned (empty file) -> acquire succeeds") { ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); - auto result = lockManager.acquireOrRoute("a1").wait(io.waitScope); + auto result = lockManager.acquireOrRoute(testId("a1")).wait(io.waitScope); KJ_ASSERT(result.is()); // The lock file should now contain our key. - auto content = tmpDir.readLockFile("a1"); + auto content = tmpDir.readLockFile(testId("a1")); KJ_ASSERT(content.size() == 32); KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); } @@ -104,13 +112,13 @@ KJ_TEST("ClusterLockManager: owned by self -> returns self-bootstrap, no wire tr ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); // First acquire to populate the lock file with our own key. - auto ownedResult = lockManager.acquireOrRoute("a2").wait(io.waitScope); + auto ownedResult = lockManager.acquireOrRoute(testId("a2")).wait(io.waitScope); KJ_ASSERT(ownedResult.is()); // Now simulate someone else also calling acquireOrRoute on the same actor (i.e. a forwarding // path). With the OwnedLock still held, the lock file is non-empty and our key is the owner. // acquireOrRoute should return a bootstrap client (routing to ourselves). - auto routeResult = lockManager.acquireOrRoute("a2").wait(io.waitScope); + auto routeResult = lockManager.acquireOrRoute(testId("a2")).wait(io.waitScope); KJ_ASSERT(routeResult.is()); } @@ -128,7 +136,7 @@ KJ_TEST("ClusterLockManager: stale owner (peer dead) -> claim path runs") { X25519PublicKey deadKey; memset(deadKey.bytes, 0xAB, sizeof(deadKey.bytes)); - tmpDir.writeLockFile("a3", kj::arrayPtr(deadKey.bytes, 32)); + tmpDir.writeLockFile(testId("a3"), kj::arrayPtr(deadKey.bytes, 32)); // Confirm isPeerDead returns true for the dead key. KJ_EXPECT(reg.isPeerDead(deadKey)); @@ -136,11 +144,11 @@ KJ_TEST("ClusterLockManager: stale owner (peer dead) -> claim path runs") { ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); // acquireOrRoute should run the claim path and succeed. - auto result = lockManager.acquireOrRoute("a3").wait(io.waitScope); + auto result = lockManager.acquireOrRoute(testId("a3")).wait(io.waitScope); KJ_ASSERT(result.is()); // The lock file should now contain *our* key. - auto content = tmpDir.readLockFile("a3"); + auto content = tmpDir.readLockFile(testId("a3")); KJ_ASSERT(content.size() == 32); KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); } @@ -158,15 +166,15 @@ KJ_TEST("ClusterLockManager: wrong-sized stale content + writer dead -> claim im // Nobody holds the lock, so the claim path should succeed. kj::byte garbage[10]; memset(garbage, 0xFF, sizeof(garbage)); - tmpDir.writeLockFile("a4", kj::arrayPtr(garbage, sizeof(garbage))); + tmpDir.writeLockFile(testId("a4"), kj::arrayPtr(garbage, sizeof(garbage))); ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); - auto result = lockManager.acquireOrRoute("a4").wait(io.waitScope); + auto result = lockManager.acquireOrRoute(testId("a4")).wait(io.waitScope); KJ_ASSERT(result.is()); // The lock file should now contain our key. - auto content = tmpDir.readLockFile("a4"); + auto content = tmpDir.readLockFile(testId("a4")); KJ_ASSERT(content.size() == 32); KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); } @@ -183,20 +191,20 @@ KJ_TEST("ClusterLockManager: OwnedLock destructor truncates and releases") { ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); { - auto result = lockManager.acquireOrRoute("a5").wait(io.waitScope); + auto result = lockManager.acquireOrRoute(testId("a5")).wait(io.waitScope); KJ_ASSERT(result.is()); // Lock file should have our key. - auto content = tmpDir.readLockFile("a5"); + auto content = tmpDir.readLockFile(testId("a5")); KJ_ASSERT(content.size() == 32); } // After OwnedLock destructor, the file should be truncated to 0 bytes (and the OFD lock // released). - auto content = tmpDir.readLockFile("a5"); + auto content = tmpDir.readLockFile(testId("a5")); KJ_EXPECT(content.size() == 0); // A second acquire should now succeed via the claim path. - auto result2 = lockManager.acquireOrRoute("a5").wait(io.waitScope); + auto result2 = lockManager.acquireOrRoute(testId("a5")).wait(io.waitScope); KJ_ASSERT(result2.is()); } @@ -214,17 +222,17 @@ KJ_TEST("ClusterLockManager: stale file naming self with no lock held -> reclaim // Simulate a claim or release that published our key but failed before clearing it: the file // names this node, but no OwnedLock (and hence no OFD lock) exists. Routing to ourselves here // would loop forever; we must reclaim instead. - tmpDir.writeLockFile("a6", kj::arrayPtr(reg.getPublicKey().bytes, 32)); + tmpDir.writeLockFile(testId("a6"), kj::arrayPtr(reg.getPublicKey().bytes, 32)); - auto result = lockManager.acquireOrRoute("a6").wait(io.waitScope); + auto result = lockManager.acquireOrRoute(testId("a6")).wait(io.waitScope); KJ_ASSERT(result.is()); - auto content = tmpDir.readLockFile("a6"); + auto content = tmpDir.readLockFile(testId("a6")); KJ_ASSERT(content.size() == 32); KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); // With the OwnedLock now held, a further lookup routes to ourselves rather than reclaiming. - auto routeResult = lockManager.acquireOrRoute("a6").wait(io.waitScope); + auto routeResult = lockManager.acquireOrRoute(testId("a6")).wait(io.waitScope); KJ_ASSERT(routeResult.is()); } @@ -246,17 +254,17 @@ KJ_TEST("ClusterLockManager: owned by live peer -> returns bootstrap client") { // and acquireOrRoute should return a bootstrap client without falling through to claim. // // Note: actor IDs aren't hex of 64 chars, so they won't collide with registry filenames. - tmpDir.writeLockFile("a7", kj::arrayPtr(reg2.getPublicKey().bytes, 32)); + tmpDir.writeLockFile(testId("a7"), kj::arrayPtr(reg2.getPublicKey().bytes, 32)); KJ_EXPECT(!reg1.isPeerDead(reg2.getPublicKey())); ClusterLockManager lockManager(tmpDir.locks(), reg1, rpc1, io.provider->getTimer()); - auto result = lockManager.acquireOrRoute("a7").wait(io.waitScope); + auto result = lockManager.acquireOrRoute(testId("a7")).wait(io.waitScope); KJ_ASSERT(result.is()); // The lock file should NOT have been modified (we did not claim). - auto content = tmpDir.readLockFile("a7"); + auto content = tmpDir.readLockFile(testId("a7")); KJ_ASSERT(content.size() == 32); KJ_EXPECT(memcmp(content.begin(), reg2.getPublicKey().bytes, 32) == 0); } @@ -273,8 +281,8 @@ KJ_TEST("ClusterLockManager: writer alive -> retries until writer releases") { // Open the lock file from outside the ClusterLockManager and take an exclusive lock. This // simulates a writer that holds the lock but hasn't yet flushed its key. The file is left // empty (matching the freshly-truncated state during the writer's claim path). - auto externalFile = - tmpDir.locks()->openFile(kj::Path({"a8"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + auto externalFile = tmpDir.locks()->openFile( + kj::Path({testId("a8")}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); int externalFd = KJ_REQUIRE_NONNULL(externalFile->getFd()); auto externalLock = KJ_ASSERT_NONNULL(OfdLock::tryLock(externalFd, OfdLock::EXCLUSIVE)); @@ -282,7 +290,7 @@ KJ_TEST("ClusterLockManager: writer alive -> retries until writer releases") { ClusterLockManager lockManager(tmpDir.locks(), reg, rpc, io.provider->getTimer()); // acquireOrRoute should see an empty file, attempt to lock, fail, then retry with backoff. - auto promise = lockManager.acquireOrRoute("a8"); + auto promise = lockManager.acquireOrRoute(testId("a8")); bool completed = false; auto trackedPromise = promise.then( @@ -323,7 +331,7 @@ KJ_TEST("ClusterLockManager: concurrent acquisitions, exactly one wins") { kj::Vector>> promises; for (uint i = 0; i < N; ++i) { - promises.add(lockManager.acquireOrRoute("a9")); + promises.add(lockManager.acquireOrRoute(testId("a9"))); } auto results = kj::joinPromises(promises.releaseAsArray()).wait(io.waitScope); diff --git a/src/workerd/server/cluster-lock.c++ b/src/workerd/server/cluster-lock.c++ index d5a9ad1335f..b6a322d14bb 100644 --- a/src/workerd/server/cluster-lock.c++ +++ b/src/workerd/server/cluster-lock.c++ @@ -18,9 +18,6 @@ namespace { // protocol is truncate(0) → write(full key) → sync, so a reader interleaving sees either size // 0 (treat as unowned, run claim path) or the full key. -constexpr kj::Duration INITIAL_BACKOFF = 10 * kj::MILLISECONDS; -constexpr kj::Duration MAX_BACKOFF = 1 * kj::SECONDS; - // Build a `cluster::VatId` Reader for a given public key, backed by an internal `MessageBuilder`. class VatIdHolder { public: @@ -38,6 +35,9 @@ class VatIdHolder { capnp::MallocMessageBuilder msg; }; +constexpr kj::Duration INITIAL_BACKOFF = 10 * kj::MILLISECONDS; +constexpr kj::Duration MAX_BACKOFF = 1 * kj::SECONDS; + // Compute the next backoff delay with ±25% jitter. `attempt` starts at 0. kj::Duration computeBackoff(uint attempt) { // Exponential: 10ms, 20ms, 40ms, ..., capped at 1s. @@ -51,7 +51,8 @@ kj::Duration computeBackoff(uint attempt) { auto nanos = (kj::systemPreciseMonotonicClock().now() - kj::origin()) / kj::NANOSECONDS; uint32_t r = kj::hashCode(nanos); - double jitter = 0.75 + (r / static_cast(0xFFFFFFFFu)) * 0.5; + constexpr decltype(r) MAX_R = kj::maxValue; + double jitter = 0.75 + (r / static_cast(MAX_R)) * 0.5; return base * static_cast(jitter * 1000) / 1000; } @@ -92,13 +93,13 @@ ClusterLockManager::ClusterLockManager(kj::Own dir, kj::Promise> ClusterLockManager::acquireOrRoute(kj::StringPtr actorId) { // Validate that the actor ID is a hex string. This is the only form of actor ID we expect on - // this path (durable actors named by a 64-char hex SHA-256 hash, or by `idFromName` which also + // this path (Durable Objects named by a 64-char hex SHA-256 hash, or by `idFromName` which also // produces hex). Enforcing hex incidentally guarantees the ID is safe to use as a filename. + KJ_REQUIRE(actorId.size() == 64, "invalid actor ID", actorId); for (char c: actorId) { KJ_REQUIRE((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'), - "actor ID is not a hex string", actorId); + "invalid actor ID", actorId); } - KJ_REQUIRE(actorId.size() > 0, "actor ID is empty"); kj::Path path({actorId}); diff --git a/src/workerd/server/cluster-registry.c++ b/src/workerd/server/cluster-registry.c++ index bf4cf767815..c5d31499406 100644 --- a/src/workerd/server/cluster-registry.c++ +++ b/src/workerd/server/cluster-registry.c++ @@ -149,9 +149,9 @@ class ClusterRegistry::ConnectionImpl final: public ClusterVatNetworkBase::Conne } cluster::VatId::Reader getPeerVatId() override { - // We can safely expected the RPC system won't call this method until there is actually some + // We can safely expect the RPC system won't call this method until there is actually some // activity on the stream that causes it to be needed. If this is an inbound stream, there - // is necessary no activity until we receive some messages, and we must have received the + // is necessarily no activity until we receive some messages, and we must have received the // peer key before that. // // This is convenient because otherwise we'd have to figure out what to do with this call diff --git a/src/workerd/server/cluster-registry.h b/src/workerd/server/cluster-registry.h index 55a249950d9..031d7b4c047 100644 --- a/src/workerd/server/cluster-registry.h +++ b/src/workerd/server/cluster-registry.h @@ -31,7 +31,7 @@ struct X25519PublicKey { static X25519PublicKey fromHex(kj::StringPtr hex); bool operator==(const X25519PublicKey& other) const { - return memcmp(bytes, other.bytes, sizeof(bytes)) == 0; + return kj::arrayPtr(bytes) == kj::arrayPtr(other.bytes); } kj::uint hashCode() const { return kj::hashCode(kj::ArrayPtr(bytes)); diff --git a/src/workerd/util/ofd-lock-test.c++ b/src/workerd/util/ofd-lock-test.c++ index aa827c444b2..1f120a7574e 100644 --- a/src/workerd/util/ofd-lock-test.c++ +++ b/src/workerd/util/ofd-lock-test.c++ @@ -69,7 +69,7 @@ KJ_TEST("shared locks do not conflict with each other") { // Both shared locks held simultaneously — no conflict. } -KJ_TEST("exclusive lock blocks while shared lock is held") { +KJ_TEST("exclusive lock fails while shared lock is held") { auto fd1 = openTempFile(); auto procPath = kj::str("/proc/self/fd/", fd1.get()); @@ -125,8 +125,8 @@ KJ_TEST("move constructor releases on destruction of target") { KJ_SYSCALL(rawFd2 = open(procPath.cStr(), O_RDWR)); auto fd2 = kj::OwnFd(rawFd2); + auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); { - auto lock1 = KJ_ASSERT_NONNULL(OfdLock::tryLock(fd1, OfdLock::EXCLUSIVE)); auto lock2 = kj::mv(lock1); // lock2 goes out of scope, releasing the lock. } diff --git a/src/workerd/util/sqlite.c++ b/src/workerd/util/sqlite.c++ index ea101db72ce..940673d8175 100644 --- a/src/workerd/util/sqlite.c++ +++ b/src/workerd/util/sqlite.c++ @@ -2426,7 +2426,7 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { KJ_IF_SOME(afterOpen, self.options.afterOpen) { afterOpen(); } else { - // sqelch spurious "dangling else" warning from clang + // squelch spurious "dangling else" warning from clang } kj::Maybe> lock; if (flags & SQLITE_OPEN_MAIN_DB) { @@ -2446,7 +2446,7 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { KJ_IF_SOME(afterOpen, self.options.afterOpen) { afterOpen(); } else { - // sqelch spurious "dangling else" warning from clang + // squelch spurious "dangling else" warning from clang } } else { kj::WriteMode mode; @@ -2466,7 +2466,7 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { KJ_IF_SOME(afterOpen, self.options.afterOpen) { afterOpen(); } else { - // sqelch spurious "dangling else" warning from clang + // squelch spurious "dangling else" warning from clang } if (flags & SQLITE_OPEN_MAIN_DB) { lock = self.lockManager.lock(path, *kjFile); From 01cf2ce5b96a655e00b010cdde7ff855b3e678e5 Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Mon, 7 Sep 2026 18:12:28 -0500 Subject: [PATCH 24/33] Address Justin's nits, part 2: FixedArray intsead of byte[32]. This actually turned out pretty nice. --- src/workerd/server/cluster-lock-test.c++ | 25 ++++++++------------ src/workerd/server/cluster-lock.c++ | 2 +- src/workerd/server/cluster-registry-test.c++ | 24 +++++++++---------- src/workerd/server/cluster-registry.c++ | 12 +++++----- src/workerd/server/cluster-registry.h | 8 +++---- 5 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/workerd/server/cluster-lock-test.c++ b/src/workerd/server/cluster-lock-test.c++ index 858c9f3cb77..d5356d59008 100644 --- a/src/workerd/server/cluster-lock-test.c++ +++ b/src/workerd/server/cluster-lock-test.c++ @@ -96,8 +96,7 @@ KJ_TEST("ClusterLockManager: unowned (empty file) -> acquire succeeds") { // The lock file should now contain our key. auto content = tmpDir.readLockFile(testId("a1")); - KJ_ASSERT(content.size() == 32); - KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); + KJ_EXPECT(content == reg.getPublicKey().bytes.asPtr()); } KJ_TEST("ClusterLockManager: owned by self -> returns self-bootstrap, no wire traffic") { @@ -134,9 +133,9 @@ KJ_TEST("ClusterLockManager: stale owner (peer dead) -> claim path runs") { // Manually create a lock file naming a non-existent peer as owner. Since `deadKey`'s registry // file doesn't exist in tmpDir, isPeerDead(deadKey) returns true and the claim path should run. X25519PublicKey deadKey; - memset(deadKey.bytes, 0xAB, sizeof(deadKey.bytes)); + deadKey.bytes.fill(0xAB); - tmpDir.writeLockFile(testId("a3"), kj::arrayPtr(deadKey.bytes, 32)); + tmpDir.writeLockFile(testId("a3"), deadKey.bytes); // Confirm isPeerDead returns true for the dead key. KJ_EXPECT(reg.isPeerDead(deadKey)); @@ -149,8 +148,7 @@ KJ_TEST("ClusterLockManager: stale owner (peer dead) -> claim path runs") { // The lock file should now contain *our* key. auto content = tmpDir.readLockFile(testId("a3")); - KJ_ASSERT(content.size() == 32); - KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); + KJ_EXPECT(content == reg.getPublicKey().bytes.asPtr()); } KJ_TEST("ClusterLockManager: wrong-sized stale content + writer dead -> claim immediately") { @@ -175,8 +173,7 @@ KJ_TEST("ClusterLockManager: wrong-sized stale content + writer dead -> claim im // The lock file should now contain our key. auto content = tmpDir.readLockFile(testId("a4")); - KJ_ASSERT(content.size() == 32); - KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); + KJ_EXPECT(content == reg.getPublicKey().bytes.asPtr()); } KJ_TEST("ClusterLockManager: OwnedLock destructor truncates and releases") { @@ -222,14 +219,13 @@ KJ_TEST("ClusterLockManager: stale file naming self with no lock held -> reclaim // Simulate a claim or release that published our key but failed before clearing it: the file // names this node, but no OwnedLock (and hence no OFD lock) exists. Routing to ourselves here // would loop forever; we must reclaim instead. - tmpDir.writeLockFile(testId("a6"), kj::arrayPtr(reg.getPublicKey().bytes, 32)); + tmpDir.writeLockFile(testId("a6"), reg.getPublicKey().bytes); auto result = lockManager.acquireOrRoute(testId("a6")).wait(io.waitScope); KJ_ASSERT(result.is()); auto content = tmpDir.readLockFile(testId("a6")); - KJ_ASSERT(content.size() == 32); - KJ_EXPECT(memcmp(content.begin(), reg.getPublicKey().bytes, 32) == 0); + KJ_EXPECT(content == reg.getPublicKey().bytes.asPtr()); // With the OwnedLock now held, a further lookup routes to ourselves rather than reclaiming. auto routeResult = lockManager.acquireOrRoute(testId("a6")).wait(io.waitScope); @@ -254,7 +250,7 @@ KJ_TEST("ClusterLockManager: owned by live peer -> returns bootstrap client") { // and acquireOrRoute should return a bootstrap client without falling through to claim. // // Note: actor IDs aren't hex of 64 chars, so they won't collide with registry filenames. - tmpDir.writeLockFile(testId("a7"), kj::arrayPtr(reg2.getPublicKey().bytes, 32)); + tmpDir.writeLockFile(testId("a7"), reg2.getPublicKey().bytes); KJ_EXPECT(!reg1.isPeerDead(reg2.getPublicKey())); @@ -265,8 +261,7 @@ KJ_TEST("ClusterLockManager: owned by live peer -> returns bootstrap client") { // The lock file should NOT have been modified (we did not claim). auto content = tmpDir.readLockFile(testId("a7")); - KJ_ASSERT(content.size() == 32); - KJ_EXPECT(memcmp(content.begin(), reg2.getPublicKey().bytes, 32) == 0); + KJ_EXPECT(content == reg2.getPublicKey().bytes.asPtr()); } KJ_TEST("ClusterLockManager: writer alive -> retries until writer releases") { @@ -308,7 +303,7 @@ KJ_TEST("ClusterLockManager: writer alive -> retries until writer releases") { // key, find the OFD lock held, and return a bootstrap client routing to the owner. (Releasing // the lock instead would leave a self-naming file with no holder, which acquireOrRoute reclaims; // see the "stale file naming self" test.) - externalFile->write(0, kj::arrayPtr(reg.getPublicKey().bytes, 32)); + externalFile->write(0, reg.getPublicKey().bytes); auto result = trackedPromise.wait(io.waitScope); KJ_EXPECT(completed); diff --git a/src/workerd/server/cluster-lock.c++ b/src/workerd/server/cluster-lock.c++ index b6a322d14bb..4c0648cffc6 100644 --- a/src/workerd/server/cluster-lock.c++ +++ b/src/workerd/server/cluster-lock.c++ @@ -23,7 +23,7 @@ class VatIdHolder { public: explicit VatIdHolder(const X25519PublicKey& key): msg(scratch) { auto vatId = msg.initRoot(); - vatId.setPublicKey(kj::ArrayPtr(key.bytes)); + vatId.setPublicKey(key.bytes.asPtr()); } cluster::VatId::Reader getReader() { diff --git a/src/workerd/server/cluster-registry-test.c++ b/src/workerd/server/cluster-registry-test.c++ index 0b7e97be563..f4adb7dcb90 100644 --- a/src/workerd/server/cluster-registry-test.c++ +++ b/src/workerd/server/cluster-registry-test.c++ @@ -120,7 +120,7 @@ class BlackholeNetwork final: public kj::Network { KJ_TEST("X25519PublicKey: hex roundtrip") { X25519PublicKey key; - memset(key.bytes, 0xAB, sizeof(key.bytes)); + key.bytes.fill(0xAB); auto hex = key.toHex(); KJ_EXPECT(hex.size() == 64); auto decoded = X25519PublicKey::fromHex(hex); @@ -129,8 +129,8 @@ KJ_TEST("X25519PublicKey: hex roundtrip") { KJ_TEST("X25519PublicKey: equality") { X25519PublicKey a, b; - memset(a.bytes, 1, sizeof(a.bytes)); - memset(b.bytes, 1, sizeof(b.bytes)); + a.bytes.fill(1); + b.bytes.fill(1); KJ_EXPECT(a == b); b.bytes[0] = 2; @@ -154,7 +154,7 @@ KJ_TEST("ClusterRegistry: self-connect returns none") { capnp::MallocMessageBuilder msg; auto vatId = msg.initRoot(); - vatId.setPublicKey(kj::arrayPtr(reg.getPublicKey().bytes, 32)); + vatId.setPublicKey(reg.getPublicKey().bytes.asPtr()); KJ_EXPECT(reg.connect(vatId.asReader()) == kj::none); } @@ -172,7 +172,7 @@ KJ_TEST("ClusterRegistry: self-bootstrap returns local capability") { capnp::MallocMessageBuilder msg; auto vatId = msg.initRoot(); - vatId.setPublicKey(kj::arrayPtr(reg.getPublicKey().bytes, 32)); + vatId.setPublicKey(reg.getPublicKey().bytes.asPtr()); auto client = rpc.bootstrap(vatId); makeCall(client, io.waitScope); @@ -212,7 +212,7 @@ KJ_TEST("ClusterRegistry: isPeerDead caches dead peers permanently") { ClusterRegistry reg(tmpDir.get(), "unix"_kj, io.provider->getNetwork(), io.provider->getTimer()); X25519PublicKey peerKey; - memset(peerKey.bytes, 0xCA, sizeof(peerKey.bytes)); + peerKey.bytes.fill(0xCA); auto peerPath = kj::Path({peerKey.toHex()}); KJ_EXPECT(reg.isPeerDead(peerKey)); @@ -245,7 +245,7 @@ KJ_TEST("ClusterRegistry: end-to-end RPC between two registries") { { capnp::MallocMessageBuilder msg; auto vatId = msg.initRoot(); - vatId.setPublicKey(kj::arrayPtr(reg2.getPublicKey().bytes, 32)); + vatId.setPublicKey(reg2.getPublicKey().bytes.asPtr()); auto client = rpc1.bootstrap(vatId); makeCall(client, io.waitScope); @@ -256,7 +256,7 @@ KJ_TEST("ClusterRegistry: end-to-end RPC between two registries") { { capnp::MallocMessageBuilder msg; auto vatId = msg.initRoot(); - vatId.setPublicKey(kj::arrayPtr(reg1.getPublicKey().bytes, 32)); + vatId.setPublicKey(reg1.getPublicKey().bytes.asPtr()); auto client = rpc2.bootstrap(vatId); makeCall(client, io.waitScope); @@ -287,7 +287,7 @@ KJ_TEST("ClusterRegistry: end-to-end RPC with binary IP registry entries") { capnp::MallocMessageBuilder msg; auto vatId = msg.initRoot(); - vatId.setPublicKey(kj::arrayPtr(reg2.getPublicKey().bytes, 32)); + vatId.setPublicKey(reg2.getPublicKey().bytes.asPtr()); auto client = rpc1.bootstrap(vatId); makeCall(client, io.waitScope); @@ -316,7 +316,7 @@ KJ_TEST("ClusterRegistry: idle timeout closes connections") { capnp::MallocMessageBuilder msg; auto vatId = msg.initRoot(); - vatId.setPublicKey(kj::arrayPtr(reg2.getPublicKey().bytes, 32)); + vatId.setPublicKey(reg2.getPublicKey().bytes.asPtr()); { auto client = rpc1.bootstrap(vatId); @@ -348,7 +348,7 @@ KJ_TEST("ClusterRegistry: hung connect is torn down and dead peer cleaned up wit // Forge a registry entry for a peer that is gone: a valid address (copied from our own entry), // not locked by anyone, and old enough to be past the new-entry grace period. X25519PublicKey peerKey; - memset(peerKey.bytes, 0xBE, sizeof(peerKey.bytes)); + peerKey.bytes.fill(0xBE); auto peerPath = kj::Path({peerKey.toHex()}); { auto dir = tmpDir.get(); @@ -364,7 +364,7 @@ KJ_TEST("ClusterRegistry: hung connect is torn down and dead peer cleaned up wit capnp::MallocMessageBuilder msg; auto vatId = msg.initRoot(); - vatId.setPublicKey(kj::arrayPtr(peerKey.bytes, 32)); + vatId.setPublicKey(peerKey.bytes.asPtr()); auto& timer = io.provider->getTimer(); auto start = timer.now(); diff --git a/src/workerd/server/cluster-registry.c++ b/src/workerd/server/cluster-registry.c++ index c5d31499406..60be9c496ef 100644 --- a/src/workerd/server/cluster-registry.c++ +++ b/src/workerd/server/cluster-registry.c++ @@ -41,7 +41,7 @@ constexpr kj::Duration PEER_LIVENESS_CACHE_TTL = 15 * kj::SECONDS; // X25519PublicKey kj::String X25519PublicKey::toHex() const { - return kj::encodeHex(kj::arrayPtr(bytes, sizeof(bytes))); + return kj::encodeHex(bytes); } X25519PublicKey X25519PublicKey::fromHex(kj::StringPtr hex) { @@ -50,7 +50,7 @@ X25519PublicKey X25519PublicKey::fromHex(kj::StringPtr hex) { KJ_REQUIRE(!decoded.hadErrors, "invalid hex in X25519 public key"); KJ_ASSERT(decoded.size() == 32); X25519PublicKey result; - memcpy(result.bytes, decoded.begin(), 32); + result.bytes.asPtr().copyFrom(decoded); return result; } @@ -164,7 +164,7 @@ class ClusterRegistry::ConnectionImpl final: public ClusterVatNetworkBase::Conne return p.getRoot().asReader(); } else { auto builder = peerVatId.emplace(peerVatIdScratch).getRoot(); - builder.setPublicKey(kj::arrayPtr(pk.bytes, sizeof(pk.bytes))); + builder.setPublicKey(pk.bytes.asPtr()); return builder.asReader(); } } @@ -453,7 +453,7 @@ ClusterRegistry::ClusterRegistry(kj::Own registryDirParam, idleTimeout(idleTimeout), connectTimeout(connectTimeout) { // Generate X25519 keypair. - X25519_keypair(publicKey.bytes, privateKey); + X25519_keypair(publicKey.bytes.begin(), privateKey.begin()); publicKeyHex = publicKey.toHex(); if (networkConfig == "unix") { @@ -577,7 +577,7 @@ void ClusterRegistry::scanDirectory() { if (decoded.hadErrors || decoded.size() != 32) continue; X25519PublicKey key; - memcpy(key.bytes, decoded.begin(), 32); + key.bytes.asPtr().copyFrom(decoded); peers.findOrCreate(key, [&]() -> decltype(peers)::Entry { return {.key = key}; }); } @@ -710,7 +710,7 @@ kj::Maybe> ClusterRegistry::connect( KJ_REQUIRE(keyData.size() == 32, "invalid VatId: publicKey must be 32 bytes"); X25519PublicKey peerKey; - memcpy(peerKey.bytes, keyData.begin(), 32); + peerKey.bytes.asPtr().copyFrom(keyData); // Self-connect returns kj::none per VatNetwork contract. if (peerKey == publicKey) { diff --git a/src/workerd/server/cluster-registry.h b/src/workerd/server/cluster-registry.h index 031d7b4c047..11494af78d6 100644 --- a/src/workerd/server/cluster-registry.h +++ b/src/workerd/server/cluster-registry.h @@ -25,16 +25,16 @@ using kj::byte; using kj::uint; struct X25519PublicKey { - kj::byte bytes[32]; + kj::FixedArray bytes; kj::String toHex() const; static X25519PublicKey fromHex(kj::StringPtr hex); bool operator==(const X25519PublicKey& other) const { - return kj::arrayPtr(bytes) == kj::arrayPtr(other.bytes); + return bytes.asPtr() == other.bytes.asPtr(); } kj::uint hashCode() const { - return kj::hashCode(kj::ArrayPtr(bytes)); + return kj::hashCode(bytes.asPtr()); } }; @@ -134,7 +134,7 @@ class ClusterRegistry final: public ClusterVatNetworkBase { kj::Duration connectTimeout; // Identity - kj::byte privateKey[32]; + kj::FixedArray privateKey; X25519PublicKey publicKey; kj::String publicKeyHex; From 34bf369ecdea9777a324b1ac707158a7ab2721cd Mon Sep 17 00:00:00 2001 From: Kenton Varda Date: Mon, 7 Sep 2026 23:15:14 -0500 Subject: [PATCH 25/33] Bugfix: Don't requineNotBroken() in getSqliteDatabase(). This proves to be too aggressive. In the edge runtime, we call getSqliteDatabase() post-broken in various cases (and we of course don't use the new close() function). This fixes broken tests in the internal build. --- src/workerd/io/actor-sqlite-test.c++ | 10 +++++----- src/workerd/io/actor-sqlite.h | 1 - src/workerd/server/server.c++ | 3 ++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/workerd/io/actor-sqlite-test.c++ b/src/workerd/io/actor-sqlite-test.c++ index f2c4b7e5dbb..60dba32d4fe 100644 --- a/src/workerd/io/actor-sqlite-test.c++ +++ b/src/workerd/io/actor-sqlite-test.c++ @@ -2637,7 +2637,7 @@ KJ_TEST("shutdown() leaves the connection open; the owner closes it afterwards") // Storage is broken, but shutdown() must not close the connection itself: embedders may call it // from inside a SQLite callback, where closing is unsafe. The owner closes it later. - KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.actor.getSqliteDatabase()); + KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.actor.getSqliteKv()); test.db.run("SELECT 1"); // The scheduled commit observes the shutdown and breaks the output gate with the shutdown @@ -2651,7 +2651,7 @@ KJ_TEST("shutdown() leaves the connection open; the owner closes it afterwards") // A second shutdown() on a closed database is a no-op. test.actor.shutdown(kj::none); - KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.actor.getSqliteDatabase()); + KJ_EXPECT_THROW_MESSAGE(ActorCache::SHUTDOWN_ERROR_MESSAGE, test.actor.getSqliteKv()); expectDatabaseClosed(test); } @@ -2664,15 +2664,15 @@ KJ_TEST("shutdown() after a critical error keeps the original exception") { auto txn = test.startTransaction(); txn->put(kj::str("uncommitted"), kj::heapArray("yes"_kj.asBytes()), {}, nullptr); breakViaCriticalError(test, *txn); - KJ_EXPECT_THROW_MESSAGE("broken.outputGateBroken", test.actor.getSqliteDatabase()); + KJ_EXPECT_THROW_MESSAGE("broken.outputGateBroken", test.actor.getSqliteKv()); auto expectOriginalBrokenException = [&]() { - KJ_IF_SOME(e, kj::runCatchingExceptions([&]() { test.actor.getSqliteDatabase(); })) { + KJ_IF_SOME(e, kj::runCatchingExceptions([&]() { test.actor.getSqliteKv(); })) { // The first exception wins: the shutdown exception must not replace the storage error. KJ_EXPECT(e.getDescription().contains("broken.outputGateBroken"), e); KJ_EXPECT(!e.getDescription().contains(ActorCache::SHUTDOWN_ERROR_MESSAGE), e); } else { - KJ_FAIL_EXPECT("getSqliteDatabase() should have thrown"); + KJ_FAIL_EXPECT("getSqliteKv() should have thrown"); } }; diff --git a/src/workerd/io/actor-sqlite.h b/src/workerd/io/actor-sqlite.h index 36ffdbffeef..527353fe2ae 100644 --- a/src/workerd/io/actor-sqlite.h +++ b/src/workerd/io/actor-sqlite.h @@ -88,7 +88,6 @@ class ActorSqlite final: public ActorCacheInterface, private kj::TaskSet::ErrorH void blockTransaction(kj::Promise promise) override; kj::Maybe getSqliteDatabase() override { - requireNotBroken(); return *db; } diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 07a2dd0dc3b..7e9f61d44aa 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -753,7 +753,8 @@ class Server::ActorNamespace final { } // Resets the actor's SQLite database while the connection is still open, - // avoiding file-locking issues on Windows. Storage that is already broken is skipped. + // avoiding file-locking issues on Windows. Storage that is already closed is skipped + // (reset() refuses to reopen a closed database). void resetStorage() { KJ_IF_SOME(a, actor) { KJ_IF_SOME(cache, a->getPersistent()) { From 7ea530a6e4c1e2de84ba0acec2813aaa262baff5 Mon Sep 17 00:00:00 2001 From: Gogo Date: Tue, 3 Nov 2026 10:25:55 +0000 Subject: [PATCH 26/33] Support Durable Object alarms in cluster mode (local multi-process) Cluster mode (#6780) refused alarms: an AlarmScheduler per instance would fight over the namespace's metadata.sqlite. This makes them work when the instances share a local disk: - Every instance writes alarm times straight into the shared _cf_ALARM table (ClusterAlarmStore, used by ClusterActorSqliteHooks). The native VFS takes real file locks, so several processes can share the file. - One instance, the alarm leader, runs the AlarmScheduler. Leadership is an OFD lock on `alarm-leader` in the namespace directory, tried every second, so it moves to another instance when the leader goes away. - The leader picks up other instances' writes every 250 ms (AlarmScheduler::syncFromDb()): new rows are scheduled, moved times reschedule a waiting alarm or queue behind a running one, deleted rows cancel a waiting alarm. - After an alarm ran, the leader deletes its row only if it still holds the time that ran, so an alarm the handler set meanwhile on another instance survives (finishAlarm()). - A due alarm is delivered like any request (ClusterActorChannel): to the instance that holds the actor, or claimed by the leader when none does. The cluster's EventDispatcher now forwards runAlarm and abandonAlarm. - Instances starting together race to create the table and switch it to WAL; SQLite answers SQLITE_BUSY at once there, so initialization retries, and a leader that cannot start its scheduler gives the lock back. Tested with 3 processes on one machine: the time moved several times (runs once, at the latest), the handler throwing (retried with backoff), the holder killed with SIGKILL (claimed by the leader, runs once), the leader killed (another takes over, runs once). Not tested across machines (NFS). Signed-off-by: Gogo --- src/workerd/server/alarm-scheduler.c++ | 77 ++++++++++- src/workerd/server/alarm-scheduler.h | 19 +++ src/workerd/server/server.c++ | 169 ++++++++++++++++++++++++- 3 files changed, 256 insertions(+), 9 deletions(-) diff --git a/src/workerd/server/alarm-scheduler.c++ b/src/workerd/server/alarm-scheduler.c++ index 529c12dd8d9..bb028070893 100644 --- a/src/workerd/server/alarm-scheduler.c++ +++ b/src/workerd/server/alarm-scheduler.c++ @@ -291,7 +291,7 @@ kj::Promise AlarmScheduler::makeAlarmTask( entry.value.task = abandonAlarm(actorRef.clone(), scheduledTime); co_return; } - deleteAlarm(actorRef); + finishAlarm(actorRef, scheduledTime); } } catch (...) { auto exception = kj::getCaughtExceptionAsKj(); @@ -319,11 +319,84 @@ kj::Promise AlarmScheduler::abandonAlarm(kj::Own actor, kj::Date KJ_IF_SOME(replacement, entry.value.queuedAlarm) { entry.value = scheduleAlarm(clock.now(), kj::mv(entry.value.actor), replacement); } else { - deleteAlarm(*actor); + finishAlarm(*actor, scheduledTime); } } } +void AlarmScheduler::enableClusterMode() { + clusterMode = true; + // Other instances write the same table (server.c++ ClusterAlarmStore). + db->run("PRAGMA busy_timeout = 2000;"); +} + +void AlarmScheduler::finishAlarm(const ActorKey& actor, kj::Date scheduledTime) { + if (!clusterMode) { + deleteAlarm(actor); + return; + } + // Another process may have written a new time for this actor while the alarm ran (the handler + // calling setAlarm() on the process that owns the actor): only delete the row that ran. + int64_t scheduledTimeNs = (scheduledTime - kj::UNIX_EPOCH) / kj::NANOSECONDS; + stmtDeleteAlarmIfTime.run(actor.actorId, scheduledTimeNs); + KJ_IF_SOME(entry, alarms.findEntry(actor)) { + KJ_IF_SOME(queued, entry.value.queuedAlarm) { + entry.value = scheduleAlarm(clock.now(), kj::mv(entry.value.actor), queued); + } else if (entry.value.status != AlarmStatus::STARTED) { + alarms.erase(entry); + } + } + // A newer row, if any, is scheduled by the next syncFromDb(). +} + +void AlarmScheduler::syncFromDb() { + KJ_REQUIRE(clusterMode); + auto now = clock.now(); + kj::HashSet present; + + auto query = db->run(R"( + SELECT actor_id, scheduled_time, actor_name FROM _cf_ALARM; + )"); + while (!query.isDone()) { + auto date = kj::UNIX_EPOCH + (kj::NANOSECONDS * query.getInt64(1)); + ActorKey probe{.actorId = query.getText(0), .name = query.getMaybeText(2)}; + present.insert(kj::str(probe.actorId)); + + KJ_IF_SOME(entry, alarms.findEntry(probe)) { + auto& alarm = entry.value; + if (alarm.status == AlarmStatus::WAITING) { + // Moved earlier or later by the actor: only the latest time counts. + if (alarm.scheduledTime != date) { + alarm = scheduleAlarm(now, kj::mv(alarm.actor), date); + } + } else if (alarm.scheduledTime != date) { + // Running, or waiting to retry: a different time is an alarm set meanwhile, which runs + // after this one (as setAlarm() does in single-process mode). + alarm.queuedAlarm = date; + } + } else { + // New, or one this process has never seen (another leader before it, or an actor that set + // its alarm on another process): schedule it. It wakes the actor wherever it lives, or + // claims it here if no process holds it. + auto actor = probe.clone(); + auto& actorRef = *actor; + alarms.insert(actorRef, scheduleAlarm(now, kj::mv(actor), date)); + } + query.nextRow(); + } + + // Rows the actors deleted: cancel those still waiting (running ones finish by themselves). + kj::Vector> gone; + for (auto& entry: alarms) { + if (entry.value.status == AlarmStatus::WAITING && !present.contains(entry.key.actorId)) { + gone.add(entry.value.actor->clone()); + } + } + for (auto& actor: gone) { + alarms.erase(*actor); + } +} + void AlarmScheduler::taskFailed(kj::Exception&& e) { KJ_LOG(WARNING, e); } diff --git a/src/workerd/server/alarm-scheduler.h b/src/workerd/server/alarm-scheduler.h index 42a914519bd..55e025ced53 100644 --- a/src/workerd/server/alarm-scheduler.h +++ b/src/workerd/server/alarm-scheduler.h @@ -83,6 +83,14 @@ class AlarmScheduler final: kj::TaskSet::ErrorHandler { // Cancels all pending alarms and removes them from persistent storage. void deleteAll(); + // Cluster mode (several workerd processes sharing one storage directory). Only one process, the + // alarm leader, runs a scheduler; every process writes alarm times straight into the shared + // `_cf_ALARM` table (ClusterAlarmStore in server.c++). The leader calls syncFromDb() regularly to + // pick up those writes, and only deletes a row after an alarm ran if the row still holds the time + // that ran, so an alarm set by the handler while it was running (on another process) survives. + void enableClusterMode(); + void syncFromDb(); + private: enum class AlarmStatus { WAITING, STARTED, FINISHED }; const kj::Clock& clock; @@ -135,12 +143,23 @@ class AlarmScheduler final: kj::TaskSet::ErrorHandler { SqliteDatabase::Statement stmtDeleteAlarm = db->prepare(R"( DELETE FROM _cf_ALARM WHERE actor_id = ? )"); + SqliteDatabase::Statement stmtDeleteAlarmIfTime = db->prepare(R"( + DELETE FROM _cf_ALARM WHERE actor_id = ? AND scheduled_time = ? + )"); + bool clusterMode = false; + // After an alarm at `scheduledTime` finished for good: forget it, and delete its row (in cluster + // mode only if the row still holds that time). + void finishAlarm(const ActorKey& actor, kj::Date scheduledTime); void taskFailed(kj::Exception&& exception) override; int maxJitterMsForDelay(kj::Duration delay); + + public: static void ensureInitialized(SqliteDatabase& db); + + private: void loadAlarmsFromDb(); }; diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 7e9f61d44aa..8075751ad30 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -60,7 +60,10 @@ #include #include +#include + #include +#include #include namespace workerd::server { @@ -417,9 +420,18 @@ class Server::ActorNamespace final { KJ_IF_SOME(d, config.tryGet()) { if (lockManager != kj::none) { - // TODO(clustering): Support alarms in cluster mode. Until then, we must skip creating - // an AlarmScheduler altogether otherwise multiple instances will fight over the alarm - // database. + // Cluster mode: every instance writes alarm times into the shared metadata.sqlite (the + // native VFS takes real file locks, so several processes can share it on a local disk); + // exactly one instance, the alarm leader, runs the AlarmScheduler. Leadership is an OFD lock + // on `alarm-leader`, so it moves to another instance when the leader goes away. + auto& as = KJ_ASSERT_NONNULL(this->actorStorage); + clusterAlarmStore = kj::heap(as.vfs); + alarmLeaderFile = as.directory->openFile( + kj::Path({"alarm-leader"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + alarmLeaderTask = alarmLeaderLoop(kj::str(d.uniqueKey)) + .eagerlyEvaluate([](kj::Exception&& e) { + KJ_LOG(ERROR, "cluster alarm leader loop failed; alarms stop on this instance", e); + }); return; } @@ -1352,7 +1364,11 @@ class Server::ActorNamespace final { kj::Own sqliteHooks; if (parent == kj::none) { if (ns.lockManager != kj::none) { - sqliteHooks = kj::heap(); + auto actorKey = ActorKey{.actorId = key, .name = actorName.map([](kj::String& n) { + return n.asPtr(); + })}.clone(); + sqliteHooks = kj::heap( + *KJ_ASSERT_NONNULL(ns.clusterAlarmStore), kj::mv(actorKey)); } else KJ_IF_SOME(a, ns.alarmScheduler) { // clone() copies the id and name into storage owned by the returned ActorKey, so // the temporary StringPtrs below only need to outlive this call. @@ -1737,6 +1753,12 @@ class Server::ActorNamespace final { kj::Maybe clusterRegistry; kj::Maybe lockManager; + // Cluster mode alarms (see alarmLeaderLoop()). + class ClusterAlarmStore; + kj::Maybe> clusterAlarmStore; + kj::Maybe> alarmLeaderFile; + kj::Maybe alarmLeaderLock; + // Tracks the canceler and cleanup promise for a Docker container's lifecycle cleanup. // Useful to await on async calls of a ContainerClient destructor when the new // one appears before they've been resolved. @@ -1770,6 +1792,8 @@ class Server::ActorNamespace final { ActorMap actors; kj::Maybe> cleanupTask; + // Declared after `actors` and the scheduler so it is torn down first. + kj::Maybe> alarmLeaderTask; kj::Timer& timer; capnp::ByteStreamFactory& byteStreamFactory; capnp::HttpOverCapnpFactory& httpOverCapnpFactory; @@ -1959,15 +1983,124 @@ class Server::ActorNamespace final { } }; + // Cluster mode: an actor's alarm time goes into the shared `_cf_ALARM` table, wherever the actor + // runs. The alarm leader's AlarmScheduler picks it up (AlarmScheduler::syncFromDb()). + class ClusterAlarmStore { + public: + explicit ClusterAlarmStore(const SqliteDatabase::Vfs& vfs) + : db([&] { + auto db = kj::heap(vfs, kj::Path({"metadata.sqlite"}), + kj::WriteMode::CREATE | kj::WriteMode::MODIFY | kj::WriteMode::CREATE_PARENT); + // Other instances write the same file: wait for their (tiny) transactions. + db->run("PRAGMA busy_timeout = 2000;"); + // Instances starting together race to create the table and switch it to WAL; SQLite + // answers SQLITE_BUSY at once in that case (busy_timeout does not cover it), so retry. + for (uint attempt = 0;; ++attempt) { + try { + AlarmScheduler::ensureInitialized(*db); + break; + } catch (...) { + auto e = kj::getCaughtExceptionAsKj(); + if (attempt >= 100 || strstr(e.getDescription().cStr(), "SQLITE_BUSY") == nullptr) { + kj::throwFatalException(kj::mv(e)); + } + usleep(20 * 1000); + } + } + return kj::mv(db); + }()) {} + + void set(const ActorKey& actor, kj::Date scheduledTime) { + int64_t scheduledTimeNs = (scheduledTime - kj::UNIX_EPOCH) / kj::NANOSECONDS; + SqliteDatabase::Query::ValuePtr nameParam = nullptr; + KJ_IF_SOME(n, actor.name) { + nameParam = n; + } + stmtSet.run(actor.actorId, scheduledTimeNs, nameParam); + } + void remove(const ActorKey& actor) { + stmtDelete.run(actor.actorId); + } + + private: + kj::Own db; + SqliteDatabase::Statement stmtSet = db->prepare(R"( + INSERT INTO _cf_ALARM VALUES(?, ?, ?) + ON CONFLICT DO UPDATE SET scheduled_time = excluded.scheduled_time, + actor_name = COALESCE(excluded.actor_name, _cf_ALARM.actor_name); + )"); + SqliteDatabase::Statement stmtDelete = db->prepare(R"( + DELETE FROM _cf_ALARM WHERE actor_id = ? + )"); + }; + class ClusterActorSqliteHooks final: public ActorSqlite::Hooks { public: + ClusterActorSqliteHooks(ClusterAlarmStore& store, kj::Own actor) + : store(store), + actor(kj::mv(actor)) {} + kj::Promise scheduleRun( kj::Maybe newAlarmTime, kj::Promise priorTask) override { - // TODO(clustering): Support alarms. - JSG_FAIL_REQUIRE(Error, "Durable Object alarms are not yet supported in cluster mode"); + KJ_IF_SOME(scheduledTime, newAlarmTime) { + store.set(*actor, scheduledTime); + } else { + store.remove(*actor); + } + return kj::READY_NOW; } + + private: + ClusterAlarmStore& store; + kj::Own actor; }; + // Cluster mode: try to become the alarm leader every second; once leader, run the + // AlarmScheduler and pick up alarm times the other instances wrote, every 250 ms. A due alarm is + // delivered like any request to the actor (ClusterActorChannel): to the instance that holds it, or + // claimed by this one when no instance does. + kj::Promise alarmLeaderLoop(kj::String uniqueKey) { + auto& as = KJ_ASSERT_NONNULL(actorStorage); + for (;;) { + if (ownAlarmScheduler == kj::none) { + auto fd = KJ_ASSERT_NONNULL(KJ_ASSERT_NONNULL(alarmLeaderFile)->getFd()); + KJ_IF_SOME(lock, OfdLock::tryLock(fd, OfdLock::EXCLUSIVE)) { + alarmLeaderLock = kj::mv(lock); + AlarmScheduler::GetActorFn getActor = + [this, idFactory = kj::heap(uniqueKey)]( + const ActorKey& actor) mutable -> kj::Own { + Worker::Actor::Id id = idFactory->idFromStringNamed( + kj::str(actor.actorId), actor.name.map([](kj::StringPtr n) { return kj::str(n); })); + return getActorChannel(kj::mv(id))->startRequest({}); + }; + try { + auto scheduler = kj::heap( + clock, timer, as.vfs, kj::Path({"metadata.sqlite"}), kj::mv(getActor)); + scheduler->enableClusterMode(); + ownAlarmScheduler = kj::mv(scheduler); + KJ_LOG(INFO, "this instance is now the alarm leader", uniqueKey); + } catch (...) { + // e.g. SQLITE_BUSY while another instance initializes the table: give up the lock and + // try again next round. + KJ_LOG(WARNING, "could not start the alarm scheduler; retrying", + kj::getCaughtExceptionAsKj()); + alarmLeaderLock = kj::none; + } + } + } + KJ_IF_SOME(scheduler, ownAlarmScheduler) { + try { + scheduler->syncFromDb(); + } catch (...) { + KJ_LOG(WARNING, "cluster alarm sync failed; retrying", kj::getCaughtExceptionAsKj()); + } + co_await timer.afterDelay(250 * kj::MILLISECONDS); + } else { + co_await timer.afterDelay(1 * kj::SECONDS); + } + } + } + // Base class for SubrequestChannels in cluster mode whose requests are routed to whichever node // currently owns some root actor in this namespace: plain actor stubs (ClusterActorChannel) and // restored stubs whose vendor chain bottoms out at such an actor @@ -6908,8 +7041,30 @@ class Server::WorkerdBootstrapImpl final: public rpc::WorkerdBootstrap::Server { throwUnsupported(); } + // Cluster mode: the alarm leader delivers a due alarm to the instance that holds the actor + // (AlarmScheduler -> ClusterActorChannel -> here). kj::Promise runAlarm(RunAlarmContext context) override { - throwUnsupported(); + auto params = context.getParams(); + auto scheduledTime = kj::UNIX_EPOCH + params.getScheduledTime() * kj::MILLISECONDS; + auto worker = getWorker(); + auto result = co_await worker->runAlarm(scheduledTime, params.getRetryCount()); + auto resp = context.getResults().initResult(); + resp.setOutcome(result.outcome); + resp.setRetry(result.retry); + resp.setRetryCountsAgainstLimit(result.retryCountsAgainstLimit); + KJ_IF_SOME(d, result.errorDescription) { + resp.setErrorDescription(d); + } + } + + kj::Promise abandonAlarm(AbandonAlarmContext context) override { + auto scheduledTime = + kj::UNIX_EPOCH + context.getParams().getScheduledTimeMs() * kj::MILLISECONDS; + auto worker = getWorker(); + auto stored = co_await worker->abandonAlarm(scheduledTime); + KJ_IF_SOME(t, stored) { + context.getResults().setStoredAlarmTimeMs((t - kj::UNIX_EPOCH) / kj::MILLISECONDS); + } } kj::Promise queue(QueueContext context) override { From ce2f51ce7b21a1579c08230191fcc1d03628bd18 Mon Sep 17 00:00:00 2001 From: Gogo Date: Tue, 3 Nov 2026 10:30:31 +0000 Subject: [PATCH 27/33] ci(botiverse): build the Linux x64 binary on a standard runner The fork has no access to upstream's 16-core runners or remote cache, so this workflow builds the cluster-alarms branch from scratch on ubuntu-24.04 (within the 6-hour job limit) and uploads the binary. Signed-off-by: Gogo --- .github/workflows/botiverse-build.yml | 55 +++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/botiverse-build.yml diff --git a/.github/workflows/botiverse-build.yml b/.github/workflows/botiverse-build.yml new file mode 100644 index 00000000000..aef41a65caa --- /dev/null +++ b/.github/workflows/botiverse-build.yml @@ -0,0 +1,55 @@ +# Botiverse fork only: build the Linux x64 workerd binary for the cluster-alarms branch on a standard +# GitHub runner. Upstream's builds use Cloudflare's 16-core runners and remote cache, which this fork +# has neither of, so this is a plain from-scratch build (a few hours; Bazel's disk cache is kept +# between runs to make later builds faster). +name: Botiverse build + +on: + push: + branches: + - botiverse/** + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: botiverse-build-${{ github.ref }} + cancel-in-progress: true + +jobs: + linux-x64: + runs-on: ubuntu-24.04 + # GitHub-hosted jobs stop at 6 hours. + timeout-minutes: 355 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + df -h / + - name: Install the toolchain (README.md: clang, libc++ and lld 19+, tcl) + run: | + sudo apt-get update -q + sudo apt-get install -y -q clang-19 libc++-19-dev libc++abi-19-dev lld-19 tcl8.6 + # Bazel's sandbox only sees /bin, /usr/bin and /usr/local/bin. + sudo ln -sf /usr/bin/clang-19 /usr/local/bin/clang + sudo ln -sf /usr/bin/clang++-19 /usr/local/bin/clang++ + sudo ln -sf /usr/bin/tclsh8.6 /usr/local/bin/tclsh + - uses: actions/cache@v5 + with: + path: ~/bazel-disk-cache + key: botiverse-bazel-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.bazelversion', '.bazelrc', 'MODULE.bazel') }}-${{ github.sha }} + restore-keys: | + botiverse-bazel-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.bazelversion', '.bazelrc', 'MODULE.bazel') }}- + - name: Build + run: bazel build //src/workerd/server:workerd --config=release --disk_cache=~/bazel-disk-cache --jobs=4 + - name: Check the binary runs + run: bazel-bin/src/workerd/server/workerd --version + - uses: actions/upload-artifact@v7 + with: + name: workerd-linux-x64-${{ github.sha }} + path: bazel-bin/src/workerd/server/workerd + retention-days: 30 From f39af5c7337673c7c7eaa72b00b304205c958b20 Mon Sep 17 00:00:00 2001 From: Gogo Date: Tue, 3 Nov 2026 10:32:30 +0000 Subject: [PATCH 28/33] ci(botiverse): fix the workflow's YAML (a colon in a step name) Signed-off-by: Gogo --- .github/workflows/botiverse-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/botiverse-build.yml b/.github/workflows/botiverse-build.yml index aef41a65caa..8c99f7c45ff 100644 --- a/.github/workflows/botiverse-build.yml +++ b/.github/workflows/botiverse-build.yml @@ -30,7 +30,7 @@ jobs: run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL df -h / - - name: Install the toolchain (README.md: clang, libc++ and lld 19+, tcl) + - name: Install the toolchain (clang, libc++ and lld 19+, tcl; see README.md) run: | sudo apt-get update -q sudo apt-get install -y -q clang-19 libc++-19-dev libc++abi-19-dev lld-19 tcl8.6 From 74110df3ce72e77573a82ba1bfee756fb9743bec Mon Sep 17 00:00:00 2001 From: Gogo Date: Tue, 3 Nov 2026 10:36:11 +0000 Subject: [PATCH 29/33] ci(botiverse): unset the Android SDK variables after removing the SDK Signed-off-by: Gogo --- .github/workflows/botiverse-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/botiverse-build.yml b/.github/workflows/botiverse-build.yml index 8c99f7c45ff..5955b1fcb07 100644 --- a/.github/workflows/botiverse-build.yml +++ b/.github/workflows/botiverse-build.yml @@ -45,7 +45,8 @@ jobs: restore-keys: | botiverse-bazel-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.bazelversion', '.bazelrc', 'MODULE.bazel') }}- - name: Build - run: bazel build //src/workerd/server:workerd --config=release --disk_cache=~/bazel-disk-cache --jobs=4 + # The Android SDK was removed above (disk space); without these the Android rules find it empty and fail. + run: env -u ANDROID_HOME -u ANDROID_SDK_ROOT -u ANDROID_NDK_HOME -u ANDROID_NDK_ROOT -u ANDROID_NDK_LATEST_HOME bazel build //src/workerd/server:workerd --config=release --disk_cache=~/bazel-disk-cache --jobs=4 - name: Check the binary runs run: bazel-bin/src/workerd/server/workerd --version - uses: actions/upload-artifact@v7 From 0074300f0e2c27f2295bce63a31c4a6fb511bb2a Mon Sep 17 00:00:00 2001 From: Gogo Date: Tue, 3 Nov 2026 11:10:03 +0000 Subject: [PATCH 30/33] Cluster alarms: address review (botiverse/workerd#1) - Only clusters on one machine get alarms. Across machines (NFS mode, an IP `network`) a shared WAL-mode SQLite file isn't safe, so setAlarm() keeps failing there, as before, with a message that says why. - The alarm leader never writes the shared table. Each actor owns its row: after a successful run ActorSqlite clears its alarm, and an alarm set by the handler is written by the instance that holds the actor, both through ClusterActorSqliteHooks. Before, the leader deleted the row of the alarm that ran, which dropped a newer alarm if the holder's write had failed. A stale row is delivered again and ActorSqlite::armAlarmHandler() cancels it (and re-requests the time it does hold); alarms that just ran are not re-delivered for 5 seconds while the actor's own deletion is on its way. - When notifying an actor that its alarm was abandoned fails, the entry is forgotten instead of staying FINISHED with no task (which swallowed later times for that actor until the leader changed); the next sync delivers it again. - A draining instance gives up alarm leadership: it can no longer claim actors, so it could not deliver alarms either. - deleteAll() (resetting this instance's actors) no longer wipes the alarm table that every instance shares. Signed-off-by: Gogo --- src/workerd/server/alarm-scheduler.c++ | 37 ++++++++++++++++--- src/workerd/server/alarm-scheduler.h | 18 ++++++---- src/workerd/server/server.c++ | 50 ++++++++++++++++++++------ 3 files changed, 82 insertions(+), 23 deletions(-) diff --git a/src/workerd/server/alarm-scheduler.c++ b/src/workerd/server/alarm-scheduler.c++ index bb028070893..a7f051eb6ed 100644 --- a/src/workerd/server/alarm-scheduler.c++ +++ b/src/workerd/server/alarm-scheduler.c++ @@ -300,11 +300,25 @@ kj::Promise AlarmScheduler::makeAlarmTask( } kj::Promise AlarmScheduler::abandonAlarm(kj::Own actor, kj::Date scheduledTime) { + bool notified = false; try { co_await getActor(*actor)->abandonAlarm(scheduledTime); + notified = true; } catch (...) { auto exception = kj::getCaughtExceptionAsKj(); KJ_LOG(WARNING, "abandonAlarm notification failed, keeping alarm in scheduler", exception); + } + if (!notified) { + if (clusterMode) { + // The row is still in the shared table: forget the entry (a FINISHED entry with no task + // would otherwise swallow later times for this actor) and let syncFromDb() deliver it again. + KJ_IF_SOME(entry, alarms.findEntry(*actor)) { + if (entry.value.status == AlarmStatus::FINISHED && entry.value.scheduledTime == scheduledTime) { + tasks.add(kj::mv(entry.value.task)); + alarms.erase(entry); + } + } + } co_return; } @@ -335,10 +349,15 @@ void AlarmScheduler::finishAlarm(const ActorKey& actor, kj::Date scheduledTime) deleteAlarm(actor); return; } - // Another process may have written a new time for this actor while the alarm ran (the handler - // calling setAlarm() on the process that owns the actor): only delete the row that ran. - int64_t scheduledTimeNs = (scheduledTime - kj::UNIX_EPOCH) / kj::NANOSECONDS; - stmtDeleteAlarmIfTime.run(actor.actorId, scheduledTimeNs); + // Cluster mode: the scheduler never writes the shared table. The actor owns its row: after a + // successful run ActorSqlite clears its alarm, which reaches ClusterActorSqliteHooks on whichever + // instance holds it, and an alarm set by the handler is written there the same way. Deleting the + // row here could drop that newer alarm if the holder's write failed. A row that is still there is + // delivered again by syncFromDb(), and ActorSqlite::armAlarmHandler() cancels a delivery that no + // longer matches its own alarm state (re-requesting the time it does hold). + recentlyFinished.upsert(kj::str(actor.actorId), + RecentlyFinished{scheduledTime, clock.now() + RECENTLY_FINISHED_GRACE}, + [](RecentlyFinished& existing, RecentlyFinished&& replacement) { existing = replacement; }); KJ_IF_SOME(entry, alarms.findEntry(actor)) { KJ_IF_SOME(queued, entry.value.queuedAlarm) { entry.value = scheduleAlarm(clock.now(), kj::mv(entry.value.actor), queued); @@ -346,7 +365,6 @@ void AlarmScheduler::finishAlarm(const ActorKey& actor, kj::Date scheduledTime) alarms.erase(entry); } } - // A newer row, if any, is scheduled by the next syncFromDb(). } void AlarmScheduler::syncFromDb() { @@ -375,6 +393,14 @@ void AlarmScheduler::syncFromDb() { alarm.queuedAlarm = date; } } else { + // The actor deletes the row of an alarm that ran, a little after the run returns: don't + // deliver that same time again while its deletion may still be on its way. + KJ_IF_SOME(f, recentlyFinished.find(probe.actorId)) { + if (f.scheduledTime == date && now < f.until) { + query.nextRow(); + continue; + } + } // New, or one this process has never seen (another leader before it, or an actor that set // its alarm on another process): schedule it. It wakes the actor wherever it lives, or // claims it here if no process holds it. @@ -395,6 +421,7 @@ void AlarmScheduler::syncFromDb() { for (auto& actor: gone) { alarms.erase(*actor); } + recentlyFinished.eraseAll([&](auto& key, RecentlyFinished& f) { return f.until <= now; }); } void AlarmScheduler::taskFailed(kj::Exception&& e) { diff --git a/src/workerd/server/alarm-scheduler.h b/src/workerd/server/alarm-scheduler.h index 55e025ced53..eeab97dba24 100644 --- a/src/workerd/server/alarm-scheduler.h +++ b/src/workerd/server/alarm-scheduler.h @@ -86,8 +86,7 @@ class AlarmScheduler final: kj::TaskSet::ErrorHandler { // Cluster mode (several workerd processes sharing one storage directory). Only one process, the // alarm leader, runs a scheduler; every process writes alarm times straight into the shared // `_cf_ALARM` table (ClusterAlarmStore in server.c++). The leader calls syncFromDb() regularly to - // pick up those writes, and only deletes a row after an alarm ran if the row still holds the time - // that ran, so an alarm set by the handler while it was running (on another process) survives. + // pick up those writes, and never writes the table itself: each actor keeps its own row. void enableClusterMode(); void syncFromDb(); @@ -143,13 +142,18 @@ class AlarmScheduler final: kj::TaskSet::ErrorHandler { SqliteDatabase::Statement stmtDeleteAlarm = db->prepare(R"( DELETE FROM _cf_ALARM WHERE actor_id = ? )"); - SqliteDatabase::Statement stmtDeleteAlarmIfTime = db->prepare(R"( - DELETE FROM _cf_ALARM WHERE actor_id = ? AND scheduled_time = ? - )"); bool clusterMode = false; - // After an alarm at `scheduledTime` finished for good: forget it, and delete its row (in cluster - // mode only if the row still holds that time). + // After an alarm at `scheduledTime` finished for good: forget it, and delete its row (cluster + // mode: leave the row to the actor, see finishAlarm()). void finishAlarm(const ActorKey& actor, kj::Date scheduledTime); + // Cluster mode: alarms that just ran, so syncFromDb() doesn't deliver them again while the actor's + // deletion of the row is still on its way. + static constexpr auto RECENTLY_FINISHED_GRACE = 5 * kj::SECONDS; + struct RecentlyFinished { + kj::Date scheduledTime; + kj::Date until; + }; + kj::HashMap recentlyFinished; void taskFailed(kj::Exception&& exception) override; diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 8075751ad30..c08fb619033 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -420,10 +420,16 @@ class Server::ActorNamespace final { KJ_IF_SOME(d, config.tryGet()) { if (lockManager != kj::none) { - // Cluster mode: every instance writes alarm times into the shared metadata.sqlite (the - // native VFS takes real file locks, so several processes can share it on a local disk); - // exactly one instance, the alarm leader, runs the AlarmScheduler. Leadership is an OFD lock - // on `alarm-leader`, so it moves to another instance when the leader goes away. + if (KJ_ASSERT_NONNULL(clusterRegistry).nfsMode()) { + // Across machines (NFS) a shared SQLite file in WAL mode isn't safe: no alarms there. + // ClusterActorSqliteHooks refuses them, as before. + return; + } + // Cluster mode on one machine: every instance writes alarm times into the shared + // metadata.sqlite (the native VFS takes real file locks, so several processes can share it + // on a local disk); exactly one instance, the alarm leader, runs the AlarmScheduler. + // Leadership is an OFD lock on `alarm-leader`, so it moves to another instance when the + // leader goes away. auto& as = KJ_ASSERT_NONNULL(this->actorStorage); clusterAlarmStore = kj::heap(as.vfs); alarmLeaderFile = as.directory->openFile( @@ -1368,7 +1374,10 @@ class Server::ActorNamespace final { return n.asPtr(); })}.clone(); sqliteHooks = kj::heap( - *KJ_ASSERT_NONNULL(ns.clusterAlarmStore), kj::mv(actorKey)); + ns.clusterAlarmStore.map([](kj::Own& s) -> ClusterAlarmStore& { + return *s; + }), + kj::mv(actorKey)); } else KJ_IF_SOME(a, ns.alarmScheduler) { // clone() copies the id and name into storage owned by the returned ActorKey, so // the temporary StringPtrs below only need to outlive this call. @@ -1723,8 +1732,12 @@ class Server::ActorNamespace final { abortAll(reason); - KJ_IF_SOME(scheduler, ownAlarmScheduler) { - scheduler->deleteAll(); + // In cluster mode the alarm table is shared by every instance, and this only resets this + // instance's actors: leave it (each reset actor cancels its stale alarm when it is delivered). + if (lockManager == kj::none) { + KJ_IF_SOME(scheduler, ownAlarmScheduler) { + scheduler->deleteAll(); + } } } @@ -2036,22 +2049,26 @@ class Server::ActorNamespace final { class ClusterActorSqliteHooks final: public ActorSqlite::Hooks { public: - ClusterActorSqliteHooks(ClusterAlarmStore& store, kj::Own actor) + // `store` is none across machines (NFS mode), where alarms are not supported. + ClusterActorSqliteHooks(kj::Maybe store, kj::Own actor) : store(store), actor(kj::mv(actor)) {} kj::Promise scheduleRun( kj::Maybe newAlarmTime, kj::Promise priorTask) override { + auto& s = JSG_REQUIRE_NONNULL(store, Error, + "Durable Object alarms are not yet supported in cluster mode across machines (NFS); " + "they are supported when all instances share a local disk (network = \"unix\")"); KJ_IF_SOME(scheduledTime, newAlarmTime) { - store.set(*actor, scheduledTime); + s.set(*actor, scheduledTime); } else { - store.remove(*actor); + s.remove(*actor); } return kj::READY_NOW; } private: - ClusterAlarmStore& store; + kj::Maybe store; kj::Own actor; }; @@ -2062,6 +2079,17 @@ class Server::ActorNamespace final { kj::Promise alarmLeaderLoop(kj::String uniqueKey) { auto& as = KJ_ASSERT_NONNULL(actorStorage); for (;;) { + if (KJ_ASSERT_NONNULL(clusterRegistry).isDraining()) { + // A draining instance can't claim actors any more, so it can't deliver alarms either: hand + // leadership to another instance (dropping the scheduler cancels its pending deliveries; the + // next leader re-reads them from the table). + if (ownAlarmScheduler != kj::none) { + KJ_LOG(INFO, "draining: giving up alarm leadership", uniqueKey); + } + ownAlarmScheduler = kj::none; + alarmLeaderLock = kj::none; + co_return; + } if (ownAlarmScheduler == kj::none) { auto fd = KJ_ASSERT_NONNULL(KJ_ASSERT_NONNULL(alarmLeaderFile)->getFd()); KJ_IF_SOME(lock, OfdLock::tryLock(fd, OfdLock::EXCLUSIVE)) { From 1980155cd717d9301f383ca39e2dac6260ed9415 Mon Sep 17 00:00:00 2001 From: Gogo Date: Tue, 3 Nov 2026 11:10:03 +0000 Subject: [PATCH 31/33] Cluster test: alarms run once in the unix variant, still rejected in the CIDR (NFS) variant The test that expected alarms to be rejected in every cluster now runs only in the CIDR variant (NFS mode). The unix variant sets an alarm through one instance and checks, through another, that it ran exactly once, left no alarm behind and was not run again, and that the shared metadata.sqlite exists. Signed-off-by: Gogo --- .../server/tests/cluster/cluster-test.mjs | 51 +++++++++++++++++-- .../server/tests/cluster/cluster-worker.js | 20 ++++++-- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index cf1c72499d9..b83c51d90fe 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -9,8 +9,9 @@ // 1. Consistent DO state across instances (single-writer correctness). // 2. Transparent forwarding when a request arrives at the "wrong" instance. // 3. Takeover after a node is killed. -// 4. Alarms are rejected with a clear error in cluster mode. -// 5. No `metadata.sqlite` is ever created (no AlarmScheduler in cluster mode). +// 4. Unix variant: an alarm runs exactly once, wherever the object lives. +// CIDR (NFS) variant: alarms are rejected with a clear error, and +// 5. no `metadata.sqlite` is ever created (no AlarmScheduler across machines). // 6. Storing/loading a DO stub without calling it takes no ownership lock. // 7. A broken DO releases its ownership lock even while stubs still pin it, // so it can be re-instantiated (on any node) instead of hanging requests. @@ -47,6 +48,9 @@ assert( const CONTROL_FD = 3; +// The CIDR variant stands in for a cluster across machines (NFS mode), where alarms stay unsupported. +const NFS_VARIANT = env.WD_TEST_CONFIG.includes('cidr'); + // A minimal harness specialized for the cluster test. Unlike server-harness.mjs // it: // - allows overriding bindings via process environment (NODE_ID), @@ -516,7 +520,7 @@ test('cluster: draining node refuses to claim an unowned DO', async () => { }); }); -test('cluster: alarms are rejected with a clear error', async () => { +test('cluster: alarms are rejected across machines (CIDR/NFS variant)', { skip: !NFS_VARIANT }, async () => { await withCluster(1, async ({ nodes, sharedPath }) => { // Fire the request. In cluster mode `setAlarm()` does not throw // synchronously to the JS code -- the rejection happens during output-gate @@ -566,7 +570,7 @@ test('cluster: alarms are rejected with a clear error', async () => { ); // No per-namespace metadata.sqlite should exist anywhere under the - // shared directory -- in cluster mode the AlarmScheduler is never + // shared directory -- across machines the AlarmScheduler is never // constructed. const nsDir = join(sharedPath, 'counter-test-namespace'); let entries = []; @@ -584,6 +588,45 @@ test('cluster: alarms are rejected with a clear error', async () => { }); }); +test('cluster: an alarm runs once, delivered across instances (unix variant)', { skip: NFS_VARIANT }, async () => { + await withCluster(2, async ({ nodes, sharedPath }) => { + // Set the alarm through node 0 (which then holds the object), read it back + // through node 1 (forwarded to the holder) once it is due. + const set = await fetch( + `http://127.0.0.1:${nodes[0].httpPort}/set-alarm?name=alarm-test&ms=1000` + ); + const setBody = await set.json(); + assert.strictEqual(set.status, 200, JSON.stringify(setBody)); + + const deadline = Date.now() + 15000; + let state; + while (Date.now() < deadline) { + const r = await fetch( + `http://127.0.0.1:${nodes[1].httpPort}/alarm-state?name=alarm-test` + ); + state = await r.json(); + if (state.fired.length > 0) break; + await sleep(200); + } + assert.strictEqual(state.fired.length, 1, `alarm should run once: ${JSON.stringify(state)}`); + assert.strictEqual(state.alarm, null, `no alarm left after the run: ${JSON.stringify(state)}`); + + // It must not run a second time (a stale table row is not re-run). + await sleep(3000); + const after = await ( + await fetch(`http://127.0.0.1:${nodes[1].httpPort}/alarm-state?name=alarm-test`) + ).json(); + assert.strictEqual(after.fired.length, 1, `alarm ran again: ${JSON.stringify(after)}`); + + // The shared alarm table exists (one per namespace, shared by all instances). + const entries = await readdir(join(sharedPath, 'counter-test-namespace')); + assert( + entries.some((name) => name === 'metadata.sqlite'), + `expected metadata.sqlite; found: ${entries.join(', ')}` + ); + }); +}); + test('cluster: registry directory is populated for each running instance', async () => { await withCluster(2, async ({ nodes, sharedPath }) => { // Give the instances a brief moment to settle. Both should have written diff --git a/src/workerd/server/tests/cluster/cluster-worker.js b/src/workerd/server/tests/cluster/cluster-worker.js index 4a6d9dcbed2..184f62883fa 100644 --- a/src/workerd/server/tests/cluster/cluster-worker.js +++ b/src/workerd/server/tests/cluster/cluster-worker.js @@ -226,10 +226,11 @@ export class Counter extends DurableObject { id: idHex, }); } else if (url.pathname === '/set-alarm') { - // Try to schedule an alarm 60s in the future. In cluster mode this should - // throw a clear error. + // Schedule an alarm `ms` milliseconds from now (default 60s). Across + // machines (the CIDR/NFS variant) this should throw a clear error. try { - await this.state.storage.setAlarm(Date.now() + 60_000); + const ms = Number(url.searchParams.get('ms') ?? 60_000); + await this.state.storage.setAlarm(Date.now() + ms); return Response.json({ ok: true, nodeId, id: idHex }); } catch (err) { return Response.json( @@ -237,6 +238,13 @@ export class Counter extends DurableObject { { status: 500 } ); } + } else if (url.pathname === '/alarm-state') { + return Response.json({ + nodeId, + id: idHex, + alarm: await this.state.storage.getAlarm(), + fired: (await this.state.storage.get('alarm-fired-on')) ?? [], + }); } else if (url.pathname === '/identity') { return Response.json({ nodeId, id: idHex }); } else if (url.pathname === '/store-stub') { @@ -343,8 +351,10 @@ export class Counter extends DurableObject { } async alarm() { - // Should never be called in cluster mode; if it ever is, surface that via - // a stored marker so the test can observe the failure mode. + // Record which instance ran it (the unix variant checks it ran exactly once). + const fired = (await this.state.storage.get('alarm-fired-on')) ?? []; + fired.push(this.env.NODE_ID ?? null); + await this.state.storage.put('alarm-fired-on', fired); await this.state.storage.put('alarm-fired', true); } From 8c995ca2cd278ec87b7d3544c73af7ea0cb8447f Mon Sep 17 00:00:00 2001 From: Gogo Date: Tue, 3 Nov 2026 11:10:03 +0000 Subject: [PATCH 32/33] test(botiverse): cluster-mode alarm cases (fresh cluster per case, exit code) Ten cases on three local processes, including the review's missing ones: setAlarm inside alarm(), the holder killed in the middle of the handler, the leader killed during a delivery, 300 alarms due at once, and 600 concurrent setAlarm calls across all processes (no SQLITE_BUSY). Signed-off-by: Gogo --- botiverse/alarm-test/README.md | 29 ++++++ botiverse/alarm-test/cases.mjs | 164 ++++++++++++++++++++++++++++++ botiverse/alarm-test/config.capnp | 16 +++ botiverse/alarm-test/worker.js | 38 +++++++ 4 files changed, 247 insertions(+) create mode 100644 botiverse/alarm-test/README.md create mode 100644 botiverse/alarm-test/cases.mjs create mode 100644 botiverse/alarm-test/config.capnp create mode 100644 botiverse/alarm-test/worker.js diff --git a/botiverse/alarm-test/README.md b/botiverse/alarm-test/README.md new file mode 100644 index 00000000000..45a26374773 --- /dev/null +++ b/botiverse/alarm-test/README.md @@ -0,0 +1,29 @@ +# Cluster-mode alarm cases (Botiverse) + +Alarm semantics with several cluster-mode workerd processes on one machine. Each case starts its own +fresh cluster of 3 processes (ports 8900-8902, a new shared directory under `/tmp`), and the script +exits 1 if any case fails. + +```sh +bazel build //src/workerd/server:workerd --config=release +node botiverse/alarm-test/cases.mjs # all cases +node botiverse/alarm-test/cases.mjs "leader" # cases whose name contains "leader" +WORKERD=/path/to/workerd node botiverse/alarm-test/cases.mjs +``` + +The cases: + +- the alarm time moved several times: it runs once, at the latest time; +- the handler throws twice: it is retried with backoff and succeeds on the third run; +- the handler sets its next alarm itself (`setAlarm()` inside `alarm()`), twice: it runs three times, and no alarm is left; +- `deleteAlarm()` before it is due: it never runs; +- the process holding the object is killed before the alarm is due: the leader claims the object and runs it once; +- the holder is killed in the middle of the handler: the alarm is re-run elsewhere and recorded once; +- the alarm leader is killed while it waits for a running handler: the alarm runs once and a new leader takes over; +- the leader is killed with an alarm pending: another process takes over and runs it once; +- 300 alarms due at the same moment: each runs exactly once; +- all processes set 600 alarms at once: no `SQLITE_BUSY`, all run once. + +Needs Node 22+. Linux only (cluster mode uses OFD locks). Upstream's own cluster test +(`src/workerd/server/tests/cluster`, `cluster-test-unix` / `cluster-test-cidr`) covers the +`network = "unix"` alarm path and the rejection across machines. diff --git a/botiverse/alarm-test/cases.mjs b/botiverse/alarm-test/cases.mjs new file mode 100644 index 00000000000..fb98d344baa --- /dev/null +++ b/botiverse/alarm-test/cases.mjs @@ -0,0 +1,164 @@ +#!/usr/bin/env node +// Cluster-mode alarm cases on one machine. Each case starts its own fresh cluster of local processes +// (ports 8900+, shared directory under /tmp), so cases don't share state. Exit code 1 if any case fails. +// node botiverse/alarm-test/cases.mjs [case ...] WORKERD= to use another build +import { spawn } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const WORKERD = process.env.WORKERD ?? join(here, "../../bazel-bin/src/workerd/server/workerd"); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +async function cluster(n) { + const dir = mkdtempSync(join(tmpdir(), "alarm-test-")); + const nodes = []; + for (let i = 0; i < n; i++) { + const log = []; + const child = spawn(WORKERD, ["serve", "--experimental", "--verbose", join(here, "config.capnp"), + `--directory-path=do-storage=${dir}`, `--socket-addr=http=127.0.0.1:${8900 + i}`], + { env: { ...process.env, NODE: `n${i}` }, stdio: ["ignore", "ignore", "pipe"] }); + child.stderr.on("data", (d) => log.push(String(d))); + nodes.push({ i, port: 8900 + i, child, log, alive: true }); + } + // up and listening + for (const nd of nodes) for (let t = 0; t < 50; t++) { try { await fetch(`http://127.0.0.1:${nd.port}/state?do=__ping`); break; } catch { await sleep(100); } } + await sleep(1500); // a leader is elected + const api = { + nodes, dir, + // Retries: right after a holder dies, the first request routed to its objects fails while the + // cluster notices (upstream behaviour, also retried by upstream's takeover test). + get: async (nd, path) => { + for (let t = 0; ; t++) { + try { const r = await fetch(`http://127.0.0.1:${nd.port}/${path}`); if (r.status < 500 || t >= 10) return await r.json(); } + catch (e) { if (t >= 10) throw e; } + await sleep(300); + } + }, + kill: (nd) => { nd.child.kill("SIGKILL"); nd.alive = false; }, + log: () => nodes.map((nd) => `--- n${nd.i}\n` + nd.log.join("").split("\n").filter((l) => /warning|error|leader/i.test(l) && !/symbolizer/.test(l)).slice(-8).join("\n")).join("\n"), + stop: () => { for (const nd of nodes) if (nd.alive) nd.child.kill("SIGKILL"); rmSync(dir, { recursive: true, force: true }); }, + }; + return api; +} +// The process that most recently logged becoming leader and is still alive. +function leaderOf(c) { + let best = null, bestAt = -1; + for (const nd of c.nodes) { + if (!nd.alive) continue; + const text = nd.log.join(""), at = text.lastIndexOf("now the alarm leader"); + if (at >= 0 && (best === null || at > bestAt)) { best = nd; bestAt = at; } + } + return best; +} +const others = (c, ex) => c.nodes.filter((nd) => nd.alive && nd !== ex); +async function waitFired(c, nd, name, want, ms) { + const end = Date.now() + ms; let s; + while (Date.now() < end) { s = await c.get(nd, `state?do=${name}`); if (s.fired.length >= want) break; await sleep(200); } + return s; +} + +const CASES = { + async "moved several times: runs once, at the latest time"(c) { + const L = leaderOf(c), [a, b] = others(c, L); + await c.get(a, "set?do=m&ms=10000"); await c.get(b, "set?do=m&ms=1000"); const last = await c.get(a, "set?do=m&ms=3000"); + await sleep(1800); + const early = await c.get(b, "state?do=m"); + if (early.fired.length) return `ran at a superseded time: ${JSON.stringify(early.fired)}`; + const s = await waitFired(c, b, "m", 1, 6000); await sleep(1500); + const f = (await c.get(b, "state?do=m")).fired; + if (f.length !== 1) return `fired ${f.length} times`; + if (f[0].at < last.set) return `ran before the latest time (${f[0].at} < ${last.set})`; + }, + async "handler throws twice: retried with backoff, then succeeds"(c) { + const L = leaderOf(c), [a] = others(c, L); + await c.get(a, "throw?do=t&n=2"); await c.get(a, "set?do=t&ms=300"); + const s = await waitFired(c, a, "t", 3, 15000); + if (s.fired.length !== 3) return `fired ${s.fired.length} times: ${JSON.stringify(s.fired)}`; + if (!s.fired[1].isRetry || !s.fired[2].isRetry) return `retries not marked: ${JSON.stringify(s.fired)}`; + }, + async "handler re-arms its own alarm (setAlarm inside alarm())"(c) { + const L = leaderOf(c), [a, b] = others(c, L); + await c.get(a, "set?do=r&ms=500&rearm=2"); + const s = await waitFired(c, b, "r", 3, 12000); await sleep(2500); + const f = (await c.get(b, "state?do=r")); + if (f.fired.length !== 3) return `want 3 runs (1 + 2 re-arms), got ${f.fired.length}`; + if (f.alarm !== null) return `alarm still set after the last run: ${f.alarm}`; + }, + async "deleteAlarm before it is due: never runs"(c) { + const L = leaderOf(c), [a, b] = others(c, L); + await c.get(a, "set?do=d&ms=1500"); await c.get(b, "del?do=d"); + await sleep(4000); + const f = (await c.get(a, "state?do=d")).fired; + if (f.length) return `ran after deleteAlarm: ${JSON.stringify(f)}`; + }, + async "holder killed before the alarm is due: leader claims the object, runs once"(c) { + const L = leaderOf(c), [a] = others(c, L); + await c.get(a, "set?do=k&ms=2000"); c.kill(a); + const s = await waitFired(c, L, "k", 1, 15000); await sleep(2500); + const f = (await c.get(L, "state?do=k")).fired; + if (f.length !== 1) return `fired ${f.length} times: ${JSON.stringify(f)}`; + }, + async "holder killed in the middle of the handler: re-run elsewhere, recorded once"(c) { + const L = leaderOf(c), [a] = others(c, L); + await c.get(a, "set?do=h&ms=500&slow=3000"); + await sleep(1500); c.kill(a); // a is inside alarm(), before it records anything + const s = await waitFired(c, L, "h", 1, 20000); await sleep(2500); + const f = (await c.get(L, "state?do=h")).fired; + if (f.length !== 1) return `recorded ${f.length} runs: ${JSON.stringify(f)}`; + }, + async "leader killed while it waits for a running handler: runs once"(c) { + const L = leaderOf(c), [a, b] = others(c, L); + await c.get(a, "set?do=l&ms=500&slow=3000"); + await sleep(1500); c.kill(L); // the delivery is in flight; a keeps running the handler + const s = await waitFired(c, b, "l", 1, 15000); await sleep(3000); + const f = (await c.get(b, "state?do=l")).fired; + if (f.length !== 1) return `ran ${f.length} times: ${JSON.stringify(f)}`; + if (!leaderOf(c)) return "no new leader"; + }, + async "leader killed with an alarm pending: another takes over, runs once"(c) { + const L = leaderOf(c), [a, b] = others(c, L); + await c.get(b, "set?do=p&ms=3000"); c.kill(L); + const s = await waitFired(c, b, "p", 1, 12000); await sleep(2500); + const f = (await c.get(b, "state?do=p")).fired; + if (f.length !== 1) return `fired ${f.length} times`; + }, + async "300 alarms due at the same time: each runs exactly once"(c) { + const live = others(c, null), N = 300; + await Promise.all(Array.from({ length: N }, (_, k) => c.get(live[k % live.length], `set?do=many${k}&ms=2500`))); + await sleep(9000); + const counts = await Promise.all(Array.from({ length: N }, (_, k) => c.get(live[(k + 1) % live.length], `state?do=many${k}`).then((s) => s.fired.length))); + const bad = counts.map((n, k) => [k, n]).filter(([, n]) => n !== 1); + if (bad.length) return `${bad.length}/${N} did not run exactly once (e.g. ${JSON.stringify(bad.slice(0, 5))})`; + }, + async "all processes set 600 alarms at once: no SQLITE_BUSY errors, all run"(c) { + const live = others(c, null), N = 600; + const res = await Promise.all(Array.from({ length: N }, (_, k) => + fetch(`http://127.0.0.1:${live[k % live.length].port}/set?do=busy${k}&ms=3000`).then((r) => r.status))); + const failed = res.filter((s) => s !== 200).length; + await sleep(10000); + const counts = await Promise.all(Array.from({ length: N }, (_, k) => c.get(live[k % live.length], `state?do=busy${k}`).then((s) => s.fired.length))); + const bad = counts.filter((n) => n !== 1).length; + const busy = c.nodes.some((nd) => nd.log.join("").includes("SQLITE_BUSY")); + if (failed || bad || busy) return `setAlarm failed ${failed}/${N}, not run exactly once ${bad}/${N}, SQLITE_BUSY in logs: ${busy}`; + }, +}; + +const only = process.argv.slice(2); +let failures = 0; +for (const [name, fn] of Object.entries(CASES)) { + if (only.length && !only.some((o) => name.includes(o))) continue; + const c = await cluster(3); + let err; + try { + if (!leaderOf(c)) err = "no alarm leader elected"; + else err = await fn(c); + } catch (e) { err = String(e?.stack ?? e); } + if (err) { failures++; console.log(`FAIL ${name}: ${err}\n${c.log()}`); } else console.log(`PASS ${name}`); + c.stop(); + await sleep(500); +} +console.log(failures ? `${failures} failed` : "all passed"); +process.exit(failures ? 1 : 0); diff --git a/botiverse/alarm-test/config.capnp b/botiverse/alarm-test/config.capnp new file mode 100644 index 00000000000..48a1cb173e9 --- /dev/null +++ b/botiverse/alarm-test/config.capnp @@ -0,0 +1,16 @@ +using Workerd = import "/workerd/workerd.capnp"; +const config :Workerd.Config = ( + services = [ + (name = "main", worker = .w), + (name = "do-storage", disk = (path = "do-storage", writable = true)), + ], + sockets = [ (name = "http", address = "127.0.0.1:8900", http = (), service = "main") ], + cluster = (sharedDirectory = "do-storage", network = "unix", key = "alarm-test-only"), +); +const w :Workerd.Worker = ( + modules = [ (name = "worker.js", esModule = embed "worker.js") ], + compatibilityDate = "2026-09-01", + durableObjectNamespaces = [ (className = "A", uniqueKey = "alarm-test-A") ], + durableObjectStorage = (localDisk = "do-storage"), + bindings = [ (name = "A", durableObjectNamespace = "A"), (name = "NODE", fromEnvironment = "NODE") ], +); diff --git a/botiverse/alarm-test/worker.js b/botiverse/alarm-test/worker.js new file mode 100644 index 00000000000..abe1f5c3093 --- /dev/null +++ b/botiverse/alarm-test/worker.js @@ -0,0 +1,38 @@ +// Alarm semantics test for cluster-mode workerd. Every process runs this same config; NODE (from the +// environment) says which process handled what. +import { DurableObject } from "cloudflare:workers"; + +export class A extends DurableObject { + async fetch(req) { + const u = new URL(req.url), s = this.ctx.storage, p = (k) => u.searchParams.get(k); + if (u.pathname === "/set") { + // rearm=N: the handler sets the next alarm itself, N more times; slow=ms: the handler takes that long + // (before it records anything); del=1: the handler deletes its alarm instead of letting it end. + if (p("rearm")) await s.put("rearm", +p("rearm")); + if (p("slow")) await s.put("slow", +p("slow")); + const at = Date.now() + +p("ms"); + await s.setAlarm(at); + return Response.json({ node: this.env.NODE, set: at }); + } + if (u.pathname === "/del") { await s.deleteAlarm(); return Response.json({ node: this.env.NODE, deleted: true }); } + if (u.pathname === "/throw") { await s.put("throws", +p("n")); return Response.json({ node: this.env.NODE }); } + if (u.pathname === "/state") return Response.json({ node: this.env.NODE, alarm: await s.getAlarm(), fired: (await s.get("fired")) ?? [] }); + return new Response("?", { status: 404 }); + } + async alarm(info) { + const s = this.ctx.storage; + const slow = (await s.get("slow")) ?? 0; + if (slow) await new Promise((r) => setTimeout(r, slow)); + const fired = (await s.get("fired")) ?? [], throws = (await s.get("throws")) ?? 0, rearm = (await s.get("rearm")) ?? 0; + fired.push({ node: this.env.NODE, at: Date.now(), retry: info?.retryCount ?? 0, isRetry: !!info?.isRetry }); + await s.put("fired", fired); + if (throws > 0) { await s.put("throws", throws - 1); throw new Error("planned failure"); } + if (rearm > 0) { await s.put("rearm", rearm - 1); await s.setAlarm(Date.now() + 1000); } + } +} +export default { + async fetch(req, env) { + const u = new URL(req.url), name = u.searchParams.get("do") ?? "x"; + return env.A.get(env.A.idFromName(name)).fetch(req); + }, +}; From ae5ca180b850fe35520b083e9fedd134fdd1a9af Mon Sep 17 00:00:00 2001 From: Gogo Date: Tue, 3 Nov 2026 11:22:21 +0000 Subject: [PATCH 33/33] Cluster alarms: format (clang-format, Prettier); yield before dropping an entry after a failed abandon Formatting from tools/cross/format.py, which the lint check asked for. Also yields once before the abandon-failure path moves the task out of its own entry (review suggestion). Signed-off-by: Gogo --- src/workerd/server/alarm-scheduler.c++ | 5 +- src/workerd/server/alarm-scheduler.h | 1 - src/workerd/server/server.c++ | 12 +- .../server/tests/cluster/cluster-test.mjs | 216 ++++++++++-------- 4 files changed, 128 insertions(+), 106 deletions(-) diff --git a/src/workerd/server/alarm-scheduler.c++ b/src/workerd/server/alarm-scheduler.c++ index a7f051eb6ed..0651528244e 100644 --- a/src/workerd/server/alarm-scheduler.c++ +++ b/src/workerd/server/alarm-scheduler.c++ @@ -310,10 +310,13 @@ kj::Promise AlarmScheduler::abandonAlarm(kj::Own actor, kj::Date } if (!notified) { if (clusterMode) { + // Don't move this task out of its own entry from inside the same turn it was resumed in. + co_await kj::yield(); // The row is still in the shared table: forget the entry (a FINISHED entry with no task // would otherwise swallow later times for this actor) and let syncFromDb() deliver it again. KJ_IF_SOME(entry, alarms.findEntry(*actor)) { - if (entry.value.status == AlarmStatus::FINISHED && entry.value.scheduledTime == scheduledTime) { + if (entry.value.status == AlarmStatus::FINISHED && + entry.value.scheduledTime == scheduledTime) { tasks.add(kj::mv(entry.value.task)); alarms.erase(entry); } diff --git a/src/workerd/server/alarm-scheduler.h b/src/workerd/server/alarm-scheduler.h index eeab97dba24..58c0b80676b 100644 --- a/src/workerd/server/alarm-scheduler.h +++ b/src/workerd/server/alarm-scheduler.h @@ -159,7 +159,6 @@ class AlarmScheduler final: kj::TaskSet::ErrorHandler { int maxJitterMsForDelay(kj::Duration delay); - public: static void ensureInitialized(SqliteDatabase& db); diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index c08fb619033..5c143f59f83 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -47,6 +47,7 @@ #include #include +#include #include #include @@ -60,8 +61,6 @@ #include #include -#include - #include #include #include @@ -434,8 +433,8 @@ class Server::ActorNamespace final { clusterAlarmStore = kj::heap(as.vfs); alarmLeaderFile = as.directory->openFile( kj::Path({"alarm-leader"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); - alarmLeaderTask = alarmLeaderLoop(kj::str(d.uniqueKey)) - .eagerlyEvaluate([](kj::Exception&& e) { + alarmLeaderTask = + alarmLeaderLoop(kj::str(d.uniqueKey)).eagerlyEvaluate([](kj::Exception&& e) { KJ_LOG(ERROR, "cluster alarm leader loop failed; alarms stop on this instance", e); }); return; @@ -1374,9 +1373,8 @@ class Server::ActorNamespace final { return n.asPtr(); })}.clone(); sqliteHooks = kj::heap( - ns.clusterAlarmStore.map([](kj::Own& s) -> ClusterAlarmStore& { - return *s; - }), + ns.clusterAlarmStore.map( + [](kj::Own& s) -> ClusterAlarmStore& { return *s; }), kj::mv(actorKey)); } else KJ_IF_SOME(a, ns.alarmScheduler) { // clone() copies the id and name into storage owned by the returned ActorKey, so diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index b83c51d90fe..6f6a21d0933 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -520,112 +520,134 @@ test('cluster: draining node refuses to claim an unowned DO', async () => { }); }); -test('cluster: alarms are rejected across machines (CIDR/NFS variant)', { skip: !NFS_VARIANT }, async () => { - await withCluster(1, async ({ nodes, sharedPath }) => { - // Fire the request. In cluster mode `setAlarm()` does not throw - // synchronously to the JS code -- the rejection happens during output-gate - // flush, after the worker's fetch handler has already returned. The - // observable behaviour is therefore a non-2xx HTTP status, while the - // canonical error message is logged to stderr where the test can detect - // it. (The spec mandates the message "Durable Object alarms are not yet - // supported in cluster mode".) - let response; - try { - response = await fetch( - `http://127.0.0.1:${nodes[0].httpPort}/set-alarm?name=alarm-test` +test( + 'cluster: alarms are rejected across machines (CIDR/NFS variant)', + { skip: !NFS_VARIANT }, + async () => { + await withCluster(1, async ({ nodes, sharedPath }) => { + // Fire the request. In cluster mode `setAlarm()` does not throw + // synchronously to the JS code -- the rejection happens during output-gate + // flush, after the worker's fetch handler has already returned. The + // observable behaviour is therefore a non-2xx HTTP status, while the + // canonical error message is logged to stderr where the test can detect + // it. (The spec mandates the message "Durable Object alarms are not yet + // supported in cluster mode".) + let response; + try { + response = await fetch( + `http://127.0.0.1:${nodes[0].httpPort}/set-alarm?name=alarm-test` + ); + } catch (err) { + // A network-level failure is acceptable too: it confirms that the alarm + // path did not silently succeed. + response = null; + } + if (response !== null) { + // Drain the body to allow logging to flush. + try { + await response.text(); + } catch (_) { + // ignore + } + assert.notStrictEqual( + Math.floor(response.status / 100), + 2, + `expected non-2xx for set-alarm in cluster mode, got ${response.status}` + ); + } + + // Wait briefly for stderr to flush the error message. + const deadline = Date.now() + 5000; + const expectedMessage = + 'Durable Object alarms are not yet supported in cluster mode'; + while ( + Date.now() < deadline && + !nodes[0].stderr.includes(expectedMessage) + ) { + await sleep(50); + } + assert( + nodes[0].stderr.includes(expectedMessage), + `expected stderr to contain the cluster alarm rejection message ` + + `("${expectedMessage}"); stderr so far:\n${nodes[0].stderr}` ); - } catch (err) { - // A network-level failure is acceptable too: it confirms that the alarm - // path did not silently succeed. - response = null; - } - if (response !== null) { - // Drain the body to allow logging to flush. + + // No per-namespace metadata.sqlite should exist anywhere under the + // shared directory -- across machines the AlarmScheduler is never + // constructed. + const nsDir = join(sharedPath, 'counter-test-namespace'); + let entries = []; try { - await response.text(); - } catch (_) { - // ignore + entries = await readdir(nsDir); + } catch (err) { + // Namespace dir might not exist if the DO was never created on disk + // because the request failed before storage was opened. That is a + // valid (and even stronger) signal that no metadata.sqlite was made. } - assert.notStrictEqual( - Math.floor(response.status / 100), - 2, - `expected non-2xx for set-alarm in cluster mode, got ${response.status}` + assert( + !entries.some((name) => name.startsWith('metadata.sqlite')), + `cluster mode must not create metadata.sqlite; found: ${entries.join(', ')}` ); - } + }); + } +); - // Wait briefly for stderr to flush the error message. - const deadline = Date.now() + 5000; - const expectedMessage = - 'Durable Object alarms are not yet supported in cluster mode'; - while ( - Date.now() < deadline && - !nodes[0].stderr.includes(expectedMessage) - ) { - await sleep(50); - } - assert( - nodes[0].stderr.includes(expectedMessage), - `expected stderr to contain the cluster alarm rejection message ` + - `("${expectedMessage}"); stderr so far:\n${nodes[0].stderr}` - ); +test( + 'cluster: an alarm runs once, delivered across instances (unix variant)', + { skip: NFS_VARIANT }, + async () => { + await withCluster(2, async ({ nodes, sharedPath }) => { + // Set the alarm through node 0 (which then holds the object), read it back + // through node 1 (forwarded to the holder) once it is due. + const set = await fetch( + `http://127.0.0.1:${nodes[0].httpPort}/set-alarm?name=alarm-test&ms=1000` + ); + const setBody = await set.json(); + assert.strictEqual(set.status, 200, JSON.stringify(setBody)); - // No per-namespace metadata.sqlite should exist anywhere under the - // shared directory -- across machines the AlarmScheduler is never - // constructed. - const nsDir = join(sharedPath, 'counter-test-namespace'); - let entries = []; - try { - entries = await readdir(nsDir); - } catch (err) { - // Namespace dir might not exist if the DO was never created on disk - // because the request failed before storage was opened. That is a - // valid (and even stronger) signal that no metadata.sqlite was made. - } - assert( - !entries.some((name) => name.startsWith('metadata.sqlite')), - `cluster mode must not create metadata.sqlite; found: ${entries.join(', ')}` - ); - }); -}); + const deadline = Date.now() + 15000; + let state; + while (Date.now() < deadline) { + const r = await fetch( + `http://127.0.0.1:${nodes[1].httpPort}/alarm-state?name=alarm-test` + ); + state = await r.json(); + if (state.fired.length > 0) break; + await sleep(200); + } + assert.strictEqual( + state.fired.length, + 1, + `alarm should run once: ${JSON.stringify(state)}` + ); + assert.strictEqual( + state.alarm, + null, + `no alarm left after the run: ${JSON.stringify(state)}` + ); -test('cluster: an alarm runs once, delivered across instances (unix variant)', { skip: NFS_VARIANT }, async () => { - await withCluster(2, async ({ nodes, sharedPath }) => { - // Set the alarm through node 0 (which then holds the object), read it back - // through node 1 (forwarded to the holder) once it is due. - const set = await fetch( - `http://127.0.0.1:${nodes[0].httpPort}/set-alarm?name=alarm-test&ms=1000` - ); - const setBody = await set.json(); - assert.strictEqual(set.status, 200, JSON.stringify(setBody)); + // It must not run a second time (a stale table row is not re-run). + await sleep(3000); + const after = await ( + await fetch( + `http://127.0.0.1:${nodes[1].httpPort}/alarm-state?name=alarm-test` + ) + ).json(); + assert.strictEqual( + after.fired.length, + 1, + `alarm ran again: ${JSON.stringify(after)}` + ); - const deadline = Date.now() + 15000; - let state; - while (Date.now() < deadline) { - const r = await fetch( - `http://127.0.0.1:${nodes[1].httpPort}/alarm-state?name=alarm-test` + // The shared alarm table exists (one per namespace, shared by all instances). + const entries = await readdir(join(sharedPath, 'counter-test-namespace')); + assert( + entries.some((name) => name === 'metadata.sqlite'), + `expected metadata.sqlite; found: ${entries.join(', ')}` ); - state = await r.json(); - if (state.fired.length > 0) break; - await sleep(200); - } - assert.strictEqual(state.fired.length, 1, `alarm should run once: ${JSON.stringify(state)}`); - assert.strictEqual(state.alarm, null, `no alarm left after the run: ${JSON.stringify(state)}`); - - // It must not run a second time (a stale table row is not re-run). - await sleep(3000); - const after = await ( - await fetch(`http://127.0.0.1:${nodes[1].httpPort}/alarm-state?name=alarm-test`) - ).json(); - assert.strictEqual(after.fired.length, 1, `alarm ran again: ${JSON.stringify(after)}`); - - // The shared alarm table exists (one per namespace, shared by all instances). - const entries = await readdir(join(sharedPath, 'counter-test-namespace')); - assert( - entries.some((name) => name === 'metadata.sqlite'), - `expected metadata.sqlite; found: ${entries.join(', ')}` - ); - }); -}); + }); + } +); test('cluster: registry directory is populated for each running instance', async () => { await withCluster(2, async ({ nodes, sharedPath }) => {