From e54554bb68915a3e983fd66e4db7e01079c52008 Mon Sep 17 00:00:00 2001 From: bvolpato Date: Wed, 28 Oct 2026 13:16:08 -0400 Subject: [PATCH] io_uring: add an opt-in for NO_IOWAIT accounting --- src/backend/io_uring.zig | 90 +++++++++++++++++++++++++++++++++++++++- src/loop.zig | 7 ++++ 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/src/backend/io_uring.zig b/src/backend/io_uring.zig index b420971b..7d2a7e44 100644 --- a/src/backend/io_uring.zig +++ b/src/backend/io_uring.zig @@ -12,6 +12,10 @@ const CallbackAction = looppkg.CallbackAction; const CompletionState = looppkg.CompletionState; const noopCallback = looppkg.NoopCallback(@This()); +// These Linux 6.15 constants are not yet defined by Zig's standard library. +const IORING_ENTER_NO_IOWAIT: u32 = 1 << 7; +const IORING_FEAT_NO_IOWAIT: u32 = 1 << 17; + pub const ShutdownHow = std.Io.net.ShutdownHow; /// True if this backend is available on this platform. @@ -55,6 +59,8 @@ pub const Loop = struct { /// Whether we're in a run or not (to prevent nested runs). in_run: bool = false, + + no_iowait: bool = false, } = .{}, /// Initialize the event loop. "entries" is the maximum number of @@ -70,6 +76,7 @@ pub const Loop = struct { // for people using the io_uring API directly to be able to set // the flags for this. .ring = try linux.IoUring.init(entries, 0), + .flags = .{ .no_iowait = options.no_iowait }, }; result.update_now(); @@ -169,7 +176,7 @@ pub const Loop = struct { // of the submit call, because then we can do exactly once syscall // to get all our events. if (self.submissions.empty()) { - _ = self.ring.submit_and_wait(wait) catch |err| switch (err) { + _ = self.submitAndWait(wait) catch |err| switch (err) { error.SignalInterrupt => continue, else => return err, }; @@ -183,7 +190,7 @@ pub const Loop = struct { } // Wait for completions... - const count = self.ring.copy_cqes(&cqes, wait) catch |err| switch (err) { + const count = self.copyCqes(&cqes, wait) catch |err| switch (err) { // EINTR means our blocking syscall was interrupted by some // process signal. We just retry when we can. See signal(7). error.SignalInterrupt => continue, @@ -209,6 +216,30 @@ pub const Loop = struct { } } + fn noIowait(self: *const Loop, wait: u32) bool { + return wait > 0 and self.flags.no_iowait and + self.ring.features & IORING_FEAT_NO_IOWAIT != 0; + } + + fn submitAndWait(self: *Loop, wait: u32) !u32 { + if (!self.noIowait(wait)) return self.ring.submit_and_wait(wait); + + const submitted = self.ring.flush_sq(); + var flags: u32 = 0; + _ = self.ring.sq_ring_needs_enter(&flags); + return self.ring.enter(submitted, wait, flags | + linux.IORING_ENTER_GETEVENTS | IORING_ENTER_NO_IOWAIT); + } + + fn copyCqes(self: *Loop, cqes: []linux.io_uring_cqe, wait: u32) !u32 { + if (!self.noIowait(wait)) return self.ring.copy_cqes(cqes, wait); + + const count = try self.ring.copy_cqes(cqes, 0); + if (count > 0) return count; + _ = try self.ring.enter(0, wait, linux.IORING_ENTER_GETEVENTS | IORING_ENTER_NO_IOWAIT); + return self.ring.copy_cqes(cqes, 0); + } + /// The errors that can come from submit. This has to be explicit /// for now since Zig 0.11 has issues with the inferred error set. /// We should try again someday. @@ -1232,6 +1263,61 @@ test "io_uring: timerfd" { try testing.expect(c.state() == .dead); } +test "io_uring: no_iowait feature gate and completion waits" { + const testing = std.testing; + const Timerfd = @import("../linux/timerfd.zig").Timerfd; + const defaults: looppkg.Options = .{}; + try testing.expect(!defaults.no_iowait); + + for ([_]bool{ false, true }) |requested| { + for ([_]bool{ false, true }) |without_support| { + // One SQ entry exercises the completion wait after the overflow + // queue is submitted, rather than the combined submit-and-wait. + for ([_]u32{ 1, 8 }) |entries| { + var loop = try Loop.init(.{ + .entries = entries, + .no_iowait = requested, + }); + defer loop.deinit(); + if (without_support) loop.ring.features &= ~IORING_FEAT_NO_IOWAIT; + const supported = loop.ring.features & IORING_FEAT_NO_IOWAIT != 0; + try testing.expectEqual(requested and supported, loop.noIowait(1)); + try testing.expect(!loop.noIowait(0)); + + var timer = try Timerfd.init(.MONOTONIC, .{}); + defer timer.deinit(); + var called: usize = 0; + var first: Completion = .{ + .op = .{ .poll = .{ .fd = timer.fd } }, + .userdata = &called, + .callback = (struct { + fn callback(ud: ?*anyopaque, _: *Loop, _: *Completion, r: Result) CallbackAction { + _ = r.poll catch unreachable; + const count: *usize = @ptrCast(@alignCast(ud.?)); + count.* += 1; + return .disarm; + } + }).callback, + }; + var second = first; + loop.add(&first); + if (entries == 1) { + loop.add(&second); + } else { + try loop.run(.no_wait); + try testing.expectEqual(@as(usize, 0), called); + } + + try timer.set(.{}, &.{ .value = .{ .nanoseconds = std.time.ns_per_ms } }, null); + try loop.run(.until_done); + try testing.expectEqual(@as(usize, if (entries == 1) 2 else 1), called); + try testing.expect(first.state() == .dead); + if (entries == 1) try testing.expect(second.state() == .dead); + } + } + } +} + test "io_uring: timer" { const testing = std.testing; diff --git a/src/loop.zig b/src/loop.zig index 5f809982..983c21a7 100644 --- a/src/loop.zig +++ b/src/loop.zig @@ -15,6 +15,13 @@ pub const Options = struct { /// Backends: io_uring entries: u32 = 256, + /// Avoid accounting for blocking waits as I/O wait. This can be useful for + /// loops waiting on non-storage events such as sockets and timers. Ignored + /// on kernels without IORING_FEAT_NO_IOWAIT, introduced in Linux 6.15. + /// + /// Backends: io_uring + no_iowait: bool = false, + /// A thread pool to use for blocking operations. If the backend doesn't /// need to perform any blocking operations then no threads will ever /// be spawned. If the backend does need to perform blocking operations