diff --git a/context/getting-started.md b/context/getting-started.md index 34581c2..0bfe9f7 100644 --- a/context/getting-started.md +++ b/context/getting-started.md @@ -21,9 +21,12 @@ require "fantail" require "io/endpoint" Sync do + configuration = Fantail::Configuration.load("config/fantail.rb") + server = Fantail::Server.new( Async::HTTP::Endpoint.parse("http://0.0.0.0:9292"), IO::Endpoint.tcp("0.0.0.0", 9293), + configuration: configuration, ) server.run.wait @@ -54,6 +57,53 @@ After connecting, the monitor performs a complete replacement. It then publishes ## Admission Semantics -Each backend has one request-processing slot and a configurable number of response exchanges. The processing slot is released as soon as upstream response headers arrive. The exchange remains reserved until the response body closes. +Each backend has a configurable number of request-processing permits and response exchanges. A processing permit is released as soon as upstream response headers arrive. The exchange remains reserved until the response body closes. This allows a worker to begin another request while an earlier response streams, without allowing an unbounded number of streaming responses to accumulate. + +The scheduler owns all permits. Request queues can decide which workers are eligible and express a soft preference between them, but cannot reserve capacity independently. If the preferred worker is unavailable, the scheduler remains work-conserving and uses another eligible worker. + +## Request Queues + +Fantail configuration is trusted application Ruby. The file's final expression must be an immutable `Fantail::Configuration`: + +~~~ ruby +# config/fantail.rb +Fantail::Configuration.define do |config| + config.queue :liquid do |queue| + queue.match{|request| request.path.start_with?("/render")} + queue.balance :spread + queue.depth_limit 500 + queue.wait_limit 0.25 + queue.shed status: 429, retry_after: 1 + end + + config.queue :grpc do |queue| + queue.match do |request| + request.headers["content-type"]&.start_with?("application/grpc") + end + + queue.balance :pack, affinity: :grpc + end + + config.default_queue :liquid + config.pending_limit 1_000 + config.permit_limit 1 +end +~~~ + +Matchers are evaluated in definition order, followed by the default queue. Across queues, the oldest eligible head request is dispatched first. If that request has no eligible worker, another queue can use the available permit. + +The built-in `:spread` policy prefers the least-active worker. The `:pack` policy prefers a worker already processing the specified affinity, while remaining bounded by its permits. An application can supply a policy object implementing `select(backends, queue:, request:)`, and can restrict hard eligibility with `queue.eligible`. + +## Load Shedding + +`depth_limit` bounds requests actually waiting in a queue; immediately dispatchable requests do not count against it. `pending_limit` provides a global bound across all queues. `wait_limit` bounds actual queue residence time in seconds. Rejected requests use the response configured by `shed`, which defaults to HTTP 429. + +Applications can add an admission policy with either a block or an object implementing `admit?(request, queue:, pending:)`: + +~~~ ruby +queue.admit do |request, queue:, pending:| + pending < application_limit_for(queue.name) +end +~~~ diff --git a/guides/getting-started/readme.md b/guides/getting-started/readme.md index 34581c2..0bfe9f7 100644 --- a/guides/getting-started/readme.md +++ b/guides/getting-started/readme.md @@ -21,9 +21,12 @@ require "fantail" require "io/endpoint" Sync do + configuration = Fantail::Configuration.load("config/fantail.rb") + server = Fantail::Server.new( Async::HTTP::Endpoint.parse("http://0.0.0.0:9292"), IO::Endpoint.tcp("0.0.0.0", 9293), + configuration: configuration, ) server.run.wait @@ -54,6 +57,53 @@ After connecting, the monitor performs a complete replacement. It then publishes ## Admission Semantics -Each backend has one request-processing slot and a configurable number of response exchanges. The processing slot is released as soon as upstream response headers arrive. The exchange remains reserved until the response body closes. +Each backend has a configurable number of request-processing permits and response exchanges. A processing permit is released as soon as upstream response headers arrive. The exchange remains reserved until the response body closes. This allows a worker to begin another request while an earlier response streams, without allowing an unbounded number of streaming responses to accumulate. + +The scheduler owns all permits. Request queues can decide which workers are eligible and express a soft preference between them, but cannot reserve capacity independently. If the preferred worker is unavailable, the scheduler remains work-conserving and uses another eligible worker. + +## Request Queues + +Fantail configuration is trusted application Ruby. The file's final expression must be an immutable `Fantail::Configuration`: + +~~~ ruby +# config/fantail.rb +Fantail::Configuration.define do |config| + config.queue :liquid do |queue| + queue.match{|request| request.path.start_with?("/render")} + queue.balance :spread + queue.depth_limit 500 + queue.wait_limit 0.25 + queue.shed status: 429, retry_after: 1 + end + + config.queue :grpc do |queue| + queue.match do |request| + request.headers["content-type"]&.start_with?("application/grpc") + end + + queue.balance :pack, affinity: :grpc + end + + config.default_queue :liquid + config.pending_limit 1_000 + config.permit_limit 1 +end +~~~ + +Matchers are evaluated in definition order, followed by the default queue. Across queues, the oldest eligible head request is dispatched first. If that request has no eligible worker, another queue can use the available permit. + +The built-in `:spread` policy prefers the least-active worker. The `:pack` policy prefers a worker already processing the specified affinity, while remaining bounded by its permits. An application can supply a policy object implementing `select(backends, queue:, request:)`, and can restrict hard eligibility with `queue.eligible`. + +## Load Shedding + +`depth_limit` bounds requests actually waiting in a queue; immediately dispatchable requests do not count against it. `pending_limit` provides a global bound across all queues. `wait_limit` bounds actual queue residence time in seconds. Rejected requests use the response configured by `shed`, which defaults to HTTP 429. + +Applications can add an admission policy with either a block or an object implementing `admit?(request, queue:, pending:)`: + +~~~ ruby +queue.admit do |request, queue:, pending:| + pending < application_limit_for(queue.name) +end +~~~ diff --git a/lib/fantail.rb b/lib/fantail.rb index 578f0ee..5becbb1 100644 --- a/lib/fantail.rb +++ b/lib/fantail.rb @@ -4,10 +4,13 @@ # Copyright, 2026, by Samuel Williams. require_relative "fantail/version" +require_relative "fantail/balance" +require_relative "fantail/configuration" require_relative "fantail/endpoint" require_relative "fantail/backend" require_relative "fantail/response_body" require_relative "fantail/registry" +require_relative "fantail/scheduler" require_relative "fantail/proxy" require_relative "fantail/control" require_relative "fantail/monitor" diff --git a/lib/fantail/backend.rb b/lib/fantail/backend.rb index c783043..f280044 100644 --- a/lib/fantail/backend.rb +++ b/lib/fantail/backend.rb @@ -10,20 +10,23 @@ class Backend # @parameter endpoint [Endpoint] The endpoint served by this backend. # @parameter client [Interface(:call, :close)] The HTTP client for the endpoint. # @parameter exchange_limit [Integer] The maximum number of outstanding response exchanges. + # @parameter permit_limit [Integer] The maximum number of concurrent processing permits. # @yields {|backend| ...} Invoked when the backend can accept another request. - def initialize(endpoint, client, exchange_limit:, &available) + def initialize(endpoint, client, exchange_limit:, permit_limit: 1, &available) raise ArgumentError, "Exchange limit must be positive!" unless exchange_limit.positive? + raise ArgumentError, "Permit limit must be positive!" unless permit_limit.positive? @endpoint = endpoint @client = client @exchange_limit = exchange_limit + @permit_limit = permit_limit @available = available @guard = Thread::Mutex.new @active = true - @processing = false + @processing = 0 + @processing_by_queue = Hash.new(0) @exchanges = 0 - @queued = false @closed = false end @@ -45,12 +48,11 @@ def start # Reserve the processing slot and one response exchange. # @returns [Boolean] Whether the backend was successfully reserved. - def reserve + def reserve(queue_name = :default) @guard.synchronize do - @queued = false - - if @active && !@processing && @exchanges < @exchange_limit - @processing = true + if @active && @processing < @permit_limit && @exchanges < @exchange_limit + @processing += 1 + @processing_by_queue[queue_name] += 1 @exchanges += 1 return true end @@ -67,20 +69,18 @@ def call(request) end # Release the request-processing slot after response headers arrive. - def processed + def processed(queue_name = :default) @guard.synchronize do - raise RuntimeError, "Backend is not processing a request!" unless @processing - @processing = false + release_processing(queue_name) end notify_available end # Release both reservations when a request fails before response headers. - def failed + def failed(queue_name = :default) close = @guard.synchronize do - raise RuntimeError, "Backend is not processing a request!" unless @processing - @processing = false + release_processing(queue_name) @exchanges -= 1 should_close? end @@ -123,26 +123,40 @@ def exchanges # @returns [Boolean] Whether a request is waiting for response headers. def processing? + @guard.synchronize{@processing.positive?} + end + + # @returns [Integer] The number of active processing permits. + def processing @guard.synchronize{@processing} end + # @returns [Integer] The number of active permits for the given queue affinity. + def processing_for(queue_name) + @guard.synchronize{@processing_by_queue[queue_name]} + end + + # @returns [Boolean] Whether another request can be admitted. + def available? + @guard.synchronize{@active && @processing < @permit_limit && @exchanges < @exchange_limit} + end + protected def notify_available - notify = @guard.synchronize do - if @active && !@processing && @exchanges < @exchange_limit && !@queued - @queued = true - true - else - false - end - end - - @available.call(self) if notify + @available.call(self) if available? end def should_close? - !@active && !@processing && @exchanges.zero? && !@closed + !@active && @processing.zero? && @exchanges.zero? && !@closed + end + + def release_processing(queue_name) + raise RuntimeError, "Backend is not processing a request!" unless @processing.positive? + raise RuntimeError, "Backend is not processing queue #{queue_name.inspect}!" unless @processing_by_queue[queue_name].positive? + + @processing -= 1 + @processing_by_queue[queue_name] -= 1 end def close_client diff --git a/lib/fantail/balance.rb b/lib/fantail/balance.rb new file mode 100644 index 0000000..8637aaf --- /dev/null +++ b/lib/fantail/balance.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2026, by Samuel Williams. + +module Fantail + # Built-in backend selection policies. + module Balance + # Prefer the backend with the fewest active requests. + class Spread + # Select the least-active backend, using its name for deterministic ties. + # @parameter backends [Array(Backend)] Eligible backends with available permits. + # @parameter queue [Configuration::Queue] The request queue being scheduled. + # @parameter request [Protocol::HTTP::Request] The pending request. + # @returns [Backend | Nil] The preferred backend. + def select(backends, queue:, request:) + backends.min_by{|backend| [backend.processing, backend.name]} + end + end + + # Prefer a backend which is already processing the same class of work. + class Pack + # @parameter affinity [Symbol | Nil] The queue affinity to pack, or the current queue by default. + def initialize(affinity: nil) + @affinity = affinity + end + + # Select the backend with the most active work for the affinity. + # @parameter backends [Array(Backend)] Eligible backends with available permits. + # @parameter queue [Configuration::Queue] The request queue being scheduled. + # @parameter request [Protocol::HTTP::Request] The pending request. + # @returns [Backend | Nil] The preferred backend. + def select(backends, queue:, request:) + affinity = @affinity || queue.name + backends.min_by do |backend| + [-backend.processing_for(affinity), backend.processing, backend.name] + end + end + end + + # Resolve a built-in policy name or validate an application policy object. + # @parameter policy [Symbol | #select] The policy name or object. + # @parameter options [Hash] Options for a built-in policy. + # @returns [#select] The resolved balance policy. + def self.coerce(policy, **options) + case policy + when :spread + Spread.new(**options) + when :pack + Pack.new(**options) + else + unless policy.respond_to?(:select) + raise ArgumentError, "Balance policy must respond to #select!" + end + + policy + end + end + end +end diff --git a/lib/fantail/configuration.rb b/lib/fantail/configuration.rb new file mode 100644 index 0000000..4db001c --- /dev/null +++ b/lib/fantail/configuration.rb @@ -0,0 +1,223 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2026, by Samuel Williams. + +require_relative "balance" + +module Fantail + # Immutable request queue and admission configuration. + class Configuration + UNDEFINED = Object.new.freeze + + # Configuration for one class of requests. + class Queue + # @parameter name [Symbol | String] The stable queue name. + def initialize(name) + @name = name.to_sym + @matcher = nil + @eligibility = nil + @admission = nil + @balance = Balance::Spread.new + @depth_limit = nil + @wait_limit = nil + @shed_status = 429 + @shed_headers = {} + end + + attr :name + attr :balance_policy + attr :shed_status + attr :shed_headers + + # Set the request classifier for this queue. + # @yields {|request| ...} Whether a request belongs to this queue. + def match(&block) + raise ArgumentError, "A matcher block is required!" unless block + @matcher = block + end + + # Restrict the backends which may serve this queue. + # @yields {|backend, request| ...} Whether the backend is eligible. + def eligible(&block) + raise ArgumentError, "An eligibility block is required!" unless block + @eligibility = block + end + + # Set an application admission policy. + # @parameter policy [#admit? | #call | Nil] The admission policy object. + # @yields {|request, queue:, pending:| ...} Whether the request can wait. + def admit(policy = nil, &block) + @admission = policy || block + raise ArgumentError, "An admission policy is required!" unless @admission + end + + # Set the soft backend balance policy. + # @parameter policy [Symbol | #select] A built-in name or application policy. + # @parameter options [Hash] Options for a built-in policy. + def balance(policy, **options) + @balance = Balance.coerce(policy, **options) + end + + # Set or get the maximum number of requests waiting in this queue. + # @parameter value [Integer] The new limit when given. + # @returns [Integer | Nil] The configured limit. + def depth_limit(value = UNDEFINED) + return @depth_limit if value.equal?(UNDEFINED) + + value = Integer(value) + raise ArgumentError, "Depth limit must not be negative!" if value.negative? + @depth_limit = value + end + + # Set or get the maximum time a request may wait for a permit. + # @parameter value [Numeric] The new limit in seconds when given. + # @returns [Float | Nil] The configured limit. + def wait_limit(value = UNDEFINED) + return @wait_limit if value.equal?(UNDEFINED) + + value = Float(value) + raise ArgumentError, "Wait limit must be positive!" unless value.positive? + @wait_limit = value + end + + # Configure the response used when admission is rejected. + # @parameter status [Integer] The HTTP response status. + # @parameter retry_after [Numeric | String | Nil] An optional Retry-After value. + # @parameter headers [Hash] Additional response headers. + def shed(status: 429, retry_after: nil, headers: {}) + @shed_status = Integer(status) + @shed_headers = headers.transform_keys(&:to_s) + @shed_headers["retry-after"] = retry_after.to_s if retry_after + end + + # @parameter request [Protocol::HTTP::Request] The request to classify. + # @returns [Boolean | Nil] Whether the request matches this queue. + def match?(request) + @matcher&.call(request) + end + + # @returns [Boolean] Whether a backend may serve the request. + def eligible?(backend, request) + !@eligibility || @eligibility.call(backend, request) + end + + # @returns [Boolean] Whether a request may enter the pending queue. + def admit?(request, pending:) + return true unless @admission + + if @admission.respond_to?(:admit?) + @admission.admit?(request, queue: self, pending: pending) + else + @admission.call(request, queue: self, pending: pending) + end + end + + # Validate and freeze this queue definition. + # @returns [Queue] The finalized queue. + def finalize + @balance_policy = @balance + @shed_headers.freeze + freeze + end + end + + # Build and finalize a configuration. + # @yields {|configuration| ...} The mutable configuration builder. + # @returns [Configuration] The immutable configuration. + def self.define + configuration = new + yield configuration if block_given? + configuration.finalize + end + + # @returns [Configuration] A single-queue, single-permit configuration. + def self.default + @default ||= define do |configuration| + configuration.queue(:default) + configuration.default_queue(:default) + end + end + + # Load trusted application configuration. The final expression must be a Configuration. + def self.load(path) + path = File.expand_path(path) + configuration = TOPLEVEL_BINDING.eval(File.read(path), path) + + unless configuration.is_a?(self) + raise TypeError, "#{path} must return a Fantail::Configuration!" + end + + configuration + end + + # Initialize an empty mutable configuration builder. + def initialize + @queues = {} + @default_queue = nil + @pending_limit = nil + @permit_limit = 1 + end + + attr :queues + attr :default_queue_name + + # Define a named request queue. Matchers are evaluated in definition order. + def queue(name) + name = name.to_sym + raise ArgumentError, "Queue #{name.inspect} is already defined!" if @queues.key?(name) + + queue = Queue.new(name) + yield queue if block_given? + @queues[name] = queue + queue + end + + # Select the fallback queue for unmatched requests. + # @parameter name [Symbol | String] A previously or subsequently defined queue. + def default_queue(name) + @default_queue = name.to_sym + end + + # Set or get the global pending request limit. + def pending_limit(value = UNDEFINED) + return @pending_limit if value.equal?(UNDEFINED) + + value = Integer(value) + raise ArgumentError, "Pending limit must not be negative!" if value.negative? + @pending_limit = value + end + + # Set or get the number of processing permits provided by each worker. + def permit_limit(value = UNDEFINED) + return @permit_limit if value.equal?(UNDEFINED) + + value = Integer(value) + raise ArgumentError, "Permit limit must be positive!" unless value.positive? + @permit_limit = value + end + + # Classify a request using matchers in definition order. + # @returns [Queue] The matching or default queue. + def classify(request) + @queues.each_value do |queue| + return queue if queue.match?(request) + end + + @queues.fetch(@default_queue) + end + + # Validate and freeze the complete configuration. + # @returns [Configuration] The finalized configuration. + def finalize + raise ArgumentError, "At least one queue must be defined!" if @queues.empty? + @default_queue ||= @queues.keys.first + raise ArgumentError, "Default queue #{@default_queue.inspect} is not defined!" unless @queues.key?(@default_queue) + + @queues.each_value(&:finalize) + @queues.freeze + @default_queue_name = @default_queue + freeze + end + end +end diff --git a/lib/fantail/proxy.rb b/lib/fantail/proxy.rb index 76a6c42..d4ffb24 100644 --- a/lib/fantail/proxy.rb +++ b/lib/fantail/proxy.rb @@ -7,34 +7,41 @@ require "protocol/http/response" require_relative "response_body" +require_relative "scheduler" module Fantail - # Routes HTTP requests through the registry's global admission queue. + # Routes HTTP requests through the configured admission queues. class Proxy # Initialize an HTTP proxy. # @parameter registry [Registry] The backend registry. - def initialize(registry) - @registry = registry + # @parameter configuration [Configuration] Request classification and scheduling policy. + def initialize(registry, configuration: Configuration.default) + @scheduler = Scheduler.new(registry, configuration) end + attr :scheduler + # Route a request to the next available backend. # @parameter request [Protocol::HTTP::Request] The downstream request. # @returns [Protocol::HTTP::Response] The upstream or generated response. def call(request) - unless backend = @registry.acquire + unless backend_reservation = @scheduler.acquire(request) return Protocol::HTTP::Response[503, {"content-type" => "text/plain"}, ["No backends available.\n"]] end + return backend_reservation.response if backend_reservation.is_a?(Scheduler::Rejection) + reservation = :processing + backend = backend_reservation.backend upstream_request = build_request(request) response = backend.call(upstream_request) - backend.processed + backend_reservation.processed reservation = :exchange if body = response.body - response.body = ResponseBody.new(body){backend.release} + response.body = ResponseBody.new(body){backend_reservation.release} else - backend.release + backend_reservation.release end reservation = nil @@ -42,9 +49,9 @@ def call(request) rescue => error case reservation when :processing - backend.failed + backend_reservation.failed when :exchange - backend.release + backend_reservation.release end return Protocol::HTTP::Response[502, {"content-type" => "text/plain"}, ["Bad Gateway: #{error.class}\n"]] diff --git a/lib/fantail/registry.rb b/lib/fantail/registry.rb index 8e4d24c..4dce9ca 100644 --- a/lib/fantail/registry.rb +++ b/lib/fantail/registry.rb @@ -10,20 +10,22 @@ require_relative "backend" module Fantail - # Maintains live backends and a global queue of available processing slots. + # Maintains live backends and notifies the scheduler when capacity changes. class Registry < Async::Bus::Controller - WAKE = Object.new.freeze - # Initialize an endpoint registry. # @parameter exchange_limit [Integer] The maximum outstanding responses per backend. - # @parameter backend_factory [Proc | Nil] An optional backend construction strategy. - def initialize(exchange_limit: 8, backend_factory: nil) + # @parameter permit_limit [Integer] The maximum active processing permits per backend. + # @parameter backend_factory [#call(endpoint, exchange_limit, permit_limit, available) | Nil] An optional backend construction strategy. + def initialize(exchange_limit: 8, permit_limit: 1, backend_factory: nil) @exchange_limit = exchange_limit + @permit_limit = permit_limit @backend_factory = backend_factory || self.method(:make_backend) @guard = Thread::Mutex.new @backends = {} - @available = Async::Queue.new + @notifications = Async::Queue.new + @waiting = 0 + @available = nil @closed = false end @@ -62,7 +64,7 @@ def update(upserted, removed) next if current&.endpoint == endpoint retired << current if current - backend = @backend_factory.call(endpoint, @exchange_limit, self.method(:offer)) + backend = @backend_factory.call(endpoint, @exchange_limit, @permit_limit, self.method(:offer)) @backends[endpoint.name] = backend started << backend end @@ -70,7 +72,7 @@ def update(upserted, removed) retired.each(&:retire) started.each(&:start) - @available.enqueue(WAKE) unless retired.empty? + notify_available unless retired.empty? self.size end @@ -78,15 +80,33 @@ def update(upserted, removed) # Acquire the next backend with processing capacity. # @returns [Backend | Nil] An admitted backend, or nil if no endpoints exist. def acquire + waiting = false + loop do return nil if self.empty? + backend = backends.find{|candidate| candidate.reserve} + return backend if backend - candidate = @available.dequeue - return nil unless candidate - next if candidate.equal?(WAKE) + unless waiting + @guard.synchronize{@waiting += 1} + waiting = true + next + end - return candidate if candidate.reserve + return nil unless @notifications.dequeue end + ensure + @guard.synchronize{@waiting -= 1} if waiting + end + + # @returns [Array(Backend)] A snapshot of active backends. + def backends + @guard.synchronize{@backends.values.dup} + end + + # Register the central scheduler capacity callback. + def on_available(&block) + @guard.synchronize{@available = block} end # @returns [Integer] The number of active endpoints. @@ -120,21 +140,27 @@ def close @backends.values.tap{@backends = {}} end - @available.close + @notifications.close backends.each(&:retire) end protected - def offer(backend) - @available.enqueue(backend) + def offer(_backend) + notify_available + end + + def notify_available + available, waiting = @guard.synchronize{[@available, @waiting]} + @notifications.enqueue(true) if waiting.positive? + available&.call rescue Async::Queue::ClosedError - # The registry is already shutting down: + # The registry is already shutting down. end - def make_backend(endpoint, exchange_limit, available) + def make_backend(endpoint, exchange_limit, permit_limit, available) client = endpoint.make_client(exchange_limit: exchange_limit) - Backend.new(endpoint, client, exchange_limit: exchange_limit, &available) + Backend.new(endpoint, client, exchange_limit: exchange_limit, permit_limit: permit_limit, &available) end end end diff --git a/lib/fantail/scheduler.rb b/lib/fantail/scheduler.rb new file mode 100644 index 0000000..c57aeb1 --- /dev/null +++ b/lib/fantail/scheduler.rb @@ -0,0 +1,185 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2026, by Samuel Williams. + +require "async/queue" +require "protocol/http/response" + +require_relative "configuration" + +module Fantail + # Matches pending requests to concrete backend permits. + class Scheduler + # A reserved processing permit and response exchange. + class Reservation + # @parameter backend [Backend] The reserved backend. + # @parameter queue_name [Symbol] The queue consuming the permit. + def initialize(backend, queue_name) + @backend = backend + @queue_name = queue_name + end + + attr :backend + + # Release the processing permit after response headers arrive. + def processed + @backend.processed(@queue_name) + end + + # Release the processing permit and exchange after an upstream failure. + def failed + @backend.failed(@queue_name) + end + + # Release the response exchange after its body closes. + def release + @backend.release + end + end + + # A queue admission rejection. + class Rejection + # @parameter queue [Configuration::Queue] The queue which rejected admission. + def initialize(queue) + @queue = queue + end + + # @returns [Protocol::HTTP::Response] The configured shedding response. + def response + headers = {"content-type" => "text/plain"}.merge(@queue.shed_headers) + Protocol::HTTP::Response[@queue.shed_status, headers, ["Request queue is full.\n"]] + end + end + + Entry = Struct.new(:request, :queue, :enqueued_at, :result, :pending, :assignment) + + # @parameter registry [Registry] The available backend registry. + # @parameter configuration [Configuration] Request and scheduling policy. + def initialize(registry, configuration = Configuration.default) + @registry = registry + @configuration = configuration + @guard = Thread::Mutex.new + @pending = configuration.queues.to_h{|name, queue| [name, []]} + @pending_count = 0 + + @registry.on_available{schedule} + end + + # Admit a request, wait for a matching permit, or return a rejection. + def acquire(request) + queue = @configuration.classify(request) + entry = nil + result = @guard.synchronize do + if reservation = reserve(queue, request) + reservation + elsif @registry.empty? + nil + elsif reject?(queue, request) + Rejection.new(queue) + else + entry = Entry.new(request, queue, now, Async::Queue.new, true, nil) + @pending.fetch(queue.name) << entry + @pending_count += 1 + schedule_locked + entry.assignment + end + end + + return result if result + return nil unless entry + return entry.assignment if entry.assignment + + if wait_limit = queue.wait_limit + remaining = wait_limit - (now - entry.enqueued_at) + result = entry.result.dequeue(timeout: remaining) if remaining.positive? + return result if result + + return cancel(entry) + else + return entry.result.dequeue + end + end + + # Try to dispatch pending requests after capacity changes. + def schedule + @guard.synchronize{schedule_locked} + end + + # @parameter queue_name [Symbol | String | Nil] An optional queue to inspect. + # @returns [Integer] The number of requests waiting for a permit. + def pending_count(queue_name = nil) + @guard.synchronize do + if queue_name + @pending.fetch(queue_name.to_sym).size + else + @pending_count + end + end + end + + protected + + def now + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + def reject?(queue, request) + return true if @configuration.pending_limit && @pending_count >= @configuration.pending_limit + pending = @pending.fetch(queue.name).size + return true if queue.depth_limit && pending >= queue.depth_limit + return true unless queue.admit?(request, pending: pending) + + false + end + + def cancel(entry) + @guard.synchronize do + return entry.assignment unless entry.pending + + @pending.fetch(entry.queue.name).delete(entry) + entry.pending = false + @pending_count -= 1 + Rejection.new(entry.queue) + end + end + + def schedule_locked + loop do + entries = @pending.each_value.filter_map(&:first).sort_by(&:enqueued_at) + matched = false + + entries.each do |entry| + if reservation = reserve(entry.queue, entry.request) + @pending.fetch(entry.queue.name).shift + @pending_count -= 1 + entry.pending = false + entry.assignment = reservation + entry.result.enqueue(reservation) + matched = true + break + end + end + + break unless matched + end + end + + def reserve(queue, request) + backends = @registry.backends.select do |backend| + backend.available? && queue.eligible?(backend, request) + end + + until backends.empty? + backend = queue.balance_policy.select(backends, queue: queue, request: request) + return nil unless backend + raise ArgumentError, "Balance policy selected an ineligible backend!" unless backends.include?(backend) + + return Reservation.new(backend, queue.name) if backend.reserve(queue.name) + backends.delete(backend) + end + + nil + end + end +end diff --git a/lib/fantail/server.rb b/lib/fantail/server.rb index 9ea23f4..66bf4a7 100644 --- a/lib/fantail/server.rb +++ b/lib/fantail/server.rb @@ -8,6 +8,7 @@ require_relative "registry" require_relative "proxy" require_relative "control" +require_relative "configuration" module Fantail # Runs the HTTP load balancer and endpoint-control server together. @@ -16,9 +17,10 @@ class Server # @parameter endpoint [Async::HTTP::Endpoint] The downstream HTTP endpoint. # @parameter control_endpoint [IO::Endpoint] The async-bus control endpoint. # @parameter exchange_limit [Integer] The maximum outstanding responses per backend. - def initialize(endpoint, control_endpoint, exchange_limit: 8) - @registry = Registry.new(exchange_limit: exchange_limit) - @proxy = Proxy.new(@registry) + # @parameter configuration [Configuration] Request classification and scheduling policy. + def initialize(endpoint, control_endpoint, exchange_limit: 8, configuration: Configuration.default) + @registry = Registry.new(exchange_limit: exchange_limit, permit_limit: configuration.permit_limit) + @proxy = Proxy.new(@registry, configuration: configuration) @http_server = Async::HTTP::Server.new(@proxy, endpoint) @control_server = Control.new(control_endpoint, @registry) end @@ -26,6 +28,11 @@ def initialize(endpoint, control_endpoint, exchange_limit: 8) # @attribute [Registry] The server's endpoint registry. attr :registry + # @attribute [Scheduler] The central request scheduler. + def scheduler + @proxy.scheduler + end + # Run the HTTP and control servers. # @parameter parent [Interface(:async)] The parent task. # @returns [Async::Task] The server task. diff --git a/readme.md b/readme.md index faf8110..65e1801 100644 --- a/readme.md +++ b/readme.md @@ -4,7 +4,7 @@ Worker-aware HTTP load balancing with a global admission queue. [![Development Status](https://github.com/socketry/fantail/workflows/Test/badge.svg)](https://github.com/socketry/fantail/actions?workflow=Test) -Fantail routes each request to a worker which is ready to process it. It separates the short-lived request-processing reservation from the potentially longer response exchange, so another request can begin after response headers arrive while the previous response body is still streaming. +Fantail routes each request to a worker which is ready to process it. Configurable request queues can express worker affinity and load-shedding policy while a central scheduler remains responsible for matching requests to worker permits. Fantail separates the short-lived request-processing reservation from the potentially longer response exchange, so another request can begin after response headers arrive while the previous response body is still streaming. ## Usage diff --git a/releases.md b/releases.md index 67fa769..98e4a2e 100644 --- a/releases.md +++ b/releases.md @@ -1,5 +1,9 @@ # Releases +## Unreleased + + - Add configurable request queues, worker affinity policies, central permit scheduling, and load shedding. + ## v0.0.1 - Initial implementation. diff --git a/test/fantail/configuration.rb b/test/fantail/configuration.rb new file mode 100644 index 0000000..26df204 --- /dev/null +++ b/test/fantail/configuration.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2026, by Samuel Williams. + +require "fantail/configuration" +require "protocol/http/request" +require "tmpdir" + +describe Fantail::Configuration do + let(:configuration) do + subject.define do |config| + config.queue :grpc do |queue| + queue.match{|request| request.headers["content-type"]&.start_with?("application/grpc")} + queue.balance :pack + end + + config.queue :liquid do |queue| + queue.balance :spread + queue.depth_limit 500 + queue.wait_limit 0.25 + queue.shed status: 429, retry_after: 1 + end + + config.default_queue :liquid + config.pending_limit 1_000 + config.permit_limit 2 + end + end + + it "classifies requests and freezes the result" do + grpc = Protocol::HTTP::Request["POST", "/rpc", {"content-type" => "application/grpc+proto"}] + liquid = Protocol::HTTP::Request["GET", "/render"] + + expect(configuration.classify(grpc).name).to be == :grpc + expect(configuration.classify(liquid).name).to be == :liquid + expect(configuration.pending_limit).to be == 1_000 + expect(configuration.permit_limit).to be == 2 + expect(configuration).to be(:frozen?) + expect(configuration.queues.fetch(:liquid)).to be(:frozen?) + end + + it "loads trusted application configuration" do + Dir.mktmpdir do |directory| + path = File.join(directory, "fantail.rb") + File.write(path, <<~RUBY) + Fantail::Configuration.define do |config| + config.queue :default + end + RUBY + + loaded = subject.load(path) + expect(loaded.default_queue_name).to be == :default + end + end + + it "rejects configuration files with the wrong result" do + Dir.mktmpdir do |directory| + path = File.join(directory, "fantail.rb") + File.write(path, "Object.new\n") + + expect{subject.load(path)}.to raise_exception(TypeError) + end + end + + it "rejects invalid balance policies" do + expect do + subject.define do |config| + config.queue(:default){|queue| queue.balance Object.new} + end + end.to raise_exception(ArgumentError) + end +end diff --git a/test/fantail/fixtures.rb b/test/fantail/fixtures.rb index 22f8add..3cfe615 100644 --- a/test/fantail/fixtures.rb +++ b/test/fantail/fixtures.rb @@ -30,17 +30,17 @@ def closed? end end - def make_registry(exchange_limit: 8, &client_factory) + def make_registry(exchange_limit: 8, permit_limit: 1, &client_factory) @clients = {} - backend_factory = proc do |endpoint, limit, available| + backend_factory = proc do |endpoint, exchange_limit, backend_permit_limit, available| client = client_factory&.call(endpoint) || Client.new @clients[endpoint.name] = client - Backend.new(endpoint, client, exchange_limit: limit, &available) + Backend.new(endpoint, client, exchange_limit: exchange_limit, permit_limit: backend_permit_limit, &available) end - Registry.new(exchange_limit: exchange_limit, backend_factory: backend_factory) + Registry.new(exchange_limit: exchange_limit, permit_limit: permit_limit, backend_factory: backend_factory) end end end diff --git a/test/fantail/registry.rb b/test/fantail/registry.rb index 093216a..19817f3 100644 --- a/test/fantail/registry.rb +++ b/test/fantail/registry.rb @@ -4,9 +4,11 @@ # Copyright, 2026, by Samuel Williams. require "fantail" +require "sus/fixtures/async/reactor_context" require_relative "fixtures" describe Fantail::Registry do + include Sus::Fixtures::Async::ReactorContext include Fantail::Fixtures let(:registry) {make_registry} @@ -70,6 +72,41 @@ expect(backend).not.to be(:processing?) end + it "wakes an acquisition when a permit is released" do + registry.replace([{name: "worker-1", url: "http://127.0.0.1:9301"}]) + first = registry.acquire + second_task = Async{registry.acquire} + Fiber.scheduler.yield + + expect(second_task).not.to be(:finished?) + first.failed + first = nil + + second = second_task.wait + expect(second.name).to be == "worker-1" + ensure + first&.failed + second&.failed + second_task&.stop + end + + it "supports multiple processing permits" do + registry.close + registry = make_registry(permit_limit: 2) + registry.replace([{name: "worker-1", url: "http://127.0.0.1:9301"}]) + backend = registry["worker-1"] + + expect(backend.reserve(:grpc)).to be_truthy + expect(backend.reserve(:grpc)).to be_truthy + expect(backend.reserve(:grpc)).to be_falsey + expect(backend.processing_for(:grpc)).to be == 2 + + backend.failed(:grpc) + backend.failed(:grpc) + ensure + registry&.close + end + it "returns nil when no endpoints exist" do expect(registry.acquire).to be_nil end diff --git a/test/fantail/scheduler.rb b/test/fantail/scheduler.rb new file mode 100644 index 0000000..ed75299 --- /dev/null +++ b/test/fantail/scheduler.rb @@ -0,0 +1,276 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2026, by Samuel Williams. + +require "fantail" +require "sus/fixtures/async/reactor_context" +require_relative "fixtures" + +describe Fantail::Scheduler do + include Sus::Fixtures::Async::ReactorContext + include Fantail::Fixtures + + def make_configuration(permit_limit: 1, pending_limit: nil, &block) + Fantail::Configuration.define do |configuration| + configuration.permit_limit permit_limit + configuration.pending_limit pending_limit if pending_limit + block.call(configuration) + end + end + + def add_workers(registry, count = 2) + registry.replace(Array.new(count) do |index| + {name: "worker-#{index + 1}", url: "http://127.0.0.1:#{9301 + index}"} + end) + end + + def finish(reservation) + reservation.processed + reservation.release + end + + it "spreads work across workers with spare permits" do + configuration = make_configuration(permit_limit: 2) do |config| + config.queue(:liquid){|queue| queue.balance :spread} + end + registry = make_registry(permit_limit: configuration.permit_limit) + add_workers(registry) + scheduler = subject.new(registry, configuration) + + first = scheduler.acquire(Protocol::HTTP::Request["GET", "/first"]) + second = scheduler.acquire(Protocol::HTTP::Request["GET", "/second"]) + + expect([first.backend.name, second.backend.name]).to be == ["worker-1", "worker-2"] + ensure + finish(first) if first + finish(second) if second + registry&.close + end + + it "packs affinity work onto an active worker" do + configuration = make_configuration(permit_limit: 2) do |config| + config.queue(:grpc){|queue| queue.balance :pack, affinity: :grpc} + end + registry = make_registry(permit_limit: configuration.permit_limit) + add_workers(registry) + scheduler = subject.new(registry, configuration) + + first = scheduler.acquire(Protocol::HTTP::Request["POST", "/first"]) + second = scheduler.acquire(Protocol::HTTP::Request["POST", "/second"]) + + expect([first.backend.name, second.backend.name]).to be == ["worker-1", "worker-1"] + ensure + finish(first) if first + finish(second) if second + registry&.close + end + + it "keeps affinity work-conserving" do + configuration = make_configuration(permit_limit: 2) do |config| + config.queue(:grpc){|queue| queue.balance :pack} + end + registry = make_registry(permit_limit: configuration.permit_limit) + add_workers(registry) + scheduler = subject.new(registry, configuration) + + first = scheduler.acquire(Protocol::HTTP::Request["POST", "/first"]) + second = scheduler.acquire(Protocol::HTTP::Request["POST", "/second"]) + third = scheduler.acquire(Protocol::HTTP::Request["POST", "/third"]) + + expect([first.backend.name, second.backend.name, third.backend.name]).to be == ["worker-1", "worker-1", "worker-2"] + ensure + finish(first) if first + finish(second) if second + finish(third) if third + registry&.close + end + + it "supports application balance policies" do + policy = Object.new + policy.define_singleton_method(:select){|backends, **| backends.last} + configuration = make_configuration do |config| + config.queue(:default){|queue| queue.balance policy} + end + registry = make_registry + add_workers(registry) + scheduler = subject.new(registry, configuration) + + reservation = scheduler.acquire(Protocol::HTTP::Request["GET", "/"]) + expect(reservation.backend.name).to be == "worker-2" + ensure + finish(reservation) if reservation + registry&.close + end + + it "retries selection when a permit is consumed concurrently" do + consumed = nil + policy = Object.new + policy.define_singleton_method(:select) do |backends, queue:, **| + unless consumed + consumed = backends.first + consumed.reserve(queue.name) + end + + backends.first + end + configuration = make_configuration do |config| + config.queue(:default){|queue| queue.balance policy} + end + registry = make_registry + add_workers(registry) + scheduler = subject.new(registry, configuration) + + reservation = scheduler.acquire(Protocol::HTTP::Request["GET", "/"]) + expect(reservation.backend.name).to be == "worker-2" + ensure + finish(reservation) if reservation + consumed&.failed(:default) + registry&.close + end + + it "uses another queue when the oldest queue has no eligible worker" do + configuration = make_configuration do |config| + config.queue :special do |queue| + queue.match{|request| request.path == "/special"} + queue.eligible{|backend, _request| backend.name == "special-worker"} + end + config.queue :default + config.default_queue :default + end + registry = make_registry + registry.replace([{name: "default-worker", url: "http://127.0.0.1:9301"}]) + scheduler = subject.new(registry, configuration) + held = scheduler.acquire(Protocol::HTTP::Request["GET", "/held"]) + + special_task = Async{scheduler.acquire(Protocol::HTTP::Request["GET", "/special"])} + Fiber.scheduler.yield + default_task = Async{scheduler.acquire(Protocol::HTTP::Request["GET", "/default"])} + Fiber.scheduler.yield + + finish(held) + held = nil + default = default_task.wait + expect(default.backend.name).to be == "default-worker" + expect(special_task).not.to be(:finished?) + ensure + finish(held) if held + finish(default) if default + special_task&.stop + default_task&.stop + registry&.close + end + + it "dispatches the oldest eligible queue head first" do + configuration = make_configuration do |config| + config.queue(:alpha){|queue| queue.match{|request| request.path == "/alpha"}} + config.queue :beta + config.default_queue :beta + end + registry = make_registry + add_workers(registry, 1) + scheduler = subject.new(registry, configuration) + held = scheduler.acquire(Protocol::HTTP::Request["GET", "/held"]) + + alpha_task = Async{scheduler.acquire(Protocol::HTTP::Request["GET", "/alpha"])} + Fiber.scheduler.yield + beta_task = Async{scheduler.acquire(Protocol::HTTP::Request["GET", "/beta"])} + Fiber.scheduler.yield + + finish(held) + held = nil + alpha = alpha_task.wait + expect(beta_task).not.to be(:finished?) + finish(alpha) + alpha = nil + beta = beta_task.wait + expect(beta.backend.name).to be == "worker-1" + ensure + finish(held) if held + finish(alpha) if alpha + finish(beta) if beta + alpha_task&.stop + beta_task&.stop + registry&.close + end + + it "sheds requests when the queue depth limit is reached" do + configuration = make_configuration do |config| + config.queue :default do |queue| + queue.depth_limit 1 + queue.shed status: 429, retry_after: 2 + end + end + registry = make_registry + add_workers(registry, 1) + scheduler = subject.new(registry, configuration) + held = scheduler.acquire(Protocol::HTTP::Request["GET", "/held"]) + waiting_task = Async{scheduler.acquire(Protocol::HTTP::Request["GET", "/waiting"])} + Fiber.scheduler.yield + expect(scheduler.pending_count(:default)).to be == 1 + + rejection = scheduler.acquire(Protocol::HTTP::Request["GET", "/rejected"]) + response = rejection.response + + expect(response.status).to be == 429 + expect(response.headers["retry-after"]).to be == "2" + ensure + response&.close + finish(held) if held + waiting = waiting_task&.wait + finish(waiting) if waiting + waiting_task&.stop + registry&.close + end + + it "sheds requests which exceed their queue wait limit" do + configuration = make_configuration do |config| + config.queue(:default){|queue| queue.wait_limit 0.01} + end + registry = make_registry + add_workers(registry, 1) + scheduler = subject.new(registry, configuration) + held = scheduler.acquire(Protocol::HTTP::Request["GET", "/held"]) + + rejection = scheduler.acquire(Protocol::HTTP::Request["GET", "/waiting"]) + expect(rejection).to be_a(Fantail::Scheduler::Rejection) + expect(scheduler.pending_count).to be == 0 + ensure + finish(held) if held + registry&.close + end + + it "supports application admission policies" do + configuration = make_configuration do |config| + config.queue(:default){|queue| queue.admit{|request, **| request.path != "/shed"}} + end + registry = make_registry + add_workers(registry, 1) + scheduler = subject.new(registry, configuration) + held = scheduler.acquire(Protocol::HTTP::Request["GET", "/held"]) + + rejection = scheduler.acquire(Protocol::HTTP::Request["GET", "/shed"]) + expect(rejection).to be_a(Fantail::Scheduler::Rejection) + ensure + finish(held) if held + registry&.close + end + + it "supports application admission policy objects" do + policy = Object.new + policy.define_singleton_method(:admit?){|request, **| request.path != "/shed"} + configuration = make_configuration do |config| + config.queue(:default){|queue| queue.admit policy} + end + registry = make_registry + add_workers(registry, 1) + scheduler = subject.new(registry, configuration) + held = scheduler.acquire(Protocol::HTTP::Request["GET", "/held"]) + + rejection = scheduler.acquire(Protocol::HTTP::Request["GET", "/shed"]) + expect(rejection).to be_a(Fantail::Scheduler::Rejection) + ensure + finish(held) if held + registry&.close + end +end diff --git a/test/fantail/server.rb b/test/fantail/server.rb index 87715f0..4ef83e0 100644 --- a/test/fantail/server.rb +++ b/test/fantail/server.rb @@ -40,6 +40,7 @@ def wait_until control_bound_endpoint = control_endpoint.bound server = subject.new(downstream_endpoint, control_bound_endpoint) + expect(server.scheduler).to be_a(Fantail::Scheduler) server_task = server.run monitor = Fantail::Monitor.new(control_endpoint)