diff --git a/spec/reeve/audit/chain/benchmark_spec.rb b/spec/reeve/audit/chain/benchmark_spec.rb index 283585d..ed9ec5f 100644 --- a/spec/reeve/audit/chain/benchmark_spec.rb +++ b/spec/reeve/audit/chain/benchmark_spec.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true require "securerandom" +require "tmpdir" +require "fileutils" require_relative "../support/ledger" # T002 / SC-005: what one ledger write costs, measured before chaining exists, so the @@ -155,9 +157,85 @@ def peak_growth_kb expect(seconds).to be < 1 end - # specs/005 SC-004: a purge of a million entries holds no lock writers queue on. A writer - # appends throughout; its slowest write is reported beside the purge's time, and verification - # afterwards walks only what the writer added. + WRITER = File.expand_path("../support/writer.rb", __dir__) + + def now = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + # A writer in a process of its own (specs/005 writer-wait.md): a thread here would share the + # GVL with the purge's HMAC work, and its slowest write would not be a database wait. + def spawn_writer(output) + env = { "REEVE_WRITER_DB" => ActiveRecord::Base.connection_db_config.configuration_hash.to_json, + "REEVE_WRITER_KEY" => Reeve.config.ledger_key } + Process.spawn(env, RbConfig.ruby, "-I", File.expand_path("../../../../lib", __dir__), WRITER, output) + end + + # Each complete write as [started, seconds]; raises with the writer's error if it wrote one. + def writes(output) + return [] unless File.exist?(output) + + lines = File.read(output).lines.select { |line| line.end_with?("\n") }.map(&:chomp) + error = lines.index { |line| line.start_with?("error ") } + raise "writer failed: #{lines[error..].join("\n")}" if error + + lines.map { |line| line.split.map { |value| Float(value) } } + end + + # Raises at once, with the writer's own error where it wrote one, if the writer has exited. + def expect_writing(pid, output) + status = Process.waitpid2(pid, Process::WNOHANG)&.last + return unless status + + writes(output) + raise "writer exited (#{status.inspect}); anything it printed is in the output above" + end + + def await_first_write(pid, output, seconds: 60) + deadline = now + seconds + while writes(output).empty? + expect_writing(pid, output) + raise "writer made no write in #{seconds} s" if now > deadline + + sleep 0.05 + end + end + + def stop_writer(pid, seconds: 30) + Process.kill("TERM", pid) + deadline = now + seconds + until (status = Process.waitpid2(pid, Process::WNOHANG)&.last) + raise "writer did not stop in #{seconds} s" if now > deadline + + sleep 0.05 + end + status + end + + # A writer left running by a failed example is killed, not left writing into the next one. + # Waited for first: a pid already reaped raises ECHILD, so one the system has since reused + # is never signalled. + def reap(pid) + return if Process.waitpid(pid, Process::WNOHANG) + + Process.kill("KILL", pid) + Process.wait(pid) + rescue Errno::ECHILD + nil + end + + def latency(writes) + sorted = writes.map { |_, seconds| seconds * 1000 }.sort + format("%d writes, median %.1f ms, p99 %.1f ms, max %.1f ms", + n: sorted.size, median: percentile(sorted, 0.5), p99: percentile(sorted, 0.99), + max: sorted.last.to_f) + end + + def percentile(sorted, fraction) + sorted.empty? ? 0.0 : sorted[((sorted.size - 1) * fraction).round] + end + + # specs/005 SC-004: a purge of a million entries holds no lock writers queue on. A writer in + # another process appends alone for a baseline, then throughout the purge; both latencies are + # reported beside the purge's time, and verification afterwards walks only what it added. it "purges 1,000,000 entries while writes continue, and verifies only the rest" do size = Integer(ENV.fetch("REEVE_PURGE_SIZE", "1000000")) build_chain(size) @@ -165,32 +243,38 @@ def peak_growth_kb Reeve.config.ledger_retention_connection = Ledger.retention_connection Reeve::Audit::Chain.checkpoint! - stop = false - waits = [] - writer = Thread.new do - chain = Reeve::Audit::Chain.new(entry_class: Reeve::Audit::Entry) - until stop - started = Process.clock_gettime(Process::CLOCK_MONOTONIC) - chain.append!(invocation_id: SecureRandom.uuid, occurred_at: Time.now, agent_id: "a", - tool_name: "t", arguments: {}, outcome: "allow", rule: "r", record_ids: [], - guard: "policy", contract_version: Reeve::Audit::CONTRACT_VERSION) - waits << (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) - end - end - started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + directory = Dir.mktmpdir("reeve-writer") + output = File.join(directory, "writes") + pid = spawn_writer(output) + await_first_write(pid, output) + sleep 3 + expect_writing(pid, output) + purge_started = now result = Reeve::Audit::Chain.purge!(through: size, no_archive: true, confirm: true) - seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started - stop = true - writer.join + purge_ended = now + expect_writing(pid, output) + status = stop_writer(pid) + pid = nil + all = writes(output) + expect(status).to be_success expect(result).to be_purged, result.to_s + # The first write opens the connection and reads the schema, so it is no baseline. + baseline = all.drop(1).select { |started, _| started < purge_started } + during = all.select { |started, _| started >= purge_started && started < purge_ended } + expect(during).not_to be_empty, "no write started during the purge; it measured nothing" after = Reeve::Audit::Chain.verify(checkpoints: []) expect(after.status).to eq(:intact) - expect(after.covered).to eq((size + 1)..(size + waits.size)) + expect(after.covered).to eq((size + 1)..(size + all.size)) RSpec.configuration.reporter.message( - format("purge (%s, %d entries): %.1f s; %d writes during it, slowest %.1f ms", - db: SpecDatabase.adapter, size: size, s: seconds, n: waits.size, ms: waits.max.to_f * 1000) + format("purge (%s, %d entries): %.1f s; writer in its own process: " \ + "baseline %s; during the purge %s", + db: SpecDatabase.adapter, size: size, s: purge_ended - purge_started, + baseline: latency(baseline), during: latency(during)) ) + ensure + reap(pid) if pid + FileUtils.rm_rf(directory) if directory end it "verifies 100,000 and 1,000,000 entries, reporting time and peak memory" do diff --git a/spec/reeve/audit/support/writer.rb b/spec/reeve/audit/support/writer.rb new file mode 100644 index 0000000..3e5269b --- /dev/null +++ b/spec/reeve/audit/support/writer.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# A ledger writer in a process of its own, for the purge benchmark (specs/005 writer-wait.md). +# It shares no GVL with the purge, so a slow write here is time spent waiting on the database. +# +# ruby writer.rb OUTPUT +# +# Takes the database configuration (JSON) from REEVE_WRITER_DB and the ledger key from +# REEVE_WRITER_KEY, then appends until SIGTERM. Each write adds a line to OUTPUT: its start +# and its duration, in seconds on CLOCK_MONOTONIC. A write that raises ends the run with an +# "error ..." line and exit status 1. +require "json" +require "securerandom" +require "active_record" +require "reeve" +require "reeve/audit" + +stop = false +Signal.trap("TERM") { stop = true } +# A parent killed outright never sends TERM; the writer stops when it is orphaned instead. +parent = Process.ppid + +File.open(ARGV.fetch(0), "w") do |output| + output.sync = true + ActiveRecord::Base.logger = nil + ActiveRecord::Base.establish_connection(JSON.parse(ENV.fetch("REEVE_WRITER_DB"), symbolize_names: true)) + Reeve.config.ledger_key = ENV.fetch("REEVE_WRITER_KEY") + chain = Reeve::Audit::Chain.new(entry_class: Reeve::Audit::Entry) + + until stop || Process.ppid != parent + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + chain.append!(invocation_id: SecureRandom.uuid, occurred_at: Time.now, agent_id: "a", + tool_name: "t", arguments: {}, outcome: "allow", rule: "r", record_ids: [], + guard: "policy", contract_version: Reeve::Audit::CONTRACT_VERSION) + output.write("#{started} #{Process.clock_gettime(Process::CLOCK_MONOTONIC) - started}\n") + end +rescue StandardError => e + output.write("error #{e.class}: #{e.message}\n") + exit 1 +end diff --git a/specs/005-ledger-retention/spec.md b/specs/005-ledger-retention/spec.md index 119af4a..5420727 100644 --- a/specs/005-ledger-retention/spec.md +++ b/specs/005-ledger-retention/spec.md @@ -302,13 +302,20 @@ from 1 to the boundary, ending at the boundary checkpoint's digest. Edit one lin - **SC-002**: Every case in User Story 2 is BROKEN in 100% of tests, each beside its legitimate twin, which is INTACT. - **SC-003**: Every refusal in User Story 3 leaves the ledger byte-identical. -- **SC-004** *(measured 2026-10-25: met as designed, with a caveat. 1,000,000 entries purged in - 331 s on SQLite and 163 s on PostgreSQL (no archive), verification INTACT afterwards over only - the entries written during the purge. The purge never takes the chain record's lock. A writer - thread in the same process kept writing throughout (95,366 writes on SQLite, slowest 441 ms; - 2,970 on PostgreSQL, slowest 1,774 ms), but it shares Ruby's GVL with the purge's HMAC work, - so those worst cases are not clean database waits; a writer in a separate process would - measure that.)*: Purging 1,000,000 entries completes without holding the chain's lock for longer +- **SC-004** *(measured 2026-10-25, writer re-measured out of process 2026-11-05: met. + The purge never takes the chain record's lock, and verification afterwards is INTACT over only + the entries written during the purge. 2026-11-05, 1,000,000 entries, no archive, a writer in + a separate process (writer-wait.md), shared machine at load 6 to 28: + SQLite: purge 194.7 s; writer alone median 1.6 ms, p99 5.1 ms, max 10.7 ms; during the purge + 80,370 writes, median 1.8 ms, p99 12.3 ms, max 207.0 ms. + PostgreSQL: purge 179.5 s; writer alone median 2.4 ms, p99 3.7 ms, max 6.0 ms; during the + purge 57,501 writes, median 2.6 ms, p99 7.1 ms, max 252.5 ms. + Writers do not queue behind the purge: the median barely moves and p99 stays within 2.5 times + the baseline. The single slowest write on each engine is not explained by this run; the delete + batch's own time was not measured. MySQL is unmeasured (no local server; CI does not run + `:benchmark_large`). The 2026-10-25 figures came from a writer thread sharing Ruby's GVL with + the purge (slowest 441 ms SQLite, 1,774 ms PostgreSQL) and are superseded.)*: Purging + 1,000,000 entries completes without holding the chain's lock for longer than one chained write at a time, and verification after it walks only the remaining entries. - **SC-005**: An operator can configure, dry-run, run and verify a purge from the README and CHANGELOG alone (checked by a cold read). diff --git a/specs/005-ledger-retention/writer-wait.md b/specs/005-ledger-retention/writer-wait.md new file mode 100644 index 0000000..0449476 --- /dev/null +++ b/specs/005-ledger-retention/writer-wait.md @@ -0,0 +1,66 @@ +# Writer-wait during a purge, measured out of process + +Follow-up to SC-004 of this spec. Frozen before build; changes are recorded below, never drifted. + +## Problem + +The SC-004 benchmark measured writer latency during a 1,000,000-entry purge with a writer +*thread* in the purge's own process. That thread shares Ruby's GVL with the purge's HMAC work, +so its slowest writes (441 ms SQLite, 1,774 ms PostgreSQL) mix GVL waits with database waits. +The claim under test, "the purge holds no lock writers queue on longer than one batch", needs a +writer that only the database can slow down. + +## Contracts + +- **C1** The example "purges 1,000,000 entries while writes continue, and verifies only the + rest" in `spec/reeve/audit/chain/benchmark_spec.rb` keeps its tag (`:benchmark_large`), its + name, and `REEVE_PURGE_SIZE`. Its in-process writer thread is removed. +- **C2** The writer is a separate OS process started with `Process.spawn` (not fork, not a + thread): its own Ruby VM, the same bundle, the same database configuration as the spec's + `ActiveRecord::Base` (SQLite: the same file; server engines: the application role, not the + retention role), the spec ledger key, and writes through `Chain#append!` as before. +- **C3** Order: start the writer; wait until it has made its first write (fail after 60 s); + let it write alone for a baseline of at least 3 s; run the purge; stop the writer with a + signal; wait for it to exit (fail after 30 s, killing it). +- **C4** The writer records, per write, its start on `CLOCK_MONOTONIC` and its duration. Writes + are classed by start time: *baseline* (before the purge started), *during* (from purge start + to purge end), *after* (counted, not reported). +- **C5** One report line: engine, size, purge seconds; for baseline and for during: count, + median, p99 and max in ms. +- **C6** Assertions only on correctness, never on latency (benchmark policy, spec_helper): + writer exited 0 with no error; the purge result is `purged`; at least one write started + during the purge; verification afterwards is INTACT and covers exactly + `(size + 1)..(size + every write the writer made)`. +- **C7** A write that raises in the writer is recorded with its message, the writer exits + non-zero, and the example fails showing that message. +- **C8** SC-004's note in `spec.md` is replaced by the out-of-process figures for SQLite and + PostgreSQL at 1,000,000 entries; MySQL is stated as unmeasured (no local server; CI does + not run `:benchmark_large`). + +## Inputs the work must cover (hard rule 6) + +- Engines: SQLite (run), PostgreSQL (run), MySQL (unmeasured, stated). +- Sizes: `REEVE_PURGE_SIZE` small (smoke) and 1,000,000 (recorded). +- Writer states: never ready, raising, refusing to stop, exiting early. +- Write classes: baseline, during, after; a run with zero writes during the purge. +- Clock: child and parent timestamps on the same monotonic clock. + +## Out of scope + +Changing the purge. If the out-of-process figures contradict SC-004 (a writer waiting longer +than one delete batch, beyond baseline noise), stop and report; a fix is a separate decision. + +## Done + +The example passes at a small `REEVE_PURGE_SIZE` on SQLite and PostgreSQL; the 1,000,000 runs +are recorded in SC-004; rubocop clean; the default suite unchanged. + +## Spec changes + +- **2026-11-05, after the first review (C3, C4):** the writer's first write opens its connection + and reads the schema, so it is left out of the baseline (still counted for coverage). A writer + that fails before or during the baseline fails the example before the purge starts, with its + own error. +- **2026-11-05, after the second round (C3, C6):** a writer that has exited by the time the purge + ends, with any status, fails the example; an exit 0 mid-purge had passed with a fraction of + the purge measured. The writer loads the repository's `lib` itself, not only through Bundler.