Roll channel and node reminders across 6-hourly buckets

.github/workflows/build.yml

@@ -20,16 +20,12 @@ jobs:
           - stable
           - 1.63.0
           - beta
-    runs-on: ubuntu-latest
+    runs-on: debian-trixie
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: https://data.forgejo.org/actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
       - name: Install Rust ${{ matrix.toolchain }} toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: ${{ matrix.toolchain }}
-          override: true
-          profile: minimal
+        run: rustup default ${{ matrix.toolchain }}
       - name: Build on Rust ${{ matrix.toolchain }}
         run: |
           cargo build --verbose --color always
@@ -39,34 +35,14 @@ jobs:
         toolchain:
           - stable
           - 1.63.0
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: postgres
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
+    runs-on: debian-trixie
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: https://data.forgejo.org/actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
       - name: Install Rust ${{ matrix.toolchain }} toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: ${{ matrix.toolchain }}
-          override: true
-          profile: minimal
+        run: rustup default ${{ matrix.toolchain }}
       - name: Run tests
         run: |
-          RUST_BACKTRACE=1 cargo test --verbose --color always -- --show-output
+          RUST_BACKTRACE=1 ./ci/ci-tests.sh --verbose --color always -- --show-output
         env:
-          RAPID_GOSSIP_TEST_DB_HOST: localhost
-          RAPID_GOSSIP_TEST_DB_NAME: postgres
-          RAPID_GOSSIP_TEST_DB_USER: postgres
-          RAPID_GOSSIP_TEST_DB_PASSWORD: postgres
           RAPID_GOSSIP_SYNC_SERVER_LOG_LEVEL: gossip

README.md

@@ -72,12 +72,23 @@ seen prior to the given timestamp, the latest known updates, and, if necessary,
 updates are collected.
 
 Then, any channel that has only had an announcement but never an update is dropped. Additionally,
-every channel whose first update was seen after the given timestamp is collected alongside its
+every channel whose updates in either direction (re)started after the given timestamp, having never
+been seen before or having been absent for at least the prune interval, is collected alongside its
 announcement.
 
+To avoid unchanging data being eventually pruned by clients, "reminder" updates are added in a
+rolling fashion.
+
 Finally, all channel update transitions are evaluated and collected into either a full or an
 incremental update.
 
+## Testing
+
+The tests require a Postgres database. `ci/ci-tests.sh` starts a throwaway Postgres instance in a
+temporary directory, runs `cargo test` against it (passing along any arguments), and deletes the
+instance afterwards. It needs the Postgres server binaries (`initdb`, `pg_ctl` and `postgres`),
+which it looks for in `$POSTGRES_BIN_DIR`, then `$PATH`, then `/usr/lib/postgresql/*/bin`.
+
 ## License
 
 [Apache 2.0](LICENSE-APACHE.md) or [MIT](LICENSE-MIT.md), [at your option](LICENSE.md).

ci/ci-tests.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -eo pipefail
+
+# Runs the test suite against a throwaway PostgreSQL instance. The database cluster lives in a
+# temporary directory, only listens on a Unix socket within that directory, and is deleted once
+# the tests complete.
+#
+# The PostgreSQL server binaries (initdb, pg_ctl and postgres) are taken from $POSTGRES_BIN_DIR if
+# set, otherwise from $PATH, falling back to Debian's /usr/lib/postgresql/<version>/bin.
+#
+# Any arguments are passed through to `cargo test`.
+
+if [ -n "$POSTGRES_BIN_DIR" ]; then
+	PATH="$POSTGRES_BIN_DIR:$PATH"
+elif ! command -v initdb > /dev/null; then
+	for DIR in /usr/lib/postgresql/*/bin; do
+		[ -x "$DIR/initdb" ] && POSTGRES_BIN_DIR="$DIR"
+	done
+	[ -n "$POSTGRES_BIN_DIR" ] && PATH="$POSTGRES_BIN_DIR:$PATH"
+fi
+for BIN in initdb pg_ctl postgres; do
+	if ! command -v "$BIN" > /dev/null; then
+		echo "Could not find $BIN, set POSTGRES_BIN_DIR to the directory containing the PostgreSQL server binaries" >&2
+		exit 1
+	fi
+done
+
+PG_DIR="$(mktemp -d)"
+cleanup() {
+	pg_ctl stop -D "$PG_DIR/data" -m immediate > /dev/null 2>&1 || true
+	rm -rf -- "$PG_DIR"
+}
+trap cleanup EXIT
+
+# Only our user can reach the socket in the (0700) temp directory, so trust auth is fine.
+initdb -D "$PG_DIR/data" -U postgres --auth=trust --encoding=UTF8 --locale=C --no-sync > /dev/null
+cat >> "$PG_DIR/data/postgresql.conf" <<EOF
+listen_addresses = ''
+unix_socket_directories = '$PG_DIR'
+fsync = off
+EOF
+if ! pg_ctl start -D "$PG_DIR/data" -l "$PG_DIR/postgres.log" -w > /dev/null; then
+	cat "$PG_DIR/postgres.log" >&2
+	exit 1
+fi
+
+export RAPID_GOSSIP_TEST_DB_HOST="$PG_DIR"
+export RAPID_GOSSIP_TEST_DB_USER=postgres
+export RAPID_GOSSIP_TEST_DB_NAME=postgres
+cargo test "$@"

src/config.rs

@@ -25,11 +25,22 @@ pub(crate) const SCHEMA_VERSION: i32 = 17;
 pub(crate) const SYMLINK_GRANULARITY_INTERVAL: u32 = 3600 * 3; // three hours
 pub(crate) const MAX_SNAPSHOT_SCOPE: u32 = 3600 * 24 * 21; // three weeks
 // generate symlinks based on a 3-hour-granularity
-/// If the last update in either direction was more than six days ago, we send a reminder
-/// That reminder may be either in the form of a channel announcement, or in the form of empty
-/// updates in both directions.
+
+/// Clients backdate everything we send them by a week and prune anything they haven't heard about
+/// in two weeks, so from the client's perspective each channel direction lives for a bit under 7
+/// days past the last snapshot that wrote it. If the reference update a client would use to apply
+/// an incremental update is older than this, it may have been pruned, and we send a full update
+/// instead. The same age is used to decide whether a node announcement warrants a reminder.
 pub(crate) const CHANNEL_REMINDER_AGE: Duration = Duration::from_secs(6 * 24 * 60 * 60);
 
+/// The reminders are sent in rolling fashion, with each channel and node assigned one of
+/// [`REMINDER_BUCKET_COUNT`] buckets. One bucket becomes due every [`REMINDER_SLOT_INTERVAL`], so
+/// every channel and node is reminded once per [`CHANNEL_REMINDER_AGE`].
+pub(crate) const REMINDER_SLOT_INTERVAL: Duration = Duration::from_secs(6 * 60 * 60);
+pub(crate) const REMINDER_BUCKET_COUNT: u64 = CHANNEL_REMINDER_AGE.as_secs() / REMINDER_SLOT_INTERVAL.as_secs();
+const _: () = assert!(CHANNEL_REMINDER_AGE.as_secs() % REMINDER_SLOT_INTERVAL.as_secs() == 0);
+const _: () = assert!(REMINDER_BUCKET_COUNT < 64, "reminder buckets are tracked in a u64 bitmask");
+
 /// The interval after which graph data gets pruned after it was first seen
 /// This should match the LDK default pruning interval, which is 14 days
 pub(crate) const PRUNE_INTERVAL: Duration = Duration::from_secs(14 * 24 * 60 * 60);