Fix local ledger downloads after snapshot recovery

tests/infra/node.py

@@ -605,17 +605,18 @@ def wait_for_ledger_chunk(self, seqno, timeout=5):
 
     def _get_local_ledger_start_seqno(self):
         with self.client() as c:
+            # Recovery input may not be locally served, even after a startup snapshot.
+            if self.remote.start_type == infra.remote.StartType.recover:
+                r = c.get("/node/network")
+                assert r.status_code == http.HTTPStatus.OK, r
+                return TxID.from_str(r.body.json()["current_service_create_txid"]).seqno
+
             r = c.get("/node/state")
             assert r.status_code == http.HTTPStatus.OK, r
             startup_seqno = r.body.json()["startup_seqno"]
             if startup_seqno != 0:
                 return startup_seqno + 1
 
-            if self.remote.start_type == infra.remote.StartType.recover:
-                r = c.get("/node/network")
-                assert r.status_code == http.HTTPStatus.OK, r
-                return TxID.from_str(r.body.json()["current_service_create_txid"]).seqno
-
             return 1
 
     def _download_ledger(

tests/recovery.py

@@ -526,6 +526,22 @@ def test_recover_service(
     )
 
 
+@reqs.description("Recover with committed ledger entries after the startup snapshot")
+def test_recover_service_with_ledger_after_snapshot(network, args):
+    primary, _ = network.find_primary()
+    snapshots_dir = network.get_committed_snapshots(primary)
+    snapshot_name = ccf.ledger.latest_snapshot(snapshots_dir)
+    assert snapshot_name is not None
+    snapshot_seqno, _ = ccf.ledger.snapshot_index_from_filename(snapshot_name)
+
+    # Keep the copied snapshot, but commit the chunk after it before recovery.
+    # get_ledger() then puts snapshot_seqno + 1 in read-only recovery input,
+    # rather than leaving it in the main directory's current chunk by chance.
+    committed_seqno = network.create_and_wait_for_ledger_chunk(primary)
+    assert committed_seqno > snapshot_seqno, (committed_seqno, snapshot_seqno)
+    return test_recover_service(network, args, snapshots_dir=snapshots_dir)
+
+
 def _recover_service(
     network,
     args,

tests/suite/test_suite.py

@@ -112,6 +112,7 @@
     # recovery:
     recovery.test_recover_service,
     recovery.test_recover_service_aborted,
+    recovery.test_recover_service_with_ledger_after_snapshot,
     # rekey:
     e2e_logging.test_rekey,
     # election:
@@ -142,6 +143,12 @@
 ]
 suites["all"] = all_tests_suite
 
+# Keep a minimal ordering for the local-ledger watermark regression.
+suites["recovery_ledger"] = [
+    recovery.test_recover_service_with_ledger_after_snapshot,
+    governance_history.test_ledger_is_readable,
+]
+
 
 # https://github.com/microsoft/CCF/issues/5236
 regression_5236_suite = [