Skip to content

Commit f6bcccb

Browse files
refactor(gc): consistent hash-side stat names (hash_paths_*, orphaned_hash_paths)
Mirror the schema-addressed naming on the hash-addressed side of the collect() report, so the two sections read in parallel: hash_referenced -> hash_paths_referenced hash_stored -> hash_paths_stored hash_orphaned -> hash_paths_orphaned hash_orphaned_bytes -> hash_paths_orphaned_bytes orphaned_hashes -> orphaned_hash_paths hash_deleted -> hash_paths_deleted Keys and the corresponding internal variables renamed; no behavior change. Tests and docs (datajoint-docs) updated. GC suite 39/39.
1 parent 6f20aef commit f6bcccb

2 files changed

Lines changed: 29 additions & 29 deletions

File tree

src/datajoint/gc.py

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -318,47 +318,47 @@ def collect(self, dry_run: bool = True, verbose: bool = False) -> dict[str, Any]
318318
319319
Returns a dict with per-section stats and the deletion outcome:
320320
321-
- hash_referenced / hash_stored / hash_orphaned / hash_orphaned_bytes
322-
- orphaned_hashes: orphaned hash items as full relative store paths
321+
- hash_paths_referenced / hash_paths_stored / hash_paths_orphaned / hash_paths_orphaned_bytes
322+
- orphaned_hash_paths: orphaned hash items as full relative store paths
323323
(NOT bare hashes — passed as-is to the deleter)
324324
- schema_paths_referenced / schema_paths_stored / schema_paths_orphaned
325325
/ schema_paths_orphaned_bytes
326326
- orphaned_paths: orphaned schema files as full relative store paths
327-
- hash_deleted / schema_paths_deleted / deleted / bytes_freed (0 when
327+
- hash_paths_deleted / schema_paths_deleted / deleted / bytes_freed (0 when
328328
dry_run) / errors / dry_run
329329
"""
330330
# References can be gathered across all schemas at once: because paths
331331
# embed the schema, a file under schema X is only ever covered by an X
332332
# reference, so combined coverage equals per-schema coverage.
333-
hash_referenced = self.hash_references(verbose=verbose)
333+
hash_paths_referenced = self.hash_references(verbose=verbose)
334334
schema_paths_referenced = self.schema_references(verbose=verbose)
335335

336-
hash_stored: dict[str, int] = {}
336+
hash_paths_stored: dict[str, int] = {}
337337
schema_paths_stored: dict[str, int] = {}
338338
for schema in self.schemas:
339-
hash_stored.update(self.list_hash_paths(schema.database))
339+
hash_paths_stored.update(self.list_hash_paths(schema.database))
340340
schema_paths_stored.update(self.list_schema_paths(schema.database))
341341

342-
orphaned_hashes = sorted(set(hash_stored.keys()) - hash_referenced)
342+
orphaned_hash_paths = sorted(set(hash_paths_stored.keys()) - hash_paths_referenced)
343343
# Coverage, not exact set difference: a referenced path may be a
344344
# directory-valued object (many files + a manifest sidecar).
345345
orphaned_paths = sorted(p for p in schema_paths_stored if not _is_covered(p, schema_paths_referenced))
346346

347-
hash_deleted = 0
347+
hash_paths_deleted = 0
348348
schema_paths_deleted = 0
349349
bytes_freed = 0
350350
errors = 0
351351

352352
if not dry_run:
353353
# The size maps from the scan above are reused for the byte tally —
354354
# no second listing pass.
355-
for path in orphaned_hashes:
355+
for path in orphaned_hash_paths:
356356
try:
357357
if delete_path(path, self.store, config=self.config):
358-
hash_deleted += 1
359-
bytes_freed += hash_stored.get(path, 0)
358+
hash_paths_deleted += 1
359+
bytes_freed += hash_paths_stored.get(path, 0)
360360
if verbose:
361-
logger.info(f"Deleted: {path} ({hash_stored.get(path, 0)} bytes)")
361+
logger.info(f"Deleted: {path} ({hash_paths_stored.get(path, 0)} bytes)")
362362
except Exception as e:
363363
errors += 1
364364
logger.warning(f"Failed to delete {path}: {e}")
@@ -376,21 +376,21 @@ def collect(self, dry_run: bool = True, verbose: bool = False) -> dict[str, Any]
376376

377377
return {
378378
# Hash-addressed storage stats
379-
"hash_referenced": len(hash_referenced),
380-
"hash_stored": len(hash_stored),
381-
"hash_orphaned": len(orphaned_hashes),
382-
"hash_orphaned_bytes": sum(hash_stored.get(h, 0) for h in orphaned_hashes),
383-
"orphaned_hashes": orphaned_hashes,
379+
"hash_paths_referenced": len(hash_paths_referenced),
380+
"hash_paths_stored": len(hash_paths_stored),
381+
"hash_paths_orphaned": len(orphaned_hash_paths),
382+
"hash_paths_orphaned_bytes": sum(hash_paths_stored.get(h, 0) for h in orphaned_hash_paths),
383+
"orphaned_hash_paths": orphaned_hash_paths,
384384
# Schema-addressed storage stats
385385
"schema_paths_referenced": len(schema_paths_referenced),
386386
"schema_paths_stored": len(schema_paths_stored),
387387
"schema_paths_orphaned": len(orphaned_paths),
388388
"schema_paths_orphaned_bytes": sum(schema_paths_stored.get(p, 0) for p in orphaned_paths),
389389
"orphaned_paths": orphaned_paths,
390390
# Deletion outcome (all zero when dry_run)
391-
"hash_deleted": hash_deleted,
391+
"hash_paths_deleted": hash_paths_deleted,
392392
"schema_paths_deleted": schema_paths_deleted,
393-
"deleted": hash_deleted + schema_paths_deleted,
393+
"deleted": hash_paths_deleted + schema_paths_deleted,
394394
"bytes_freed": bytes_freed,
395395
"errors": errors,
396396
"dry_run": dry_run,

tests/integration/test_gc.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -228,8 +228,8 @@ def test_dry_run_reports_stats_and_deletes_nothing(self):
228228
assert stats["deleted"] == 0 and stats["bytes_freed"] == 0
229229
mock_delete.assert_not_called()
230230
# full report is present
231-
assert stats["hash_orphaned"] == 1 and stats["orphaned_hashes"] == ["_hash/s/path3"]
232-
assert stats["hash_orphaned_bytes"] == 200
231+
assert stats["hash_paths_orphaned"] == 1 and stats["orphaned_hash_paths"] == ["_hash/s/path3"]
232+
assert stats["hash_paths_orphaned_bytes"] == 200
233233
assert stats["schema_paths_orphaned"] == 1 and stats["orphaned_paths"] == ["s/t/pk2/f"]
234234
assert stats["schema_paths_orphaned_bytes"] == 300
235235
# no combined totals
@@ -246,7 +246,7 @@ def test_delete_removes_orphans_via_bound_store(self):
246246
collector = _mock_collector()
247247
stats = collector.collect(dry_run=False)
248248

249-
assert stats["hash_deleted"] == 1 and stats["schema_paths_deleted"] == 1
249+
assert stats["hash_paths_deleted"] == 1 and stats["schema_paths_deleted"] == 1
250250
assert stats["deleted"] == 2
251251
assert stats["bytes_freed"] == 500 # 200 (hash path3) + 300 (schema pk2)
252252
assert stats["dry_run"] is False
@@ -303,7 +303,7 @@ def schema_object(self, connection_test, prefix, mock_stores):
303303
schema.drop()
304304

305305
def test_scan_finds_active_blob_reference(self, schema_blob):
306-
"""scan() must report hash_referenced >= 1 for a populated <blob@> column.
306+
"""scan() must report hash_paths_referenced >= 1 for a populated <blob@> column.
307307
308308
Decoded value type returned by BlobCodec.decode is numpy.ndarray, which
309309
does not satisfy the raw-metadata dict/JSON-string check in referenced_paths — this
@@ -314,7 +314,7 @@ def test_scan_finds_active_blob_reference(self, schema_blob):
314314
collector = _gc(schema_blob)
315315
stats = collector.collect()
316316

317-
assert stats["hash_referenced"] >= 1, f"scan should find the active <blob@> reference; got {stats}"
317+
assert stats["hash_paths_referenced"] >= 1, f"scan should find the active <blob@> reference; got {stats}"
318318

319319
def test_scan_finds_active_npy_reference(self, schema_npy):
320320
"""scan() must report schema_paths_referenced >= 1 for a populated <npy@> column.
@@ -607,8 +607,8 @@ def test_custom_sections_written_scanned_and_collected(self, schema_prefixed):
607607

608608
# GC scans the same sections: everything referenced, nothing falsely orphaned
609609
stats = collector.collect()
610-
assert stats["hash_referenced"] >= 1
611-
assert not (hash_refs & set(stats["orphaned_hashes"]))
610+
assert stats["hash_paths_referenced"] >= 1
611+
assert not (hash_refs & set(stats["orphaned_hash_paths"]))
612612
assert not (obj_refs & set(stats["orphaned_paths"]))
613613

614614
# And reclamation works within the configured sections
@@ -663,7 +663,7 @@ def test_live_hash_object_survives_prefix_change(self, schema_pfx):
663663
assert hash_ref not in set(
664664
stats["orphaned_paths"]
665665
), "live hash object outside the current hash section must not be a schema orphan"
666-
assert hash_ref not in set(stats["orphaned_hashes"])
666+
assert hash_ref not in set(stats["orphaned_hash_paths"])
667667

668668
# End-to-end: collect must not destroy the live hash object.
669669
collector.collect(dry_run=False)
@@ -735,8 +735,8 @@ def test_scan_one_schema_ignores_the_other(self, two_schemas):
735735

736736
stats = collector.collect()
737737
# b's live objects are outside a's subtree → not counted, not orphaned
738-
assert stats["hash_orphaned"] == 0 and stats["schema_paths_orphaned"] == 0
739-
assert not any(b.database in p for p in stats["orphaned_paths"] + stats["orphaned_hashes"])
738+
assert stats["hash_paths_orphaned"] == 0 and stats["schema_paths_orphaned"] == 0
739+
assert not any(b.database in p for p in stats["orphaned_paths"] + stats["orphaned_hash_paths"])
740740

741741
def test_collect_one_schema_never_touches_the_other(self, two_schemas):
742742
a, b, b_blob, b_obj = two_schemas

0 commit comments

Comments
 (0)