From 6a33bf11562f249588bd6a120de98072c41504d0 Mon Sep 17 00:00:00 2001 From: mkuchenbecker Date: Wed, 2 Sep 2026 12:50:58 -0700 Subject: [PATCH 1/2] fix: delete files during snapshot expiration Enable Iceberg's synchronous file cleanup when expiring snapshots so files are removed as part of the operation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../main/java/com/linkedin/openhouse/jobs/spark/Operations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/spark/src/main/java/com/linkedin/openhouse/jobs/spark/Operations.java b/apps/spark/src/main/java/com/linkedin/openhouse/jobs/spark/Operations.java index 6d15821aa..838895ed5 100644 --- a/apps/spark/src/main/java/com/linkedin/openhouse/jobs/spark/Operations.java +++ b/apps/spark/src/main/java/com/linkedin/openhouse/jobs/spark/Operations.java @@ -272,7 +272,7 @@ public void expireSnapshots(String fqtn, int maxAge, String granularity, int ver * number of snapshots younger than the maxAge */ public void expireSnapshots(Table table, int maxAge, String granularity, int versions) { - ExpireSnapshots expireSnapshotsCommand = table.expireSnapshots().cleanExpiredFiles(false); + ExpireSnapshots expireSnapshotsCommand = table.expireSnapshots().cleanExpiredFiles(true); // maxAge will always be defined ChronoUnit timeUnitGranularity = From 5f1af8fdc73225289ef73034262c28179e393e3d Mon Sep 17 00:00:00 2001 From: mkuchenbecker Date: Wed, 2 Sep 2026 13:01:08 -0700 Subject: [PATCH 2/2] test: verify snapshot expiration deletes files Guard the synchronous cleanup setting by verifying that snapshot expiration configures Iceberg to clean expired files. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../openhouse/jobs/spark/OperationsTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/spark/src/test/java/com/linkedin/openhouse/jobs/spark/OperationsTest.java b/apps/spark/src/test/java/com/linkedin/openhouse/jobs/spark/OperationsTest.java index db8ff7e98..309d81c88 100644 --- a/apps/spark/src/test/java/com/linkedin/openhouse/jobs/spark/OperationsTest.java +++ b/apps/spark/src/test/java/com/linkedin/openhouse/jobs/spark/OperationsTest.java @@ -34,6 +34,7 @@ import org.apache.commons.lang3.tuple.Triple; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.ExpireSnapshots; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; @@ -509,6 +510,18 @@ public void testOrphanFilesDeletionDeleteDataWhenDataManifestNotExists() throws } } + @Test + public void testExpireSnapshotsCleansExpiredFilesSynchronously() throws Exception { + Table table = Mockito.mock(Table.class); + ExpireSnapshots expireSnapshots = + Mockito.mock(ExpireSnapshots.class, Mockito.RETURNS_SELF); + Mockito.when(table.expireSnapshots()).thenReturn(expireSnapshots); + + Operations.of(getSparkSession(), otelEmitter).expireSnapshots(table, 3, "DAYS", 0); + + Mockito.verify(expireSnapshots).cleanExpiredFiles(true); + } + @Test public void testSnapshotsExpirationMaxAge() throws Exception { final String tableName = "db.test_es_maxage_java";