diff --git a/apps/spark/src/main/java/com/linkedin/openhouse/jobs/spark/Operations.java b/apps/spark/src/main/java/com/linkedin/openhouse/jobs/spark/Operations.java index 6d15821aa..838895ed5 100644 --- a/apps/spark/src/main/java/com/linkedin/openhouse/jobs/spark/Operations.java +++ b/apps/spark/src/main/java/com/linkedin/openhouse/jobs/spark/Operations.java @@ -272,7 +272,7 @@ public void expireSnapshots(String fqtn, int maxAge, String granularity, int ver * number of snapshots younger than the maxAge */ public void expireSnapshots(Table table, int maxAge, String granularity, int versions) { - ExpireSnapshots expireSnapshotsCommand = table.expireSnapshots().cleanExpiredFiles(false); + ExpireSnapshots expireSnapshotsCommand = table.expireSnapshots().cleanExpiredFiles(true); // maxAge will always be defined ChronoUnit timeUnitGranularity = diff --git a/apps/spark/src/test/java/com/linkedin/openhouse/jobs/spark/OperationsTest.java b/apps/spark/src/test/java/com/linkedin/openhouse/jobs/spark/OperationsTest.java index db8ff7e98..309d81c88 100644 --- a/apps/spark/src/test/java/com/linkedin/openhouse/jobs/spark/OperationsTest.java +++ b/apps/spark/src/test/java/com/linkedin/openhouse/jobs/spark/OperationsTest.java @@ -34,6 +34,7 @@ import org.apache.commons.lang3.tuple.Triple; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.ExpireSnapshots; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; @@ -509,6 +510,18 @@ public void testOrphanFilesDeletionDeleteDataWhenDataManifestNotExists() throws } } + @Test + public void testExpireSnapshotsCleansExpiredFilesSynchronously() throws Exception { + Table table = Mockito.mock(Table.class); + ExpireSnapshots expireSnapshots = + Mockito.mock(ExpireSnapshots.class, Mockito.RETURNS_SELF); + Mockito.when(table.expireSnapshots()).thenReturn(expireSnapshots); + + Operations.of(getSparkSession(), otelEmitter).expireSnapshots(table, 3, "DAYS", 0); + + Mockito.verify(expireSnapshots).cleanExpiredFiles(true); + } + @Test public void testSnapshotsExpirationMaxAge() throws Exception { final String tableName = "db.test_es_maxage_java";