HIVE-29853: Iceberg CompactionEvaluator double-counts input files

iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java

@@ -73,7 +73,6 @@ public CompactionEvaluator(Table table, CompactionInfo ci, Map<String, String> p
     );
     this.table = table;
     this.ci = ci;
-    addFiles();
   }
 
   public boolean isEligibleForCompaction() {

iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q

@@ -42,6 +42,11 @@ alter table ice_orc create tag v1;
 
 alter table ice_orc set partition spec(company_id, dept_id);
 insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100);
+insert into ice_orc VALUES ('fn41','ln41', 1, 11, 100);
+insert into ice_orc VALUES ('fn42','ln42', 1, 11, 100);
+insert into ice_orc VALUES ('fn43','ln43', 1, 11, 100);
+insert into ice_orc VALUES ('fn44','ln44', 1, 11, 100);
+insert into ice_orc VALUES ('fn45','ln45', 1, 11, 100);
 insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100);
 insert into ice_orc VALUES ('fn6','ln6', 2, 20, 100);
 insert into ice_orc VALUES (null,null, null, null, null);

iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q

@@ -38,15 +38,21 @@ insert into ice_orc VALUES ('fn1','ln1', 1);
 insert into ice_orc VALUES ('fn2','ln2', 1);
 insert into ice_orc VALUES ('fn3','ln3', 1);
 insert into ice_orc VALUES ('fn4','ln4', 1);
-delete from ice_orc where last_name in ('ln3', 'ln4');
+insert into ice_orc VALUES ('fn5','ln5', 1);
+insert into ice_orc VALUES ('fn6','ln6', 1);
+insert into ice_orc VALUES ('fn7','ln7', 1);
+delete from ice_orc where last_name in ('ln6', 'ln7');
 
 alter table ice_orc set partition spec(dept_id);
 
-insert into ice_orc PARTITION(dept_id=2) VALUES ('fn5','ln5');
-insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6');
-insert into ice_orc PARTITION(dept_id=2) VALUES ('fn7','ln7');
 insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8');
-delete from ice_orc where last_name in ('ln7', 'ln8');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn9','ln9');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn10','ln10');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn11','ln11');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn12','ln12');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn13','ln13');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn14','ln14');
+delete from ice_orc where last_name in ('ln13', 'ln14');
 
 select * from ice_orc;
 describe formatted ice_orc;
@@ -60,8 +66,13 @@ show compactions order by 'partition';
 
 alter table ice_orc set partition spec();
 
-insert into ice_orc VALUES ('fn9','ln9', 3);
-insert into ice_orc VALUES ('fn10','ln10', 3);
+insert into ice_orc VALUES ('fn15','ln15', 3);
+insert into ice_orc VALUES ('fn16','ln16', 3);
+insert into ice_orc VALUES ('fn17','ln17', 3);
+insert into ice_orc VALUES ('fn18','ln18', 3);
+insert into ice_orc VALUES ('fn19','ln19', 3);
+insert into ice_orc VALUES ('fn20','ln20', 3);
+insert into ice_orc VALUES ('fn21','ln21', 3);
 
 select * from ice_orc;
 describe formatted ice_orc;

iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q

@@ -49,6 +49,11 @@ insert into ice_orc VALUES
 insert into ice_orc VALUES 
 ('fn7','ln7', 2, 20, 101),
 ('fn8','ln8', 2, 20, 101);
+insert into ice_orc VALUES ('fn81','ln81', 2, 20, 101);
+insert into ice_orc VALUES ('fn82','ln82', 2, 20, 101);
+insert into ice_orc VALUES ('fn83','ln83', 2, 20, 101);
+insert into ice_orc VALUES ('fn84','ln84', 2, 20, 101);
+insert into ice_orc VALUES ('fn85','ln85', 2, 20, 101);
 
 insert into ice_orc VALUES 
 ('fn9', 'ln9',  3, 20, 101),

iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q

@@ -25,65 +25,62 @@ set hive.vectorized.execution.enabled=true;
 set hive.optimize.shared.work.merge.ts.schema=true;
 
 create table ice_orc (
-    event_id int, 
+    event_id int,
     event_time timestamp with local time zone,
-    event_src string    
+    event_src string
 )
 partitioned by spec(truncate(3, event_src))
 stored by iceberg stored as orc
 tblproperties ('compactor.threshold.target.size'='1500');
 
-insert into ice_orc values 
+insert into ice_orc values
     (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'),
     (2, cast('2023-07-21 00:00:00' as timestamp with local time zone), 'AAA_2');
-insert into ice_orc values 
+insert into ice_orc values
     (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1'),
     (4, cast('2023-07-24 00:00:00' as timestamp with local time zone), 'AAA_2');
-insert into ice_orc values 
+insert into ice_orc values
     (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1'),
     (6, cast('2023-08-05 00:00:00' as timestamp with local time zone), 'BBB_2');
-insert into ice_orc values 
+insert into ice_orc values
     (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1'),
     (8, cast('2023-08-27 00:00:00' as timestamp with local time zone), 'BBB_2');
-    
+
 alter table ice_orc set partition spec(truncate(3, event_src), month(event_time));
 
-insert into ice_orc values 
+insert into ice_orc values
     (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'),
     (10, cast('2024-07-21 00:00:00' as timestamp with local time zone), 'AAA_2');
-insert into ice_orc values     
-    (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1'),
-    (12, cast('2024-08-23 00:00:00' as timestamp with local time zone), 'AAA_2');
-insert into ice_orc values     
-    (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1'),
-    (14, cast('2024-08-25 00:00:00' as timestamp with local time zone), 'AAA_2');
-insert into ice_orc values 
+insert into ice_orc values (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1');
+insert into ice_orc values (12, cast('2024-08-23 00:00:00' as timestamp with local time zone), 'AAA_2');
+insert into ice_orc values (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1');
+insert into ice_orc values (14, cast('2024-08-25 00:00:00' as timestamp with local time zone), 'AAA_2');
+insert into ice_orc values
     (15, cast('2024-09-24 00:00:00' as timestamp with local time zone), 'AAA_1'),
     (16, cast('2024-09-25 00:00:00' as timestamp with local time zone), 'AAA_2');
-insert into ice_orc values 
-    (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1'),
-    (18, cast('2024-07-02 00:00:00' as timestamp with local time zone), 'BBB_2');
-insert into ice_orc values 
-    (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1'),
-    (20, cast('2024-07-04 00:00:00' as timestamp with local time zone), 'BBB_2');
-insert into ice_orc values 
-    (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1'),
-    (22, cast('2024-08-04 00:00:00' as timestamp with local time zone), 'BBB_2');
-insert into ice_orc values 
-    (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1'),
-    (24, cast('2024-08-06 00:00:00' as timestamp with local time zone), 'BBB_2');
+insert into ice_orc values
+    (151, cast('2024-09-26 00:00:00' as timestamp with local time zone), 'AAA_1'),
+    (161, cast('2024-09-27 00:00:00' as timestamp with local time zone), 'AAA_2');
+insert into ice_orc values (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1');
+insert into ice_orc values (18, cast('2024-07-02 00:00:00' as timestamp with local time zone), 'BBB_2');
+insert into ice_orc values (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1');
+insert into ice_orc values (20, cast('2024-07-04 00:00:00' as timestamp with local time zone), 'BBB_2');
+insert into ice_orc values (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1');
+insert into ice_orc values (22, cast('2024-08-04 00:00:00' as timestamp with local time zone), 'BBB_2');
+insert into ice_orc values (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1');
+insert into ice_orc values (24, cast('2024-08-06 00:00:00' as timestamp with local time zone), 'BBB_2');
 insert into ice_orc values
     (25, cast('2024-09-05 00:00:00' as timestamp with local time zone), 'BBB_1'),
     (26, cast('2024-09-06 00:00:00' as timestamp with local time zone), 'BBB_2');
 
 select * from ice_orc;
 describe formatted ice_orc;
 
-alter table ice_orc COMPACT 'major' and wait 
+alter table ice_orc COMPACT 'major' and wait
 where (event_src in ('BBB_1', 'BBB_2') and event_time < '2024-09-01 00:00:00') or
       (event_src in ('AAA_1', 'AAA_2') and event_time > '2024-08-01 00:00:00');
 
 select * from ice_orc;
 describe formatted ice_orc;
 
-show compactions order by 'partition';
+show compactions order by 'partition';

iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q

@@ -24,16 +24,13 @@ CREATE TABLE x (name VARCHAR(50), age TINYINT, num_clicks BIGINT)
 stored by iceberg stored as orc 
 TBLPROPERTIES ('external.table.purge'='true', 'format-version'='2', 'compactor.threshold.target.size'='1500');
 
-insert into x values 
-('amy', 35, 123412344),
-('adxfvy', 36, 123412534),
-('amsdfyy', 37, 123417234),
-('asafmy', 38, 123412534);
-
-insert into x values 
-('amerqwy', 39, 123441234),
-('amyxzcv', 40, 123341234),
-('erweramy', 45, 122341234);
+insert into x values ('amy', 35, 123412344);
+insert into x values ('adxfvy', 36, 123412534);
+insert into x values ('amsdfyy', 37, 123417234);
+insert into x values ('asafmy', 38, 123412534);
+insert into x values ('amerqwy', 39, 123441234);
+insert into x values ('amyxzcv', 40, 123341234);
+insert into x values ('erweramy', 45, 122341234);
 
 select * from default.x.data_files;
 select count(*) from default.x.data_files;

iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q

@@ -38,13 +38,17 @@ tblproperties ('format-version'='2');
 
 insert into ice_orc VALUES ('fn2','ln2', 1), ('fn1','ln1', 1);
 insert into ice_orc VALUES ('fn4','ln4', 1), ('fn3','ln3', 1);
+insert into ice_orc VALUES ('fn15','ln15', 1);
+insert into ice_orc VALUES ('fn16','ln16', 1);
 
 delete from ice_orc where last_name in ('ln4');
 
 alter table ice_orc set partition spec(dept_id);
 
 insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6'), ('fn5','ln5');
 insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8'), ('fn7','ln7');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn17','ln17');
+insert into ice_orc PARTITION(dept_id=2) VALUES ('fn18','ln18');
 
 delete from ice_orc where last_name in ('ln8');
 
@@ -73,6 +77,8 @@ alter table ice_orc set partition spec();
 
 insert into ice_orc VALUES ('fn9','ln9', 3), ('fn10','ln10', 3);
 insert into ice_orc VALUES ('fn11','ln11', 3), ('fn12','ln12', 3);
+insert into ice_orc VALUES ('fn19','ln19', 3);
+insert into ice_orc VALUES ('fn20','ln20', 3);
 
 select * from ice_orc;
 describe formatted ice_orc;

iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q

@@ -33,8 +33,12 @@ stored by iceberg stored as orc
 tblproperties ('format-version'='2');
 
 insert into ice_orc VALUES ('fn1','ln1');
-insert into ice_orc VALUES ('fn5','ln5'), ('fn7','ln7'), ('fn4','ln4'), ('fn6','ln6');
-insert into ice_orc VALUES ('fn3','ln3'), ('fn2','ln2');
+insert into ice_orc VALUES ('fn5','ln5');
+insert into ice_orc VALUES ('fn7','ln7');
+insert into ice_orc VALUES ('fn4','ln4');
+insert into ice_orc VALUES ('fn6','ln6');
+insert into ice_orc VALUES ('fn3','ln3');
+insert into ice_orc VALUES ('fn2','ln2');
 
 -- File size threshold = fragment size = target_size/fragment_ratio = 2920/8 = 365.
 alter table ice_orc set tblproperties ('compactor.threshold.target.size'='2920', 'compactor.threshold.min.input.files'='2');

iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q

@@ -74,6 +74,8 @@ alter table part_tbl set tblproperties ('compactor.threshold.target.size'='8000'
 alter table part_tbl set partition spec(dept_id, id);
 
 insert into part_tbl values (5,'p5', 10);
+insert into part_tbl values (5,'p5', 10);
+insert into part_tbl values (6,'p6', 20);
 insert into part_tbl values (6,'p6', 20);
 
 SELECT *, ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER