diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java index e3f177a842c8..78c3a08e9d28 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java @@ -73,7 +73,6 @@ public CompactionEvaluator(Table table, CompactionInfo ci, Map p ); this.table = table; this.ci = ci; - addFiles(); } public boolean isEligibleForCompaction() { diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q index a38a856f6430..10c17d7eb42b 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q @@ -42,6 +42,11 @@ alter table ice_orc create tag v1; alter table ice_orc set partition spec(company_id, dept_id); insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100); +insert into ice_orc VALUES ('fn41','ln41', 1, 11, 100); +insert into ice_orc VALUES ('fn42','ln42', 1, 11, 100); +insert into ice_orc VALUES ('fn43','ln43', 1, 11, 100); +insert into ice_orc VALUES ('fn44','ln44', 1, 11, 100); +insert into ice_orc VALUES ('fn45','ln45', 1, 11, 100); insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100); insert into ice_orc VALUES ('fn6','ln6', 2, 20, 100); insert into ice_orc VALUES (null,null, null, null, null); diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q index 4ca441ff0b14..faa3933d7286 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q @@ -38,15 +38,21 @@ insert into ice_orc VALUES ('fn1','ln1', 1); insert into ice_orc VALUES ('fn2','ln2', 1); insert into ice_orc VALUES ('fn3','ln3', 1); insert into ice_orc VALUES ('fn4','ln4', 1); -delete from ice_orc where last_name in ('ln3', 'ln4'); +insert into ice_orc VALUES ('fn5','ln5', 1); +insert into ice_orc VALUES ('fn6','ln6', 1); +insert into ice_orc VALUES ('fn7','ln7', 1); +delete from ice_orc where last_name in ('ln6', 'ln7'); alter table ice_orc set partition spec(dept_id); -insert into ice_orc PARTITION(dept_id=2) VALUES ('fn5','ln5'); -insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6'); -insert into ice_orc PARTITION(dept_id=2) VALUES ('fn7','ln7'); insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8'); -delete from ice_orc where last_name in ('ln7', 'ln8'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn9','ln9'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn10','ln10'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn11','ln11'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn12','ln12'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn13','ln13'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn14','ln14'); +delete from ice_orc where last_name in ('ln13', 'ln14'); select * from ice_orc; describe formatted ice_orc; @@ -60,8 +66,13 @@ show compactions order by 'partition'; alter table ice_orc set partition spec(); -insert into ice_orc VALUES ('fn9','ln9', 3); -insert into ice_orc VALUES ('fn10','ln10', 3); +insert into ice_orc VALUES ('fn15','ln15', 3); +insert into ice_orc VALUES ('fn16','ln16', 3); +insert into ice_orc VALUES ('fn17','ln17', 3); +insert into ice_orc VALUES ('fn18','ln18', 3); +insert into ice_orc VALUES ('fn19','ln19', 3); +insert into ice_orc VALUES ('fn20','ln20', 3); +insert into ice_orc VALUES ('fn21','ln21', 3); select * from ice_orc; describe formatted ice_orc; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q index 0dfe28140421..852d978cd87c 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q @@ -49,6 +49,11 @@ insert into ice_orc VALUES insert into ice_orc VALUES ('fn7','ln7', 2, 20, 101), ('fn8','ln8', 2, 20, 101); +insert into ice_orc VALUES ('fn81','ln81', 2, 20, 101); +insert into ice_orc VALUES ('fn82','ln82', 2, 20, 101); +insert into ice_orc VALUES ('fn83','ln83', 2, 20, 101); +insert into ice_orc VALUES ('fn84','ln84', 2, 20, 101); +insert into ice_orc VALUES ('fn85','ln85', 2, 20, 101); insert into ice_orc VALUES ('fn9', 'ln9', 3, 20, 101), diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q index 896f07d5bdaf..830c4970c83b 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q @@ -33,47 +33,60 @@ partitioned by spec(truncate(3, event_src)) stored by iceberg stored as orc tblproperties ('compactor.threshold.target.size'='1500'); -insert into ice_orc values - (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), +insert into ice_orc values + (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'); +insert into ice_orc values (2, cast('2023-07-21 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values - (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1'), +insert into ice_orc values + (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1'); +insert into ice_orc values (4, cast('2023-07-24 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values - (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1'), +insert into ice_orc values + (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1'); +insert into ice_orc values (6, cast('2023-08-05 00:00:00' as timestamp with local time zone), 'BBB_2'); -insert into ice_orc values - (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1'), +insert into ice_orc values + (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1'); +insert into ice_orc values (8, cast('2023-08-27 00:00:00' as timestamp with local time zone), 'BBB_2'); alter table ice_orc set partition spec(truncate(3, event_src), month(event_time)); -insert into ice_orc values - (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), +insert into ice_orc values + (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'); +insert into ice_orc values (10, cast('2024-07-21 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values - (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1'), +insert into ice_orc values + (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1'); +insert into ice_orc values (12, cast('2024-08-23 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values - (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1'), +insert into ice_orc values + (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1'); +insert into ice_orc values (14, cast('2024-08-25 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values - (15, cast('2024-09-24 00:00:00' as timestamp with local time zone), 'AAA_1'), +insert into ice_orc values + (15, cast('2024-09-24 00:00:00' as timestamp with local time zone), 'AAA_1'); +insert into ice_orc values (16, cast('2024-09-25 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values - (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1'), +insert into ice_orc values + (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1'); +insert into ice_orc values (18, cast('2024-07-02 00:00:00' as timestamp with local time zone), 'BBB_2'); -insert into ice_orc values - (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1'), +insert into ice_orc values + (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1'); +insert into ice_orc values (20, cast('2024-07-04 00:00:00' as timestamp with local time zone), 'BBB_2'); -insert into ice_orc values - (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1'), +insert into ice_orc values + (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1'); +insert into ice_orc values (22, cast('2024-08-04 00:00:00' as timestamp with local time zone), 'BBB_2'); -insert into ice_orc values - (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1'), +insert into ice_orc values + (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1'); +insert into ice_orc values (24, cast('2024-08-06 00:00:00' as timestamp with local time zone), 'BBB_2'); insert into ice_orc values - (25, cast('2024-09-05 00:00:00' as timestamp with local time zone), 'BBB_1'), + (25, cast('2024-09-05 00:00:00' as timestamp with local time zone), 'BBB_1'); +insert into ice_orc values (26, cast('2024-09-06 00:00:00' as timestamp with local time zone), 'BBB_2'); select * from ice_orc; @@ -86,4 +99,4 @@ where (event_src in ('BBB_1', 'BBB_2') and event_time < '2024-09-01 00:00:00') o select * from ice_orc; describe formatted ice_orc; -show compactions order by 'partition'; \ No newline at end of file +show compactions order by 'partition'; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q index ecdeacffbbf1..24b23769a365 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q @@ -24,16 +24,13 @@ CREATE TABLE x (name VARCHAR(50), age TINYINT, num_clicks BIGINT) stored by iceberg stored as orc TBLPROPERTIES ('external.table.purge'='true', 'format-version'='2', 'compactor.threshold.target.size'='1500'); -insert into x values -('amy', 35, 123412344), -('adxfvy', 36, 123412534), -('amsdfyy', 37, 123417234), -('asafmy', 38, 123412534); - -insert into x values -('amerqwy', 39, 123441234), -('amyxzcv', 40, 123341234), -('erweramy', 45, 122341234); +insert into x values ('amy', 35, 123412344); +insert into x values ('adxfvy', 36, 123412534); +insert into x values ('amsdfyy', 37, 123417234); +insert into x values ('asafmy', 38, 123412534); +insert into x values ('amerqwy', 39, 123441234); +insert into x values ('amyxzcv', 40, 123341234); +insert into x values ('erweramy', 45, 122341234); select * from default.x.data_files; select count(*) from default.x.data_files; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q index f89fbf519e0d..313103da1b2f 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q @@ -38,6 +38,8 @@ tblproperties ('format-version'='2'); insert into ice_orc VALUES ('fn2','ln2', 1), ('fn1','ln1', 1); insert into ice_orc VALUES ('fn4','ln4', 1), ('fn3','ln3', 1); +insert into ice_orc VALUES ('fn15','ln15', 1); +insert into ice_orc VALUES ('fn16','ln16', 1); delete from ice_orc where last_name in ('ln4'); @@ -45,6 +47,8 @@ alter table ice_orc set partition spec(dept_id); insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6'), ('fn5','ln5'); insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8'), ('fn7','ln7'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn17','ln17'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn18','ln18'); delete from ice_orc where last_name in ('ln8'); @@ -73,6 +77,8 @@ alter table ice_orc set partition spec(); insert into ice_orc VALUES ('fn9','ln9', 3), ('fn10','ln10', 3); insert into ice_orc VALUES ('fn11','ln11', 3), ('fn12','ln12', 3); +insert into ice_orc VALUES ('fn19','ln19', 3); +insert into ice_orc VALUES ('fn20','ln20', 3); select * from ice_orc; describe formatted ice_orc; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q index 61e3a82525d1..3f7570c0310f 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q @@ -33,11 +33,15 @@ stored by iceberg stored as orc tblproperties ('format-version'='2'); insert into ice_orc VALUES ('fn1','ln1'); -insert into ice_orc VALUES ('fn5','ln5'), ('fn7','ln7'), ('fn4','ln4'), ('fn6','ln6'); -insert into ice_orc VALUES ('fn3','ln3'), ('fn2','ln2'); +insert into ice_orc VALUES ('fn5','ln5'); +insert into ice_orc VALUES ('fn7','ln7'); +insert into ice_orc VALUES ('fn4','ln4'); +insert into ice_orc VALUES ('fn6','ln6'); +insert into ice_orc VALUES ('fn3','ln3'); +insert into ice_orc VALUES ('fn2','ln2'); --- File size threshold = fragment size = target_size/fragment_ratio = 2920/8 = 365. -alter table ice_orc set tblproperties ('compactor.threshold.target.size'='2920', 'compactor.threshold.min.input.files'='2'); +-- File size threshold = fragment size = target_size/fragment_ratio = 4000/8 = 365. +alter table ice_orc set tblproperties ('compactor.threshold.target.size'='4000', 'compactor.threshold.min.input.files'='2'); explain alter table ice_orc COMPACT 'minor' and wait pool 'iceberg'; describe formatted ice_orc; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q index 8f802d75b39a..5a7de18b0a0a 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q @@ -74,6 +74,8 @@ alter table part_tbl set tblproperties ('compactor.threshold.target.size'='8000' alter table part_tbl set partition spec(dept_id, id); insert into part_tbl values (5,'p5', 10); +insert into part_tbl values (5,'p5', 10); +insert into part_tbl values (6,'p6', 20); insert into part_tbl values (6,'p6', 20); SELECT *, ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out index 8e77ed6fd0a9..5afab5819057 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out @@ -75,6 +75,46 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn41','ln41', 1, 11, 100) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn41','ln41', 1, 11, 100) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn42','ln42', 1, 11, 100) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn42','ln42', 1, 11, 100) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn43','ln43', 1, 11, 100) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn43','ln43', 1, 11, 100) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn44','ln44', 1, 11, 100) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn44','ln44', 1, 11, 100) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn45','ln45', 1, 11, 100) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn45','ln45', 1, 11, 100) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table @@ -257,6 +297,11 @@ NULL NULL NULL NULL NULL fn2 ln2a 1 10 100 fn3 ln3a 1 11 100 fn4 ln4a 1 11 100 +fn41 ln41 1 11 100 +fn42 ln42 1 11 100 +fn43 ln43 1 11 100 +fn44 ln44 1 11 100 +fn45 ln45 1 11 100 fn5 ln5a 2 20 100 fn6 ln6a 2 20 100 fn7 ln7a 2 21 100 @@ -294,19 +339,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"5\",\"manifests-replaced\":\"6\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"17\",\"total-delete-files\":\"8\",\"total-position-deletes\":\"8\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"5\",\"manifests-replaced\":\"6\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"22\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"22\",\"total-delete-files\":\"8\",\"total-position-deletes\":\"8\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 hive.compactor.worker.pool iceberg #### A masked pattern was here #### - numFiles 17 - numRows 17 + numFiles 22 + numRows 22 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 20 + snapshot-count 25 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -340,6 +385,11 @@ POSTHOOK: Input: default@ice_orc {"company_id":100,"dept_id":1,"team_id":11} 2 0 1 {"company_id":100,"dept_id":1,"team_id":11} 2 0 1 {"company_id":100,"dept_id":1,"team_id":null} 1 0 1 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 1 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 1 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 1 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 1 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 1 {"company_id":100,"dept_id":1,"team_id":null} 1 1 1 {"company_id":100,"dept_id":2,"team_id":21} 2 0 1 {"company_id":100,"dept_id":2,"team_id":21} 2 0 1 @@ -426,14 +476,14 @@ STAGE PLANS: TableScan alias: ice_orc.tag_v2 Snapshot ref: tag_v2 - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -469,14 +519,14 @@ STAGE PLANS: TableScan alias: ice_orc.tag_v3 Snapshot ref: tag_v3 - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -512,14 +562,14 @@ STAGE PLANS: TableScan alias: ice_orc.tag_v4 Snapshot ref: tag_v4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -554,14 +604,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_orc - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -645,17 +695,17 @@ STAGE PLANS: alias: ice_orc.tag_v2 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v2 - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -692,17 +742,17 @@ STAGE PLANS: alias: ice_orc.tag_v3 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v3 - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -739,17 +789,17 @@ STAGE PLANS: alias: ice_orc.tag_v4 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -785,17 +835,17 @@ STAGE PLANS: TableScan alias: ice_orc filterExpr: company_id is not null (type: boolean) - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -851,6 +901,11 @@ NULL NULL NULL NULL NULL fn2 ln2a 1 10 100 fn3 ln3a 1 11 100 fn4 ln4a 1 11 100 +fn41 ln41 1 11 100 +fn42 ln42 1 11 100 +fn43 ln43 1 11 100 +fn44 ln44 1 11 100 +fn45 ln45 1 11 100 fn5 ln5a 2 20 100 fn6 ln6a 2 20 100 fn7 ln7a 2 21 100 @@ -888,20 +943,20 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"6\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"5\",\"added-data-files\":\"2\",\"deleted-data-files\":\"10\",\"removed-position-delete-files\":\"5\",\"removed-delete-files\":\"5\",\"added-records\":\"5\",\"deleted-records\":\"10\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"5\",\"changed-partition-count\":\"8\",\"total-records\":\"9\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"6\",\"manifests-kept\":\"3\",\"manifests-replaced\":\"5\",\"added-data-files\":\"2\",\"deleted-data-files\":\"10\",\"removed-position-delete-files\":\"5\",\"removed-delete-files\":\"5\",\"added-records\":\"5\",\"deleted-records\":\"10\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"5\",\"changed-partition-count\":\"8\",\"total-records\":\"14\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"5\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 hive.compactor.worker.pool iceberg #### A masked pattern was here #### - numFiles 4 + numFiles 5 numPartitions 3 - numRows 9 + numRows 14 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 23 + snapshot-count 28 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -941,6 +996,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_orc #### A masked pattern was here #### {"company_id":100,"dept_id":1,"team_id":null} 1 0 3 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 5 {"company_id":100,"dept_id":2,"team_id":null} 1 0 3 {"company_id":null,"dept_id":null,"team_id":null} 1 0 1 {"company_id":null,"dept_id":null,"team_id":null} 1 0 2 @@ -965,14 +1021,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_orc - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 14 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 14 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 14 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1008,14 +1064,14 @@ STAGE PLANS: TableScan alias: ice_orc filterExpr: company_id is not null (type: boolean) - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out index 60ad0af240d3..307e5d4bc47a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out @@ -62,11 +62,35 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc -PREHOOK: query: delete from ice_orc where last_name in ('ln3', 'ln4') +PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn6','ln6', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn6','ln6', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn7','ln7', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn7','ln7', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: delete from ice_orc where last_name in ('ln6', 'ln7') PREHOOK: type: QUERY PREHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### -POSTHOOK: query: delete from ice_orc where last_name in ('ln3', 'ln4') +POSTHOOK: query: delete from ice_orc where last_name in ('ln6', 'ln7') POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### @@ -77,43 +101,67 @@ POSTHOOK: query: alter table ice_orc set partition spec(dept_id) POSTHOOK: type: ALTERTABLE_SETPARTSPEC POSTHOOK: Input: ice_comp@ice_orc POSTHOOK: Output: ice_comp@ice_orc -PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn5','ln5') +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc@dept_id=2 -POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn5','ln5') +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 -PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6') +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn9','ln9') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc@dept_id=2 -POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6') +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn9','ln9') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 -PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn7','ln7') +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn10','ln10') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc@dept_id=2 -POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn7','ln7') +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn10','ln10') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 -PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8') +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn11','ln11') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc@dept_id=2 -POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8') +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn11','ln11') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn12','ln12') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc@dept_id=2 +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn12','ln12') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn13','ln13') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc@dept_id=2 +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn13','ln13') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn14','ln14') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc@dept_id=2 +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn14','ln14') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 -PREHOOK: query: delete from ice_orc where last_name in ('ln7', 'ln8') +PREHOOK: query: delete from ice_orc where last_name in ('ln13', 'ln14') PREHOOK: type: QUERY PREHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### -POSTHOOK: query: delete from ice_orc where last_name in ('ln7', 'ln8') +POSTHOOK: query: delete from ice_orc where last_name in ('ln13', 'ln14') POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### @@ -126,9 +174,15 @@ POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### fn1 ln1 1 +fn10 ln10 2 +fn11 ln11 2 +fn12 ln12 2 fn2 ln2 1 -fn5 ln5 2 -fn6 ln6 2 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 1 +fn8 ln8 2 +fn9 ln9 2 PREHOOK: query: describe formatted ice_orc PREHOOK: type: DESCTABLE PREHOOK: Input: ice_comp@ice_orc @@ -160,18 +214,18 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"3\",\"manifests-replaced\":\"2\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"2\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"10\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 #### A masked pattern was here #### - numFiles 4 - numRows 4 + numFiles 10 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 10 + snapshot-count 16 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -211,9 +265,15 @@ POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### fn1 ln1 1 +fn10 ln10 2 +fn11 ln11 2 +fn12 ln12 2 fn2 ln2 1 -fn5 ln5 2 -fn6 ln6 2 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 1 +fn8 ln8 2 +fn9 ln9 2 PREHOOK: query: describe formatted ice_orc PREHOOK: type: DESCTABLE PREHOOK: Input: ice_comp@ice_orc @@ -245,19 +305,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"1\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"2\",\"deleted-records\":\"2\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"1\",\"added-data-files\":\"1\",\"deleted-data-files\":\"5\",\"added-records\":\"5\",\"deleted-records\":\"5\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 #### A masked pattern was here #### numFiles 2 numPartitions 2 - numRows 4 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 12 + snapshot-count 18 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -289,19 +349,59 @@ POSTHOOK: query: alter table ice_orc set partition spec() POSTHOOK: type: ALTERTABLE_SETPARTSPEC POSTHOOK: Input: ice_comp@ice_orc POSTHOOK: Output: ice_comp@ice_orc -PREHOOK: query: insert into ice_orc VALUES ('fn9','ln9', 3) +PREHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn17','ln17', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn17','ln17', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn18','ln18', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn18','ln18', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc -POSTHOOK: query: insert into ice_orc VALUES ('fn9','ln9', 3) +POSTHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc -PREHOOK: query: insert into ice_orc VALUES ('fn10','ln10', 3) +PREHOOK: query: insert into ice_orc VALUES ('fn21','ln21', 3) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc -POSTHOOK: query: insert into ice_orc VALUES ('fn10','ln10', 3) +POSTHOOK: query: insert into ice_orc VALUES ('fn21','ln21', 3) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc @@ -314,11 +414,22 @@ POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### fn1 ln1 1 -fn10 ln10 3 +fn10 ln10 2 +fn11 ln11 2 +fn12 ln12 2 +fn15 ln15 3 +fn16 ln16 3 +fn17 ln17 3 +fn18 ln18 3 +fn19 ln19 3 fn2 ln2 1 -fn5 ln5 2 -fn6 ln6 2 -fn9 ln9 3 +fn20 ln20 3 +fn21 ln21 3 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 1 +fn8 ln8 2 +fn9 ln9 2 PREHOOK: query: describe formatted ice_orc PREHOOK: type: DESCTABLE PREHOOK: Input: ice_comp@ice_orc @@ -343,17 +454,17 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"7\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"9\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### - numFiles 4 - numRows 6 + numFiles 9 + numRows 17 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 14 + snapshot-count 25 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -388,11 +499,22 @@ POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### fn1 ln1 1 -fn10 ln10 3 +fn10 ln10 2 +fn11 ln11 2 +fn12 ln12 2 +fn15 ln15 3 +fn16 ln16 3 +fn17 ln17 3 +fn18 ln18 3 +fn19 ln19 3 fn2 ln2 1 -fn5 ln5 2 -fn6 ln6 2 -fn9 ln9 3 +fn20 ln20 3 +fn21 ln21 3 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 1 +fn8 ln8 2 +fn9 ln9 2 PREHOOK: query: describe formatted ice_orc PREHOOK: type: DESCTABLE PREHOOK: Input: ice_comp@ice_orc @@ -417,17 +539,17 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"4\",\"added-records\":\"6\",\"deleted-records\":\"6\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"9\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"8\",\"added-data-files\":\"1\",\"deleted-data-files\":\"9\",\"added-records\":\"17\",\"deleted-records\":\"17\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### numFiles 1 - numRows 6 + numRows 17 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 15 + snapshot-count 26 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out index b59449ea645f..c4acdcaade3f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out @@ -69,6 +69,46 @@ POSTHOOK: query: insert into ice_orc VALUES POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn81','ln81', 2, 20, 101) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn81','ln81', 2, 20, 101) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn82','ln82', 2, 20, 101) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn82','ln82', 2, 20, 101) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn83','ln83', 2, 20, 101) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn83','ln83', 2, 20, 101) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn84','ln84', 2, 20, 101) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn84','ln84', 2, 20, 101) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn85','ln85', 2, 20, 101) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn85','ln85', 2, 20, 101) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc PREHOOK: query: insert into ice_orc VALUES ('fn9', 'ln9', 3, 20, 101), ('fn10','ln10', 3, 20, 101), @@ -109,6 +149,11 @@ fn5 ln5 2 20 101 fn6 ln6 2 20 101 fn7 ln7 2 20 101 fn8 ln8 2 20 101 +fn81 ln81 2 20 101 +fn82 ln82 2 20 101 +fn83 ln83 2 20 101 +fn84 ln84 2 20 101 +fn85 ln85 2 20 101 PREHOOK: query: select * from ice_orc where dept_id = 3 order by first_name PREHOOK: type: QUERY PREHOOK: Input: default@ice_orc @@ -154,19 +199,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"3\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"4\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"8\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"4\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"9\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 #### A masked pattern was here #### - numFiles 4 + numFiles 9 numPartitions 3 - numRows 12 + numRows 17 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 4 + snapshot-count 9 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -253,6 +298,11 @@ POSTHOOK: query: select * from ice_orc where dept_id = 2 POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_orc #### A masked pattern was here #### +fn85 ln85 2 20 101 +fn84 ln84 2 20 101 +fn83 ln83 2 20 101 +fn82 ln82 2 20 101 +fn81 ln81 2 20 101 fn8 ln8 2 20 101 fn7 ln7 2 20 101 fn6 ln6 2 20 101 @@ -302,19 +352,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"3\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"2\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"4\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"8\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"7\",\"added-data-files\":\"1\",\"deleted-data-files\":\"7\",\"added-records\":\"9\",\"deleted-records\":\"9\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 #### A masked pattern was here #### numFiles 3 numPartitions 3 - numRows 12 + numRows 17 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 5 + snapshot-count 10 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out index 49cff71970f1..2734f041c349 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out @@ -20,50 +20,82 @@ tblproperties ('compactor.threshold.target.size'='1500') POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), +PREHOOK: query: insert into ice_orc values + (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (2, cast('2023-07-21 00:00:00' as timestamp with local time zone), 'AAA_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), +POSTHOOK: query: insert into ice_orc values (2, cast('2023-07-21 00:00:00' as timestamp with local time zone), 'AAA_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1'), +PREHOOK: query: insert into ice_orc values + (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (4, cast('2023-07-24 00:00:00' as timestamp with local time zone), 'AAA_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1'), +POSTHOOK: query: insert into ice_orc values (4, cast('2023-07-24 00:00:00' as timestamp with local time zone), 'AAA_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1'), +PREHOOK: query: insert into ice_orc values + (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (6, cast('2023-08-05 00:00:00' as timestamp with local time zone), 'BBB_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1'), +POSTHOOK: query: insert into ice_orc values (6, cast('2023-08-05 00:00:00' as timestamp with local time zone), 'BBB_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1'), +PREHOOK: query: insert into ice_orc values + (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (8, cast('2023-08-27 00:00:00' as timestamp with local time zone), 'BBB_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1'), +POSTHOOK: query: insert into ice_orc values (8, cast('2023-08-27 00:00:00' as timestamp with local time zone), 'BBB_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table @@ -75,110 +107,182 @@ POSTHOOK: query: alter table ice_orc set partition spec(truncate(3, event_src), POSTHOOK: type: ALTERTABLE_SETPARTSPEC POSTHOOK: Input: default@ice_orc POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), +PREHOOK: query: insert into ice_orc values + (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (10, cast('2024-07-21 00:00:00' as timestamp with local time zone), 'AAA_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), +POSTHOOK: query: insert into ice_orc values (10, cast('2024-07-21 00:00:00' as timestamp with local time zone), 'AAA_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1'), +PREHOOK: query: insert into ice_orc values + (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (12, cast('2024-08-23 00:00:00' as timestamp with local time zone), 'AAA_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1'), +POSTHOOK: query: insert into ice_orc values (12, cast('2024-08-23 00:00:00' as timestamp with local time zone), 'AAA_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1'), +PREHOOK: query: insert into ice_orc values + (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (14, cast('2024-08-25 00:00:00' as timestamp with local time zone), 'AAA_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1'), +POSTHOOK: query: insert into ice_orc values (14, cast('2024-08-25 00:00:00' as timestamp with local time zone), 'AAA_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (15, cast('2024-09-24 00:00:00' as timestamp with local time zone), 'AAA_1'), +PREHOOK: query: insert into ice_orc values + (15, cast('2024-09-24 00:00:00' as timestamp with local time zone), 'AAA_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (15, cast('2024-09-24 00:00:00' as timestamp with local time zone), 'AAA_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (16, cast('2024-09-25 00:00:00' as timestamp with local time zone), 'AAA_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (15, cast('2024-09-24 00:00:00' as timestamp with local time zone), 'AAA_1'), +POSTHOOK: query: insert into ice_orc values (16, cast('2024-09-25 00:00:00' as timestamp with local time zone), 'AAA_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1'), +PREHOOK: query: insert into ice_orc values + (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (18, cast('2024-07-02 00:00:00' as timestamp with local time zone), 'BBB_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1'), +POSTHOOK: query: insert into ice_orc values (18, cast('2024-07-02 00:00:00' as timestamp with local time zone), 'BBB_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1'), +PREHOOK: query: insert into ice_orc values + (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (20, cast('2024-07-04 00:00:00' as timestamp with local time zone), 'BBB_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1'), +POSTHOOK: query: insert into ice_orc values (20, cast('2024-07-04 00:00:00' as timestamp with local time zone), 'BBB_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1'), +PREHOOK: query: insert into ice_orc values + (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (22, cast('2024-08-04 00:00:00' as timestamp with local time zone), 'BBB_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1'), +POSTHOOK: query: insert into ice_orc values (22, cast('2024-08-04 00:00:00' as timestamp with local time zone), 'BBB_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1'), +PREHOOK: query: insert into ice_orc values + (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (24, cast('2024-08-06 00:00:00' as timestamp with local time zone), 'BBB_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1'), +POSTHOOK: query: insert into ice_orc values (24, cast('2024-08-06 00:00:00' as timestamp with local time zone), 'BBB_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc PREHOOK: query: insert into ice_orc values - (25, cast('2024-09-05 00:00:00' as timestamp with local time zone), 'BBB_1'), + (25, cast('2024-09-05 00:00:00' as timestamp with local time zone), 'BBB_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (25, cast('2024-09-05 00:00:00' as timestamp with local time zone), 'BBB_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (26, cast('2024-09-06 00:00:00' as timestamp with local time zone), 'BBB_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc POSTHOOK: query: insert into ice_orc values - (25, cast('2024-09-05 00:00:00' as timestamp with local time zone), 'BBB_1'), (26, cast('2024-09-06 00:00:00' as timestamp with local time zone), 'BBB_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table @@ -249,19 +353,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"event_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"event_time\",\"required\":false,\"type\":\"timestamptz\"},{\"id\":3,\"name\":\"event_src\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"9\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"13\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"18\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"26\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]} format-version 2 #### A masked pattern was here #### - numFiles 13 + numFiles 26 numPartitions 8 numRows 26 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 13 + snapshot-count 26 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -357,19 +461,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"event_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"event_time\",\"required\":false,\"type\":\"timestamptz\"},{\"id\":3,\"name\":\"event_src\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"1\",\"added-data-files\":\"2\",\"deleted-data-files\":\"4\",\"added-records\":\"8\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"4\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"9\",\"manifests-replaced\":\"1\",\"added-data-files\":\"2\",\"deleted-data-files\":\"8\",\"added-records\":\"8\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"4\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"11\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]} format-version 2 #### A masked pattern was here #### - numFiles 8 + numFiles 11 numPartitions 8 numRows 26 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 17 + snapshot-count 30 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out index e12af3d3b662..6da52474f3e6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out @@ -10,33 +10,59 @@ TBLPROPERTIES ('external.table.purge'='true', 'format-version'='2', 'compactor.t POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@x -PREHOOK: query: insert into x values -('amy', 35, 123412344), -('adxfvy', 36, 123412534), -('amsdfyy', 37, 123417234), -('asafmy', 38, 123412534) +PREHOOK: query: insert into x values ('amy', 35, 123412344) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@x -POSTHOOK: query: insert into x values -('amy', 35, 123412344), -('adxfvy', 36, 123412534), -('amsdfyy', 37, 123417234), -('asafmy', 38, 123412534) +POSTHOOK: query: insert into x values ('amy', 35, 123412344) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@x -PREHOOK: query: insert into x values -('amerqwy', 39, 123441234), -('amyxzcv', 40, 123341234), -('erweramy', 45, 122341234) +PREHOOK: query: insert into x values ('adxfvy', 36, 123412534) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@x -POSTHOOK: query: insert into x values -('amerqwy', 39, 123441234), -('amyxzcv', 40, 123341234), -('erweramy', 45, 122341234) +POSTHOOK: query: insert into x values ('adxfvy', 36, 123412534) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@x +PREHOOK: query: insert into x values ('amsdfyy', 37, 123417234) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@x +POSTHOOK: query: insert into x values ('amsdfyy', 37, 123417234) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@x +PREHOOK: query: insert into x values ('asafmy', 38, 123412534) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@x +POSTHOOK: query: insert into x values ('asafmy', 38, 123412534) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@x +PREHOOK: query: insert into x values ('amerqwy', 39, 123441234) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@x +POSTHOOK: query: insert into x values ('amerqwy', 39, 123441234) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@x +PREHOOK: query: insert into x values ('amyxzcv', 40, 123341234) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@x +POSTHOOK: query: insert into x values ('amyxzcv', 40, 123341234) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@x +PREHOOK: query: insert into x values ('erweramy', 45, 122341234) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@x +POSTHOOK: query: insert into x values ('erweramy', 45, 122341234) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@x @@ -56,7 +82,7 @@ POSTHOOK: query: select count(*) from default.x.data_files POSTHOOK: type: QUERY POSTHOOK: Input: default@x #### A masked pattern was here #### -2 +7 PREHOOK: query: alter table x compact 'major' and wait PREHOOK: type: ALTERTABLE_COMPACT PREHOOK: Input: default@x @@ -95,7 +121,7 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"num_clicks\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"3\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"2\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"8\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"7\",\"added-data-files\":\"1\",\"deleted-data-files\":\"7\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# external.table.purge true format-version 2 @@ -106,7 +132,7 @@ Table Parameters: #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 3 + snapshot-count 8 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out index 466c234e72c5..689a22a6433f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out @@ -46,6 +46,22 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1), ('fn3','ln3', 1) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc PREHOOK: query: delete from ice_orc where last_name in ('ln4') PREHOOK: type: QUERY PREHOOK: Input: ice_comp@ice_orc @@ -77,6 +93,22 @@ POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8'), POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn17','ln17') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc@dept_id=2 +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn17','ln17') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn18','ln18') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc@dept_id=2 +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn18','ln18') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 PREHOOK: query: delete from ice_orc where last_name in ('ln8') PREHOOK: type: QUERY PREHOOK: Input: ice_comp@ice_orc @@ -115,19 +147,19 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"4\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"8\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 #### A masked pattern was here #### - numFiles 4 + numFiles 8 numPartitions 1 - numRows 6 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 6 + snapshot-count 10 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -198,19 +230,19 @@ Table Parameters: compactor.threshold.target.size 3200 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"4\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"8\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 #### A masked pattern was here #### - numFiles 4 + numFiles 8 numPartitions 1 - numRows 6 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 6 + snapshot-count 10 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -260,6 +292,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### fn1 ln1 1 +fn15 ln15 1 +fn16 ln16 1 fn2 ln2 1 fn3 ln3 1 PREHOOK: query: select * from ice_orc where dept_id=2 @@ -270,6 +304,8 @@ POSTHOOK: query: select * from ice_orc where dept_id=2 POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### +fn17 ln17 2 +fn18 ln18 2 fn5 ln5 2 fn6 ln6 2 fn7 ln7 2 @@ -305,19 +341,19 @@ Table Parameters: compactor.threshold.target.size 4000 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"removed-position-delete-files\":\"2\",\"removed-delete-files\":\"2\",\"added-records\":\"3\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"2\",\"changed-partition-count\":\"3\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"4\",\"removed-position-delete-files\":\"2\",\"removed-delete-files\":\"2\",\"added-records\":\"5\",\"deleted-records\":\"6\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"2\",\"changed-partition-count\":\"3\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 #### A masked pattern was here #### numFiles 2 numPartitions 2 - numRows 6 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 8 + snapshot-count 12 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -367,6 +403,22 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn11','ln11', 3), ('fn12','ln12', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc PREHOOK: query: select * from ice_orc PREHOOK: type: QUERY PREHOOK: Input: ice_comp@ice_orc @@ -379,7 +431,13 @@ fn1 ln1 1 fn10 ln10 3 fn11 ln11 3 fn12 ln12 3 +fn15 ln15 1 +fn16 ln16 1 +fn17 ln17 2 +fn18 ln18 2 +fn19 ln19 3 fn2 ln2 1 +fn20 ln20 3 fn3 ln3 1 fn5 ln5 2 fn6 ln6 2 @@ -410,17 +468,17 @@ Table Parameters: compactor.threshold.target.size 4000 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"4\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"16\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"6\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### - numFiles 4 - numRows 10 + numFiles 6 + numRows 16 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 10 + snapshot-count 16 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -466,7 +524,13 @@ fn1 ln1 1 fn10 ln10 3 fn11 ln11 3 fn12 ln12 3 +fn15 ln15 1 +fn16 ln16 1 +fn17 ln17 2 +fn18 ln18 2 +fn19 ln19 3 fn2 ln2 1 +fn20 ln20 3 fn3 ln3 1 fn5 ln5 2 fn6 ln6 2 @@ -497,17 +561,17 @@ Table Parameters: compactor.threshold.target.size 4000 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"4\",\"added-records\":\"10\",\"deleted-records\":\"10\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"6\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"5\",\"added-data-files\":\"1\",\"deleted-data-files\":\"6\",\"added-records\":\"16\",\"deleted-records\":\"16\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"16\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### numFiles 1 - numRows 10 + numRows 16 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 11 + snapshot-count 17 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out index 79536df6215d..10e9f2c0958b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out @@ -24,27 +24,59 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn1','ln1') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5'), ('fn7','ln7'), ('fn4','ln4'), ('fn6','ln6') +PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5'), ('fn7','ln7'), ('fn4','ln4'), ('fn6','ln6') +POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc VALUES ('fn3','ln3'), ('fn2','ln2') +PREHOOK: query: insert into ice_orc VALUES ('fn7','ln7') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc VALUES ('fn3','ln3'), ('fn2','ln2') +POSTHOOK: query: insert into ice_orc VALUES ('fn7','ln7') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: alter table ice_orc set tblproperties ('compactor.threshold.target.size'='2920', 'compactor.threshold.min.input.files'='2') +PREHOOK: query: insert into ice_orc VALUES ('fn4','ln4') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn6','ln6') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn6','ln6') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn3','ln3') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn3','ln3') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn2','ln2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn2','ln2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: alter table ice_orc set tblproperties ('compactor.threshold.target.size'='4000', 'compactor.threshold.min.input.files'='2') PREHOOK: type: ALTERTABLE_PROPERTIES PREHOOK: Input: default@ice_orc PREHOOK: Output: default@ice_orc -POSTHOOK: query: alter table ice_orc set tblproperties ('compactor.threshold.target.size'='2920', 'compactor.threshold.min.input.files'='2') +POSTHOOK: query: alter table ice_orc set tblproperties ('compactor.threshold.target.size'='4000', 'compactor.threshold.min.input.files'='2') POSTHOOK: type: ALTERTABLE_PROPERTIES POSTHOOK: Input: default@ice_orc POSTHOOK: Output: default@ice_orc @@ -90,20 +122,20 @@ Table Parameters: EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 2 - compactor.threshold.target.size 2920 + compactor.threshold.target.size 4000 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"7\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### - numFiles 3 + numFiles 7 numRows 7 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 3 + snapshot-count 7 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -173,10 +205,10 @@ Table Parameters: EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 2 - compactor.threshold.target.size 2920 + compactor.threshold.target.size 4000 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"3\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"8\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"7\",\"added-data-files\":\"1\",\"deleted-data-files\":\"7\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### @@ -186,7 +218,7 @@ Table Parameters: #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 4 + snapshot-count 8 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -270,7 +302,7 @@ Table Parameters: EXTERNAL TRUE bucketing_version 2 compactor.threshold.min.input.files 2 - compactor.threshold.target.size 2920 + compactor.threshold.target.size 4000 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"removed-position-delete-files\":\"1\",\"removed-delete-files\":\"1\",\"added-records\":\"7\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} @@ -283,7 +315,7 @@ Table Parameters: #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 7 + snapshot-count 11 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out index b4d8dcd592d5..7f593a3a577d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out @@ -229,6 +229,22 @@ POSTHOOK: query: insert into part_tbl values (5,'p5', 10) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp_all@part_tbl +PREHOOK: query: insert into part_tbl values (5,'p5', 10) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp_all@part_tbl +POSTHOOK: query: insert into part_tbl values (5,'p5', 10) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp_all@part_tbl +PREHOOK: query: insert into part_tbl values (6,'p6', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp_all@part_tbl +POSTHOOK: query: insert into part_tbl values (6,'p6', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp_all@part_tbl PREHOOK: query: insert into part_tbl values (6,'p6', 20) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table @@ -254,7 +270,9 @@ POSTHOOK: Input: ice_comp_all@part_tbl 3 p3 20 2 3 4 p4 20 3 4 5 p5 10 11 9 -6 p6 20 16 10 +5 p5 10 16 10 +6 p6 20 17 11 +6 p6 20 18 12 PREHOOK: query: alter table part_tbl compact 'minor' and wait pool 'iceberg' PREHOOK: type: ALTERTABLE_COMPACT PREHOOK: Input: ice_comp_all@part_tbl @@ -280,7 +298,9 @@ POSTHOOK: Input: ice_comp_all@part_tbl 3 p3 20 2 3 4 p4 20 3 4 5 p5 10 11 9 -6 p6 20 16 10 +5 p5 10 16 10 +6 p6 20 17 11 +6 p6 20 18 12 PREHOOK: query: show compactions PREHOOK: type: SHOW COMPACTIONS POSTHOOK: query: show compactions