From e184a602291bc5e864623eed189a9e9d93ed5e92 Mon Sep 17 00:00:00 2001 From: Raghav Aggarwal Date: Sat, 29 Aug 2026 00:32:29 +0530 Subject: [PATCH] HIVE-29853: Iceberg CompactionEvaluator double-counts input files --- .../evaluator/CompactionEvaluator.java | 1 - ...erg_major_compaction_partition_evolution.q | 5 + ...rg_major_compaction_partition_evolution2.q | 25 ++- ...r_compaction_partition_evolution_ordered.q | 5 + ..._partition_evolution_w_dyn_spec_w_filter.q | 55 +++-- .../iceberg_major_compaction_query_metadata.q | 17 +- ...erg_minor_compaction_partition_evolution.q | 6 + .../iceberg_minor_compaction_unpartitioned.q | 8 +- .../iceberg_row_lineage_compactions.q | 2 + ...major_compaction_partition_evolution.q.out | 140 ++++++++---- ...ajor_compaction_partition_evolution2.q.out | 206 ++++++++++++++---- ...mpaction_partition_evolution_ordered.q.out | 64 +++++- ...tition_evolution_w_dyn_spec_w_filter.q.out | 164 ++++++++------ ...berg_major_compaction_query_metadata.q.out | 68 ++++-- ...minor_compaction_partition_evolution.q.out | 100 +++++++-- ...eberg_minor_compaction_unpartitioned.q.out | 60 +++-- .../iceberg_row_lineage_compactions.q.out | 24 +- 17 files changed, 693 insertions(+), 257 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java index e3f177a842c8..78c3a08e9d28 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/evaluator/CompactionEvaluator.java @@ -73,7 +73,6 @@ public CompactionEvaluator(Table table, CompactionInfo ci, Map p ); this.table = table; this.ci = ci; - addFiles(); } public boolean isEligibleForCompaction() { diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q index a38a856f6430..10c17d7eb42b 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution.q @@ -42,6 +42,11 @@ alter table ice_orc create tag v1; alter table ice_orc set partition spec(company_id, dept_id); insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100); +insert into ice_orc VALUES ('fn41','ln41', 1, 11, 100); +insert into ice_orc VALUES ('fn42','ln42', 1, 11, 100); +insert into ice_orc VALUES ('fn43','ln43', 1, 11, 100); +insert into ice_orc VALUES ('fn44','ln44', 1, 11, 100); +insert into ice_orc VALUES ('fn45','ln45', 1, 11, 100); insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100); insert into ice_orc VALUES ('fn6','ln6', 2, 20, 100); insert into ice_orc VALUES (null,null, null, null, null); diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q index 4ca441ff0b14..faa3933d7286 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution2.q @@ -38,15 +38,21 @@ insert into ice_orc VALUES ('fn1','ln1', 1); insert into ice_orc VALUES ('fn2','ln2', 1); insert into ice_orc VALUES ('fn3','ln3', 1); insert into ice_orc VALUES ('fn4','ln4', 1); -delete from ice_orc where last_name in ('ln3', 'ln4'); +insert into ice_orc VALUES ('fn5','ln5', 1); +insert into ice_orc VALUES ('fn6','ln6', 1); +insert into ice_orc VALUES ('fn7','ln7', 1); +delete from ice_orc where last_name in ('ln6', 'ln7'); alter table ice_orc set partition spec(dept_id); -insert into ice_orc PARTITION(dept_id=2) VALUES ('fn5','ln5'); -insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6'); -insert into ice_orc PARTITION(dept_id=2) VALUES ('fn7','ln7'); insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8'); -delete from ice_orc where last_name in ('ln7', 'ln8'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn9','ln9'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn10','ln10'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn11','ln11'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn12','ln12'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn13','ln13'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn14','ln14'); +delete from ice_orc where last_name in ('ln13', 'ln14'); select * from ice_orc; describe formatted ice_orc; @@ -60,8 +66,13 @@ show compactions order by 'partition'; alter table ice_orc set partition spec(); -insert into ice_orc VALUES ('fn9','ln9', 3); -insert into ice_orc VALUES ('fn10','ln10', 3); +insert into ice_orc VALUES ('fn15','ln15', 3); +insert into ice_orc VALUES ('fn16','ln16', 3); +insert into ice_orc VALUES ('fn17','ln17', 3); +insert into ice_orc VALUES ('fn18','ln18', 3); +insert into ice_orc VALUES ('fn19','ln19', 3); +insert into ice_orc VALUES ('fn20','ln20', 3); +insert into ice_orc VALUES ('fn21','ln21', 3); select * from ice_orc; describe formatted ice_orc; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q index 0dfe28140421..852d978cd87c 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_ordered.q @@ -49,6 +49,11 @@ insert into ice_orc VALUES insert into ice_orc VALUES ('fn7','ln7', 2, 20, 101), ('fn8','ln8', 2, 20, 101); +insert into ice_orc VALUES ('fn81','ln81', 2, 20, 101); +insert into ice_orc VALUES ('fn82','ln82', 2, 20, 101); +insert into ice_orc VALUES ('fn83','ln83', 2, 20, 101); +insert into ice_orc VALUES ('fn84','ln84', 2, 20, 101); +insert into ice_orc VALUES ('fn85','ln85', 2, 20, 101); insert into ice_orc VALUES ('fn9', 'ln9', 3, 20, 101), diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q index 896f07d5bdaf..9edb1373add2 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q @@ -25,53 +25,50 @@ set hive.vectorized.execution.enabled=true; set hive.optimize.shared.work.merge.ts.schema=true; create table ice_orc ( - event_id int, + event_id int, event_time timestamp with local time zone, - event_src string + event_src string ) partitioned by spec(truncate(3, event_src)) stored by iceberg stored as orc tblproperties ('compactor.threshold.target.size'='1500'); -insert into ice_orc values +insert into ice_orc values (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), (2, cast('2023-07-21 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values +insert into ice_orc values (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1'), (4, cast('2023-07-24 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values +insert into ice_orc values (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1'), (6, cast('2023-08-05 00:00:00' as timestamp with local time zone), 'BBB_2'); -insert into ice_orc values +insert into ice_orc values (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1'), (8, cast('2023-08-27 00:00:00' as timestamp with local time zone), 'BBB_2'); - + alter table ice_orc set partition spec(truncate(3, event_src), month(event_time)); -insert into ice_orc values +insert into ice_orc values (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), (10, cast('2024-07-21 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values - (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1'), - (12, cast('2024-08-23 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values - (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1'), - (14, cast('2024-08-25 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values +insert into ice_orc values (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1'); +insert into ice_orc values (12, cast('2024-08-23 00:00:00' as timestamp with local time zone), 'AAA_2'); +insert into ice_orc values (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1'); +insert into ice_orc values (14, cast('2024-08-25 00:00:00' as timestamp with local time zone), 'AAA_2'); +insert into ice_orc values (15, cast('2024-09-24 00:00:00' as timestamp with local time zone), 'AAA_1'), (16, cast('2024-09-25 00:00:00' as timestamp with local time zone), 'AAA_2'); -insert into ice_orc values - (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1'), - (18, cast('2024-07-02 00:00:00' as timestamp with local time zone), 'BBB_2'); -insert into ice_orc values - (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1'), - (20, cast('2024-07-04 00:00:00' as timestamp with local time zone), 'BBB_2'); -insert into ice_orc values - (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1'), - (22, cast('2024-08-04 00:00:00' as timestamp with local time zone), 'BBB_2'); -insert into ice_orc values - (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1'), - (24, cast('2024-08-06 00:00:00' as timestamp with local time zone), 'BBB_2'); +insert into ice_orc values + (151, cast('2024-09-26 00:00:00' as timestamp with local time zone), 'AAA_1'), + (161, cast('2024-09-27 00:00:00' as timestamp with local time zone), 'AAA_2'); +insert into ice_orc values (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1'); +insert into ice_orc values (18, cast('2024-07-02 00:00:00' as timestamp with local time zone), 'BBB_2'); +insert into ice_orc values (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1'); +insert into ice_orc values (20, cast('2024-07-04 00:00:00' as timestamp with local time zone), 'BBB_2'); +insert into ice_orc values (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1'); +insert into ice_orc values (22, cast('2024-08-04 00:00:00' as timestamp with local time zone), 'BBB_2'); +insert into ice_orc values (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1'); +insert into ice_orc values (24, cast('2024-08-06 00:00:00' as timestamp with local time zone), 'BBB_2'); insert into ice_orc values (25, cast('2024-09-05 00:00:00' as timestamp with local time zone), 'BBB_1'), (26, cast('2024-09-06 00:00:00' as timestamp with local time zone), 'BBB_2'); @@ -79,11 +76,11 @@ insert into ice_orc values select * from ice_orc; describe formatted ice_orc; -alter table ice_orc COMPACT 'major' and wait +alter table ice_orc COMPACT 'major' and wait where (event_src in ('BBB_1', 'BBB_2') and event_time < '2024-09-01 00:00:00') or (event_src in ('AAA_1', 'AAA_2') and event_time > '2024-08-01 00:00:00'); select * from ice_orc; describe formatted ice_orc; -show compactions order by 'partition'; \ No newline at end of file +show compactions order by 'partition'; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q index ecdeacffbbf1..24b23769a365 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q @@ -24,16 +24,13 @@ CREATE TABLE x (name VARCHAR(50), age TINYINT, num_clicks BIGINT) stored by iceberg stored as orc TBLPROPERTIES ('external.table.purge'='true', 'format-version'='2', 'compactor.threshold.target.size'='1500'); -insert into x values -('amy', 35, 123412344), -('adxfvy', 36, 123412534), -('amsdfyy', 37, 123417234), -('asafmy', 38, 123412534); - -insert into x values -('amerqwy', 39, 123441234), -('amyxzcv', 40, 123341234), -('erweramy', 45, 122341234); +insert into x values ('amy', 35, 123412344); +insert into x values ('adxfvy', 36, 123412534); +insert into x values ('amsdfyy', 37, 123417234); +insert into x values ('asafmy', 38, 123412534); +insert into x values ('amerqwy', 39, 123441234); +insert into x values ('amyxzcv', 40, 123341234); +insert into x values ('erweramy', 45, 122341234); select * from default.x.data_files; select count(*) from default.x.data_files; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q index f89fbf519e0d..313103da1b2f 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_partition_evolution.q @@ -38,6 +38,8 @@ tblproperties ('format-version'='2'); insert into ice_orc VALUES ('fn2','ln2', 1), ('fn1','ln1', 1); insert into ice_orc VALUES ('fn4','ln4', 1), ('fn3','ln3', 1); +insert into ice_orc VALUES ('fn15','ln15', 1); +insert into ice_orc VALUES ('fn16','ln16', 1); delete from ice_orc where last_name in ('ln4'); @@ -45,6 +47,8 @@ alter table ice_orc set partition spec(dept_id); insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6'), ('fn5','ln5'); insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8'), ('fn7','ln7'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn17','ln17'); +insert into ice_orc PARTITION(dept_id=2) VALUES ('fn18','ln18'); delete from ice_orc where last_name in ('ln8'); @@ -73,6 +77,8 @@ alter table ice_orc set partition spec(); insert into ice_orc VALUES ('fn9','ln9', 3), ('fn10','ln10', 3); insert into ice_orc VALUES ('fn11','ln11', 3), ('fn12','ln12', 3); +insert into ice_orc VALUES ('fn19','ln19', 3); +insert into ice_orc VALUES ('fn20','ln20', 3); select * from ice_orc; describe formatted ice_orc; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q index 61e3a82525d1..5235d76317df 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_minor_compaction_unpartitioned.q @@ -33,8 +33,12 @@ stored by iceberg stored as orc tblproperties ('format-version'='2'); insert into ice_orc VALUES ('fn1','ln1'); -insert into ice_orc VALUES ('fn5','ln5'), ('fn7','ln7'), ('fn4','ln4'), ('fn6','ln6'); -insert into ice_orc VALUES ('fn3','ln3'), ('fn2','ln2'); +insert into ice_orc VALUES ('fn5','ln5'); +insert into ice_orc VALUES ('fn7','ln7'); +insert into ice_orc VALUES ('fn4','ln4'); +insert into ice_orc VALUES ('fn6','ln6'); +insert into ice_orc VALUES ('fn3','ln3'); +insert into ice_orc VALUES ('fn2','ln2'); -- File size threshold = fragment size = target_size/fragment_ratio = 2920/8 = 365. alter table ice_orc set tblproperties ('compactor.threshold.target.size'='2920', 'compactor.threshold.min.input.files'='2'); diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q index 8f802d75b39a..5a7de18b0a0a 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_lineage_compactions.q @@ -74,6 +74,8 @@ alter table part_tbl set tblproperties ('compactor.threshold.target.size'='8000' alter table part_tbl set partition spec(dept_id, id); insert into part_tbl values (5,'p5', 10); +insert into part_tbl values (5,'p5', 10); +insert into part_tbl values (6,'p6', 20); insert into part_tbl values (6,'p6', 20); SELECT *, ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out index 8e77ed6fd0a9..5afab5819057 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out @@ -75,6 +75,46 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1, 11, 100) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn41','ln41', 1, 11, 100) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn41','ln41', 1, 11, 100) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn42','ln42', 1, 11, 100) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn42','ln42', 1, 11, 100) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn43','ln43', 1, 11, 100) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn43','ln43', 1, 11, 100) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn44','ln44', 1, 11, 100) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn44','ln44', 1, 11, 100) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn45','ln45', 1, 11, 100) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn45','ln45', 1, 11, 100) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 2, 20, 100) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table @@ -257,6 +297,11 @@ NULL NULL NULL NULL NULL fn2 ln2a 1 10 100 fn3 ln3a 1 11 100 fn4 ln4a 1 11 100 +fn41 ln41 1 11 100 +fn42 ln42 1 11 100 +fn43 ln43 1 11 100 +fn44 ln44 1 11 100 +fn45 ln45 1 11 100 fn5 ln5a 2 20 100 fn6 ln6a 2 20 100 fn7 ln7a 2 21 100 @@ -294,19 +339,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"5\",\"manifests-replaced\":\"6\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"17\",\"total-delete-files\":\"8\",\"total-position-deletes\":\"8\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"5\",\"manifests-replaced\":\"6\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"22\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"22\",\"total-delete-files\":\"8\",\"total-position-deletes\":\"8\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 hive.compactor.worker.pool iceberg #### A masked pattern was here #### - numFiles 17 - numRows 17 + numFiles 22 + numRows 22 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 20 + snapshot-count 25 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -340,6 +385,11 @@ POSTHOOK: Input: default@ice_orc {"company_id":100,"dept_id":1,"team_id":11} 2 0 1 {"company_id":100,"dept_id":1,"team_id":11} 2 0 1 {"company_id":100,"dept_id":1,"team_id":null} 1 0 1 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 1 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 1 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 1 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 1 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 1 {"company_id":100,"dept_id":1,"team_id":null} 1 1 1 {"company_id":100,"dept_id":2,"team_id":21} 2 0 1 {"company_id":100,"dept_id":2,"team_id":21} 2 0 1 @@ -426,14 +476,14 @@ STAGE PLANS: TableScan alias: ice_orc.tag_v2 Snapshot ref: tag_v2 - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -469,14 +519,14 @@ STAGE PLANS: TableScan alias: ice_orc.tag_v3 Snapshot ref: tag_v3 - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -512,14 +562,14 @@ STAGE PLANS: TableScan alias: ice_orc.tag_v4 Snapshot ref: tag_v4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -554,14 +604,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_orc - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -645,17 +695,17 @@ STAGE PLANS: alias: ice_orc.tag_v2 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v2 - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 3136 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 5096 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -692,17 +742,17 @@ STAGE PLANS: alias: ice_orc.tag_v3 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v3 - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 11 Data size: 4312 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 6272 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -739,17 +789,17 @@ STAGE PLANS: alias: ice_orc.tag_v4 filterExpr: company_id is not null (type: boolean) Snapshot ref: tag_v4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -785,17 +835,17 @@ STAGE PLANS: TableScan alias: ice_orc filterExpr: company_id is not null (type: boolean) - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: company_id is not null (type: boolean) - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 3528 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 14 Data size: 5488 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -851,6 +901,11 @@ NULL NULL NULL NULL NULL fn2 ln2a 1 10 100 fn3 ln3a 1 11 100 fn4 ln4a 1 11 100 +fn41 ln41 1 11 100 +fn42 ln42 1 11 100 +fn43 ln43 1 11 100 +fn44 ln44 1 11 100 +fn45 ln45 1 11 100 fn5 ln5a 2 20 100 fn6 ln6a 2 20 100 fn7 ln7a 2 21 100 @@ -888,20 +943,20 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"6\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"5\",\"added-data-files\":\"2\",\"deleted-data-files\":\"10\",\"removed-position-delete-files\":\"5\",\"removed-delete-files\":\"5\",\"added-records\":\"5\",\"deleted-records\":\"10\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"5\",\"changed-partition-count\":\"8\",\"total-records\":\"9\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"6\",\"manifests-kept\":\"3\",\"manifests-replaced\":\"5\",\"added-data-files\":\"2\",\"deleted-data-files\":\"10\",\"removed-position-delete-files\":\"5\",\"removed-delete-files\":\"5\",\"added-records\":\"5\",\"deleted-records\":\"10\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"5\",\"changed-partition-count\":\"8\",\"total-records\":\"14\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"5\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 hive.compactor.worker.pool iceberg #### A masked pattern was here #### - numFiles 4 + numFiles 5 numPartitions 3 - numRows 9 + numRows 14 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 23 + snapshot-count 28 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -941,6 +996,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_orc #### A masked pattern was here #### {"company_id":100,"dept_id":1,"team_id":null} 1 0 3 +{"company_id":100,"dept_id":1,"team_id":null} 1 0 5 {"company_id":100,"dept_id":2,"team_id":null} 1 0 3 {"company_id":null,"dept_id":null,"team_id":null} 1 0 1 {"company_id":null,"dept_id":null,"team_id":null} 1 0 2 @@ -965,14 +1021,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: ice_orc - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 14 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 14 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 9 Data size: 1592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 14 Data size: 2600 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1008,14 +1064,14 @@ STAGE PLANS: TableScan alias: ice_orc filterExpr: company_id is not null (type: boolean) - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: first_name (type: string), last_name (type: string), dept_id (type: bigint), team_id (type: bigint), company_id (type: bigint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 1194 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 2200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out index 60ad0af240d3..307e5d4bc47a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out @@ -62,11 +62,35 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc -PREHOOK: query: delete from ice_orc where last_name in ('ln3', 'ln4') +PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn6','ln6', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn6','ln6', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn7','ln7', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn7','ln7', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: delete from ice_orc where last_name in ('ln6', 'ln7') PREHOOK: type: QUERY PREHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### -POSTHOOK: query: delete from ice_orc where last_name in ('ln3', 'ln4') +POSTHOOK: query: delete from ice_orc where last_name in ('ln6', 'ln7') POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### @@ -77,43 +101,67 @@ POSTHOOK: query: alter table ice_orc set partition spec(dept_id) POSTHOOK: type: ALTERTABLE_SETPARTSPEC POSTHOOK: Input: ice_comp@ice_orc POSTHOOK: Output: ice_comp@ice_orc -PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn5','ln5') +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc@dept_id=2 -POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn5','ln5') +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 -PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6') +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn9','ln9') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc@dept_id=2 -POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn6','ln6') +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn9','ln9') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 -PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn7','ln7') +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn10','ln10') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc@dept_id=2 -POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn7','ln7') +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn10','ln10') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 -PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8') +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn11','ln11') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc@dept_id=2 -POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8') +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn11','ln11') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn12','ln12') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc@dept_id=2 +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn12','ln12') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn13','ln13') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc@dept_id=2 +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn13','ln13') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn14','ln14') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc@dept_id=2 +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn14','ln14') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 -PREHOOK: query: delete from ice_orc where last_name in ('ln7', 'ln8') +PREHOOK: query: delete from ice_orc where last_name in ('ln13', 'ln14') PREHOOK: type: QUERY PREHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### -POSTHOOK: query: delete from ice_orc where last_name in ('ln7', 'ln8') +POSTHOOK: query: delete from ice_orc where last_name in ('ln13', 'ln14') POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### @@ -126,9 +174,15 @@ POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### fn1 ln1 1 +fn10 ln10 2 +fn11 ln11 2 +fn12 ln12 2 fn2 ln2 1 -fn5 ln5 2 -fn6 ln6 2 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 1 +fn8 ln8 2 +fn9 ln9 2 PREHOOK: query: describe formatted ice_orc PREHOOK: type: DESCTABLE PREHOOK: Input: ice_comp@ice_orc @@ -160,18 +214,18 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"3\",\"manifests-replaced\":\"2\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"2\",\"deleted-data-files\":\"2\",\"deleted-records\":\"2\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"10\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 #### A masked pattern was here #### - numFiles 4 - numRows 4 + numFiles 10 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 10 + snapshot-count 16 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -211,9 +265,15 @@ POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### fn1 ln1 1 +fn10 ln10 2 +fn11 ln11 2 +fn12 ln12 2 fn2 ln2 1 -fn5 ln5 2 -fn6 ln6 2 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 1 +fn8 ln8 2 +fn9 ln9 2 PREHOOK: query: describe formatted ice_orc PREHOOK: type: DESCTABLE PREHOOK: Input: ice_comp@ice_orc @@ -245,19 +305,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"1\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"2\",\"deleted-records\":\"2\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"4\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"1\",\"added-data-files\":\"1\",\"deleted-data-files\":\"5\",\"added-records\":\"5\",\"deleted-records\":\"5\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 #### A masked pattern was here #### numFiles 2 numPartitions 2 - numRows 4 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 12 + snapshot-count 18 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -289,19 +349,59 @@ POSTHOOK: query: alter table ice_orc set partition spec() POSTHOOK: type: ALTERTABLE_SETPARTSPEC POSTHOOK: Input: ice_comp@ice_orc POSTHOOK: Output: ice_comp@ice_orc -PREHOOK: query: insert into ice_orc VALUES ('fn9','ln9', 3) +PREHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn17','ln17', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn17','ln17', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn18','ln18', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn18','ln18', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc -POSTHOOK: query: insert into ice_orc VALUES ('fn9','ln9', 3) +POSTHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc -PREHOOK: query: insert into ice_orc VALUES ('fn10','ln10', 3) +PREHOOK: query: insert into ice_orc VALUES ('fn21','ln21', 3) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: ice_comp@ice_orc -POSTHOOK: query: insert into ice_orc VALUES ('fn10','ln10', 3) +POSTHOOK: query: insert into ice_orc VALUES ('fn21','ln21', 3) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc @@ -314,11 +414,22 @@ POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### fn1 ln1 1 -fn10 ln10 3 +fn10 ln10 2 +fn11 ln11 2 +fn12 ln12 2 +fn15 ln15 3 +fn16 ln16 3 +fn17 ln17 3 +fn18 ln18 3 +fn19 ln19 3 fn2 ln2 1 -fn5 ln5 2 -fn6 ln6 2 -fn9 ln9 3 +fn20 ln20 3 +fn21 ln21 3 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 1 +fn8 ln8 2 +fn9 ln9 2 PREHOOK: query: describe formatted ice_orc PREHOOK: type: DESCTABLE PREHOOK: Input: ice_comp@ice_orc @@ -343,17 +454,17 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"7\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"9\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### - numFiles 4 - numRows 6 + numFiles 9 + numRows 17 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 14 + snapshot-count 25 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -388,11 +499,22 @@ POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### fn1 ln1 1 -fn10 ln10 3 +fn10 ln10 2 +fn11 ln11 2 +fn12 ln12 2 +fn15 ln15 3 +fn16 ln16 3 +fn17 ln17 3 +fn18 ln18 3 +fn19 ln19 3 fn2 ln2 1 -fn5 ln5 2 -fn6 ln6 2 -fn9 ln9 3 +fn20 ln20 3 +fn21 ln21 3 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 1 +fn8 ln8 2 +fn9 ln9 2 PREHOOK: query: describe formatted ice_orc PREHOOK: type: DESCTABLE PREHOOK: Input: ice_comp@ice_orc @@ -417,17 +539,17 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"4\",\"added-records\":\"6\",\"deleted-records\":\"6\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"9\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"8\",\"added-data-files\":\"1\",\"deleted-data-files\":\"9\",\"added-records\":\"17\",\"deleted-records\":\"17\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### numFiles 1 - numRows 6 + numRows 17 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 15 + snapshot-count 26 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out index b59449ea645f..c4acdcaade3f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out @@ -69,6 +69,46 @@ POSTHOOK: query: insert into ice_orc VALUES POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn81','ln81', 2, 20, 101) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn81','ln81', 2, 20, 101) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn82','ln82', 2, 20, 101) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn82','ln82', 2, 20, 101) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn83','ln83', 2, 20, 101) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn83','ln83', 2, 20, 101) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn84','ln84', 2, 20, 101) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn84','ln84', 2, 20, 101) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn85','ln85', 2, 20, 101) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn85','ln85', 2, 20, 101) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc PREHOOK: query: insert into ice_orc VALUES ('fn9', 'ln9', 3, 20, 101), ('fn10','ln10', 3, 20, 101), @@ -109,6 +149,11 @@ fn5 ln5 2 20 101 fn6 ln6 2 20 101 fn7 ln7 2 20 101 fn8 ln8 2 20 101 +fn81 ln81 2 20 101 +fn82 ln82 2 20 101 +fn83 ln83 2 20 101 +fn84 ln84 2 20 101 +fn85 ln85 2 20 101 PREHOOK: query: select * from ice_orc where dept_id = 3 order by first_name PREHOOK: type: QUERY PREHOOK: Input: default@ice_orc @@ -154,19 +199,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"3\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"4\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"8\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"4\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"9\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 #### A masked pattern was here #### - numFiles 4 + numFiles 9 numPartitions 3 - numRows 12 + numRows 17 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 4 + snapshot-count 9 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -253,6 +298,11 @@ POSTHOOK: query: select * from ice_orc where dept_id = 2 POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_orc #### A masked pattern was here #### +fn85 ln85 2 20 101 +fn84 ln84 2 20 101 +fn83 ln83 2 20 101 +fn82 ln82 2 20 101 +fn81 ln81 2 20 101 fn8 ln8 2 20 101 fn7 ln7 2 20 101 fn6 ln6 2 20 101 @@ -302,19 +352,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"3\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"2\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"4\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"8\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"7\",\"added-data-files\":\"1\",\"deleted-data-files\":\"7\",\"added-records\":\"9\",\"deleted-records\":\"9\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"17\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 #### A masked pattern was here #### numFiles 3 numPartitions 3 - numRows 12 + numRows 17 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 5 + snapshot-count 10 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out index 49cff71970f1..b60266000dea 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out @@ -1,7 +1,7 @@ PREHOOK: query: create table ice_orc ( - event_id int, + event_id int, event_time timestamp with local time zone, - event_src string + event_src string ) partitioned by spec(truncate(3, event_src)) stored by iceberg stored as orc @@ -10,9 +10,9 @@ PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@ice_orc POSTHOOK: query: create table ice_orc ( - event_id int, + event_id int, event_time timestamp with local time zone, - event_src string + event_src string ) partitioned by spec(truncate(3, event_src)) stored by iceberg stored as orc @@ -20,49 +20,49 @@ tblproperties ('compactor.threshold.target.size'='1500') POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values +PREHOOK: query: insert into ice_orc values (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), (2, cast('2023-07-21 00:00:00' as timestamp with local time zone), 'AAA_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values +POSTHOOK: query: insert into ice_orc values (1, cast('2023-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), (2, cast('2023-07-21 00:00:00' as timestamp with local time zone), 'AAA_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values +PREHOOK: query: insert into ice_orc values (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1'), (4, cast('2023-07-24 00:00:00' as timestamp with local time zone), 'AAA_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values +POSTHOOK: query: insert into ice_orc values (3, cast('2023-07-23 00:00:00' as timestamp with local time zone), 'AAA_1'), (4, cast('2023-07-24 00:00:00' as timestamp with local time zone), 'AAA_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values +PREHOOK: query: insert into ice_orc values (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1'), (6, cast('2023-08-05 00:00:00' as timestamp with local time zone), 'BBB_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values +POSTHOOK: query: insert into ice_orc values (5, cast('2023-08-04 00:00:00' as timestamp with local time zone), 'BBB_1'), (6, cast('2023-08-05 00:00:00' as timestamp with local time zone), 'BBB_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values +PREHOOK: query: insert into ice_orc values (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1'), (8, cast('2023-08-27 00:00:00' as timestamp with local time zone), 'BBB_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values +POSTHOOK: query: insert into ice_orc values (7, cast('2023-08-06 00:00:00' as timestamp with local time zone), 'BBB_1'), (8, cast('2023-08-27 00:00:00' as timestamp with local time zone), 'BBB_2') POSTHOOK: type: QUERY @@ -75,99 +75,135 @@ POSTHOOK: query: alter table ice_orc set partition spec(truncate(3, event_src), POSTHOOK: type: ALTERTABLE_SETPARTSPEC POSTHOOK: Input: default@ice_orc POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values +PREHOOK: query: insert into ice_orc values (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), (10, cast('2024-07-21 00:00:00' as timestamp with local time zone), 'AAA_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values +POSTHOOK: query: insert into ice_orc values (9, cast('2024-07-20 00:00:00' as timestamp with local time zone), 'AAA_1'), (10, cast('2024-07-21 00:00:00' as timestamp with local time zone), 'AAA_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1'), - (12, cast('2024-08-23 00:00:00' as timestamp with local time zone), 'AAA_2') +PREHOOK: query: insert into ice_orc values (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (12, cast('2024-08-23 00:00:00' as timestamp with local time zone), 'AAA_2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values (12, cast('2024-08-23 00:00:00' as timestamp with local time zone), 'AAA_2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (11, cast('2024-08-22 00:00:00' as timestamp with local time zone), 'AAA_1'), - (12, cast('2024-08-23 00:00:00' as timestamp with local time zone), 'AAA_2') +POSTHOOK: query: insert into ice_orc values (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1'), - (14, cast('2024-08-25 00:00:00' as timestamp with local time zone), 'AAA_2') +PREHOOK: query: insert into ice_orc values (14, cast('2024-08-25 00:00:00' as timestamp with local time zone), 'AAA_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (13, cast('2024-08-24 00:00:00' as timestamp with local time zone), 'AAA_1'), - (14, cast('2024-08-25 00:00:00' as timestamp with local time zone), 'AAA_2') +POSTHOOK: query: insert into ice_orc values (14, cast('2024-08-25 00:00:00' as timestamp with local time zone), 'AAA_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values +PREHOOK: query: insert into ice_orc values (15, cast('2024-09-24 00:00:00' as timestamp with local time zone), 'AAA_1'), (16, cast('2024-09-25 00:00:00' as timestamp with local time zone), 'AAA_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values +POSTHOOK: query: insert into ice_orc values (15, cast('2024-09-24 00:00:00' as timestamp with local time zone), 'AAA_1'), (16, cast('2024-09-25 00:00:00' as timestamp with local time zone), 'AAA_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1'), - (18, cast('2024-07-02 00:00:00' as timestamp with local time zone), 'BBB_2') +PREHOOK: query: insert into ice_orc values + (151, cast('2024-09-26 00:00:00' as timestamp with local time zone), 'AAA_1'), + (161, cast('2024-09-27 00:00:00' as timestamp with local time zone), 'AAA_2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values + (151, cast('2024-09-26 00:00:00' as timestamp with local time zone), 'AAA_1'), + (161, cast('2024-09-27 00:00:00' as timestamp with local time zone), 'AAA_2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (18, cast('2024-07-02 00:00:00' as timestamp with local time zone), 'BBB_2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values (18, cast('2024-07-02 00:00:00' as timestamp with local time zone), 'BBB_2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (20, cast('2024-07-04 00:00:00' as timestamp with local time zone), 'BBB_2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc values (20, cast('2024-07-04 00:00:00' as timestamp with local time zone), 'BBB_2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc values (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (17, cast('2024-07-01 00:00:00' as timestamp with local time zone), 'BBB_1'), - (18, cast('2024-07-02 00:00:00' as timestamp with local time zone), 'BBB_2') +POSTHOOK: query: insert into ice_orc values (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1'), - (20, cast('2024-07-04 00:00:00' as timestamp with local time zone), 'BBB_2') +PREHOOK: query: insert into ice_orc values (22, cast('2024-08-04 00:00:00' as timestamp with local time zone), 'BBB_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (19, cast('2024-07-03 00:00:00' as timestamp with local time zone), 'BBB_1'), - (20, cast('2024-07-04 00:00:00' as timestamp with local time zone), 'BBB_2') +POSTHOOK: query: insert into ice_orc values (22, cast('2024-08-04 00:00:00' as timestamp with local time zone), 'BBB_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1'), - (22, cast('2024-08-04 00:00:00' as timestamp with local time zone), 'BBB_2') +PREHOOK: query: insert into ice_orc values (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (21, cast('2024-08-03 00:00:00' as timestamp with local time zone), 'BBB_1'), - (22, cast('2024-08-04 00:00:00' as timestamp with local time zone), 'BBB_2') +POSTHOOK: query: insert into ice_orc values (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc values - (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1'), - (24, cast('2024-08-06 00:00:00' as timestamp with local time zone), 'BBB_2') +PREHOOK: query: insert into ice_orc values (24, cast('2024-08-06 00:00:00' as timestamp with local time zone), 'BBB_2') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc values - (23, cast('2024-08-05 00:00:00' as timestamp with local time zone), 'BBB_1'), - (24, cast('2024-08-06 00:00:00' as timestamp with local time zone), 'BBB_2') +POSTHOOK: query: insert into ice_orc values (24, cast('2024-08-06 00:00:00' as timestamp with local time zone), 'BBB_2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc @@ -212,8 +248,10 @@ POSTHOOK: Input: default@ice_orc #Masked#4 2024-08-25 00:00:00.0 US/Pacific AAA_2 #Masked#5 2024-09-05 00:00:00.0 US/Pacific BBB_1 #Masked#5 2024-09-24 00:00:00.0 US/Pacific AAA_1 +#Masked#51 2024-09-26 00:00:00.0 US/Pacific AAA_1 #Masked#6 2024-09-06 00:00:00.0 US/Pacific BBB_2 #Masked#6 2024-09-25 00:00:00.0 US/Pacific AAA_2 +#Masked#61 2024-09-27 00:00:00.0 US/Pacific AAA_2 #Masked#7 2024-07-01 00:00:00.0 US/Pacific BBB_1 #Masked#8 2024-07-02 00:00:00.0 US/Pacific BBB_2 #Masked#9 2024-07-03 00:00:00.0 US/Pacific BBB_1 @@ -249,19 +287,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"event_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"event_time\",\"required\":false,\"type\":\"timestamptz\"},{\"id\":3,\"name\":\"event_src\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"9\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"13\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"16\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"28\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"20\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]} format-version 2 #### A masked pattern was here #### - numFiles 13 + numFiles 20 numPartitions 8 - numRows 26 + numRows 28 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 13 + snapshot-count 20 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -279,13 +317,13 @@ InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat Compressed: No Sort Columns: [] -PREHOOK: query: alter table ice_orc COMPACT 'major' and wait +PREHOOK: query: alter table ice_orc COMPACT 'major' and wait where (event_src in ('BBB_1', 'BBB_2') and event_time < '2024-09-01 00:00:00') or (event_src in ('AAA_1', 'AAA_2') and event_time > '2024-08-01 00:00:00') PREHOOK: type: ALTERTABLE_COMPACT PREHOOK: Input: default@ice_orc PREHOOK: Output: default@ice_orc -POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait +POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait where (event_src in ('BBB_1', 'BBB_2') and event_time < '2024-09-01 00:00:00') or (event_src in ('AAA_1', 'AAA_2') and event_time > '2024-08-01 00:00:00') POSTHOOK: type: ALTERTABLE_COMPACT @@ -320,8 +358,10 @@ POSTHOOK: Input: default@ice_orc #Masked#4 2024-08-25 00:00:00.0 US/Pacific AAA_2 #Masked#5 2024-09-05 00:00:00.0 US/Pacific BBB_1 #Masked#5 2024-09-24 00:00:00.0 US/Pacific AAA_1 +#Masked#51 2024-09-26 00:00:00.0 US/Pacific AAA_1 #Masked#6 2024-09-06 00:00:00.0 US/Pacific BBB_2 #Masked#6 2024-09-25 00:00:00.0 US/Pacific AAA_2 +#Masked#61 2024-09-27 00:00:00.0 US/Pacific AAA_2 #Masked#7 2024-07-01 00:00:00.0 US/Pacific BBB_1 #Masked#8 2024-07-02 00:00:00.0 US/Pacific BBB_2 #Masked#9 2024-07-03 00:00:00.0 US/Pacific BBB_1 @@ -357,19 +397,19 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"event_id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"event_time\",\"required\":false,\"type\":\"timestamptz\"},{\"id\":3,\"name\":\"event_src\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"1\",\"added-data-files\":\"2\",\"deleted-data-files\":\"4\",\"added-records\":\"8\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"4\",\"total-records\":\"26\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"2\",\"manifests-kept\":\"7\",\"manifests-replaced\":\"1\",\"added-data-files\":\"2\",\"deleted-data-files\":\"4\",\"added-records\":\"8\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"4\",\"total-records\":\"28\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"9\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"event_src_trunc\",\"transform\":\"truncate[3]\",\"source-id\":3,\"field-id\":1000},{\"name\":\"event_time_month\",\"transform\":\"month\",\"source-id\":2,\"field-id\":1001}]} format-version 2 #### A masked pattern was here #### - numFiles 8 + numFiles 9 numPartitions 8 - numRows 26 + numRows 28 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 17 + snapshot-count 24 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out index e12af3d3b662..6da52474f3e6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out @@ -10,33 +10,59 @@ TBLPROPERTIES ('external.table.purge'='true', 'format-version'='2', 'compactor.t POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@x -PREHOOK: query: insert into x values -('amy', 35, 123412344), -('adxfvy', 36, 123412534), -('amsdfyy', 37, 123417234), -('asafmy', 38, 123412534) +PREHOOK: query: insert into x values ('amy', 35, 123412344) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@x -POSTHOOK: query: insert into x values -('amy', 35, 123412344), -('adxfvy', 36, 123412534), -('amsdfyy', 37, 123417234), -('asafmy', 38, 123412534) +POSTHOOK: query: insert into x values ('amy', 35, 123412344) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@x -PREHOOK: query: insert into x values -('amerqwy', 39, 123441234), -('amyxzcv', 40, 123341234), -('erweramy', 45, 122341234) +PREHOOK: query: insert into x values ('adxfvy', 36, 123412534) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@x -POSTHOOK: query: insert into x values -('amerqwy', 39, 123441234), -('amyxzcv', 40, 123341234), -('erweramy', 45, 122341234) +POSTHOOK: query: insert into x values ('adxfvy', 36, 123412534) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@x +PREHOOK: query: insert into x values ('amsdfyy', 37, 123417234) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@x +POSTHOOK: query: insert into x values ('amsdfyy', 37, 123417234) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@x +PREHOOK: query: insert into x values ('asafmy', 38, 123412534) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@x +POSTHOOK: query: insert into x values ('asafmy', 38, 123412534) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@x +PREHOOK: query: insert into x values ('amerqwy', 39, 123441234) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@x +POSTHOOK: query: insert into x values ('amerqwy', 39, 123441234) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@x +PREHOOK: query: insert into x values ('amyxzcv', 40, 123341234) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@x +POSTHOOK: query: insert into x values ('amyxzcv', 40, 123341234) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@x +PREHOOK: query: insert into x values ('erweramy', 45, 122341234) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@x +POSTHOOK: query: insert into x values ('erweramy', 45, 122341234) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@x @@ -56,7 +82,7 @@ POSTHOOK: query: select count(*) from default.x.data_files POSTHOOK: type: QUERY POSTHOOK: Input: default@x #### A masked pattern was here #### -2 +7 PREHOOK: query: alter table x compact 'major' and wait PREHOOK: type: ALTERTABLE_COMPACT PREHOOK: Input: default@x @@ -95,7 +121,7 @@ Table Parameters: compactor.threshold.target.size 1500 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"num_clicks\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"3\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"2\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"8\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"7\",\"added-data-files\":\"1\",\"deleted-data-files\":\"7\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# external.table.purge true format-version 2 @@ -106,7 +132,7 @@ Table Parameters: #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 3 + snapshot-count 8 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out index 466c234e72c5..689a22a6433f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out @@ -46,6 +46,22 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4', 1), ('fn3','ln3', 1) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn15','ln15', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn16','ln16', 1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc PREHOOK: query: delete from ice_orc where last_name in ('ln4') PREHOOK: type: QUERY PREHOOK: Input: ice_comp@ice_orc @@ -77,6 +93,22 @@ POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn8','ln8'), POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn17','ln17') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc@dept_id=2 +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn17','ln17') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 +PREHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn18','ln18') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc@dept_id=2 +POSTHOOK: query: insert into ice_orc PARTITION(dept_id=2) VALUES ('fn18','ln18') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc@dept_id=2 PREHOOK: query: delete from ice_orc where last_name in ('ln8') PREHOOK: type: QUERY PREHOOK: Input: ice_comp@ice_orc @@ -115,19 +147,19 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"4\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"8\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 #### A masked pattern was here #### - numFiles 4 + numFiles 8 numPartitions 1 - numRows 6 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 6 + snapshot-count 10 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -198,19 +230,19 @@ Table Parameters: compactor.threshold.target.size 3200 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"4\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"8\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"0\",\"added-position-delete-files\":\"1\",\"added-delete-files\":\"1\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"12\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"2\",\"total-position-deletes\":\"2\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 #### A masked pattern was here #### - numFiles 4 + numFiles 8 numPartitions 1 - numRows 6 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 6 + snapshot-count 10 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -260,6 +292,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### fn1 ln1 1 +fn15 ln15 1 +fn16 ln16 1 fn2 ln2 1 fn3 ln3 1 PREHOOK: query: select * from ice_orc where dept_id=2 @@ -270,6 +304,8 @@ POSTHOOK: query: select * from ice_orc where dept_id=2 POSTHOOK: type: QUERY POSTHOOK: Input: ice_comp@ice_orc #### A masked pattern was here #### +fn17 ln17 2 +fn18 ln18 2 fn5 ln5 2 fn6 ln6 2 fn7 ln7 2 @@ -305,19 +341,19 @@ Table Parameters: compactor.threshold.target.size 4000 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"removed-position-delete-files\":\"2\",\"removed-delete-files\":\"2\",\"added-records\":\"3\",\"deleted-records\":\"4\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"2\",\"changed-partition-count\":\"3\",\"total-records\":\"6\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"1\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"4\",\"removed-position-delete-files\":\"2\",\"removed-delete-files\":\"2\",\"added-records\":\"5\",\"deleted-records\":\"6\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"2\",\"changed-partition-count\":\"3\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 #### A masked pattern was here #### numFiles 2 numPartitions 2 - numRows 6 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 8 + snapshot-count 12 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -367,6 +403,22 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn11','ln11', 3), ('fn12','ln12', POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn19','ln19', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn20','ln20', 3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp@ice_orc PREHOOK: query: select * from ice_orc PREHOOK: type: QUERY PREHOOK: Input: ice_comp@ice_orc @@ -379,7 +431,13 @@ fn1 ln1 1 fn10 ln10 3 fn11 ln11 3 fn12 ln12 3 +fn15 ln15 1 +fn16 ln16 1 +fn17 ln17 2 +fn18 ln18 2 +fn19 ln19 3 fn2 ln2 1 +fn20 ln20 3 fn3 ln3 1 fn5 ln5 2 fn6 ln6 2 @@ -410,17 +468,17 @@ Table Parameters: compactor.threshold.target.size 4000 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"4\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"4\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"16\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"6\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### - numFiles 4 - numRows 10 + numFiles 6 + numRows 16 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 10 + snapshot-count 16 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -466,7 +524,13 @@ fn1 ln1 1 fn10 ln10 3 fn11 ln11 3 fn12 ln12 3 +fn15 ln15 1 +fn16 ln16 1 +fn17 ln17 2 +fn18 ln18 2 +fn19 ln19 3 fn2 ln2 1 +fn20 ln20 3 fn3 ln3 1 fn5 ln5 2 fn6 ln6 2 @@ -497,17 +561,17 @@ Table Parameters: compactor.threshold.target.size 4000 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"4\",\"added-records\":\"10\",\"deleted-records\":\"10\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"6\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"5\",\"added-data-files\":\"1\",\"deleted-data-files\":\"6\",\"added-records\":\"16\",\"deleted-records\":\"16\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"3\",\"total-records\":\"16\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### numFiles 1 - numRows 10 + numRows 16 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 11 + snapshot-count 17 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out index 79536df6215d..ac170e6dfac1 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_unpartitioned.q.out @@ -24,19 +24,51 @@ POSTHOOK: query: insert into ice_orc VALUES ('fn1','ln1') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5'), ('fn7','ln7'), ('fn4','ln4'), ('fn6','ln6') +PREHOOK: query: insert into ice_orc VALUES ('fn5','ln5') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5'), ('fn7','ln7'), ('fn4','ln4'), ('fn6','ln6') +POSTHOOK: query: insert into ice_orc VALUES ('fn5','ln5') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc VALUES ('fn3','ln3'), ('fn2','ln2') +PREHOOK: query: insert into ice_orc VALUES ('fn7','ln7') PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc VALUES ('fn3','ln3'), ('fn2','ln2') +POSTHOOK: query: insert into ice_orc VALUES ('fn7','ln7') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn4','ln4') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn4','ln4') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn6','ln6') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn6','ln6') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn3','ln3') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn3','ln3') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: insert into ice_orc VALUES ('fn2','ln2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: insert into ice_orc VALUES ('fn2','ln2') POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_orc @@ -93,17 +125,17 @@ Table Parameters: compactor.threshold.target.size 2920 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"3\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"6\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"7\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### - numFiles 3 + numFiles 7 numRows 7 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 3 + snapshot-count 7 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -176,7 +208,7 @@ Table Parameters: compactor.threshold.target.size 2920 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"3\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"8\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"7\",\"added-data-files\":\"1\",\"deleted-data-files\":\"7\",\"added-records\":\"7\",\"deleted-records\":\"7\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### @@ -186,7 +218,7 @@ Table Parameters: #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 4 + snapshot-count 8 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -242,13 +274,13 @@ POSTHOOK: query: select * from ice_orc POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_orc #### A masked pattern was here #### +fn8 ln8 fn1 ln1 fn2 ln2 fn3 ln3 fn4 ln4 fn5 ln5 fn6 ln6 -fn8 ln8 PREHOOK: query: describe formatted ice_orc PREHOOK: type: DESCTABLE PREHOOK: Input: default@ice_orc @@ -273,17 +305,17 @@ Table Parameters: compactor.threshold.target.size 2920 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"manifests-created\":\"4\",\"manifests-kept\":\"0\",\"manifests-replaced\":\"3\",\"added-data-files\":\"1\",\"deleted-data-files\":\"2\",\"removed-position-delete-files\":\"1\",\"removed-delete-files\":\"1\",\"added-records\":\"7\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"1\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"manifests-created\":\"1\",\"manifests-kept\":\"2\",\"manifests-replaced\":\"0\",\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"8\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"1\",\"total-position-deletes\":\"1\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# format-version 2 #### A masked pattern was here #### - numFiles 1 + numFiles 2 numRows 7 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 7 + snapshot-count 10 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -307,4 +339,4 @@ POSTHOOK: query: show compactions POSTHOOK: type: SHOW COMPACTIONS CompactionId Database Table Partition Type State Worker host Worker Enqueue Time Start Time Duration(ms) HadoopJobId Error message Initiator host Initiator Pool name TxnId Next TxnId Commit Time Highest WriteId #Masked# default ice_orc --- MINOR succeeded #Masked# manual iceberg 0 0 0 --- -#Masked# default ice_orc --- MINOR succeeded #Masked# manual iceberg 0 0 0 --- +#Masked# default ice_orc --- MINOR refused #Masked# manual iceberg 0 0 0 --- diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out index b4d8dcd592d5..7f593a3a577d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_lineage_compactions.q.out @@ -229,6 +229,22 @@ POSTHOOK: query: insert into part_tbl values (5,'p5', 10) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: ice_comp_all@part_tbl +PREHOOK: query: insert into part_tbl values (5,'p5', 10) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp_all@part_tbl +POSTHOOK: query: insert into part_tbl values (5,'p5', 10) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp_all@part_tbl +PREHOOK: query: insert into part_tbl values (6,'p6', 20) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_comp_all@part_tbl +POSTHOOK: query: insert into part_tbl values (6,'p6', 20) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_comp_all@part_tbl PREHOOK: query: insert into part_tbl values (6,'p6', 20) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table @@ -254,7 +270,9 @@ POSTHOOK: Input: ice_comp_all@part_tbl 3 p3 20 2 3 4 p4 20 3 4 5 p5 10 11 9 -6 p6 20 16 10 +5 p5 10 16 10 +6 p6 20 17 11 +6 p6 20 18 12 PREHOOK: query: alter table part_tbl compact 'minor' and wait pool 'iceberg' PREHOOK: type: ALTERTABLE_COMPACT PREHOOK: Input: ice_comp_all@part_tbl @@ -280,7 +298,9 @@ POSTHOOK: Input: ice_comp_all@part_tbl 3 p3 20 2 3 4 p4 20 3 4 5 p5 10 11 9 -6 p6 20 16 10 +5 p5 10 16 10 +6 p6 20 17 11 +6 p6 20 18 12 PREHOOK: query: show compactions PREHOOK: type: SHOW COMPACTIONS POSTHOOK: query: show compactions