You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ay...@apache.org on 2022/09/30 09:17:43 UTC
[hive] branch master updated: HIVE-26529: Iceberg: Fix VectorizedSupport support for DECIMAL_64 in HiveIcebergInputFormat. (#3607). (Ayush Saxena, reviewed by Adam Szita)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 3f80d99f869 HIVE-26529: Iceberg: Fix VectorizedSupport support for DECIMAL_64 in HiveIcebergInputFormat. (#3607). (Ayush Saxena, reviewed by Adam Szita)
3f80d99f869 is described below
commit 3f80d99f869529b5d0354263038949ecefc689b0
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Fri Sep 30 14:47:30 2022 +0530
HIVE-26529: Iceberg: Fix VectorizedSupport support for DECIMAL_64 in HiveIcebergInputFormat. (#3607). (Ayush Saxena, reviewed by Adam Szita)
---
.../iceberg/mr/hive/HiveIcebergInputFormat.java | 17 +-
.../iceberg/mr/hive/HiveIcebergMetaHook.java | 32 ++
.../mr/hive/vector/HiveVectorizedReader.java | 2 -
.../hive/TestHiveIcebergStorageHandlerNoScan.java | 8 +-
.../positive/vectorized_iceberg_read_mixed.q | 12 +
.../queries/positive/vectorized_iceberg_read_orc.q | 14 +-
.../positive/vectorized_iceberg_read_parquet.q | 6 +
.../alter_multi_part_table_to_iceberg.q.out | 3 +
.../positive/alter_part_table_to_iceberg.q.out | 3 +
.../results/positive/alter_table_to_iceberg.q.out | 3 +
.../results/positive/create_iceberg_table.q.out | 1 +
...create_iceberg_table_stored_as_fileformat.q.out | 5 +
.../create_iceberg_table_stored_by_iceberg.q.out | 1 +
...le_stored_by_iceberg_with_serdeproperties.q.out | 1 +
.../results/positive/describe_iceberg_table.q.out | 4 +
.../llap/vectorized_iceberg_read_mixed.q.out | 465 +++++++++++++++++
.../llap/vectorized_iceberg_read_orc.q.out | 548 +++++++++++++++++++++
.../llap/vectorized_iceberg_read_parquet.q.out | 277 +++++++++++
.../positive/show_create_iceberg_table.q.out | 4 +
.../positive/truncate_force_iceberg_table.q.out | 2 +
.../results/positive/truncate_iceberg_table.q.out | 5 +
.../truncate_partitioned_iceberg_table.q.out | 2 +
.../positive/vectorized_iceberg_read_mixed.q.out | 460 +++++++++++++++++
.../positive/vectorized_iceberg_read_orc.q.out | 542 ++++++++++++++++++++
.../positive/vectorized_iceberg_read_parquet.q.out | 274 +++++++++++
.../vector/VectorizedInputFormatInterface.java | 8 +
.../hive/ql/optimizer/physical/Vectorizer.java | 19 +-
27 files changed, 2698 insertions(+), 20 deletions(-)
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
index 23d76d4260d..532f8eda06b 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
@@ -21,6 +21,7 @@ package org.apache.iceberg.mr.hive;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.io.DataCache;
@@ -177,9 +178,19 @@ public class HiveIcebergInputFormat extends MapredIcebergInputFormat<Record>
@Override
public VectorizedSupport.Support[] getSupportedFeatures() {
- // disabling VectorizedSupport.Support.DECIMAL_64 as Parquet doesn't support it, and we have no way of telling
- // beforehand what kind of file format we're going to hit later
- return new VectorizedSupport.Support[]{ };
+ throw new UnsupportedOperationException("This overload of getSupportedFeatures should never be called");
+ }
+
+ @Override
+ public VectorizedSupport.Support[] getSupportedFeatures(HiveConf hiveConf, Properties properties) {
+ // disabling VectorizedSupport.Support.DECIMAL_64 for Parquet as it doesn't support it
+ boolean isORCOnly = Boolean.parseBoolean(properties.getProperty(HiveIcebergMetaHook.ORC_FILES_ONLY)) &&
+ org.apache.iceberg.FileFormat.ORC.name().equalsIgnoreCase(properties.getProperty("write.format.default"));
+ if (!isORCOnly) {
+ HiveConf.setVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED, "");
+ return new VectorizedSupport.Support[] {};
+ }
+ return new VectorizedSupport.Support[] { VectorizedSupport.Support.DECIMAL_64 };
}
@Override
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index c22ac6db110..067057def75 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -125,6 +125,7 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
private static final List<org.apache.commons.lang3.tuple.Pair<Integer, byte[]>> EMPTY_FILTER =
Lists.newArrayList(org.apache.commons.lang3.tuple.Pair.of(1, new byte[0]));
static final String MIGRATED_TO_ICEBERG = "MIGRATED_TO_ICEBERG";
+ static final String ORC_FILES_ONLY = "iceberg.orc.files.only";
private final Configuration conf;
private Table icebergTable = null;
@@ -211,6 +212,9 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
}
assertFileFormat(catalogProperties.getProperty(TableProperties.DEFAULT_FILE_FORMAT));
+
+ // Set whether the format is ORC, to be used during vectorization.
+ setOrcOnlyFilesParam(hmsTable);
}
@Override
@@ -304,6 +308,8 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
// If so, we will create the iceberg table in commitAlterTable and go ahead with the migration
assertTableCanBeMigrated(hmsTable);
isTableMigration = true;
+ // Set whether the format is ORC, to be used during vectorization.
+ setOrcOnlyFilesParam(hmsTable);
StorageDescriptor sd = hmsTable.getSd();
preAlterTableProperties = new PreAlterTableProperties();
@@ -364,6 +370,14 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
} else if (AlterTableType.ADDPROPS.equals(currentAlterTableOp)) {
assertNotCrossTableMetadataLocationChange(hmsTable.getParameters());
}
+
+ // Migration case is already handled above, in case of migration we don't have all the properties set till this
+ // point.
+ if (!isTableMigration) {
+ // Set whether the format is ORC, to be used during vectorization.
+ setOrcOnlyFilesParam(hmsTable);
+ }
+
}
/**
@@ -826,6 +840,24 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
return (Type.PrimitiveType) newType;
}
+ private void setOrcOnlyFilesParam(org.apache.hadoop.hive.metastore.api.Table hmsTable) {
+ if (isOrcOnlyFiles(hmsTable)) {
+ hmsTable.getParameters().put(ORC_FILES_ONLY, "true");
+ } else {
+ hmsTable.getParameters().put(ORC_FILES_ONLY, "false");
+ }
+ }
+
+ private boolean isOrcOnlyFiles(org.apache.hadoop.hive.metastore.api.Table hmsTable) {
+ return !"FALSE".equalsIgnoreCase(hmsTable.getParameters().get(ORC_FILES_ONLY)) &&
+ ((hmsTable.getSd().getInputFormat() != null &&
+ hmsTable.getSd().getInputFormat().toUpperCase().contains(org.apache.iceberg.FileFormat.ORC.name())) ||
+ org.apache.iceberg.FileFormat.ORC.name()
+ .equalsIgnoreCase(hmsTable.getSd().getSerdeInfo().getParameters().get("write.format.default")) ||
+ org.apache.iceberg.FileFormat.ORC.name()
+ .equalsIgnoreCase(hmsTable.getParameters().get("write.format.default")));
+ }
+
private class PreAlterTableProperties {
private String tableLocation;
private String format;
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
index 372a43376e2..2e0d11e59c5 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
@@ -173,8 +173,6 @@ public class HiveVectorizedReader {
// If LLAP enabled, try to retrieve an LLAP record reader - this might yield to null in some special cases
if (HiveConf.getBoolVar(job, HiveConf.ConfVars.LLAP_IO_ENABLED, LlapProxy.isDaemon()) &&
LlapProxy.getIo() != null) {
- // Required to prevent LLAP from dealing with decimal64, HiveIcebergInputFormat.getSupportedFeatures()
- HiveConf.setVar(job, HiveConf.ConfVars.HIVE_VECTORIZED_INPUT_FORMAT_SUPPORTS_ENABLED, "");
recordReader = LlapProxy.getIo().llapVectorizedOrcReaderForPath(fileId, path, null, readColumnIds,
job, start, length, reporter);
}
diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
index 7890d32e918..bfb5997d2bd 100644
--- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
+++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
@@ -945,7 +945,7 @@ public class TestHiveIcebergStorageHandlerNoScan {
Assert.assertEquals(expectedIcebergProperties, icebergTable.properties());
if (Catalogs.hiveCatalog(shell.getHiveConf(), tableProperties)) {
- Assert.assertEquals(10, hmsParams.size());
+ Assert.assertEquals(11, hmsParams.size());
Assert.assertEquals("initial_val", hmsParams.get("custom_property"));
Assert.assertEquals("TRUE", hmsParams.get("EXTERNAL"));
Assert.assertEquals("true", hmsParams.get(TableProperties.ENGINE_HIVE_ENABLED));
@@ -959,7 +959,7 @@ public class TestHiveIcebergStorageHandlerNoScan {
Assert.assertNotNull(hmsParams.get(hive_metastoreConstants.DDL_TIME));
Assert.assertNotNull(hmsParams.get(serdeConstants.SERIALIZATION_FORMAT));
} else {
- Assert.assertEquals(6, hmsParams.size());
+ Assert.assertEquals(7, hmsParams.size());
Assert.assertNull(hmsParams.get(TableProperties.ENGINE_HIVE_ENABLED));
}
@@ -982,7 +982,7 @@ public class TestHiveIcebergStorageHandlerNoScan {
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
if (Catalogs.hiveCatalog(shell.getHiveConf(), tableProperties)) {
- Assert.assertEquals(13, hmsParams.size()); // 2 newly-added properties + previous_metadata_location prop
+ Assert.assertEquals(14, hmsParams.size()); // 2 newly-added properties + previous_metadata_location prop
Assert.assertEquals("true", hmsParams.get("new_prop_1"));
Assert.assertEquals("false", hmsParams.get("new_prop_2"));
Assert.assertEquals("new_val", hmsParams.get("custom_property"));
@@ -992,7 +992,7 @@ public class TestHiveIcebergStorageHandlerNoScan {
Assert.assertEquals(hmsParams.get(BaseMetastoreTableOperations.PREVIOUS_METADATA_LOCATION_PROP), prevSnapshot);
Assert.assertEquals(hmsParams.get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP), newSnapshot);
} else {
- Assert.assertEquals(6, hmsParams.size());
+ Assert.assertEquals(7, hmsParams.size());
}
// Remove some Iceberg props and see if they're removed from HMS table props as well
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_mixed.q b/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_mixed.q
index 0b6393b64f5..be127d0d53d 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_mixed.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_mixed.q
@@ -3,12 +3,17 @@ set hive.vectorized.execution.enabled=true;
drop table if exists tbl_ice_mixed;
create external table tbl_ice_mixed(a int, b string) stored by iceberg stored as orc;
insert into table tbl_ice_mixed values (1, 'one'), (2, 'two'), (3, 'three'), (4, 'four'), (5, 'five'), (111, 'one'), (22, 'two'), (11, 'one'), (44444, 'four'), (44, 'four');
+
+explain vectorization only detail select b, max(a) from tbl_ice_mixed group by b;
+
alter table tbl_ice_mixed set tblproperties ('write.format.default'='parquet');
insert into table tbl_ice_mixed values (10, 'ten'), (20, 'twenty'), (30, 'thirty'), (40, 'fourty'), (50, 'fifty'), (1110, 'ten'), (220, 'twenty'), (44445, 'four'), (10, 'one');
analyze table tbl_ice_mixed compute statistics for columns;
explain select b, max(a) from tbl_ice_mixed group by b;
+
+explain vectorization only detail select b, max(a) from tbl_ice_mixed group by b;
select b, max(a) from tbl_ice_mixed group by b;
create external table tbl_ice_mixed_all_types (
@@ -25,9 +30,15 @@ create external table tbl_ice_mixed_all_types (
) stored by iceberg stored as orc;
insert into tbl_ice_mixed_all_types values (1.1, 1.2, false, 4, 567890123456789, '6', "col7", cast('2012-10-03 19:58:08' as timestamp), date('1234-09-09'), cast('10.01' as decimal(4,2)));
+explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string,
+ t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal;
alter table tbl_ice_mixed_all_types set tblproperties ('write.format.default'='parquet');
insert into tbl_ice_mixed_all_types values (5.1, 6.2, true, 40, 567890123456780, '8', "col07", cast('2012-10-03 19:58:09' as timestamp), date('1234-09-10'), cast('10.02' as decimal(4,2)));
+explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string,
+t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal;
explain select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal;
select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
@@ -68,6 +79,7 @@ alter table tbl_ice_mixed_parted change column a a int after b;
describe tbl_ice_mixed_parted;
-- should yield to the same result as previously
+explain vectorization only detail select p1, a, min(b) from tbl_ice_mixed_parted group by p1, a;
select p1, a, min(b) from tbl_ice_mixed_parted group by p1, a;
insert into tbl_ice_mixed_parted values ('Europe', 'cc', 3, 'Italy');
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_orc.q b/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_orc.q
index ee91b9f64e6..a61aa7ab85e 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_orc.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_orc.q
@@ -8,6 +8,8 @@ analyze table tbl_ice_orc compute statistics for columns;
explain select b, max(a) from tbl_ice_orc group by b;
select b, max(a) from tbl_ice_orc group by b;
+explain vectorization only detail select b, max(a) from tbl_ice_orc group by b;
+
create external table tbl_ice_orc_all_types (
t_float FLOAT,
t_double DOUBLE,
@@ -20,7 +22,6 @@ create external table tbl_ice_orc_all_types (
t_date DATE,
t_decimal DECIMAL(4,2)
) stored by iceberg stored as orc;
-
insert into tbl_ice_orc_all_types values (1.1, 1.2, false, 4, 567890123456789, '6', "col7", cast('2012-10-03 19:58:08' as timestamp), date('1234-09-09'), cast('10.01' as decimal(4,2)));
explain select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_orc_all_types
@@ -28,6 +29,9 @@ explain select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_s
select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_orc_all_types
group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal;
+explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_orc_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal;
+
create external table tbl_ice_orc_parted (
a int,
b string
@@ -48,6 +52,8 @@ alter table tbl_ice_orc_parted change column p1 p1 string after a;
-- should yield to the same result as previously
select p1, a, min(b) from tbl_ice_orc_parted group by p1, a;
+explain vectorization only detail select p1, a, min(b) from tbl_ice_orc_parted group by p1, a;
+
-- move non-partition columns
alter table tbl_ice_orc_parted change column a a int after b;
@@ -56,11 +62,15 @@ describe tbl_ice_orc_parted;
-- should yield to the same result as previously
select p1, a, min(b) from tbl_ice_orc_parted group by p1, a;
+explain vectorization only detail select p1, a, min(b) from tbl_ice_orc_parted group by p1, a;
+
insert into tbl_ice_orc_parted values ('Europe', 'cc', 3, 'Austria');
-- projecting all columns
select p1, p2, a, min(b) from tbl_ice_orc_parted group by p1, p2, a;
+explain vectorization only detail select p1, p2, a, min(b) from tbl_ice_orc_parted group by p1, p2, a;
+
-- create iceberg table with complex types
create external table tbl_ice_orc_complex (
a int,
@@ -94,6 +104,8 @@ insert into tbl_ice_orc_complex values (
select * from tbl_ice_orc_complex order by a;
+explain vectorization only detail select * from tbl_ice_orc_complex order by a;
+
drop table tbl_ice_orc;
drop table tbl_ice_orc_all_types;
drop table tbl_ice_orc_parted;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_parquet.q b/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_parquet.q
index a5bbe547a0f..989d6fcb718 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_parquet.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_parquet.q
@@ -6,6 +6,7 @@ insert into table tbl_ice_parquet values (1, 'one'), (2, 'two'), (3, 'three'), (
analyze table tbl_ice_parquet compute statistics for columns;
explain select b, max(a) from tbl_ice_parquet group by b;
+explain vectorization only detail select b, max(a) from tbl_ice_parquet group by b;
select b, max(a) from tbl_ice_parquet group by b;
create external table tbl_ice_parquet_all_types (
@@ -25,6 +26,9 @@ insert into tbl_ice_parquet_all_types values (1.1, 1.2, false, 4, 56789012345678
explain select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_parquet_all_types
group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal;
+
+explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_parquet_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal;
select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_parquet_all_types
group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal;
@@ -56,6 +60,8 @@ describe tbl_ice_parquet_parted;
-- should yield to the same result as previously
select p1, a, min(b) from tbl_ice_parquet_parted group by p1, a;
+explain vectorization only detail select p1, a, min(b) from tbl_ice_parquet_parted group by p1, a;
+
insert into tbl_ice_parquet_parted values ('Europe', 'cc', 3, 'Austria');
-- projecting all columns
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
index 5f7af3f68cf..85ec00aa376 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
@@ -186,6 +186,7 @@ Table Parameters:
MIGRATED_TO_ICEBERG true
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only true
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 7
@@ -433,6 +434,7 @@ Table Parameters:
MIGRATED_TO_ICEBERG true
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 7
@@ -680,6 +682,7 @@ Table Parameters:
MIGRATED_TO_ICEBERG true
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 7
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
index ca580ca464b..35ad102b8f5 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
@@ -144,6 +144,7 @@ Table Parameters:
MIGRATED_TO_ICEBERG true
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only true
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 4
@@ -340,6 +341,7 @@ Table Parameters:
MIGRATED_TO_ICEBERG true
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 4
@@ -536,6 +538,7 @@ Table Parameters:
MIGRATED_TO_ICEBERG true
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 4
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out
index bde0774838b..b14d1133cfd 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out
@@ -99,6 +99,7 @@ Table Parameters:
MIGRATED_TO_ICEBERG true
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only true
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 1
@@ -247,6 +248,7 @@ Table Parameters:
MIGRATED_TO_ICEBERG true
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 1
@@ -395,6 +397,7 @@ Table Parameters:
MIGRATED_TO_ICEBERG true
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 1
diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out
index 5df4e76e51f..d8952502e64 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out
@@ -29,6 +29,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out
index 2bc658d6885..82a76919e09 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out
@@ -33,6 +33,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only true
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
@@ -94,6 +95,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
@@ -155,6 +157,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
@@ -216,6 +219,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
@@ -274,6 +278,7 @@ Table Parameters:
bucketing_version 2
dummy dummy_value
engine.hive.enabled true
+ iceberg.orc.files.only true
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out
index 5df4e76e51f..d8952502e64 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out
@@ -29,6 +29,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out
index 59603b64115..610eb67907f 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out
@@ -29,6 +29,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only true
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
diff --git a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
index 99936486cb8..c1251f81f62 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
@@ -69,6 +69,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
@@ -122,6 +123,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
@@ -176,6 +178,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
@@ -218,6 +221,7 @@ Table Parameters:
EXTERNAL TRUE
bucketing_version 2
engine.hive.enabled true
+ iceberg.orc.files.only false
metadata_location hdfs://### HDFS PATH ###
numFiles 0
numRows 0
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out
index b3d9d7f5e80..3e4b1f8806b 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out
@@ -18,6 +18,95 @@ POSTHOOK: query: insert into table tbl_ice_mixed values (1, 'one'), (2, 'two'),
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice_mixed
+PREHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_mixed group by b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_mixed
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_mixed group by b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_mixed
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:b:string]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 0:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: a:int, b:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: z
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY._col0:string, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 1:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: alter table tbl_ice_mixed set tblproperties ('write.format.default'='parquet')
PREHOOK: type: ALTERTABLE_PROPERTIES
PREHOOK: Input: default@tbl_ice_mixed
@@ -108,6 +197,95 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_mixed group by b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_mixed
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_mixed group by b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_mixed
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:b:string]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 0:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: a:int, b:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: z
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY._col0:string, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 1:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: select b, max(a) from tbl_ice_mixed group by b
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice_mixed
@@ -164,6 +342,103 @@ POSTHOOK: query: insert into tbl_ice_mixed_all_types values (1.1, 1.2, false, 4,
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice_mixed_all_types
+PREHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string,
+ t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_mixed_all_types
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string,
+ t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_mixed_all_types
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 0:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 10:decimal(4,2)
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:double, 1:boolean, 2:int, 3:bigint, 4:binary, 5:string, 6:timestamp, 7:date, 8:decimal(4,2)
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 9:float
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)/DECIMAL_64
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [decimal(4,2)]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zzzzzzzzz
+ reduceColumnSortOrder: +++++++++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ dataColumns: KEY._col0:double, KEY._col1:boolean, KEY._col2:int, KEY._col3:bigint, KEY._col4:binary, KEY._col5:string, KEY._col6:timestamp, KEY._col7:date, KEY._col8:decimal(4,2), VALUE._col0:float
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 9:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:double, col 1:boolean, col 2:int, col 3:bigint, col 4:binary, col 5:string, col 6:timestamp, col 7:date, col 8:decimal(4,2)
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [9, 0, 1, 2, 3, 4, 5, 6, 7, 8]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: alter table tbl_ice_mixed_all_types set tblproperties ('write.format.default'='parquet')
PREHOOK: type: ALTERTABLE_PROPERTIES
PREHOOK: Input: default@tbl_ice_mixed_all_types
@@ -180,6 +455,103 @@ POSTHOOK: query: insert into tbl_ice_mixed_all_types values (5.1, 6.2, true, 40,
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice_mixed_all_types
+PREHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string,
+t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_mixed_all_types
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string,
+t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_mixed_all_types
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 0:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, col 9:decimal(4,2)
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:double, 1:boolean, 2:int, 3:bigint, 4:binary, 5:string, 6:timestamp, 7:date, 8:decimal(4,2)
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 9:float
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zzzzzzzzz
+ reduceColumnSortOrder: +++++++++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ dataColumns: KEY._col0:double, KEY._col1:boolean, KEY._col2:int, KEY._col3:bigint, KEY._col4:binary, KEY._col5:string, KEY._col6:timestamp, KEY._col7:date, KEY._col8:decimal(4,2), VALUE._col0:float
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 9:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:double, col 1:boolean, col 2:int, col 3:bigint, col 4:binary, col 5:string, col 6:timestamp, col 7:date, col 8:decimal(4,2)
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [9, 0, 1, 2, 3, 4, 5, 6, 7, 8]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: explain select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
PREHOOK: type: QUERY
@@ -375,6 +747,99 @@ p2 string
# col_name transform_type
p1 IDENTITY
p2 IDENTITY
+PREHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_mixed_parted group by p1, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_mixed_parted
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_mixed_parted group by p1, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_mixed_parted
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 1:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 2:int
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:string, 1:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 2:string
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2]
+ dataColumns: p1:string, b:string, a:int, p2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY._col0:string, KEY._col1:int, VALUE._col0:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 2:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:int
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: select p1, a, min(b) from tbl_ice_mixed_parted group by p1, a
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice_mixed_parted
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out
index dbfa9b6c901..d675c242e67 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out
@@ -105,6 +105,95 @@ four 44444
one 111
three 3
two 22
+PREHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_orc group by b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_orc group by b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:b:string]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 0:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: a:int, b:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: z
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY._col0:string, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 1:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: create external table tbl_ice_orc_all_types (
t_float FLOAT,
t_double DOUBLE,
@@ -224,6 +313,101 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tbl_ice_orc_all_types
#### A masked pattern was here ####
1.1 1.2 false 4 567890123456789 6 col7 2012-10-03 19:58:08 1234-09-02 10.01
+PREHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_orc_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc_all_types
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_orc_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc_all_types
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 0:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 10:decimal(4,2)
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:double, 1:boolean, 2:int, 3:bigint, 4:binary, 5:string, 6:timestamp, 7:date, 8:decimal(4,2)
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 9:float
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)/DECIMAL_64
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [decimal(4,2)]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zzzzzzzzz
+ reduceColumnSortOrder: +++++++++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ dataColumns: KEY._col0:double, KEY._col1:boolean, KEY._col2:int, KEY._col3:bigint, KEY._col4:binary, KEY._col5:string, KEY._col6:timestamp, KEY._col7:date, KEY._col8:decimal(4,2), VALUE._col0:float
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 9:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:double, col 1:boolean, col 2:int, col 3:bigint, col 4:binary, col 5:string, col 6:timestamp, col 7:date, col 8:decimal(4,2)
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [9, 0, 1, 2, 3, 4, 5, 6, 7, 8]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: create external table tbl_ice_orc_parted (
a int,
b string
@@ -286,6 +470,103 @@ POSTHOOK: Input: default@tbl_ice_orc_parted
#### A masked pattern was here ####
Europe 1 aa
America 2 aa
+PREHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_orc_parted group by p1, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc_parted
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_orc_parted group by p1, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc_parted
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 2:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:int, col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:int, 1:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 2:string
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2]
+ dataColumns: a:int, p1:string, b:string, p2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY._col0:int, KEY._col1:string, VALUE._col0:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 2:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:int, col 1:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 0, 2]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: alter table tbl_ice_orc_parted change column a a int after b
PREHOOK: type: ALTERTABLE_RENAMECOL
PREHOOK: Input: default@tbl_ice_orc_parted
@@ -319,6 +600,99 @@ POSTHOOK: Input: default@tbl_ice_orc_parted
#### A masked pattern was here ####
America 2 aa
Europe 1 aa
+PREHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_orc_parted group by p1, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc_parted
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_orc_parted group by p1, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc_parted
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 1:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 2:int
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:string, 1:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 2:string
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2]
+ dataColumns: p1:string, b:string, a:int, p2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY._col0:string, KEY._col1:int, VALUE._col0:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 2:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:int
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: insert into tbl_ice_orc_parted values ('Europe', 'cc', 3, 'Austria')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
@@ -339,6 +713,99 @@ America Canada 2 bb
America USA 2 aa
Europe Hungary 1 aa
Europe Austria 3 cc
+PREHOOK: query: explain vectorization only detail select p1, p2, a, min(b) from tbl_ice_orc_parted group by p1, p2, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc_parted
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select p1, p2, a, min(b) from tbl_ice_orc_parted group by p1, p2, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc_parted
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string]
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 1:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 2:int, col 3:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:string, 1:int, 2:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 3:string
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2, 3]
+ dataColumns: p1:string, b:string, a:int, p2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zzz
+ reduceColumnSortOrder: +++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY._col0:string, KEY._col1:int, KEY._col2:string, VALUE._col0:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 3:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:int, col 2:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1, 3]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: create external table tbl_ice_orc_complex (
a int,
arrayofprimitives array<string>,
@@ -414,6 +881,87 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tbl_ice_orc_complex
#### A masked pattern was here ####
1 ["a","b","c"] [["a"],["b","c"]] [{"a":"b"},{"e":"f"}] [{"something":"a","someone":"b","somewhere":"c"},{"something":"e","someone":"f","somewhere":"g"}] {"a":"b"} {"a":["b","c"]} {"a":{"b":"c"}} {"a":{"something":"b","someone":"c","somewhere":"d"}} {"something":"a","somewhere":"b"} {"names":["a","b"],"birthdays":["c","d","e"]} {"map1":{"a":"b"},"map2":{"c":"d"}}
+PREHOOK: query: explain vectorization only detail select * from tbl_ice_orc_complex order by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc_complex
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select * from tbl_ice_orc_complex order by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc_complex
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array<string>, 2:arrayofarrays:array<array<string>>, 3:arrayofmaps:array<map<string,string>>, 4:arrayofstructs:array<struct<something:string,someone:string,somewhere:string>>, 5:mapofprimitives:map<string,string>, 6:mapofarrays:map<string,array<string>>, 7:mapofmaps:map<string,map<string,string>>, 8:mapofstructs:map<string,struct<something:string,someone:string,somewhere:string>>, 9:structofprimitives:struct< [...]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:array<string>, 2:array<array<string>>, 3:array<map<string,string>>, 4:array<struct<something:string,someone:string,somewhere:string>>, 5:map<string,string>, 6:map<string,array<string>>, 7:map<string,map<string,string>>, 8:map<string,struct<something:string,someone:string,somewhere:string>>, 9:struct<something:string,somewhere:string>, 10:struct<names:array<string>,birthdays:array<string>>, 11:struct<map1:map<string,string>,map2:map<string,string>>
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ dataColumns: a:int, arrayofprimitives:array<string>, arrayofarrays:array<array<string>>, arrayofmaps:array<map<string,string>>, arrayofstructs:array<struct<something:string,someone:string,somewhere:string>>, mapofprimitives:map<string,string>, mapofarrays:map<string,array<string>>, mapofmaps:map<string,map<string,string>>, mapofstructs:map<string,struct<something:string,someone:string,somewhere:string>>, structofprimitives:struct<something:string,somewhere:string>, st [...]
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: z
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ dataColumns: KEY.reducesinkkey0:int, VALUE._col0:array<string>, VALUE._col1:array<array<string>>, VALUE._col2:array<map<string,string>>, VALUE._col3:array<struct<something:string,someone:string,somewhere:string>>, VALUE._col4:map<string,string>, VALUE._col5:map<string,array<string>>, VALUE._col6:map<string,map<string,string>>, VALUE._col7:map<string,struct<something:string,someone:string,somewhere:string>>, VALUE._col8:struct<something:string,somewhere:string>, VALUE. [...]
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: drop table tbl_ice_orc
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@tbl_ice_orc
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
index ba31ed21cbe..0361b3b292c 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
@@ -92,6 +92,95 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_parquet group by b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_parquet
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_parquet group by b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_parquet
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:b:string]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 0:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: a:int, b:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: z
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY._col0:string, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 1:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: select b, max(a) from tbl_ice_parquet group by b
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice_parquet
@@ -213,6 +302,101 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_parquet_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_parquet_all_types
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_parquet_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_parquet_all_types
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 0:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, col 9:decimal(4,2)
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:double, 1:boolean, 2:int, 3:bigint, 4:binary, 5:string, 6:timestamp, 7:date, 8:decimal(4,2)
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 9:float
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zzzzzzzzz
+ reduceColumnSortOrder: +++++++++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ dataColumns: KEY._col0:double, KEY._col1:boolean, KEY._col2:int, KEY._col3:bigint, KEY._col4:binary, KEY._col5:string, KEY._col6:timestamp, KEY._col7:date, KEY._col8:decimal(4,2), VALUE._col0:float
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 9:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:double, col 1:boolean, col 2:int, col 3:bigint, col 4:binary, col 5:string, col 6:timestamp, col 7:date, col 8:decimal(4,2)
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [9, 0, 1, 2, 3, 4, 5, 6, 7, 8]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_parquet_all_types
group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
PREHOOK: type: QUERY
@@ -319,6 +503,99 @@ POSTHOOK: Input: default@tbl_ice_parquet_parted
#### A masked pattern was here ####
America 2 aa
Europe 1 aa
+PREHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_parquet_parted group by p1, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_parquet_parted
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_parquet_parted group by p1, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_parquet_parted
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 1:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 2:int
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:string, 1:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 2:string
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2]
+ dataColumns: p1:string, b:string, a:int, p2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY._col0:string, KEY._col1:int, VALUE._col0:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 2:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:int
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: insert into tbl_ice_parquet_parted values ('Europe', 'cc', 3, 'Austria')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
diff --git a/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out
index 257b5a9c7b9..75722edaf9c 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out
@@ -31,6 +31,7 @@ LOCATION
TBLPROPERTIES (
'bucketing_version'='2',
'engine.hive.enabled'='true',
+ 'iceberg.orc.files.only'='false',
'metadata_location'='hdfs://### HDFS PATH ###',
'serialization.format'='1',
'table_type'='ICEBERG',
@@ -80,6 +81,7 @@ LOCATION
TBLPROPERTIES (
'bucketing_version'='2',
'engine.hive.enabled'='true',
+ 'iceberg.orc.files.only'='false',
'metadata_location'='hdfs://### HDFS PATH ###',
'serialization.format'='1',
'table_type'='ICEBERG',
@@ -130,6 +132,7 @@ LOCATION
TBLPROPERTIES (
'bucketing_version'='2',
'engine.hive.enabled'='true',
+ 'iceberg.orc.files.only'='false',
'metadata_location'='hdfs://### HDFS PATH ###',
'serialization.format'='1',
'table_type'='ICEBERG',
@@ -168,6 +171,7 @@ LOCATION
TBLPROPERTIES (
'bucketing_version'='2',
'engine.hive.enabled'='true',
+ 'iceberg.orc.files.only'='false',
'metadata_location'='hdfs://### HDFS PATH ###',
'serialization.format'='1',
'table_type'='ICEBERG',
diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
index 79c5f540b9d..b470f127792 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
@@ -90,6 +90,7 @@ Table Parameters:
bucketing_version 2
engine.hive.enabled true
external.table.purge false
+ iceberg.orc.files.only false
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 3
@@ -155,6 +156,7 @@ Table Parameters:
bucketing_version 2
engine.hive.enabled true
external.table.purge false
+ iceberg.orc.files.only false
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 0
diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
index a66b376f9bc..135923e446f 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
@@ -90,6 +90,7 @@ Table Parameters:
bucketing_version 2
engine.hive.enabled true
external.table.purge true
+ iceberg.orc.files.only true
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 3
@@ -155,6 +156,7 @@ Table Parameters:
bucketing_version 2
engine.hive.enabled true
external.table.purge true
+ iceberg.orc.files.only true
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 0
@@ -218,6 +220,7 @@ Table Parameters:
bucketing_version 2
engine.hive.enabled true
external.table.purge true
+ iceberg.orc.files.only true
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 1
@@ -283,6 +286,7 @@ Table Parameters:
bucketing_version 2
engine.hive.enabled true
external.table.purge true
+ iceberg.orc.files.only true
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 0
@@ -364,6 +368,7 @@ Table Parameters:
bucketing_version 2
engine.hive.enabled true
external.table.purge false
+ iceberg.orc.files.only true
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 0
diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
index 82923c519de..0723a7b8603 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
@@ -97,6 +97,7 @@ Table Parameters:
bucketing_version 2
engine.hive.enabled true
external.table.purge true
+ iceberg.orc.files.only false
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 4
@@ -189,6 +190,7 @@ Table Parameters:
bucketing_version 2
engine.hive.enabled true
external.table.purge true
+ iceberg.orc.files.only false
#### A masked pattern was here ####
metadata_location hdfs://### HDFS PATH ###
numFiles 0
diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out
index e6d990caa41..97b5349dd45 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out
@@ -18,6 +18,94 @@ POSTHOOK: query: insert into table tbl_ice_mixed values (1, 'one'), (2, 'two'),
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice_mixed
+PREHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_mixed group by b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_mixed
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_mixed group by b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_mixed
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:b:string]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 0:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: a:int, b:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: z
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY._col0:string, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 1:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: alter table tbl_ice_mixed set tblproperties ('write.format.default'='parquet')
PREHOOK: type: ALTERTABLE_PROPERTIES
PREHOOK: Input: default@tbl_ice_mixed
@@ -73,6 +161,94 @@ Stage-0
TableScan [TS_0] (rows=19 width=92)
default@tbl_ice_mixed,tbl_ice_mixed,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"]
+PREHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_mixed group by b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_mixed
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_mixed group by b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_mixed
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:b:string]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 0:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: a:int, b:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: z
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY._col0:string, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 1:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: select b, max(a) from tbl_ice_mixed group by b
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice_mixed
@@ -129,6 +305,102 @@ POSTHOOK: query: insert into tbl_ice_mixed_all_types values (1.1, 1.2, false, 4,
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice_mixed_all_types
+PREHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string,
+ t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_mixed_all_types
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string,
+ t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_mixed_all_types
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 0:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 10:decimal(4,2)
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:double, 1:boolean, 2:int, 3:bigint, 4:binary, 5:string, 6:timestamp, 7:date, 8:decimal(4,2)
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 9:float
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)/DECIMAL_64
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [decimal(4,2)]
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zzzzzzzzz
+ reduceColumnSortOrder: +++++++++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ dataColumns: KEY._col0:double, KEY._col1:boolean, KEY._col2:int, KEY._col3:bigint, KEY._col4:binary, KEY._col5:string, KEY._col6:timestamp, KEY._col7:date, KEY._col8:decimal(4,2), VALUE._col0:float
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 9:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:double, col 1:boolean, col 2:int, col 3:bigint, col 4:binary, col 5:string, col 6:timestamp, col 7:date, col 8:decimal(4,2)
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [9, 0, 1, 2, 3, 4, 5, 6, 7, 8]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: alter table tbl_ice_mixed_all_types set tblproperties ('write.format.default'='parquet')
PREHOOK: type: ALTERTABLE_PROPERTIES
PREHOOK: Input: default@tbl_ice_mixed_all_types
@@ -145,6 +417,102 @@ POSTHOOK: query: insert into tbl_ice_mixed_all_types values (5.1, 6.2, true, 40,
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice_mixed_all_types
+PREHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string,
+t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_mixed_all_types
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string,
+t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_mixed_all_types
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 0:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, col 9:decimal(4,2)
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:double, 1:boolean, 2:int, 3:bigint, 4:binary, 5:string, 6:timestamp, 7:date, 8:decimal(4,2)
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 9:float
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zzzzzzzzz
+ reduceColumnSortOrder: +++++++++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ dataColumns: KEY._col0:double, KEY._col1:boolean, KEY._col2:int, KEY._col3:bigint, KEY._col4:binary, KEY._col5:string, KEY._col6:timestamp, KEY._col7:date, KEY._col8:decimal(4,2), VALUE._col0:float
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 9:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:double, col 1:boolean, col 2:int, col 3:bigint, col 4:binary, col 5:string, col 6:timestamp, col 7:date, col 8:decimal(4,2)
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [9, 0, 1, 2, 3, 4, 5, 6, 7, 8]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: explain select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_mixed_all_types
group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
PREHOOK: type: QUERY
@@ -303,6 +671,98 @@ p2 string
# col_name transform_type
p1 IDENTITY
p2 IDENTITY
+PREHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_mixed_parted group by p1, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_mixed_parted
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_mixed_parted group by p1, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_mixed_parted
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 1:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 2:int
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:string, 1:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 2:string
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2]
+ dataColumns: p1:string, b:string, a:int, p2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY._col0:string, KEY._col1:int, VALUE._col0:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 2:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:int
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: select p1, a, min(b) from tbl_ice_mixed_parted group by p1, a
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice_mixed_parted
diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out
index 91c7bb2c54a..fba9bd1e8a0 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out
@@ -70,6 +70,94 @@ four 44444
one 111
three 3
two 22
+PREHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_orc group by b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_orc group by b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:b:string]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 0:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: a:int, b:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: z
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY._col0:string, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 1:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: create external table tbl_ice_orc_all_types (
t_float FLOAT,
t_double DOUBLE,
@@ -152,6 +240,100 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tbl_ice_orc_all_types
POSTHOOK: Output: hdfs://### HDFS PATH ###
1.1 1.2 false 4 567890123456789 6 col7 2012-10-03 19:58:08 1234-09-02 10.01
+PREHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_orc_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc_all_types
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_orc_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc_all_types
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)/DECIMAL_64]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 0:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, ConvertDecimal64ToDecimal(col 9:decimal(4,2)/DECIMAL_64) -> 10:decimal(4,2)
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:double, 1:boolean, 2:int, 3:bigint, 4:binary, 5:string, 6:timestamp, 7:date, 8:decimal(4,2)
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 9:float
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)/DECIMAL_64
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [decimal(4,2)]
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zzzzzzzzz
+ reduceColumnSortOrder: +++++++++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ dataColumns: KEY._col0:double, KEY._col1:boolean, KEY._col2:int, KEY._col3:bigint, KEY._col4:binary, KEY._col5:string, KEY._col6:timestamp, KEY._col7:date, KEY._col8:decimal(4,2), VALUE._col0:float
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 9:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:double, col 1:boolean, col 2:int, col 3:bigint, col 4:binary, col 5:string, col 6:timestamp, col 7:date, col 8:decimal(4,2)
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [9, 0, 1, 2, 3, 4, 5, 6, 7, 8]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: create external table tbl_ice_orc_parted (
a int,
b string
@@ -214,6 +396,102 @@ POSTHOOK: Input: default@tbl_ice_orc_parted
POSTHOOK: Output: hdfs://### HDFS PATH ###
Europe 1 aa
America 2 aa
+PREHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_orc_parted group by p1, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc_parted
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_orc_parted group by p1, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc_parted
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:p1:string, 2:b:string, 3:p2:string]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 2:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:int, col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:int, 1:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 2:string
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2]
+ dataColumns: a:int, p1:string, b:string, p2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY._col0:int, KEY._col1:string, VALUE._col0:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 2:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:int, col 1:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [1, 0, 2]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: alter table tbl_ice_orc_parted change column a a int after b
PREHOOK: type: ALTERTABLE_RENAMECOL
PREHOOK: Input: default@tbl_ice_orc_parted
@@ -247,6 +525,98 @@ POSTHOOK: Input: default@tbl_ice_orc_parted
POSTHOOK: Output: hdfs://### HDFS PATH ###
America 2 aa
Europe 1 aa
+PREHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_orc_parted group by p1, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc_parted
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_orc_parted group by p1, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc_parted
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 1:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 2:int
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:string, 1:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 2:string
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2]
+ dataColumns: p1:string, b:string, a:int, p2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY._col0:string, KEY._col1:int, VALUE._col0:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 2:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:int
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: insert into tbl_ice_orc_parted values ('Europe', 'cc', 3, 'Austria')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
@@ -267,6 +637,98 @@ America Canada 2 bb
America USA 2 aa
Europe Hungary 1 aa
Europe Austria 3 cc
+PREHOOK: query: explain vectorization only detail select p1, p2, a, min(b) from tbl_ice_orc_parted group by p1, p2, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc_parted
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select p1, p2, a, min(b) from tbl_ice_orc_parted group by p1, p2, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc_parted
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string]
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 1:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 2:int, col 3:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:string, 1:int, 2:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 3:string
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2, 3]
+ dataColumns: p1:string, b:string, a:int, p2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zzz
+ reduceColumnSortOrder: +++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ dataColumns: KEY._col0:string, KEY._col1:int, KEY._col2:string, VALUE._col0:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 3:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:int, col 2:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 2, 1, 3]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: create external table tbl_ice_orc_complex (
a int,
arrayofprimitives array<string>,
@@ -342,6 +804,86 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tbl_ice_orc_complex
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 ["a","b","c"] [["a"],["b","c"]] [{"a":"b"},{"e":"f"}] [{"something":"a","someone":"b","somewhere":"c"},{"something":"e","someone":"f","somewhere":"g"}] {"a":"b"} {"a":["b","c"]} {"a":{"b":"c"}} {"a":{"something":"b","someone":"c","somewhere":"d"}} {"something":"a","somewhere":"b"} {"names":["a","b"],"birthdays":["c","d","e"]} {"map1":{"a":"b"},"map2":{"c":"d"}}
+PREHOOK: query: explain vectorization only detail select * from tbl_ice_orc_complex order by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_orc_complex
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select * from tbl_ice_orc_complex order by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_orc_complex
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:arrayofprimitives:array<string>, 2:arrayofarrays:array<array<string>>, 3:arrayofmaps:array<map<string,string>>, 4:arrayofstructs:array<struct<something:string,someone:string,somewhere:string>>, 5:mapofprimitives:map<string,string>, 6:mapofarrays:map<string,array<string>>, 7:mapofmaps:map<string,map<string,string>>, 8:mapofstructs:map<string,struct<something:string,someone:string,somewhere:string>>, 9:structofprimitives:struct< [...]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 0:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:array<string>, 2:array<array<string>>, 3:array<map<string,string>>, 4:array<struct<something:string,someone:string,somewhere:string>>, 5:map<string,string>, 6:map<string,array<string>>, 7:map<string,map<string,string>>, 8:map<string,struct<something:string,someone:string,somewhere:string>>, 9:struct<something:string,somewhere:string>, 10:struct<names:array<string>,birthdays:array<string>>, 11:struct<map1:map<string,string>,map2:map<string,string>>
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ dataColumns: a:int, arrayofprimitives:array<string>, arrayofarrays:array<array<string>>, arrayofmaps:array<map<string,string>>, arrayofstructs:array<struct<something:string,someone:string,somewhere:string>>, mapofprimitives:map<string,string>, mapofarrays:map<string,array<string>>, mapofmaps:map<string,map<string,string>>, mapofstructs:map<string,struct<something:string,someone:string,somewhere:string>>, structofprimitives:struct<something:string,somewhere:string>, st [...]
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: z
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ dataColumns: KEY.reducesinkkey0:int, VALUE._col0:array<string>, VALUE._col1:array<array<string>>, VALUE._col2:array<map<string,string>>, VALUE._col3:array<struct<something:string,someone:string,somewhere:string>>, VALUE._col4:map<string,string>, VALUE._col5:map<string,array<string>>, VALUE._col6:map<string,map<string,string>>, VALUE._col7:map<string,struct<something:string,someone:string,somewhere:string>>, VALUE._col8:struct<something:string,somewhere:string>, VALUE. [...]
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: drop table tbl_ice_orc
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@tbl_ice_orc
diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
index 43da19476b4..8bdbc62fc5d 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
@@ -57,6 +57,94 @@ Stage-0
TableScan [TS_0] (rows=10 width=92)
default@tbl_ice_parquet,tbl_ice_parquet,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"]
+PREHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_parquet group by b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_parquet
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select b, max(a) from tbl_ice_parquet group by b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_parquet
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:a:int, 1:b:string]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 0:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:string
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkStringOperator
+ keyColumns: 0:string
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 1:int
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: a:int, b:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: z
+ reduceColumnSortOrder: +
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ dataColumns: KEY._col0:string, VALUE._col0:int
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxLong(col 1:int) -> int
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: select b, max(a) from tbl_ice_parquet group by b
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice_parquet
@@ -141,6 +229,100 @@ Stage-0
TableScan [TS_0] (rows=1 width=372)
default@tbl_ice_parquet_all_types,tbl_ice_parquet_all_types,Tbl:COMPLETE,Col:COMPLETE,Output:["t_float","t_double","t_boolean","t_int","t_bigint","t_binary","t_string","t_timestamp","t_date","t_decimal"]
+PREHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_parquet_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_parquet_all_types
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_parquet_all_types
+group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_parquet_all_types
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:t_float:float, 1:t_double:double, 2:t_boolean:boolean, 3:t_int:int, 4:t_bigint:bigint, 5:t_binary:binary, 6:t_string:string, 7:t_timestamp:timestamp, 8:t_date:date, 9:t_decimal:decimal(4,2)]
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 0:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 1:double, col 2:boolean, col 3:int, col 4:bigint, col 5:binary, col 6:string, col 7:timestamp, col 8:date, col 9:decimal(4,2)
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:double, 1:boolean, 2:int, 3:bigint, 4:binary, 5:string, 6:timestamp, 7:date, 8:decimal(4,2)
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 9:float
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ includeColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ dataColumns: t_float:float, t_double:double, t_boolean:boolean, t_int:int, t_bigint:bigint, t_binary:binary, t_string:string, t_timestamp:timestamp, t_date:date, t_decimal:decimal(4,2)
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zzzzzzzzz
+ reduceColumnSortOrder: +++++++++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 10
+ dataColumns: KEY._col0:double, KEY._col1:boolean, KEY._col2:int, KEY._col3:bigint, KEY._col4:binary, KEY._col5:string, KEY._col6:timestamp, KEY._col7:date, KEY._col8:decimal(4,2), VALUE._col0:float
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMaxDouble(col 9:float) -> float
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:double, col 1:boolean, col 2:int, col 3:bigint, col 4:binary, col 5:string, col 6:timestamp, col 7:date, col 8:decimal(4,2)
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [9, 0, 1, 2, 3, 4, 5, 6, 7, 8]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: select max(t_float), t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal from tbl_ice_parquet_all_types
group by t_double, t_boolean, t_int, t_bigint, t_binary, t_string, t_timestamp, t_date, t_decimal
PREHOOK: type: QUERY
@@ -247,6 +429,98 @@ POSTHOOK: Input: default@tbl_ice_parquet_parted
POSTHOOK: Output: hdfs://### HDFS PATH ###
America 2 aa
Europe 1 aa
+PREHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_parquet_parted group by p1, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_parquet_parted
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only detail select p1, a, min(b) from tbl_ice_parquet_parted group by p1, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_parquet_parted
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:p1:string, 1:b:string, 2:a:int, 3:p2:string]
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [0, 1, 2]
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 1:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: HASH
+ keyExpressions: col 0:string, col 2:int
+ native: false
+ vectorProcessingMode: HASH
+ projectedOutputColumnNums: [0]
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumns: 0:string, 1:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumns: 2:string
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2]
+ dataColumns: p1:string, b:string, a:int, p2:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: zz
+ reduceColumnSortOrder: ++
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 3
+ dataColumns: KEY._col0:string, KEY._col1:int, VALUE._col0:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: []
+ Reduce Operator Tree:
+ Group By Vectorization:
+ aggregators: VectorUDAFMinString(col 2:string) -> string
+ className: VectorGroupByOperator
+ groupByMode: MERGEPARTIAL
+ keyExpressions: col 0:string, col 1:int
+ native: false
+ vectorProcessingMode: MERGE_PARTIAL
+ projectedOutputColumnNums: [0]
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+
+ Stage: Stage-0
+ Fetch Operator
+
PREHOOK: query: insert into tbl_ice_parquet_parted values ('Europe', 'cc', 3, 'Austria')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedInputFormatInterface.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedInputFormatInterface.java
index 8ee59e4173f..4aa16dde7ac 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedInputFormatInterface.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedInputFormatInterface.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.exec.vector;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import java.util.Properties;
+
/**
* Marker interface to indicate a given input format supports
* vectorization input.
@@ -25,4 +29,8 @@ package org.apache.hadoop.hive.ql.exec.vector;
public interface VectorizedInputFormatInterface {
VectorizedSupport.Support[] getSupportedFeatures();
+
+ default VectorizedSupport.Support[] getSupportedFeatures(HiveConf hiveConf, Properties properties) {
+ return getSupportedFeatures();
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 875fe4c53b7..54258b63eeb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -1258,16 +1258,16 @@ public class Vectorizer implements PhysicalPlanResolver {
}
}
- private Support[] getVectorizedInputFormatSupports(
- Class<? extends InputFormat> inputFileFormatClass) {
+ private Support[] getVectorizedInputFormatSupports(Class<? extends InputFormat> inputFileFormatClass,
+ Properties properties) {
try {
InputFormat inputFormat = FetchOperator.getInputFormatFromCache(inputFileFormatClass, hiveConf);
if (inputFormat instanceof VectorizedInputFormatInterface) {
- return ((VectorizedInputFormatInterface) inputFormat).getSupportedFeatures();
+ return ((VectorizedInputFormatInterface) inputFormat).getSupportedFeatures(hiveConf, properties);
}
} catch (IOException e) {
- LOG.error("Unable to instantiate {} input format class. Cannot determine vectorization support.", e);
+ LOG.error("Unable to instantiate input format class. Cannot determine vectorization support.", e);
}
// FUTURE: Decide how to ask an input file format what vectorization features it supports.
return null;
@@ -1276,13 +1276,12 @@ public class Vectorizer implements PhysicalPlanResolver {
/*
* Add the support of the VectorizedInputFileFormatInterface.
*/
- private void addVectorizedInputFileFormatSupport(
- Set<Support> newSupportSet,
- boolean isInputFileFormatVectorized, Class<? extends InputFormat>inputFileFormatClass) {
+ private void addVectorizedInputFileFormatSupport(Set<Support> newSupportSet, boolean isInputFileFormatVectorized,
+ Class<? extends InputFormat> inputFileFormatClass, Properties properties) {
final Support[] supports;
if (isInputFileFormatVectorized) {
- supports = getVectorizedInputFormatSupports(inputFileFormatClass);
+ supports = getVectorizedInputFormatSupports(inputFileFormatClass, properties);
} else {
supports = null;
}
@@ -1374,7 +1373,7 @@ public class Vectorizer implements PhysicalPlanResolver {
}
addVectorizedInputFileFormatSupport(
- newSupportSet, isInputFileFormatVectorized, inputFileFormatClass);
+ newSupportSet, isInputFileFormatVectorized, inputFileFormatClass, pd.getTableDesc().getProperties());
addVectorPartitionDesc(
pd,
@@ -1402,7 +1401,7 @@ public class Vectorizer implements PhysicalPlanResolver {
allTypeInfoList)) {
addVectorizedInputFileFormatSupport(
- newSupportSet, isInputFileFormatVectorized, inputFileFormatClass);
+ newSupportSet, isInputFileFormatVectorized, inputFileFormatClass, pd.getTableDesc().getProperties());
addVectorPartitionDesc(
pd,