You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2026/07/09 16:25:15 UTC
(hive) branch master updated: HIVE-29413: Generalize Table column APIs to support non-native partition columns (#6413)
This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5dae4635e9f HIVE-29413: Generalize Table column APIs to support non-native partition columns (#6413)
5dae4635e9f is described below
commit 5dae4635e9f235ce5d7f6d951428598a27b3fd23
Author: ramitg254 <97...@users.noreply.github.com>
AuthorDate: Thu Jul 9 21:55:08 2026 +0530
HIVE-29413: Generalize Table column APIs to support non-native partition columns (#6413)
---
.../iceberg/mr/hive/BaseHiveIcebergMetaHook.java | 15 +-
.../iceberg/mr/hive/HiveIcebergMetaHook.java | 6 +-
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 35 +-
.../org/apache/iceberg/mr/hive/HiveTableUtil.java | 21 +
.../hive/TestHiveIcebergStorageHandlerNoScan.java | 14 +-
.../positive/update_iceberg_partitioned_avro.q | 7 +
.../positive/update_iceberg_partitioned_orc.q | 8 +
.../positive/update_iceberg_partitioned_parquet.q | 9 +
.../alter_multi_part_table_to_iceberg.q.out | 15 -
.../positive/alter_part_table_to_iceberg.q.out | 5 -
.../test/results/positive/bucket_map_join_9.q.out | 16 +-
.../src/test/results/positive/col_stats.q.out | 4 +-
.../positive/ctas_iceberg_partitioned_orc.q.out | 2 -
.../delete_iceberg_copy_on_write_partitioned.q.out | 154 +++---
.../results/positive/desc_ice_tbl_part_spec.q.out | 15 +-
.../results/positive/describe_iceberg_table.q.out | 17 +-
.../positive/dynamic_partition_pruning.q.out | 38 +-
.../positive/dynamic_partition_writes.q.out | 16 +-
.../results/positive/iceberg_drop_column.q.out | 4 +-
.../positive/iceberg_insert_into_partition.q.out | 8 -
.../iceberg_insert_into_partition_transforms.q.out | 70 ++-
...berg_insert_into_partition_with_evolution.q.out | 1 -
.../iceberg_insert_overwrite_partition.q.out | 7 -
...erg_insert_overwrite_partition_transforms.q.out | 56 +--
.../test/results/positive/iceberg_load_data.q.out | 44 +-
.../llap/hadoop_catalog_create_table.q.out | 10 -
.../positive/llap/iceberg_bucket_map_join_1.q.out | 124 +++--
.../positive/llap/iceberg_bucket_map_join_2.q.out | 64 +--
.../positive/llap/iceberg_bucket_map_join_3.q.out | 44 +-
.../positive/llap/iceberg_bucket_map_join_4.q.out | 52 +-
.../positive/llap/iceberg_bucket_map_join_5.q.out | 44 +-
.../positive/llap/iceberg_bucket_map_join_6.q.out | 32 +-
.../positive/llap/iceberg_bucket_map_join_7.q.out | 168 +++----
.../positive/llap/iceberg_bucket_map_join_8.q.out | 100 ++--
.../iceberg_create_locally_zordered_table.q.out | 1 -
...berg_major_compaction_partition_evolution.q.out | 4 -
...erg_major_compaction_partition_evolution2.q.out | 2 -
...or_compaction_partition_evolution_ordered.q.out | 2 -
...n_partition_evolution_w_dyn_spec_w_filter.q.out | 4 -
...on_partition_evolution_w_id_spec_w_filter.q.out | 4 -
.../iceberg_major_compaction_partitioned.q.out | 4 -
...iceberg_major_compaction_schema_evolution.q.out | 2 -
...iceberg_major_compaction_single_partition.q.out | 9 -
...ompaction_single_partition_with_evolution.q.out | 15 -
...mpaction_single_partition_with_evolution2.q.out | 4 -
.../llap/iceberg_minor_compaction_bucket.q.out | 2 -
...berg_minor_compaction_partition_evolution.q.out | 3 -
.../llap/vectorized_iceberg_read_mixed.q.out | 4 +-
.../llap/vectorized_iceberg_read_orc.q.out | 4 +-
.../llap/vectorized_iceberg_read_parquet.q.out | 4 +-
.../positive/mv_iceberg_partitioned_orc.q.out | 2 -
.../positive/mv_iceberg_partitioned_orc2.q.out | 4 -
.../src/test/results/positive/row_count.q.out | 4 -
.../results/positive/show_partitions_test.q.out | 5 -
.../truncate_partitioned_iceberg_table.q.out | 2 -
.../update_iceberg_copy_on_write_partitioned.q.out | 556 +++++++++++----------
.../positive/update_iceberg_partitioned_avro.q.out | 43 ++
.../positive/update_iceberg_partitioned_orc.q.out | 43 ++
.../update_iceberg_partitioned_parquet.q.out | 43 ++
.../positive/vectorized_iceberg_merge_mixed.q.out | 54 +-
.../positive/vectorized_iceberg_read_mixed.q.out | 4 +-
.../positive/vectorized_iceberg_read_orc.q.out | 4 +-
.../positive/vectorized_iceberg_read_parquet.q.out | 4 +-
.../hive/llap/io/api/impl/LlapInputFormat.java | 3 +-
.../ddl/table/AbstractBaseAlterTableAnalyzer.java | 2 +-
.../table/column/show/ShowColumnsOperation.java | 13 +-
.../ql/ddl/table/create/CreateTableOperation.java | 3 +-
.../ql/ddl/table/info/desc/DescTableOperation.java | 2 +-
.../desc/formatter/TextDescTableFormatter.java | 36 +-
.../table/misc/truncate/TruncateTableAnalyzer.java | 21 +-
.../ql/ddl/table/partition/PartitionUtils.java | 1 -
.../storage/compact/AlterTableCompactAnalyzer.java | 7 +-
.../apache/hadoop/hive/ql/exec/ArchiveUtils.java | 3 +-
.../org/apache/hadoop/hive/ql/exec/ColumnInfo.java | 13 +-
.../apache/hadoop/hive/ql/exec/DDLPlanUtils.java | 6 +-
.../org/apache/hadoop/hive/ql/exec/MoveTask.java | 10 +-
.../hadoop/hive/ql/exec/repl/ReplLoadTask.java | 4 +-
.../hive/ql/exec/vector/VectorizedRowBatchCtx.java | 3 +-
.../hadoop/hive/ql/metadata/DummyPartition.java | 5 +-
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 12 +-
.../ql/metadata/HiveMaterializedViewsRegistry.java | 7 +-
.../apache/hadoop/hive/ql/metadata/Partition.java | 35 +-
.../org/apache/hadoop/hive/ql/metadata/Table.java | 162 ++++--
.../hive/ql/optimizer/ColumnPrunerProcFactory.java | 19 +-
.../DynamicPartitionPruningOptimization.java | 18 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 2 +-
.../hive/ql/optimizer/ppr/PartExprEvalUtils.java | 7 +-
.../hive/ql/parse/AcidExportSemanticAnalyzer.java | 3 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 9 +-
.../ql/parse/ColumnStatsAutoGatherContext.java | 77 ++-
.../hive/ql/parse/ColumnStatsSemanticAnalyzer.java | 11 +-
.../hive/ql/parse/ImportSemanticAnalyzer.java | 2 -
.../hadoop/hive/ql/parse/LoadSemanticAnalyzer.java | 2 +-
.../hive/ql/parse/MergeSemanticAnalyzer.java | 37 +-
.../apache/hadoop/hive/ql/parse/ParseUtils.java | 29 +-
.../hive/ql/parse/RewriteSemanticAnalyzer.java | 11 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 76 +--
.../ql/parse/rewrite/CopyOnWriteMergeRewriter.java | 20 +-
.../parse/rewrite/CopyOnWriteUpdateRewriter.java | 9 +-
.../hive/ql/parse/rewrite/MergeRewriter.java | 39 +-
.../rewrite/PartitionSplitUpdateRewriter.java | 51 ++
.../hive/ql/parse/rewrite/SplitMergeRewriter.java | 15 +-
.../hive/ql/parse/rewrite/SplitUpdateRewriter.java | 74 +--
.../ql/parse/rewrite/UpdateRewriterFactory.java | 4 +-
.../parse/rewrite/sql/MultiInsertSqlGenerator.java | 13 +-
.../sql/NativeAcidMultiInsertSqlGenerator.java | 7 +
.../rewrite/sql/PartitionSetValuesClause.java | 43 ++
.../hive/ql/parse/rewrite/sql/SetValuesClause.java | 40 ++
.../ql/parse/rewrite/sql/SetValuesClauseBase.java | 59 +++
.../apache/hadoop/hive/ql/plan/PartitionDesc.java | 8 +-
.../hadoop/hive/ql/session/LineageState.java | 14 +
.../org/apache/hadoop/hive/common/io/CacheTag.java | 4 +-
112 files changed, 1715 insertions(+), 1397 deletions(-)
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java
index 917829ac304..ab6eecae7c1 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java
@@ -34,7 +34,6 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
-import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.CreateTableRequest;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -117,16 +116,6 @@ public BaseHiveIcebergMetaHook(Configuration conf) {
this.conf = conf;
}
- public static boolean isIcebergView(org.apache.hadoop.hive.metastore.api.Table hmsTable) {
- if (hmsTable == null ||
- hmsTable.getParameters() == null ||
- !TableType.VIRTUAL_VIEW.toString().equals(hmsTable.getTableType())) {
- return false;
- }
- String storageHandler = hmsTable.getParameters().get(hive_metastoreConstants.META_TABLE_STORAGE);
- return HiveMetaHook.HIVE_ICEBERG_STORAGE_HANDLER.equals(storageHandler);
- }
-
@Override
public void preCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) {
CreateTableRequest request = new CreateTableRequest(hmsTable);
@@ -139,7 +128,7 @@ public void preCreateTable(CreateTableRequest request) {
if (hmsTable.isTemporary()) {
throw new UnsupportedOperationException("Creation of temporary iceberg tables is not supported.");
}
- if (isIcebergView(hmsTable)) {
+ if (HiveTableUtil.isIcebergView(hmsTable)) {
preCreateIcebergView(request);
return;
}
@@ -522,7 +511,7 @@ protected void setWriteModeDefaults(Table icebergTbl, Map<String, String> newPro
public void postGetTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) {
if (hmsTable != null) {
try {
- if (isIcebergView(hmsTable)) {
+ if (HiveTableUtil.isIcebergView(hmsTable)) {
IcebergViewSupport.enrichHmsTableFromIcebergView(hmsTable, conf);
return;
}
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index 271ce74bfb7..92070d64528 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -179,7 +179,7 @@ public HiveIcebergMetaHook(Configuration conf) {
@Override
public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) {
- if (isIcebergView(hmsTable)) {
+ if (HiveTableUtil.isIcebergView(hmsTable)) {
tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf);
Map<String, String> tblProps =
hmsTable.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(hmsTable.getParameters());
@@ -266,7 +266,7 @@ public void commitDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable,
@Override
public void preAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, EnvironmentContext context)
throws MetaException {
- if (BaseHiveIcebergMetaHook.isIcebergView(hmsTable)) {
+ if (HiveTableUtil.isIcebergView(hmsTable)) {
currentAlterTableOp = null;
if (commitLock == null) {
commitLock = new NoLock();
@@ -494,7 +494,7 @@ public void commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable
if (commitLock == null) {
throw new IllegalStateException("Hive commit lock should already be set");
}
- if (BaseHiveIcebergMetaHook.isIcebergView(hmsTable)) {
+ if (HiveTableUtil.isIcebergView(hmsTable)) {
tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf);
Map<String, String> tblProps =
hmsTable.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(hmsTable.getParameters());
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index fbc704a3619..0fa610e6f3d 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -433,7 +433,7 @@ public boolean canProvidePartitionStatistics(org.apache.hadoop.hive.ql.metadata.
if (!getStatsSource().equals(HiveMetaHook.ICEBERG)) {
return false;
}
- if (BaseHiveIcebergMetaHook.isIcebergView(hmsTable.getTTable())) {
+ if (HiveTableUtil.isIcebergView(hmsTable.getTTable())) {
return false;
}
Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
@@ -899,7 +899,7 @@ public boolean supportsPartitionTransform() {
@Override
public List<TransformSpec> getPartitionTransformSpec(org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
- if (BaseHiveIcebergMetaHook.isIcebergView(hmsTable.getTTable())) {
+ if (HiveTableUtil.isIcebergView(hmsTable.getTTable())) {
return Collections.emptyList();
}
Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
@@ -916,7 +916,7 @@ public List<TransformSpec> getPartitionTransformSpec(org.apache.hadoop.hive.ql.m
@Override
public Map<Integer, List<TransformSpec>> getPartitionTransformSpecs(
org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
- if (BaseHiveIcebergMetaHook.isIcebergView(hmsTable.getTTable())) {
+ if (HiveTableUtil.isIcebergView(hmsTable.getTTable())) {
return Collections.emptyMap();
}
Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
@@ -1550,10 +1550,10 @@ public List<FieldSchema> acidSelectColumns(org.apache.hadoop.hive.ql.metadata.Ta
case DELETE ->
// TODO: make it configurable whether we want to include the table columns in the select query.
// It might make delete writes faster if we don't have to write out the row object
- ListUtils.union(ACID_VIRTUAL_COLS_AS_FIELD_SCHEMA, table.getCols());
+ ListUtils.union(ACID_VIRTUAL_COLS_AS_FIELD_SCHEMA, table.getAllCols());
case UPDATE -> shouldOverwrite(table, operation) ?
ACID_VIRTUAL_COLS_AS_FIELD_SCHEMA :
- ListUtils.union(ACID_VIRTUAL_COLS_AS_FIELD_SCHEMA, table.getCols());
+ ListUtils.union(ACID_VIRTUAL_COLS_AS_FIELD_SCHEMA, table.getAllCols());
case MERGE -> ACID_VIRTUAL_COLS_AS_FIELD_SCHEMA;
default -> ImmutableList.of();
};
@@ -1584,7 +1584,7 @@ public boolean supportsSortColumns() {
@Override
public List<FieldSchema> sortColumns(org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
- if (BaseHiveIcebergMetaHook.isIcebergView(hmsTable.getTTable())) {
+ if (HiveTableUtil.isIcebergView(hmsTable.getTTable())) {
return Collections.emptyList();
}
TableDesc tableDesc = Utilities.getTableDesc(hmsTable);
@@ -1989,8 +1989,9 @@ public void setTableParametersForCTLT(org.apache.hadoop.hive.ql.metadata.Table t
desc.setIsExternal(true);
}
- // If source is Iceberg table set the schema and the partition spec
- if (MetaStoreUtils.isIcebergTable(origParams)) {
+ // parameter table_type is set to "ICEBERG" in case of Iceberg tables
+ // set the schema and the partition spec accordingly
+ if (HiveTableUtil.isTableTypeSet(origParams)) {
tbl.getParameters()
.put(InputFormatConfig.TABLE_SCHEMA, origParams.get(InputFormatConfig.TABLE_SCHEMA));
tbl.getParameters()
@@ -2147,13 +2148,12 @@ public List<Partition> getPartitions(org.apache.hadoop.hive.ql.metadata.Table hm
}
public boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
- if (BaseHiveIcebergMetaHook.isIcebergView(hmsTable.getTTable())) {
- List<FieldSchema> partCols = hmsTable.getPartCols();
- return partCols != null && !partCols.isEmpty();
- }
- if (!hmsTable.getTTable().isSetId()) {
+ if (hmsTable.getMetaTable() != null) {
return false;
}
+ if (!HiveTableUtil.isRegistered(hmsTable) || HiveTableUtil.isIcebergView(hmsTable.getTTable())) {
+ return !hmsTable.getPartitionKeys().isEmpty();
+ }
Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
Snapshot snapshot = IcebergTableUtil.getTableSnapshot(table, hmsTable);
@@ -2296,13 +2296,12 @@ public boolean canPerformMetadataDelete(org.apache.hadoop.hive.ql.metadata.Table
@Override
public List<FieldSchema> getPartitionKeys(org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
- if (BaseHiveIcebergMetaHook.isIcebergView(hmsTable.getTTable())) {
- List<FieldSchema> partCols = hmsTable.getPartCols();
- return partCols != null ? partCols : Collections.emptyList();
- }
- if (!hmsTable.getTTable().isSetId()) {
+ if (hmsTable.getMetaTable() != null) {
return Collections.emptyList();
}
+ if (!HiveTableUtil.isRegistered(hmsTable) || HiveTableUtil.isIcebergView(hmsTable.getTTable())) {
+ return hmsTable.getPartitionKeys();
+ }
Table icebergTable = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
return MetastoreUtil.getPartitionKeys(icebergTable, icebergTable.spec().specId());
}
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java
index 69580e35872..a129266f6db 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java
@@ -40,6 +40,8 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -380,4 +382,23 @@ public static boolean isCtas(Properties properties) {
return Boolean.parseBoolean(properties.getProperty(hive_metastoreConstants.TABLE_IS_CTAS));
}
+ public static boolean isTableTypeSet(Map<String, String> params) {
+ return params != null &&
+ HiveMetaHook.ICEBERG.equalsIgnoreCase(params.get(HiveMetaHook.TABLE_TYPE));
+ }
+
+ public static boolean isIcebergView(org.apache.hadoop.hive.metastore.api.Table hmsTable) {
+ if (hmsTable == null ||
+ hmsTable.getParameters() == null ||
+ !TableType.VIRTUAL_VIEW.toString().equals(hmsTable.getTableType())) {
+ return false;
+ }
+ String storageHandler = hmsTable.getParameters().get(hive_metastoreConstants.META_TABLE_STORAGE);
+ return HiveMetaHook.HIVE_ICEBERG_STORAGE_HANDLER.equals(storageHandler);
+ }
+
+ public static boolean isRegistered(org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
+ return hmsTable.getTTable().isSetId() && isTableTypeSet(hmsTable.getParameters());
+ }
+
}
diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
index a320148858e..4284ce7a26e 100644
--- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
+++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
@@ -946,7 +946,8 @@ public void testCreateTableWithoutColumnComments() {
@Test
public void testCreatePartitionedTableWithColumnComments() {
TableIdentifier identifier = TableIdentifier.of("default", "partitioned_with_comment_table");
- String[] expectedDoc = new String[] {"int column", "string column", null, "partition column", null};
+ String[] expectedDoc = new String[] {"int column", "string column", null, "partition column",
+ "Transform: identity"};
shell.executeStatement("CREATE EXTERNAL TABLE partitioned_with_comment_table (" +
"t_int INT COMMENT 'int column', " +
"t_string STRING COMMENT 'string column', " +
@@ -959,13 +960,18 @@ public void testCreatePartitionedTableWithColumnComments() {
List<Object[]> rows = shell.executeStatement("DESCRIBE default.partitioned_with_comment_table");
List<Types.NestedField> columns = icebergTable.schema().columns();
+ List<String> partitionColumns = List.of("t_string_3", "t_string_4");
// The partition transform information and partition information is 6 extra lines, and 4 more line for the columns
Assert.assertEquals(columns.size() + 10, rows.size());
for (int i = 0; i < columns.size(); i++) {
Types.NestedField field = columns.get(i);
- Assert.assertArrayEquals(new Object[] {field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(),
- field.doc() != null ? field.doc() : ""}, rows.get(i));
- Assert.assertEquals(expectedDoc[i], field.doc());
+ String fieldDoc = field.doc();
+ if (fieldDoc == null && partitionColumns.contains(field.name())) {
+ fieldDoc = "Transform: identity";
+ }
+ Assert.assertArrayEquals(new Object[]{field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(),
+ fieldDoc != null ? fieldDoc : ""}, rows.get(i));
+ Assert.assertEquals(expectedDoc[i], fieldDoc);
}
}
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_avro.q b/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_avro.q
index 648b422527e..0bd8e6b609c 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_avro.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_avro.q
@@ -24,3 +24,10 @@ create external table tbl_standard_other(a int, b string) stored as orc;
insert into tbl_standard_other values (10, 'ten'), (444, 'tutu');
update tbl_ice set b='The last one' where a in (select t1.a from tbl_ice t1 join tbl_standard_other t2 on t1.a = t2.a);
select * from tbl_ice order by a, b, c;
+
+-- update when all table columns are partition columns
+drop table if exists tbl_ice_all_part;
+create external table tbl_ice_all_part(a int, b string) partitioned by spec (a, b) stored by iceberg stored as avro tblproperties ('format-version'='2');
+insert into tbl_ice_all_part values (1, 'one'), (2, 'two'), (3, 'three');
+update tbl_ice_all_part set b='updated' where a = 2;
+select * from tbl_ice_all_part order by a, b;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_orc.q b/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_orc.q
index a8ddb7fc3e6..f367ff03e50 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_orc.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_orc.q
@@ -24,3 +24,11 @@ create external table tbl_standard_other(a int, b string) stored as orc;
insert into tbl_standard_other values (10, 'ten'), (444, 'tutu');
update tbl_ice set b='The last one' where a in (select t1.a from tbl_ice t1 join tbl_standard_other t2 on t1.a = t2.a);
select * from tbl_ice order by a, b, c;
+
+-- update when all table columns are partition columns
+drop table if exists tbl_ice_all_part;
+create external table tbl_ice_all_part(a int, b string) partitioned by spec (a, b) stored by iceberg stored as orc tblproperties ('format-version'='2');
+
+insert into tbl_ice_all_part values (1, 'one'), (2, 'two'), (3, 'three');
+update tbl_ice_all_part set b='updated' where a = 2;
+select * from tbl_ice_all_part order by a, b;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_parquet.q b/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_parquet.q
index c207f956c53..3b144920e2a 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_parquet.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/update_iceberg_partitioned_parquet.q
@@ -24,3 +24,12 @@ create external table tbl_standard_other(a int, b string) stored as orc;
insert into tbl_standard_other values (10, 'ten'), (444, 'tutu');
update tbl_ice set b='The last one' where a in (select t1.a from tbl_ice t1 join tbl_standard_other t2 on t1.a = t2.a);
select * from tbl_ice order by a, b, c;
+
+-- update when all table columns are partition columns
+-- Disable vectorization: vectorized Parquet read fails for partition-only tables.
+set hive.vectorized.execution.enabled=false;
+drop table if exists tbl_ice_all_part;
+create external table tbl_ice_all_part(a int, b string) partitioned by spec (a, b) stored by iceberg tblproperties ('format-version'='2');
+insert into tbl_ice_all_part values (1, 'one'), (2, 'two'), (3, 'three');
+update tbl_ice_all_part set b='updated' where a = 2;
+select * from tbl_ice_all_part order by a, b;
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
index 346f8b8cd10..dd6068f39ba 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
@@ -177,8 +177,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_orc
# col_name data_type comment
a int
-b string
-c string
# Partition Information
# col_name data_type comment
@@ -452,8 +450,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_parquet
# col_name data_type comment
a int
-b string
-c string
# Partition Information
# col_name data_type comment
@@ -727,8 +723,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_avro
# col_name data_type comment
a int
-b string
-c string
# Partition Information
# col_name data_type comment
@@ -1063,9 +1057,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_orc_mixed
# col_name data_type comment
a int
-b double
-c int
-d string
# Partition Information
# col_name data_type comment
@@ -1509,9 +1500,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_parquet_mixed
# col_name data_type comment
a int
-b double
-c int
-d string
# Partition Information
# col_name data_type comment
@@ -1955,9 +1943,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_avro_mixed
# col_name data_type comment
a int
-b double
-c int
-d string
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
index 813ec51708e..3f4409e0e83 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
@@ -137,7 +137,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_orc
# col_name data_type comment
a int
-b string
# Partition Information
# col_name data_type comment
@@ -414,7 +413,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_parquet
# col_name data_type comment
a int
-b string
# Partition Information
# col_name data_type comment
@@ -768,7 +766,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_parquet_int
# col_name data_type comment
a int
-b int
# Partition Information
# col_name data_type comment
@@ -1122,7 +1119,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_parquet_double
# col_name data_type comment
a int
-b double
# Partition Information
# col_name data_type comment
@@ -1422,7 +1418,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_avro
# col_name data_type comment
a int
-b string
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out b/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out
index 8153bdd697f..23b2d41d3dd 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/bucket_map_join_9.q.out
@@ -34,21 +34,21 @@ Stage-0
limit:-1
Stage-1
Map 1 vectorized
- File Output Operator [FS_53]
- Map Join Operator [MAPJOIN_52] (rows=2 width=530)
- BucketMapJoin:true,Conds:SEL_51._col1, _col2=RS_49._col1, _col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ File Output Operator [FS_23]
+ Map Join Operator [MAPJOIN_22] (rows=2 width=530)
+ BucketMapJoin:true,Conds:SEL_21._col1, _col2=RS_19._col1, _col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
<-Map 2 [CUSTOM_EDGE] vectorized
- MULTICAST [RS_49]
+ MULTICAST [RS_19]
PartitionCols:_col2, _col1
- Select Operator [SEL_48] (rows=2 width=265)
+ Select Operator [SEL_18] (rows=2 width=265)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_47] (rows=2 width=265)
+ Filter Operator [FIL_17] (rows=2 width=265)
predicate:(id is not null and part is not null)
TableScan [TS_3] (rows=2 width=265)
default@tbl,tbl2,Tbl:COMPLETE,Col:COMPLETE,Output:["foid","part","id"]
- <-Select Operator [SEL_51] (rows=2 width=265)
+ <-Select Operator [SEL_21] (rows=2 width=265)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_50] (rows=2 width=265)
+ Filter Operator [FIL_20] (rows=2 width=265)
predicate:(id is not null and part is not null)
TableScan [TS_0] (rows=2 width=265)
default@tbl,tbl,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:100,Grouping Partition Columns:["id","part"],Output:["foid","part","id"]
diff --git a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out
index 6810c72c93e..ef5c4410fbb 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out
@@ -347,7 +347,7 @@ PREHOOK: Input: default@tbl_ice_puffin
POSTHOOK: query: desc formatted tbl_ice_puffin C
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_puffin
-col_name C
+col_name c
data_type int
min 52
max 56
@@ -358,7 +358,7 @@ max_col_len
num_trues
num_falses
bit_vector HL
-comment
+comment Transform: identity
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\"}}
PREHOOK: query: EXPLAIN select count(*) from src_ice t1 join tbl_ice_puffin t2 on (t1.a = t2.a)
PREHOOK: type: QUERY
diff --git a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out
index 0f60cd08c28..4d7606ac8ba 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out
@@ -273,8 +273,6 @@ POSTHOOK: query: describe formatted tbl_ice
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice
# col_name data_type comment
-a int
-b string
c int
# Partition Information
diff --git a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out
index 6e6d2da48e8..a531c87a685 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_copy_on_write_partitioned.q.out
@@ -67,12 +67,12 @@ POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice
Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
Warning: Shuffle Join MERGEJOIN[63][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product
-Warning: Shuffle Join MERGEJOIN[225][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
-Warning: Shuffle Join MERGEJOIN[227][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product
-Warning: Shuffle Join MERGEJOIN[223][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 9' is a cross product
-Warning: Shuffle Join MERGEJOIN[229][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product
-Warning: Shuffle Join MERGEJOIN[221][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 14' is a cross product
-Warning: Shuffle Join MERGEJOIN[231][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 16' is a cross product
+Warning: Shuffle Join MERGEJOIN[220][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
+Warning: Shuffle Join MERGEJOIN[222][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product
+Warning: Shuffle Join MERGEJOIN[218][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 9' is a cross product
+Warning: Shuffle Join MERGEJOIN[224][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product
+Warning: Shuffle Join MERGEJOIN[216][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 14' is a cross product
+Warning: Shuffle Join MERGEJOIN[226][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 16' is a cross product
PREHOOK: query: explain delete from tbl_ice where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
@@ -216,33 +216,29 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
- Select Operator
- expressions: a (type: int)
- outputColumnNames: a
+ Group By Operator
+ keys: a (type: int)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: a (type: int)
- minReductionHashAggr: 0.99
- mode: hash
- outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(), count(a)
- minReductionHashAggr: 0.99
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(), count(a)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ null sort order:
+ sort order:
Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- null sort order:
- sort order:
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
Select Operator
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -753,12 +749,12 @@ STAGE PLANS:
Warning: Shuffle Join MERGEJOIN[61][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
Warning: Shuffle Join MERGEJOIN[63][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product
-Warning: Shuffle Join MERGEJOIN[225][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
-Warning: Shuffle Join MERGEJOIN[227][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product
-Warning: Shuffle Join MERGEJOIN[223][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 9' is a cross product
-Warning: Shuffle Join MERGEJOIN[229][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product
-Warning: Shuffle Join MERGEJOIN[221][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 14' is a cross product
-Warning: Shuffle Join MERGEJOIN[231][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 16' is a cross product
+Warning: Shuffle Join MERGEJOIN[220][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
+Warning: Shuffle Join MERGEJOIN[222][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product
+Warning: Shuffle Join MERGEJOIN[218][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 9' is a cross product
+Warning: Shuffle Join MERGEJOIN[224][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 11' is a cross product
+Warning: Shuffle Join MERGEJOIN[216][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 14' is a cross product
+Warning: Shuffle Join MERGEJOIN[226][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 16' is a cross product
PREHOOK: query: delete from tbl_ice where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
@@ -799,7 +795,7 @@ POSTHOOK: query: insert into tbl_ice_other values (10, 'ten'), (333, 'hundred')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice_other
-Warning: Shuffle Join MERGEJOIN[217][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
+Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
PREHOOK: query: explain delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
@@ -1175,12 +1171,24 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
null sort order: z
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
null sort order: z
@@ -1207,7 +1215,7 @@ STAGE PLANS:
Stats Work
Basic Stats Work:
-Warning: Shuffle Join MERGEJOIN[217][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
+Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
PREHOOK: query: delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
@@ -1251,7 +1259,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_standard_other
POSTHOOK: Lineage: tbl_standard_other.a SCRIPT []
POSTHOOK: Lineage: tbl_standard_other.b SCRIPT []
-Warning: Shuffle Join MERGEJOIN[217][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
+Warning: Shuffle Join MERGEJOIN[177][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
PREHOOK: query: explain delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
@@ -1273,13 +1281,13 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Reducer 9 (CUSTOM_SIMPLE_EDGE)
- Reducer 11 <- Reducer 9 (SIMPLE_EDGE)
- Reducer 12 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
- Reducer 13 <- Reducer 12 (SIMPLE_EDGE)
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE)
- Reducer 3 <- Reducer 10 (XPROD_EDGE), Reducer 2 (XPROD_EDGE)
- Reducer 4 <- Reducer 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS)
+ Reducer 10 <- Reducer 9 (SIMPLE_EDGE)
+ Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
+ Reducer 12 <- Reducer 11 (SIMPLE_EDGE)
+ Reducer 13 <- Reducer 9 (CUSTOM_SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 13 (XPROD_EDGE), Reducer 2 (XPROD_EDGE)
+ Reducer 4 <- Reducer 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS)
Reducer 6 <- Union 5 (SIMPLE_EDGE)
Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
Reducer 8 <- Reducer 7 (SIMPLE_EDGE), Union 5 (CONTAINS)
@@ -1365,19 +1373,6 @@ STAGE PLANS:
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized
Reducer 10
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0), count(VALUE._col1)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- null sort order:
- sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint), _col1 (type: bigint)
- Reducer 11
Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
@@ -1396,7 +1391,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: boolean)
- Reducer 12
+ Reducer 11
Reduce Operator Tree:
Merge Join Operator
condition map:
@@ -1412,7 +1407,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reducer 13
+ Reducer 12
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
@@ -1458,6 +1453,19 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reducer 13
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
Reducer 2
Reduce Operator Tree:
Merge Join Operator
@@ -1604,17 +1612,6 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col1
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(), count(_col1)
- minReductionHashAggr: 0.99
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- null sort order:
- sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint), _col1 (type: bigint)
Group By Operator
keys: _col1 (type: int)
minReductionHashAggr: 0.99
@@ -1643,6 +1640,17 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(), count(_col1)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
Select Operator
expressions: _col1 (type: int)
outputColumnNames: _col0
@@ -1679,7 +1687,7 @@ STAGE PLANS:
Stats Work
Basic Stats Work:
-Warning: Shuffle Join MERGEJOIN[180][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
+Warning: Shuffle Join MERGEJOIN[155][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
PREHOOK: query: delete from tbl_ice where a in (select t1.a from tbl_ice t1 join tbl_standard_other t2 on t1.a = t2.a)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
diff --git a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out
index 9ae5ea93e5f..a4ae2cf81f1 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out
@@ -72,8 +72,8 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t
a int
b string
-c int
-d string
+c int Transform: identity
+d string Transform: identity
# Partition Information
# col_name data_type comment
@@ -97,8 +97,6 @@ POSTHOOK: Input: default@ice_t
# col_name data_type comment
a int
b string
-c int
-d string
# Partition Information
# col_name data_type comment
@@ -218,9 +216,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t
# col_name data_type comment
a int
-b string
-c int
-d string
# Partition Information
# col_name data_type comment
@@ -260,9 +255,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t
# col_name data_type comment
a int
-b string
-c int
-d string
# Partition Information
# col_name data_type comment
@@ -302,9 +294,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t
# col_name data_type comment
a int
-b string
-c int
-d string
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
index 185fd2a2732..21bf7bd8f90 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
@@ -108,13 +108,6 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_t_transform
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t_transform
# col_name data_type comment
-year_field date
-month_field date
-day_field date
-hour_field timestamp
-truncate_field string
-bucket_field int
-identity_field int
# Partition Information
# col_name data_type comment
@@ -180,13 +173,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t_transform_prop
# col_name data_type comment
id int
-year_field date
-month_field date
-day_field date
-hour_field timestamp
-truncate_field string
-bucket_field int
-identity_field int
# Partition Information
# col_name data_type comment
@@ -252,7 +238,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t_identity_part
# col_name data_type comment
a int
-b string
# Partition Information
# col_name data_type comment
@@ -334,7 +319,7 @@ max_col_len
num_trues
num_falses
bit_vector
-comment
+comment Transform: year
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bucket_field\":\"true\",\"day_field\":\"true\",\"hour_field\":\"true\",\"identity_field\":\"true\",\"month_field\":\"true\",\"truncate_field\":\"true\",\"year_field\":\"true\"}}
PREHOOK: query: DESCRIBE FORMATTED ice_t_transform_prop id
PREHOOK: type: DESCTABLE
diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out
index a8920ac102d..563564b4013 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_pruning.q.out
@@ -2628,28 +2628,24 @@ STAGE PLANS:
alias: srcpart_iceberg
filterExpr: ds is not null (type: boolean)
Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: ds (type: string)
- outputColumnNames: ds
- Statistics: Num rows: 2000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: ds (type: string)
- minReductionHashAggr: 0.99
- mode: hash
- outputColumnNames: _col0
+ Group By Operator
+ keys: ds (type: string)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized
Map 6
Map Operator Tree:
diff --git a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out
index 0c58a118a7c..cd09db32773 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/dynamic_partition_writes.q.out
@@ -806,7 +806,7 @@ Stage-3
Group By Operator [GBY_15] (rows=21 width=584)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"],aggregations:["min(id)","max(id)","count(1)","count(id)","compute_bit_vector_hll(id)","max(length(ccy))","avg(COALESCE(length(ccy),0))","count(ccy)","compute_bit_vector_hll(ccy)"],keys:iceberg_truncate(id, 2)
Select Operator [SEL_14] (rows=22 width=87)
- Output:["id","ccy"]
+ Output:["ccy","id"]
Please refer to the previous Select Operator [SEL_12]
PREHOOK: query: insert into table tbl_target_truncate_int select a, b from tbl_src
@@ -1101,7 +1101,7 @@ Stage-3
Group By Operator [GBY_8] (rows=1 width=672)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],k [...]
Select Operator [SEL_7] (rows=1 width=240)
- Output:["id","date_time_date","year_partition"]
+ Output:["id","year_partition","date_time_date"]
Please refer to the previous Select Operator [SEL_3]
PREHOOK: query: insert into tbl_year_date values (88669, '2018-05-27', 2018), (40568, '2018-02-12', 2018), (40568, '2018-07-03', 2018)
@@ -1187,7 +1187,7 @@ Stage-3
Group By Operator [GBY_8] (rows=1 width=640)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll [...]
Select Operator [SEL_7] (rows=1 width=224)
- Output:["id","date_time_timestamp","year_partition"]
+ Output:["id","year_partition","date_time_timestamp"]
Please refer to the previous Select Operator [SEL_3]
PREHOOK: query: insert into tbl_year_timestamp values (88669, '2018-05-27 11:12:00', 2018), (40568, '2018-02-12 12:45:56', 2018), (40568, '2018-07-03 06:07:56', 2018)
@@ -1273,7 +1273,7 @@ Stage-3
Group By Operator [GBY_8] (rows=1 width=672)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],k [...]
Select Operator [SEL_7] (rows=1 width=240)
- Output:["id","date_time_date","year_partition"]
+ Output:["id","year_partition","date_time_date"]
Please refer to the previous Select Operator [SEL_3]
PREHOOK: query: insert into tbl_month_date values (88669, '2018-05-27', 2018), (40568, '2018-02-12', 2018), (40568, '2018-07-03', 2018)
@@ -1359,7 +1359,7 @@ Stage-3
Group By Operator [GBY_8] (rows=1 width=640)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll [...]
Select Operator [SEL_7] (rows=1 width=224)
- Output:["id","date_time_timestamp","year_partition"]
+ Output:["id","year_partition","date_time_timestamp"]
Please refer to the previous Select Operator [SEL_3]
PREHOOK: query: insert into tbl_month_timestamp values (88669, '2018-05-27 11:12:00', 2018), (40568, '2018-02-12 12:45:56', 2018), (40568, '2018-07-03 06:07:56', 2018)
@@ -1445,7 +1445,7 @@ Stage-3
Group By Operator [GBY_8] (rows=1 width=672)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_date)","max(date_time_date)","count(date_time_date)","compute_bit_vector_hll(date_time_date)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll(year_partition)"],k [...]
Select Operator [SEL_7] (rows=1 width=240)
- Output:["id","date_time_date","year_partition"]
+ Output:["id","year_partition","date_time_date"]
Please refer to the previous Select Operator [SEL_3]
PREHOOK: query: insert into tbl_day_date values (88669, '2018-05-27', 2018), (40568, '2018-02-12', 2018), (40568, '2018-07-03', 2018)
@@ -1531,7 +1531,7 @@ Stage-3
Group By Operator [GBY_8] (rows=1 width=640)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll [...]
Select Operator [SEL_7] (rows=1 width=224)
- Output:["id","date_time_timestamp","year_partition"]
+ Output:["id","year_partition","date_time_timestamp"]
Please refer to the previous Select Operator [SEL_3]
PREHOOK: query: insert into tbl_day_timestamp values (88669, '2018-05-27 11:12:00', 2018), (40568, '2018-02-12 12:45:56', 2018), (40568, '2018-07-03 06:07:56', 2018)
@@ -1617,7 +1617,7 @@ Stage-3
Group By Operator [GBY_8] (rows=1 width=640)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"],aggregations:["max(length(id))","avg(COALESCE(length(id),0))","count(1)","count(id)","compute_bit_vector_hll(id)","min(date_time_timestamp)","max(date_time_timestamp)","count(date_time_timestamp)","compute_bit_vector_hll(date_time_timestamp)","min(year_partition)","max(year_partition)","count(year_partition)","compute_bit_vector_hll [...]
Select Operator [SEL_7] (rows=1 width=224)
- Output:["id","date_time_timestamp","year_partition"]
+ Output:["id","year_partition","date_time_timestamp"]
Please refer to the previous Select Operator [SEL_3]
PREHOOK: query: insert into tbl_hour_timestamp values (88669, '2018-05-27 11:12:00', 2018), (40568, '2018-02-12 12:45:56', 2018), (40568, '2018-07-03 06:07:56', 2018)
diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out
index 74d891eefc1..430591d0d01 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out
@@ -27,8 +27,8 @@ POSTHOOK: Input: default@ice_tbl
col_name data_type comment
strcol string
intcol int
-pcol string
-datecol date
+pcol string Transform: identity
+datecol date Transform: identity
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out
index 1f417296d01..d30e06eab60 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out
@@ -479,7 +479,6 @@ POSTHOOK: Input: default@ice_parquet_int
# col_name data_type comment
strcol string
intcol int
-pcol int
# Partition Information
# col_name data_type comment
@@ -1675,8 +1674,6 @@ POSTHOOK: Input: default@ice_parquet_string
# col_name data_type comment
name string
age int
-country string
-state string
# Partition Information
# col_name data_type comment
@@ -2731,8 +2728,6 @@ POSTHOOK: Input: default@ice_parquet_string
# col_name data_type comment
name string
age int
-country string
-state string
# Partition Information
# col_name data_type comment
@@ -3427,7 +3422,6 @@ POSTHOOK: Input: default@ice_parquet_bigint
# col_name data_type comment
datecol date
intcol int
-pcol bigint
# Partition Information
# col_name data_type comment
@@ -4120,7 +4114,6 @@ POSTHOOK: Input: default@ice_parquet_double
# col_name data_type comment
datecol date
intcol int
-pcol double
# Partition Information
# col_name data_type comment
@@ -4813,7 +4806,6 @@ POSTHOOK: Input: default@ice_parquet_decimal
# col_name data_type comment
datecol date
intcol int
-pcol decimal(10,6)
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out
index 758c774405d..e7dd12e81cb 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out
@@ -610,7 +610,6 @@ POSTHOOK: Input: default@ice_parquet_date_transform_year
# col_name data_type comment
bigintcol bigint
intcol int
-pcol date
# Partition Information
# col_name data_type comment
@@ -759,8 +758,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_month
Select Operator
- expressions: _col0 (type: bigint), _col1 (type: date), _col2 (type: int)
- outputColumnNames: bigintcol, pcol, intcol
+ expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -876,8 +875,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_month
Select Operator
- expressions: _col0 (type: bigint), _col1 (type: date), _col2 (type: int)
- outputColumnNames: bigintcol, pcol, intcol
+ expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -993,8 +992,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_month
Select Operator
- expressions: _col0 (type: bigint), _col1 (type: date), _col2 (type: int)
- outputColumnNames: bigintcol, pcol, intcol
+ expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1116,8 +1115,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_month
Select Operator
- expressions: _col0 (type: bigint), _col1 (type: date), _col2 (type: int)
- outputColumnNames: bigintcol, pcol, intcol
+ expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1226,8 +1225,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_month
Select Operator
- expressions: _col0 (type: bigint), _col1 (type: date), _col2 (type: int)
- outputColumnNames: bigintcol, pcol, intcol
+ expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1304,7 +1303,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_parquet_date_transform_month
# col_name data_type comment
bigintcol bigint
-pcol date
intcol int
# Partition Information
@@ -1454,8 +1452,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_day
Select Operator
- expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1571,8 +1569,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_day
Select Operator
- expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1688,8 +1686,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_day
Select Operator
- expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1811,8 +1809,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_day
Select Operator
- expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1921,8 +1919,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_day
Select Operator
- expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1998,7 +1996,6 @@ POSTHOOK: query: describe formatted ice_parquet_date_transform_day
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_parquet_date_transform_day
# col_name data_type comment
-pcol date
bigintcol bigint
intcol int
@@ -2149,8 +2146,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_truncate
Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -2266,8 +2263,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_truncate
Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -2375,8 +2372,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_truncate
Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -2452,7 +2449,6 @@ POSTHOOK: query: describe formatted ice_parquet_date_transform_truncate
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_parquet_date_transform_truncate
# col_name data_type comment
-pcol string
bigintcol bigint
intcol int
@@ -2587,8 +2583,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_bucket
Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -2704,8 +2700,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_bucket
Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -2813,8 +2809,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_bucket
Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 4 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -2890,7 +2886,6 @@ POSTHOOK: query: describe formatted ice_parquet_date_transform_bucket
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_parquet_date_transform_bucket
# col_name data_type comment
-pcol string
bigintcol bigint
intcol int
@@ -3118,7 +3113,6 @@ POSTHOOK: query: describe formatted ice_parquet_decimal_transform_bucket
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_parquet_decimal_transform_bucket
# col_name data_type comment
-pcol decimal(38,0)
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out
index 956d89494fc..c444d538e49 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out
@@ -162,7 +162,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@testice1000
# col_name data_type comment
a int
-b string
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out
index c19c6331c07..db70eaa6503 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out
@@ -261,7 +261,6 @@ POSTHOOK: Input: default@ice_parquet_int
# col_name data_type comment
strcol string
intcol int
-pcol int
# Partition Information
# col_name data_type comment
@@ -1231,8 +1230,6 @@ POSTHOOK: Input: default@ice_parquet_string
# col_name data_type comment
name string
age int
-country string
-state string
# Partition Information
# col_name data_type comment
@@ -1713,7 +1710,6 @@ POSTHOOK: Input: default@ice_parquet_date
# col_name data_type comment
bigintcol bigint
intcol int
-pcol date
# Partition Information
# col_name data_type comment
@@ -2180,7 +2176,6 @@ POSTHOOK: Input: default@ice_parquet_bigint
# col_name data_type comment
datecol date
intcol int
-pcol bigint
# Partition Information
# col_name data_type comment
@@ -2647,7 +2642,6 @@ POSTHOOK: Input: default@ice_parquet_double
# col_name data_type comment
datecol date
intcol int
-pcol double
# Partition Information
# col_name data_type comment
@@ -3114,7 +3108,6 @@ POSTHOOK: Input: default@ice_parquet_decimal
# col_name data_type comment
datecol date
intcol int
-pcol decimal(10,6)
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out
index 09bff4eaf55..937da612148 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out
@@ -606,7 +606,6 @@ POSTHOOK: Input: default@ice_parquet_date_transform_year
# col_name data_type comment
bigintcol bigint
intcol int
-pcol date
# Partition Information
# col_name data_type comment
@@ -733,8 +732,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_month
Select Operator
- expressions: _col0 (type: bigint), _col1 (type: date), _col2 (type: int)
- outputColumnNames: bigintcol, pcol, intcol
+ expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -850,8 +849,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_month
Select Operator
- expressions: _col0 (type: bigint), _col1 (type: date), _col2 (type: int)
- outputColumnNames: bigintcol, pcol, intcol
+ expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -967,8 +966,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_month
Select Operator
- expressions: _col0 (type: bigint), _col1 (type: date), _col2 (type: int)
- outputColumnNames: bigintcol, pcol, intcol
+ expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1086,8 +1085,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_month
Select Operator
- expressions: _col0 (type: bigint), _col1 (type: date), _col2 (type: int)
- outputColumnNames: bigintcol, pcol, intcol
+ expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1196,8 +1195,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_month
Select Operator
- expressions: _col0 (type: bigint), _col1 (type: date), _col2 (type: int)
- outputColumnNames: bigintcol, pcol, intcol
+ expressions: _col0 (type: bigint), _col2 (type: int), _col1 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: min(bigintcol), max(bigintcol), count(1), count(bigintcol), compute_bit_vector_hll(bigintcol), min(pcol), max(pcol), count(pcol), compute_bit_vector_hll(pcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1274,7 +1273,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_parquet_date_transform_month
# col_name data_type comment
bigintcol bigint
-pcol date
intcol int
# Partition Information
@@ -1402,8 +1400,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_day
Select Operator
- expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1519,8 +1517,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_day
Select Operator
- expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1636,8 +1634,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_day
Select Operator
- expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1759,8 +1757,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_day
Select Operator
- expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1869,8 +1867,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_day
Select Operator
- expressions: _col0 (type: date), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: date)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: min(pcol), max(pcol), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -1946,7 +1944,6 @@ POSTHOOK: query: describe formatted ice_parquet_date_transform_day
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_parquet_date_transform_day
# col_name data_type comment
-pcol date
bigintcol bigint
intcol int
@@ -2097,8 +2094,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_truncate
Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -2214,8 +2211,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_truncate
Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -2323,8 +2320,8 @@ STAGE PLANS:
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
name: default.ice_parquet_date_transform_truncate
Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int)
- outputColumnNames: pcol, bigintcol, intcol
+ expressions: _col1 (type: bigint), _col2 (type: int), _col0 (type: string)
+ outputColumnNames: bigintcol, intcol, pcol
Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: max(length(pcol)), avg(COALESCE(length(pcol),0)), count(1), count(pcol), compute_bit_vector_hll(pcol), min(bigintcol), max(bigintcol), count(bigintcol), compute_bit_vector_hll(bigintcol), min(intcol), max(intcol), count(intcol), compute_bit_vector_hll(intcol)
@@ -2400,7 +2397,6 @@ POSTHOOK: query: describe formatted ice_parquet_date_transform_truncate
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_parquet_date_transform_truncate
# col_name data_type comment
-pcol string
bigintcol bigint
intcol int
diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out
index a364be2dedf..bba70b438c0 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_load_data.q.out
@@ -35,31 +35,31 @@ Stage-3
Dependency Collection{}
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_18]
+ File Output Operator [FS_20]
table:{"name:":"default.ice_parquet"}
- Select Operator [SEL_17]
+ Select Operator [SEL_19]
Output:["_col0","_col1","_col2","_col2"]
<-Map 1 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_13]
+ SHUFFLE [RS_15]
PartitionCols:_col2
- Select Operator [SEL_12] (rows=77 width=187)
+ Select Operator [SEL_14] (rows=77 width=187)
Output:["_col0","_col1","_col2"]
TableScan [TS_0] (rows=77 width=187)
default@ice_parquet__temp_table_for_load_data__,ice_parquet__temp_table_for_load_data__,Tbl:COMPLETE,Col:NONE,Output:["strcol","intcol","pcol"]
Reducer 3 vectorized
- File Output Operator [FS_21]
- Select Operator [SEL_20] (rows=38 width=187)
+ File Output Operator [FS_23]
+ Select Operator [SEL_22] (rows=38 width=187)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"]
- Group By Operator [GBY_19] (rows=38 width=187)
+ Group By Operator [GBY_21] (rows=38 width=187)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0
<-Map 1 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_16]
+ SHUFFLE [RS_18]
PartitionCols:_col0
- Group By Operator [GBY_15] (rows=77 width=187)
+ Group By Operator [GBY_17] (rows=77 width=187)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(length(strcol))","avg(COALESCE(length(strcol),0))","count(1)","count(strcol)","compute_bit_vector_hll(strcol)","min(intcol)","max(intcol)","count(intcol)","compute_bit_vector_hll(intcol)","min(pcol)","max(pcol)","count(pcol)","compute_bit_vector_hll(pcol)"],keys:pcol
- Select Operator [SEL_14] (rows=77 width=187)
+ Select Operator [SEL_16] (rows=77 width=187)
Output:["strcol","intcol","pcol"]
- Please refer to the previous Select Operator [SEL_12]
+ Please refer to the previous Select Operator [SEL_14]
PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_partition' OVERWRITE INTO TABLE `ice_parquet`
PREHOOK: type: QUERY
@@ -90,31 +90,31 @@ Stage-3
Dependency Collection{}
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_18]
+ File Output Operator [FS_20]
table:{"name:":"default.ice_parquet"}
- Select Operator [SEL_17]
+ Select Operator [SEL_19]
Output:["_col0","_col1","_col2","_col2"]
<-Map 1 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_13]
+ SHUFFLE [RS_15]
PartitionCols:_col2
- Select Operator [SEL_12] (rows=77/6 width=187)
+ Select Operator [SEL_14] (rows=77/6 width=187)
Output:["_col0","_col1","_col2"]
TableScan [TS_0] (rows=77/6 width=187)
default@ice_parquet__temp_table_for_load_data__,ice_parquet__temp_table_for_load_data__,Tbl:COMPLETE,Col:NONE,Output:["strcol","intcol","pcol"]
Reducer 3 vectorized
- File Output Operator [FS_21]
- Select Operator [SEL_20] (rows=38/3 width=187)
+ File Output Operator [FS_23]
+ Select Operator [SEL_22] (rows=38/3 width=187)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18"]
- Group By Operator [GBY_19] (rows=38/3 width=187)
+ Group By Operator [GBY_21] (rows=38/3 width=187)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","min(VALUE._col5)","max(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","min(VALUE._col9)","max(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"],keys:KEY._col0
<-Map 1 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_16]
+ SHUFFLE [RS_18]
PartitionCols:_col0
- Group By Operator [GBY_15] (rows=77/3 width=187)
+ Group By Operator [GBY_17] (rows=77/3 width=187)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13"],aggregations:["max(length(strcol))","avg(COALESCE(length(strcol),0))","count(1)","count(strcol)","compute_bit_vector_hll(strcol)","min(intcol)","max(intcol)","count(intcol)","compute_bit_vector_hll(intcol)","min(pcol)","max(pcol)","count(pcol)","compute_bit_vector_hll(pcol)"],keys:pcol
- Select Operator [SEL_14] (rows=77/6 width=187)
+ Select Operator [SEL_16] (rows=77/6 width=187)
Output:["strcol","intcol","pcol"]
- Please refer to the previous Select Operator [SEL_12]
+ Please refer to the previous Select Operator [SEL_14]
PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_partition' OVERWRITE INTO TABLE ice_parquet
PREHOOK: type: QUERY
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out
index 146cda56a50..3e3f7761f05 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out
@@ -89,8 +89,6 @@ orderid int
quantity int
itemid int
tradets timestamp
-p1 string
-p2 string
# Partition Information
# col_name data_type comment
@@ -171,8 +169,6 @@ orderid int
quantity int
itemid int
tradets timestamp
-p1 string
-p2 string
# Partition Information
# col_name data_type comment
@@ -261,8 +257,6 @@ orderid int
quantity int
itemid int
tradets timestamp
-p1 string
-p2 string
# Partition Information
# col_name data_type comment
@@ -350,8 +344,6 @@ orderid int
quantity int
itemid int
tradets timestamp
-p1 string
-p2 string
# Partition Information
# col_name data_type comment
@@ -424,8 +416,6 @@ orderid int
quantity int
itemid int
tradets timestamp
-p1 string
-p2 string
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out
index f8dfb22e5fa..4e2b9476d8b 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out
@@ -113,7 +113,7 @@ Stage-0
File Output Operator [FS_10]
Select Operator [SEL_9] (rows=3 width=520)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Map Join Operator [MAPJOIN_45] (rows=3 width=336)
+ Map Join Operator [MAPJOIN_25] (rows=3 width=336)
BucketMapJoin:true,Conds:SEL_2._col0, _col1=RS_7._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
<-Map 2 [CUSTOM_EDGE] llap
MULTICAST [RS_7]
@@ -177,7 +177,7 @@ Stage-0
File Output Operator [FS_14]
Select Operator [SEL_13] (rows=3 width=520)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Map Join Operator [MAPJOIN_49] (rows=3 width=336)
+ Map Join Operator [MAPJOIN_39] (rows=3 width=336)
BucketMapJoin:true,Conds:SEL_2._col0, _col1=RS_11._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
<-Reducer 3 [CUSTOM_EDGE] llap
MULTICAST [RS_11]
@@ -189,12 +189,10 @@ Stage-0
PartitionCols:_col0, _col1
Group By Operator [GBY_6] (rows=3 width=168)
Output:["_col0","_col1"],keys:date_col, decimal_col
- Select Operator [SEL_5] (rows=3 width=168)
- Output:["date_col","decimal_col"]
- Filter Operator [FIL_18] (rows=3 width=168)
- predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
- TableScan [TS_3] (rows=7 width=168)
- default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ Filter Operator [FIL_18] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
<-Select Operator [SEL_2] (rows=10 width=168)
Output:["_col0","_col1"]
Filter Operator [FIL_17] (rows=10 width=260)
@@ -247,7 +245,7 @@ Stage-0
File Output Operator [FS_10]
Select Operator [SEL_9] (rows=3 width=520)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Map Join Operator [MAPJOIN_45] (rows=3 width=336)
+ Map Join Operator [MAPJOIN_25] (rows=3 width=336)
Conds:SEL_2._col0, _col1=RS_7._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
<-Map 2 [BROADCAST_EDGE] llap
BROADCAST [RS_7]
@@ -311,7 +309,7 @@ Stage-0
File Output Operator [FS_14]
Select Operator [SEL_13] (rows=3 width=520)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Map Join Operator [MAPJOIN_49] (rows=3 width=336)
+ Map Join Operator [MAPJOIN_39] (rows=3 width=336)
Conds:SEL_2._col0, _col1=RS_11._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
<-Reducer 3 [BROADCAST_EDGE] llap
BROADCAST [RS_11]
@@ -323,12 +321,10 @@ Stage-0
PartitionCols:_col0, _col1
Group By Operator [GBY_6] (rows=3 width=168)
Output:["_col0","_col1"],keys:date_col, decimal_col
- Select Operator [SEL_5] (rows=3 width=168)
- Output:["date_col","decimal_col"]
- Filter Operator [FIL_18] (rows=3 width=168)
- predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
- TableScan [TS_3] (rows=7 width=168)
- default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ Filter Operator [FIL_18] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
<-Select Operator [SEL_2] (rows=10 width=168)
Output:["_col0","_col1"]
Filter Operator [FIL_17] (rows=10 width=260)
@@ -378,23 +374,23 @@ Stage-0
limit:-1
Stage-1
Map 1 vectorized, llap
- File Output Operator [FS_54]
- Select Operator [SEL_53] (rows=3 width=520)
+ File Output Operator [FS_34]
+ Select Operator [SEL_33] (rows=3 width=520)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Map Join Operator [MAPJOIN_52] (rows=3 width=336)
- BucketMapJoin:true,Conds:SEL_51._col0, _col1=RS_49._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ Map Join Operator [MAPJOIN_32] (rows=3 width=336)
+ BucketMapJoin:true,Conds:SEL_31._col0, _col1=RS_29._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
<-Map 2 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_49]
+ MULTICAST [RS_29]
PartitionCols:_col1
- Select Operator [SEL_48] (rows=3 width=168)
+ Select Operator [SEL_28] (rows=3 width=168)
Output:["_col0","_col1"]
- Filter Operator [FIL_47] (rows=3 width=168)
+ Filter Operator [FIL_27] (rows=3 width=168)
predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
TableScan [TS_3] (rows=7 width=168)
default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
- <-Select Operator [SEL_51] (rows=10 width=168)
+ <-Select Operator [SEL_31] (rows=10 width=168)
Output:["_col0","_col1"]
- Filter Operator [FIL_50] (rows=10 width=260)
+ Filter Operator [FIL_30] (rows=10 width=260)
predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
TableScan [TS_0] (rows=20 width=260)
default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"]
@@ -442,30 +438,28 @@ Stage-0
limit:-1
Stage-1
Map 1 vectorized, llap
- File Output Operator [FS_61]
- Select Operator [SEL_60] (rows=3 width=520)
+ File Output Operator [FS_50]
+ Select Operator [SEL_49] (rows=3 width=520)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Map Join Operator [MAPJOIN_59] (rows=3 width=336)
- BucketMapJoin:true,Conds:SEL_58._col0, _col1=RS_56._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ Map Join Operator [MAPJOIN_48] (rows=3 width=336)
+ BucketMapJoin:true,Conds:SEL_47._col0, _col1=RS_45._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
<-Reducer 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_56]
+ MULTICAST [RS_45]
PartitionCols:_col1
- Group By Operator [GBY_55] (rows=3 width=168)
+ Group By Operator [GBY_44] (rows=3 width=168)
Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
<-Map 2 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_54]
+ SHUFFLE [RS_43]
PartitionCols:_col0, _col1
- Group By Operator [GBY_53] (rows=3 width=168)
+ Group By Operator [GBY_42] (rows=3 width=168)
Output:["_col0","_col1"],keys:date_col, decimal_col
- Select Operator [SEL_52] (rows=3 width=168)
- Output:["date_col","decimal_col"]
- Filter Operator [FIL_51] (rows=3 width=168)
- predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
- TableScan [TS_3] (rows=7 width=168)
- default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
- <-Select Operator [SEL_58] (rows=10 width=168)
+ Filter Operator [FIL_41] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ <-Select Operator [SEL_47] (rows=10 width=168)
Output:["_col0","_col1"]
- Filter Operator [FIL_57] (rows=10 width=260)
+ Filter Operator [FIL_46] (rows=10 width=260)
predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
TableScan [TS_0] (rows=20 width=260)
default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"]
@@ -512,23 +506,23 @@ Stage-0
limit:-1
Stage-1
Map 1 vectorized, llap
- File Output Operator [FS_54]
- Select Operator [SEL_53] (rows=3 width=520)
+ File Output Operator [FS_34]
+ Select Operator [SEL_33] (rows=3 width=520)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Map Join Operator [MAPJOIN_52] (rows=3 width=336)
- Conds:SEL_51._col0, _col1=RS_49._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ Map Join Operator [MAPJOIN_32] (rows=3 width=336)
+ Conds:SEL_31._col0, _col1=RS_29._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
<-Map 2 [BROADCAST_EDGE] vectorized, llap
- BROADCAST [RS_49]
+ BROADCAST [RS_29]
PartitionCols:_col0, _col1
- Select Operator [SEL_48] (rows=3 width=168)
+ Select Operator [SEL_28] (rows=3 width=168)
Output:["_col0","_col1"]
- Filter Operator [FIL_47] (rows=3 width=168)
+ Filter Operator [FIL_27] (rows=3 width=168)
predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
TableScan [TS_3] (rows=7 width=168)
default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
- <-Select Operator [SEL_51] (rows=10 width=168)
+ <-Select Operator [SEL_31] (rows=10 width=168)
Output:["_col0","_col1"]
- Filter Operator [FIL_50] (rows=10 width=260)
+ Filter Operator [FIL_30] (rows=10 width=260)
predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
TableScan [TS_0] (rows=20 width=260)
default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"]
@@ -576,30 +570,28 @@ Stage-0
limit:-1
Stage-1
Map 1 vectorized, llap
- File Output Operator [FS_61]
- Select Operator [SEL_60] (rows=3 width=520)
+ File Output Operator [FS_50]
+ Select Operator [SEL_49] (rows=3 width=520)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Map Join Operator [MAPJOIN_59] (rows=3 width=336)
- Conds:SEL_58._col0, _col1=RS_56._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ Map Join Operator [MAPJOIN_48] (rows=3 width=336)
+ Conds:SEL_47._col0, _col1=RS_45._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
<-Reducer 3 [BROADCAST_EDGE] vectorized, llap
- BROADCAST [RS_56]
+ BROADCAST [RS_45]
PartitionCols:_col0, _col1
- Group By Operator [GBY_55] (rows=3 width=168)
+ Group By Operator [GBY_44] (rows=3 width=168)
Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
<-Map 2 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_54]
+ SHUFFLE [RS_43]
PartitionCols:_col0, _col1
- Group By Operator [GBY_53] (rows=3 width=168)
+ Group By Operator [GBY_42] (rows=3 width=168)
Output:["_col0","_col1"],keys:date_col, decimal_col
- Select Operator [SEL_52] (rows=3 width=168)
- Output:["date_col","decimal_col"]
- Filter Operator [FIL_51] (rows=3 width=168)
- predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
- TableScan [TS_3] (rows=7 width=168)
- default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
- <-Select Operator [SEL_58] (rows=10 width=168)
+ Filter Operator [FIL_41] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ <-Select Operator [SEL_47] (rows=10 width=168)
Output:["_col0","_col1"]
- Filter Operator [FIL_57] (rows=10 width=260)
+ Filter Operator [FIL_46] (rows=10 width=260)
predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
TableScan [TS_0] (rows=20 width=260)
default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"]
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out
index 413fd81aa95..7ce866f2b03 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out
@@ -251,39 +251,39 @@ Stage-3
Dependency Collection{}
Stage-1
Reducer 3 vectorized, llap
- File Output Operator [FS_48]
- Select Operator [SEL_47] (rows=1 width=798)
+ File Output Operator [FS_38]
+ Select Operator [SEL_37] (rows=1 width=798)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"]
- Group By Operator [GBY_46] (rows=1 width=500)
+ Group By Operator [GBY_36] (rows=1 width=500)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"]
<-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- File Output Operator [FS_42]
+ File Output Operator [FS_32]
table:{"name:":"default.bucketmapjoin_tmp_result"}
- Select Operator [SEL_41] (rows=785 width=366)
+ Select Operator [SEL_31] (rows=785 width=366)
Output:["_col0","_col1","_col2"]
- Map Join Operator [MAPJOIN_40] (rows=785 width=186)
- BucketMapJoin:true,Conds:RS_37._col0=SEL_39._col0(Inner),Output:["_col0","_col1","_col3"]
+ Map Join Operator [MAPJOIN_30] (rows=785 width=186)
+ BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"]
<-Map 1 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_37]
+ MULTICAST [RS_27]
PartitionCols:_col0
- Select Operator [SEL_36] (rows=238 width=95)
+ Select Operator [SEL_26] (rows=238 width=95)
Output:["_col0","_col1"]
- Filter Operator [FIL_35] (rows=238 width=95)
+ Filter Operator [FIL_25] (rows=238 width=95)
predicate:key is not null
TableScan [TS_0] (rows=238 width=95)
default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Select Operator [SEL_39] (rows=1000 width=95)
+ <-Select Operator [SEL_29] (rows=1000 width=95)
Output:["_col0","_col1"]
- Filter Operator [FIL_38] (rows=1000 width=95)
+ Filter Operator [FIL_28] (rows=1000 width=95)
predicate:key is not null
TableScan [TS_3] (rows=1000 width=95)
default@srcbucket_mapjoin_part_n0,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key"],Output:["key","value"]
- PARTITION_ONLY_SHUFFLE [RS_45]
- Group By Operator [GBY_44] (rows=1 width=704)
+ PARTITION_ONLY_SHUFFLE [RS_35]
+ Group By Operator [GBY_34] (rows=1 width=704)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"]
- Select Operator [SEL_43] (rows=785 width=366)
+ Select Operator [SEL_33] (rows=785 width=366)
Output:["key","value1","value2"]
- Please refer to the previous Select Operator [SEL_41]
+ Please refer to the previous Select Operator [SEL_31]
PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result
select /*+mapjoin(a)*/ a.key, a.value, b.value
@@ -415,39 +415,39 @@ Stage-3
Dependency Collection{}
Stage-1
Reducer 3 vectorized, llap
- File Output Operator [FS_48]
- Select Operator [SEL_47] (rows=1 width=798)
+ File Output Operator [FS_38]
+ Select Operator [SEL_37] (rows=1 width=798)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"]
- Group By Operator [GBY_46] (rows=1 width=500)
+ Group By Operator [GBY_36] (rows=1 width=500)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"]
<-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- File Output Operator [FS_42]
+ File Output Operator [FS_32]
table:{"name:":"default.bucketmapjoin_tmp_result"}
- Select Operator [SEL_41] (rows=809 width=366)
+ Select Operator [SEL_31] (rows=809 width=366)
Output:["_col0","_col1","_col2"]
- Map Join Operator [MAPJOIN_40] (rows=809 width=186)
- BucketMapJoin:true,Conds:RS_37._col0=SEL_39._col0(Inner),Output:["_col0","_col1","_col3"]
+ Map Join Operator [MAPJOIN_30] (rows=809 width=186)
+ BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"]
<-Map 1 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_37]
+ MULTICAST [RS_27]
PartitionCols:_col0
- Select Operator [SEL_36] (rows=238 width=95)
+ Select Operator [SEL_26] (rows=238 width=95)
Output:["_col0","_col1"]
- Filter Operator [FIL_35] (rows=238 width=95)
+ Filter Operator [FIL_25] (rows=238 width=95)
predicate:key is not null
TableScan [TS_0] (rows=238 width=95)
default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Select Operator [SEL_39] (rows=524 width=95)
+ <-Select Operator [SEL_29] (rows=524 width=95)
Output:["_col0","_col1"]
- Filter Operator [FIL_38] (rows=524 width=95)
+ Filter Operator [FIL_28] (rows=524 width=95)
predicate:key is not null
TableScan [TS_3] (rows=524 width=95)
default@srcbucket_mapjoin_part_2,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"]
- PARTITION_ONLY_SHUFFLE [RS_45]
- Group By Operator [GBY_44] (rows=1 width=704)
+ PARTITION_ONLY_SHUFFLE [RS_35]
+ Group By Operator [GBY_34] (rows=1 width=704)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"]
- Select Operator [SEL_43] (rows=809 width=366)
+ Select Operator [SEL_33] (rows=809 width=366)
Output:["key","value1","value2"]
- Please refer to the previous Select Operator [SEL_41]
+ Please refer to the previous Select Operator [SEL_31]
PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result
select /*+mapjoin(a)*/ a.key, a.value, b.value
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out
index bd82280c9c0..f21a3c84da2 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out
@@ -122,27 +122,27 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_40]
- Group By Operator [GBY_39] (rows=1 width=8)
+ File Output Operator [FS_30]
+ Group By Operator [GBY_29] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- PARTITION_ONLY_SHUFFLE [RS_38]
- Group By Operator [GBY_37] (rows=1 width=8)
+ PARTITION_ONLY_SHUFFLE [RS_28]
+ Group By Operator [GBY_27] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
- Map Join Operator [MAPJOIN_36] (rows=372 width=8)
- BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner)
+ Map Join Operator [MAPJOIN_26] (rows=372 width=8)
+ BucketMapJoin:true,Conds:SEL_25._col0=RS_23._col0(Inner)
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_33]
+ MULTICAST [RS_23]
PartitionCols:_col0
- Select Operator [SEL_32] (rows=238 width=4)
+ Select Operator [SEL_22] (rows=238 width=4)
Output:["_col0"]
- Filter Operator [FIL_31] (rows=238 width=89)
+ Filter Operator [FIL_21] (rows=238 width=89)
predicate:((part = '1') and key is not null)
TableScan [TS_3] (rows=238 width=89)
default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
- <-Select Operator [SEL_35] (rows=238 width=4)
+ <-Select Operator [SEL_25] (rows=238 width=4)
Output:["_col0"]
- Filter Operator [FIL_34] (rows=238 width=89)
+ Filter Operator [FIL_24] (rows=238 width=89)
predicate:((part = '1') and key is not null)
TableScan [TS_0] (rows=238 width=89)
default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"]
@@ -196,27 +196,27 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_40]
- Group By Operator [GBY_39] (rows=1 width=8)
+ File Output Operator [FS_35]
+ Group By Operator [GBY_34] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- PARTITION_ONLY_SHUFFLE [RS_38]
- Group By Operator [GBY_37] (rows=1 width=8)
+ PARTITION_ONLY_SHUFFLE [RS_33]
+ Group By Operator [GBY_32] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
- Map Join Operator [MAPJOIN_36] (rows=372 width=8)
- BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner)
+ Map Join Operator [MAPJOIN_31] (rows=372 width=8)
+ BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner)
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_33]
+ MULTICAST [RS_28]
PartitionCols:_col0
- Select Operator [SEL_32] (rows=238 width=4)
+ Select Operator [SEL_27] (rows=238 width=4)
Output:["_col0"]
- Filter Operator [FIL_31] (rows=238 width=89)
+ Filter Operator [FIL_26] (rows=238 width=89)
predicate:((part = '1') and key is not null)
TableScan [TS_3] (rows=238 width=89)
default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
- <-Select Operator [SEL_35] (rows=238 width=4)
+ <-Select Operator [SEL_30] (rows=238 width=4)
Output:["_col0"]
- Filter Operator [FIL_34] (rows=238 width=89)
+ Filter Operator [FIL_29] (rows=238 width=89)
predicate:((part = '1') and key is not null)
TableScan [TS_0] (rows=238 width=89)
default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"]
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out
index 4ddd445ecbf..dd93fcf3185 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out
@@ -235,27 +235,27 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_40]
- Group By Operator [GBY_39] (rows=1 width=8)
+ File Output Operator [FS_30]
+ Group By Operator [GBY_29] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- PARTITION_ONLY_SHUFFLE [RS_38]
- Group By Operator [GBY_37] (rows=1 width=8)
+ PARTITION_ONLY_SHUFFLE [RS_28]
+ Group By Operator [GBY_27] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
- Map Join Operator [MAPJOIN_36] (rows=1797 width=8)
- Conds:SEL_35._col0=RS_33._col0(Inner)
+ Map Join Operator [MAPJOIN_26] (rows=1797 width=8)
+ Conds:SEL_25._col0=RS_23._col0(Inner)
<-Map 3 [BROADCAST_EDGE] vectorized, llap
- BROADCAST [RS_33]
+ BROADCAST [RS_23]
PartitionCols:_col0
- Select Operator [SEL_32] (rows=738 width=4)
+ Select Operator [SEL_22] (rows=738 width=4)
Output:["_col0"]
- Filter Operator [FIL_31] (rows=738 width=89)
+ Filter Operator [FIL_21] (rows=738 width=89)
predicate:(part is not null and key is not null)
TableScan [TS_3] (rows=738 width=89)
default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
- <-Select Operator [SEL_35] (rows=738 width=4)
+ <-Select Operator [SEL_25] (rows=738 width=4)
Output:["_col0"]
- Filter Operator [FIL_34] (rows=738 width=89)
+ Filter Operator [FIL_24] (rows=738 width=89)
predicate:(part is not null and key is not null)
TableScan [TS_0] (rows=738 width=89)
default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
@@ -302,33 +302,33 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_49]
- Group By Operator [GBY_48] (rows=1 width=8)
+ File Output Operator [FS_39]
+ Group By Operator [GBY_38] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- PARTITION_ONLY_SHUFFLE [RS_47]
- Group By Operator [GBY_46] (rows=1 width=8)
+ PARTITION_ONLY_SHUFFLE [RS_37]
+ Group By Operator [GBY_36] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
- Map Join Operator [MAPJOIN_45] (rows=1797 width=8)
- Conds:SEL_44._col0, _col1=RS_39._col0, _col1(Inner)
+ Map Join Operator [MAPJOIN_35] (rows=1797 width=8)
+ Conds:SEL_34._col0, _col1=RS_29._col0, _col1(Inner)
<-Map 3 [BROADCAST_EDGE] vectorized, llap
- BROADCAST [RS_39]
+ BROADCAST [RS_29]
PartitionCols:_col0, _col1
- Select Operator [SEL_38] (rows=738 width=89)
+ Select Operator [SEL_28] (rows=738 width=89)
Output:["_col0","_col1"]
- Filter Operator [FIL_37] (rows=738 width=89)
+ Filter Operator [FIL_27] (rows=738 width=89)
predicate:(part is not null and key is not null)
TableScan [TS_3] (rows=738 width=89)
default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
- Dynamic Partitioning Event Operator [EVENT_42] (rows=2 width=85)
- Group By Operator [GBY_41] (rows=2 width=85)
+ Dynamic Partitioning Event Operator [EVENT_32] (rows=2 width=85)
+ Group By Operator [GBY_31] (rows=2 width=85)
Output:["_col0"],keys:_col0
- Select Operator [SEL_40] (rows=738 width=85)
+ Select Operator [SEL_30] (rows=738 width=85)
Output:["_col0"]
- Please refer to the previous Select Operator [SEL_38]
- <-Select Operator [SEL_44] (rows=738 width=89)
+ Please refer to the previous Select Operator [SEL_28]
+ <-Select Operator [SEL_34] (rows=738 width=89)
Output:["_col0","_col1"]
- Filter Operator [FIL_43] (rows=738 width=89)
+ Filter Operator [FIL_33] (rows=738 width=89)
predicate:(part is not null and key is not null)
TableScan [TS_0] (rows=738 width=89)
default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out
index 4b92f571e02..8ff5e300aa4 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out
@@ -174,27 +174,27 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_40]
- Group By Operator [GBY_39] (rows=1 width=8)
+ File Output Operator [FS_35]
+ Group By Operator [GBY_34] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- PARTITION_ONLY_SHUFFLE [RS_38]
- Group By Operator [GBY_37] (rows=1 width=8)
+ PARTITION_ONLY_SHUFFLE [RS_33]
+ Group By Operator [GBY_32] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
- Map Join Operator [MAPJOIN_36] (rows=372 width=8)
- BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner)
+ Map Join Operator [MAPJOIN_31] (rows=372 width=8)
+ BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner)
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_33]
+ MULTICAST [RS_28]
PartitionCols:_col0
- Select Operator [SEL_32] (rows=238 width=4)
+ Select Operator [SEL_27] (rows=238 width=4)
Output:["_col0"]
- Filter Operator [FIL_31] (rows=238 width=89)
+ Filter Operator [FIL_26] (rows=238 width=89)
predicate:((part = '1') and key is not null)
TableScan [TS_3] (rows=238 width=89)
default@srcbucket_mapjoin_part_2_n0,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
- <-Select Operator [SEL_35] (rows=238 width=4)
+ <-Select Operator [SEL_30] (rows=238 width=4)
Output:["_col0"]
- Filter Operator [FIL_34] (rows=238 width=89)
+ Filter Operator [FIL_29] (rows=238 width=89)
predicate:((part = '1') and key is not null)
TableScan [TS_0] (rows=238 width=89)
default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"]
@@ -241,27 +241,27 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_40]
- Group By Operator [GBY_39] (rows=1 width=8)
+ File Output Operator [FS_30]
+ Group By Operator [GBY_29] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- PARTITION_ONLY_SHUFFLE [RS_38]
- Group By Operator [GBY_37] (rows=1 width=8)
+ PARTITION_ONLY_SHUFFLE [RS_28]
+ Group By Operator [GBY_27] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
- Map Join Operator [MAPJOIN_36] (rows=372 width=8)
- BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner)
+ Map Join Operator [MAPJOIN_26] (rows=372 width=8)
+ BucketMapJoin:true,Conds:SEL_25._col0=RS_23._col0(Inner)
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_33]
+ MULTICAST [RS_23]
PartitionCols:_col0
- Select Operator [SEL_32] (rows=238 width=4)
+ Select Operator [SEL_22] (rows=238 width=4)
Output:["_col0"]
- Filter Operator [FIL_31] (rows=238 width=89)
+ Filter Operator [FIL_21] (rows=238 width=89)
predicate:((part = '1') and key is not null)
TableScan [TS_3] (rows=238 width=89)
default@srcbucket_mapjoin_part_3,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
- <-Select Operator [SEL_35] (rows=238 width=4)
+ <-Select Operator [SEL_25] (rows=238 width=4)
Output:["_col0"]
- Filter Operator [FIL_34] (rows=238 width=89)
+ Filter Operator [FIL_24] (rows=238 width=89)
predicate:((part = '1') and key is not null)
TableScan [TS_0] (rows=238 width=89)
default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"]
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out
index 11477c98ecc..024fde7f157 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out
@@ -141,37 +141,37 @@ Stage-3
Dependency Collection{}
Stage-1
Reducer 3 vectorized, llap
- File Output Operator [FS_48]
- Select Operator [SEL_47] (rows=1 width=798)
+ File Output Operator [FS_38]
+ Select Operator [SEL_37] (rows=1 width=798)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"]
- Group By Operator [GBY_46] (rows=1 width=500)
+ Group By Operator [GBY_36] (rows=1 width=500)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"]
<-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- File Output Operator [FS_42]
+ File Output Operator [FS_32]
table:{"name:":"default.bucketmapjoin_tmp_result_n3"}
- Select Operator [SEL_41] (rows=809 width=366)
+ Select Operator [SEL_31] (rows=809 width=366)
Output:["_col0","_col1","_col2"]
- Map Join Operator [MAPJOIN_40] (rows=809 width=186)
- BucketMapJoin:true,Conds:RS_37._col0=SEL_39._col0(Inner),Output:["_col0","_col1","_col3"]
+ Map Join Operator [MAPJOIN_30] (rows=809 width=186)
+ BucketMapJoin:true,Conds:RS_27._col0=SEL_29._col0(Inner),Output:["_col0","_col1","_col3"]
<-Map 1 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_37]
+ MULTICAST [RS_27]
PartitionCols:_col0
- Select Operator [SEL_36] (rows=238 width=95)
+ Select Operator [SEL_26] (rows=238 width=95)
Output:["_col0","_col1"]
- Filter Operator [FIL_35] (rows=238 width=95)
+ Filter Operator [FIL_25] (rows=238 width=95)
predicate:key is not null
TableScan [TS_0] (rows=238 width=95)
default@srcbucket_mapjoin_n5,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Select Operator [SEL_39] (rows=524 width=95)
+ <-Select Operator [SEL_29] (rows=524 width=95)
Output:["_col0","_col1"]
- Filter Operator [FIL_38] (rows=524 width=95)
+ Filter Operator [FIL_28] (rows=524 width=95)
predicate:key is not null
TableScan [TS_3] (rows=524 width=95)
default@srcbucket_mapjoin_part_2_n7,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"]
- PARTITION_ONLY_SHUFFLE [RS_45]
- Group By Operator [GBY_44] (rows=1 width=704)
+ PARTITION_ONLY_SHUFFLE [RS_35]
+ Group By Operator [GBY_34] (rows=1 width=704)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"]
- Select Operator [SEL_43] (rows=809 width=366)
+ Select Operator [SEL_33] (rows=809 width=366)
Output:["key","value1","value2"]
- Please refer to the previous Select Operator [SEL_41]
+ Please refer to the previous Select Operator [SEL_31]
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out
index 6701fbaf410..7b486db6769 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out
@@ -49,29 +49,29 @@ Stage-0
limit:20
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_61]
- Limit [LIM_60] (rows=20 width=447)
+ File Output Operator [FS_46]
+ Limit [LIM_45] (rows=20 width=447)
Number of rows:20
- Select Operator [SEL_59] (rows=791 width=447)
+ Select Operator [SEL_44] (rows=791 width=447)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_58]
- Top N Key Operator [TNK_57] (rows=791 width=447)
+ SHUFFLE [RS_43]
+ Top N Key Operator [TNK_42] (rows=791 width=447)
keys:_col0,top n:20
- Map Join Operator [MAPJOIN_56] (rows=791 width=447)
- BucketMapJoin:true,Conds:SEL_55._col0, _col1=RS_53._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ Map Join Operator [MAPJOIN_41] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_40._col0, _col1=RS_38._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_53]
+ MULTICAST [RS_38]
PartitionCols:_col0, _col1
- Select Operator [SEL_52] (rows=500 width=178)
+ Select Operator [SEL_37] (rows=500 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_51] (rows=500 width=178)
+ Filter Operator [FIL_36] (rows=500 width=178)
predicate:(key is not null and value is not null)
TableScan [TS_3] (rows=500 width=178)
default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Select Operator [SEL_55] (rows=500 width=269)
+ <-Select Operator [SEL_40] (rows=500 width=269)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_54] (rows=500 width=269)
+ Filter Operator [FIL_39] (rows=500 width=269)
predicate:(key1 is not null and key2 is not null)
TableScan [TS_0] (rows=500 width=269)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"]
@@ -147,29 +147,29 @@ Stage-0
limit:20
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_61]
- Limit [LIM_60] (rows=20 width=447)
+ File Output Operator [FS_46]
+ Limit [LIM_45] (rows=20 width=447)
Number of rows:20
- Select Operator [SEL_59] (rows=791 width=447)
+ Select Operator [SEL_44] (rows=791 width=447)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_58]
- Top N Key Operator [TNK_57] (rows=791 width=447)
+ SHUFFLE [RS_43]
+ Top N Key Operator [TNK_42] (rows=791 width=447)
keys:_col0,top n:20
- Map Join Operator [MAPJOIN_56] (rows=791 width=447)
- BucketMapJoin:true,Conds:SEL_55._col0, _col1=RS_53._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ Map Join Operator [MAPJOIN_41] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_40._col0, _col1=RS_38._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_53]
+ MULTICAST [RS_38]
PartitionCols:_col0, _col1
- Select Operator [SEL_52] (rows=500 width=178)
+ Select Operator [SEL_37] (rows=500 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_51] (rows=500 width=178)
+ Filter Operator [FIL_36] (rows=500 width=178)
predicate:((key <> '0') and (key <> '100') and value is not null)
TableScan [TS_3] (rows=500 width=178)
default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Select Operator [SEL_55] (rows=500 width=269)
+ <-Select Operator [SEL_40] (rows=500 width=269)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_54] (rows=500 width=269)
+ Filter Operator [FIL_39] (rows=500 width=269)
predicate:((key1 <> '0') and (key1 <> '100') and key2 is not null)
TableScan [TS_0] (rows=500 width=269)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"]
@@ -245,29 +245,29 @@ Stage-0
limit:20
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_41]
- Limit [LIM_40] (rows=20 width=447)
+ File Output Operator [FS_36]
+ Limit [LIM_35] (rows=20 width=447)
Number of rows:20
- Select Operator [SEL_39] (rows=791 width=447)
+ Select Operator [SEL_34] (rows=791 width=447)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_38]
- Top N Key Operator [TNK_37] (rows=791 width=447)
+ SHUFFLE [RS_33]
+ Top N Key Operator [TNK_32] (rows=791 width=447)
keys:_col0,top n:20
- Map Join Operator [MAPJOIN_36] (rows=791 width=447)
- BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ Map Join Operator [MAPJOIN_31] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_33]
+ MULTICAST [RS_28]
PartitionCols:_col0
- Select Operator [SEL_32] (rows=500 width=178)
+ Select Operator [SEL_27] (rows=500 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_31] (rows=500 width=178)
+ Filter Operator [FIL_26] (rows=500 width=178)
predicate:key is not null
TableScan [TS_3] (rows=500 width=178)
default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Select Operator [SEL_35] (rows=500 width=269)
+ <-Select Operator [SEL_30] (rows=500 width=269)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_34] (rows=500 width=269)
+ Filter Operator [FIL_29] (rows=500 width=269)
predicate:key1 is not null
TableScan [TS_0] (rows=500 width=269)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"]
@@ -343,29 +343,29 @@ Stage-0
limit:20
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_41]
- Limit [LIM_40] (rows=20 width=447)
+ File Output Operator [FS_36]
+ Limit [LIM_35] (rows=20 width=447)
Number of rows:20
- Select Operator [SEL_39] (rows=791 width=447)
+ Select Operator [SEL_34] (rows=791 width=447)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_38]
- Top N Key Operator [TNK_37] (rows=791 width=447)
+ SHUFFLE [RS_33]
+ Top N Key Operator [TNK_32] (rows=791 width=447)
keys:_col0,top n:20
- Map Join Operator [MAPJOIN_36] (rows=791 width=447)
- BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ Map Join Operator [MAPJOIN_31] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_33]
+ MULTICAST [RS_28]
PartitionCols:_col0
- Select Operator [SEL_32] (rows=500 width=178)
+ Select Operator [SEL_27] (rows=500 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_31] (rows=500 width=178)
+ Filter Operator [FIL_26] (rows=500 width=178)
predicate:((key <> '0') and (key <> '100'))
TableScan [TS_3] (rows=500 width=178)
default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Select Operator [SEL_35] (rows=500 width=269)
+ <-Select Operator [SEL_30] (rows=500 width=269)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_34] (rows=500 width=269)
+ Filter Operator [FIL_29] (rows=500 width=269)
predicate:((key1 <> '0') and (key1 <> '100'))
TableScan [TS_0] (rows=500 width=269)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"]
@@ -443,29 +443,29 @@ Stage-0
limit:20
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_41]
- Limit [LIM_40] (rows=20 width=447)
+ File Output Operator [FS_36]
+ Limit [LIM_35] (rows=20 width=447)
Number of rows:20
- Select Operator [SEL_39] (rows=791 width=447)
+ Select Operator [SEL_34] (rows=791 width=447)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_38]
- Top N Key Operator [TNK_37] (rows=791 width=447)
+ SHUFFLE [RS_33]
+ Top N Key Operator [TNK_32] (rows=791 width=447)
keys:_col0,top n:20
- Map Join Operator [MAPJOIN_36] (rows=791 width=447)
- BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ Map Join Operator [MAPJOIN_31] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_30._col0=RS_28._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_33]
+ MULTICAST [RS_28]
PartitionCols:_col0
- Select Operator [SEL_32] (rows=500 width=178)
+ Select Operator [SEL_27] (rows=500 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_31] (rows=500 width=178)
+ Filter Operator [FIL_26] (rows=500 width=178)
predicate:key is not null
TableScan [TS_3] (rows=500 width=178)
default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Select Operator [SEL_35] (rows=500 width=269)
+ <-Select Operator [SEL_30] (rows=500 width=269)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_34] (rows=500 width=269)
+ Filter Operator [FIL_29] (rows=500 width=269)
predicate:((key2 <> 'val_0') and (key2 <> 'val_100') and key1 is not null)
TableScan [TS_0] (rows=500 width=269)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"]
@@ -541,29 +541,29 @@ Stage-0
limit:20
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_41]
- Limit [LIM_40] (rows=20 width=447)
+ File Output Operator [FS_36]
+ Limit [LIM_35] (rows=20 width=447)
Number of rows:20
- Select Operator [SEL_39] (rows=814 width=447)
+ Select Operator [SEL_34] (rows=814 width=447)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_38]
- Top N Key Operator [TNK_37] (rows=814 width=447)
+ SHUFFLE [RS_33]
+ Top N Key Operator [TNK_32] (rows=814 width=447)
keys:_col0,top n:20
- Map Join Operator [MAPJOIN_36] (rows=814 width=447)
- Conds:SEL_35._col1=RS_33._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ Map Join Operator [MAPJOIN_31] (rows=814 width=447)
+ Conds:SEL_30._col1=RS_28._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 3 [BROADCAST_EDGE] vectorized, llap
- BROADCAST [RS_33]
+ BROADCAST [RS_28]
PartitionCols:_col1
- Select Operator [SEL_32] (rows=500 width=178)
+ Select Operator [SEL_27] (rows=500 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_31] (rows=500 width=178)
+ Filter Operator [FIL_26] (rows=500 width=178)
predicate:value is not null
TableScan [TS_3] (rows=500 width=178)
default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Select Operator [SEL_35] (rows=500 width=269)
+ <-Select Operator [SEL_30] (rows=500 width=269)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_34] (rows=500 width=269)
+ Filter Operator [FIL_29] (rows=500 width=269)
predicate:key2 is not null
TableScan [TS_0] (rows=500 width=269)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"]
@@ -637,29 +637,29 @@ Stage-0
limit:20
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_66]
- Limit [LIM_65] (rows=20 width=447)
+ File Output Operator [FS_51]
+ Limit [LIM_50] (rows=20 width=447)
Number of rows:20
- Select Operator [SEL_64] (rows=791 width=447)
+ Select Operator [SEL_49] (rows=791 width=447)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_63]
- Top N Key Operator [TNK_62] (rows=791 width=447)
+ SHUFFLE [RS_48]
+ Top N Key Operator [TNK_47] (rows=791 width=447)
keys:_col0,top n:20
- Map Join Operator [MAPJOIN_61] (rows=791 width=447)
- BucketMapJoin:true,Conds:SEL_60._col0, _col1, _col2=RS_58._col0, _col1, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ Map Join Operator [MAPJOIN_46] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_45._col0, _col1, _col2=RS_43._col0, _col1, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_58]
+ MULTICAST [RS_43]
PartitionCols:_col0, _col1
- Select Operator [SEL_57] (rows=500 width=178)
+ Select Operator [SEL_42] (rows=500 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_56] (rows=500 width=178)
+ Filter Operator [FIL_41] (rows=500 width=178)
predicate:(key is not null and value is not null)
TableScan [TS_3] (rows=500 width=178)
default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Select Operator [SEL_60] (rows=500 width=269)
+ <-Select Operator [SEL_45] (rows=500 width=269)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_59] (rows=500 width=269)
+ Filter Operator [FIL_44] (rows=500 width=269)
predicate:(key1 is not null and key2 is not null and value is not null)
TableScan [TS_0] (rows=500 width=269)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"]
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out
index e7a677f0628..148f2d89ee2 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out
@@ -147,25 +147,25 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_37]
- Select Operator [SEL_36] (rows=16 width=8)
+ File Output Operator [FS_32]
+ Select Operator [SEL_31] (rows=16 width=8)
Output:["_col0","_col1"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_35]
- Map Join Operator [MAPJOIN_34] (rows=16 width=8)
- BucketMapJoin:true,Conds:SEL_33._col0=RS_31._col0(Inner),Output:["_col0","_col1"]
+ SHUFFLE [RS_30]
+ Map Join Operator [MAPJOIN_29] (rows=16 width=8)
+ BucketMapJoin:true,Conds:SEL_28._col0=RS_26._col0(Inner),Output:["_col0","_col1"]
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_31]
+ MULTICAST [RS_26]
PartitionCols:_col0
- Select Operator [SEL_30] (rows=4 width=4)
+ Select Operator [SEL_25] (rows=4 width=4)
Output:["_col0"]
- Filter Operator [FIL_29] (rows=4 width=4)
+ Filter Operator [FIL_24] (rows=4 width=4)
predicate:key1 is not null
TableScan [TS_3] (rows=6 width=3)
default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1"]
- <-Select Operator [SEL_33] (rows=16 width=8)
+ <-Select Operator [SEL_28] (rows=16 width=8)
Output:["_col0","_col1"]
- Filter Operator [FIL_32] (rows=16 width=8)
+ Filter Operator [FIL_27] (rows=16 width=8)
predicate:key1 is not null
TableScan [TS_0] (rows=16 width=8)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","id"]
@@ -231,25 +231,25 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_37]
- Select Operator [SEL_36] (rows=16 width=92)
+ File Output Operator [FS_32]
+ Select Operator [SEL_31] (rows=16 width=92)
Output:["_col0","_col1"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_35]
- Map Join Operator [MAPJOIN_34] (rows=16 width=92)
- BucketMapJoin:true,Conds:SEL_33._col0=RS_31._col0(Inner),Output:["_col0","_col1"]
+ SHUFFLE [RS_30]
+ Map Join Operator [MAPJOIN_29] (rows=16 width=92)
+ BucketMapJoin:true,Conds:SEL_28._col0=RS_26._col0(Inner),Output:["_col0","_col1"]
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_31]
+ MULTICAST [RS_26]
PartitionCols:_col0
- Select Operator [SEL_30] (rows=4 width=87)
+ Select Operator [SEL_25] (rows=4 width=87)
Output:["_col0"]
- Filter Operator [FIL_29] (rows=4 width=87)
+ Filter Operator [FIL_24] (rows=4 width=87)
predicate:key2 is not null
TableScan [TS_3] (rows=6 width=72)
default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key2"]
- <-Select Operator [SEL_33] (rows=16 width=92)
+ <-Select Operator [SEL_28] (rows=16 width=92)
Output:["_col0","_col1"]
- Filter Operator [FIL_32] (rows=16 width=92)
+ Filter Operator [FIL_27] (rows=16 width=92)
predicate:key2 is not null
TableScan [TS_0] (rows=16 width=92)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key2"],Output:["key2","id"]
@@ -399,25 +399,25 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_57]
- Select Operator [SEL_56] (rows=12 width=96)
+ File Output Operator [FS_42]
+ Select Operator [SEL_41] (rows=12 width=96)
Output:["_col0","_col1","_col2"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_55]
- Map Join Operator [MAPJOIN_54] (rows=12 width=96)
- BucketMapJoin:true,Conds:SEL_53._col0, _col1=RS_51._col0, _col1(Inner),Output:["_col0","_col1","_col2"]
+ SHUFFLE [RS_40]
+ Map Join Operator [MAPJOIN_39] (rows=12 width=96)
+ BucketMapJoin:true,Conds:SEL_38._col0, _col1=RS_36._col0, _col1(Inner),Output:["_col0","_col1","_col2"]
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_51]
+ MULTICAST [RS_36]
PartitionCols:_col0, _col1
- Select Operator [SEL_50] (rows=3 width=91)
+ Select Operator [SEL_35] (rows=3 width=91)
Output:["_col0","_col1"]
- Filter Operator [FIL_49] (rows=3 width=91)
+ Filter Operator [FIL_34] (rows=3 width=91)
predicate:(key1 is not null and key2 is not null)
TableScan [TS_3] (rows=6 width=75)
default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2"]
- <-Select Operator [SEL_53] (rows=12 width=96)
+ <-Select Operator [SEL_38] (rows=12 width=96)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_52] (rows=12 width=96)
+ Filter Operator [FIL_37] (rows=12 width=96)
predicate:(key1 is not null and key2 is not null)
TableScan [TS_0] (rows=12 width=96)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","id"]
@@ -479,25 +479,25 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_57]
- Select Operator [SEL_56] (rows=9 width=78)
+ File Output Operator [FS_47]
+ Select Operator [SEL_46] (rows=9 width=78)
Output:["_col0","_col1","_col2"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_55]
- Map Join Operator [MAPJOIN_54] (rows=9 width=78)
- BucketMapJoin:true,Conds:SEL_53._col0, _col1=RS_51._col0, _col1(Inner),Output:["_col0","_col1","_col2"]
+ SHUFFLE [RS_45]
+ Map Join Operator [MAPJOIN_44] (rows=9 width=78)
+ BucketMapJoin:true,Conds:SEL_43._col0, _col1=RS_41._col0, _col1(Inner),Output:["_col0","_col1","_col2"]
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_51]
+ MULTICAST [RS_41]
PartitionCols:_col0
- Select Operator [SEL_50] (rows=3 width=93)
+ Select Operator [SEL_40] (rows=3 width=93)
Output:["_col0","_col1"]
- Filter Operator [FIL_49] (rows=3 width=93)
+ Filter Operator [FIL_39] (rows=3 width=93)
predicate:(key1 is not null and value is not null)
TableScan [TS_3] (rows=6 width=77)
default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","value"]
- <-Select Operator [SEL_53] (rows=12 width=83)
+ <-Select Operator [SEL_43] (rows=12 width=83)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_52] (rows=12 width=83)
+ Filter Operator [FIL_42] (rows=12 width=83)
predicate:(value is not null and key1 is not null)
TableScan [TS_0] (rows=16 width=81)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","value","id"]
@@ -559,25 +559,25 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized, llap
- File Output Operator [FS_67]
- Select Operator [SEL_66] (rows=5 width=150)
+ File Output Operator [FS_52]
+ Select Operator [SEL_51] (rows=5 width=150)
Output:["_col0","_col1","_col2","_col3"]
<-Map 1 [SIMPLE_EDGE] vectorized, llap
- SHUFFLE [RS_65]
- Map Join Operator [MAPJOIN_64] (rows=5 width=150)
- BucketMapJoin:true,Conds:SEL_63._col0, _col1, _col2=RS_61._col0, _col1, _col2(Inner),Output:["_col0","_col1","_col2","_col3"]
+ SHUFFLE [RS_50]
+ Map Join Operator [MAPJOIN_49] (rows=5 width=150)
+ BucketMapJoin:true,Conds:SEL_48._col0, _col1, _col2=RS_46._col0, _col1, _col2(Inner),Output:["_col0","_col1","_col2","_col3"]
<-Map 3 [CUSTOM_EDGE] vectorized, llap
- MULTICAST [RS_61]
+ MULTICAST [RS_46]
PartitionCols:_col0, _col1
- Select Operator [SEL_60] (rows=2 width=180)
+ Select Operator [SEL_45] (rows=2 width=180)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_59] (rows=2 width=180)
+ Filter Operator [FIL_44] (rows=2 width=180)
predicate:(key1 is not null and key2 is not null and value is not null)
TableScan [TS_3] (rows=6 width=150)
default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"]
- <-Select Operator [SEL_63] (rows=8 width=164)
+ <-Select Operator [SEL_48] (rows=8 width=164)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_62] (rows=8 width=164)
+ Filter Operator [FIL_47] (rows=8 width=164)
predicate:(value is not null and key1 is not null and key2 is not null)
TableScan [TS_0] (rows=12 width=164)
default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:32,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value","id"]
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out
index 3636b845c56..26a5363a943 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_create_locally_zordered_table.q.out
@@ -636,7 +636,6 @@ POSTHOOK: Input: default@zorder_tsdl_test
# col_name data_type comment
ts timestamp
dd double
-ll int
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
index 6a28128817c..4a858d03531 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out
@@ -269,9 +269,7 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
team_id bigint
-company_id bigint
# Partition Information
# col_name data_type comment
@@ -865,9 +863,7 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
team_id bigint
-company_id bigint
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out
index a541f9350bf..ceaa9ee22cb 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out
@@ -138,7 +138,6 @@ POSTHOOK: Input: ice_comp@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
# Partition Information
# col_name data_type comment
@@ -224,7 +223,6 @@ POSTHOOK: Input: ice_comp@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out
index 835a2af5e57..c5ab2c7aa92 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out
@@ -130,7 +130,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
team_id bigint
company_id bigint
@@ -279,7 +278,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
team_id bigint
company_id bigint
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
index cf41867c1d2..cc3f6ec1f66 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out
@@ -225,8 +225,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_orc
# col_name data_type comment
event_id int
-event_time timestamp with local time zone
-event_src string
# Partition Information
# col_name data_type comment
@@ -335,8 +333,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_orc
# col_name data_type comment
event_id int
-event_time timestamp with local time zone
-event_src string
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out
index d391b9ebd3d..a9018e08418 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out
@@ -176,9 +176,7 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
team_id bigint
-company_id bigint
# Partition Information
# col_name data_type comment
@@ -288,9 +286,7 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
team_id bigint
-company_id bigint
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out
index 435a56d2329..b8583a68f88 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out
@@ -173,7 +173,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
# Partition Information
# col_name data_type comment
@@ -275,7 +274,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
# Partition Information
# col_name data_type comment
@@ -512,7 +510,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
# Partition Information
# col_name data_type comment
@@ -618,7 +615,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out
index 365e446af9b..a7edef40fc3 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out
@@ -208,7 +208,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
fname string
last_name string
-dept_id bigint
address string
# Partition Information
@@ -311,7 +310,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
fname string
last_name string
-dept_id bigint
address string
# Partition Information
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out
index 5efe2d0f536..d3721c029a1 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out
@@ -177,9 +177,6 @@ POSTHOOK: Input: default@ice_orc_wo_evo
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
-city string
-registration_date date
# Partition Information
# col_name data_type comment
@@ -288,9 +285,6 @@ POSTHOOK: Input: default@ice_orc_wo_evo
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
-city string
-registration_date date
# Partition Information
# col_name data_type comment
@@ -406,9 +400,6 @@ POSTHOOK: Input: default@ice_orc_wo_evo
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
-city string
-registration_date date
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out
index 8b9310b71a3..8bdf21bce32 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out
@@ -158,9 +158,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-registration_date date
-dept_id bigint
-city string
# Partition Information
# col_name data_type comment
@@ -274,9 +271,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-registration_date date
-dept_id bigint
-city string
# Partition Information
# col_name data_type comment
@@ -390,9 +384,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-registration_date date
-dept_id bigint
-city string
# Partition Information
# col_name data_type comment
@@ -505,9 +496,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-registration_date date
-dept_id bigint
-city string
# Partition Information
# col_name data_type comment
@@ -620,9 +608,6 @@ POSTHOOK: Input: default@ice_orc
# col_name data_type comment
first_name string
last_name string
-registration_date date
-dept_id bigint
-city string
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out
index 0eb2ba637a4..f6619549e26 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out
@@ -106,7 +106,6 @@ POSTHOOK: query: describe formatted ice_orc
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_orc
# col_name data_type comment
-a string
b bigint
# Partition Information
@@ -199,7 +198,6 @@ POSTHOOK: query: describe formatted ice_orc
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_orc
# col_name data_type comment
-a string
b bigint
# Partition Information
@@ -306,7 +304,6 @@ POSTHOOK: query: describe formatted ice_orc
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_orc
# col_name data_type comment
-a string
b bigint
# Partition Information
@@ -413,7 +410,6 @@ POSTHOOK: query: describe formatted ice_orc
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_orc
# col_name data_type comment
-a string
b bigint
# Partition Information
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out
index a177af309c6..fd318d589ff 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out
@@ -65,7 +65,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@srcbucket_big
# col_name data_type comment
id string
-key int
value string
# Partition Information
@@ -182,7 +181,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@srcbucket_big
# col_name data_type comment
id string
-key int
value string
# Partition Information
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out
index c372ea96c22..97c2dff7c39 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out
@@ -94,7 +94,6 @@ POSTHOOK: Input: ice_comp@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
# Partition Information
# col_name data_type comment
@@ -176,7 +175,6 @@ POSTHOOK: Input: ice_comp@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
# Partition Information
# col_name data_type comment
@@ -284,7 +282,6 @@ POSTHOOK: Input: ice_comp@ice_orc
# col_name data_type comment
first_name string
last_name string
-dept_id bigint
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out
index 5d197263b37..37bff732939 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out
@@ -876,10 +876,10 @@ PREHOOK: Input: default@tbl_ice_mixed_parted
POSTHOOK: query: describe tbl_ice_mixed_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_mixed_parted
-p1 string
b string
a int
-p2 string
+p1 string Transform: identity
+p2 string Transform: identity
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out
index f1a01721563..c215c4005a4 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out
@@ -605,10 +605,10 @@ PREHOOK: Input: default@tbl_ice_orc_parted
POSTHOOK: query: describe tbl_ice_orc_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_orc_parted
-p1 string
b string
a int
-p2 string
+p1 string Transform: identity
+p2 string Transform: identity
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
index 3b0d1940c1b..b2ea3e67b86 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
@@ -508,10 +508,10 @@ PREHOOK: Input: default@tbl_ice_parquet_parted
POSTHOOK: query: describe tbl_ice_parquet_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_parquet_parted
-p1 string
b string
a int
-p2 string
+p1 string Transform: identity
+p2 string Transform: identity
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out
index e58b7ffd846..b7a399fcbb4 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out
@@ -48,7 +48,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@mat1
# col_name data_type comment
c int
-b string
# Partition Information
# col_name data_type comment
@@ -139,7 +138,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@mat2
# col_name data_type comment
c int
-b string
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out
index c9b3f1af8d8..26706b57f40 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out
@@ -47,8 +47,6 @@ POSTHOOK: query: describe formatted mat1
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@mat1
# col_name data_type comment
-b string
-c int
# Partition Information
# col_name data_type comment
@@ -140,8 +138,6 @@ POSTHOOK: query: describe formatted mat2
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@mat2
# col_name data_type comment
-b string
-c int
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out
index e46dfab9914..24c3d72f7d2 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out
@@ -82,8 +82,6 @@ orderid int
quantity int
itemid int
tradets timestamp
-p1 string
-p2 string
# Partition Information
# col_name data_type comment
@@ -173,8 +171,6 @@ orderid int
quantity int
itemid int
tradets timestamp
-p1 string
-p2 string
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out b/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out
index 24fff78df88..c3c309b656a 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out
@@ -37,11 +37,6 @@ equality_delete_record_count bigint Count of records in equality d
equality_delete_file_count int Count of equality delete files
last_updated_at timestamp with local time zone Commit time of snapshot that last updated this partition
last_updated_snapshot_id bigint Id of snapshot that last updated this partition
-
-# Partition Information
-# col_name data_type comment
-d_part int Transform: identity
-e_part int Transform: identity
PREHOOK: query: select * from default.ice1.partitions
PREHOOK: type: QUERY
PREHOOK: Input: default@ice1
diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
index f28166ec3b6..62b89a109d7 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
@@ -79,7 +79,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@test_truncate
# col_name data_type comment
a int
-b string
# Partition Information
# col_name data_type comment
@@ -186,7 +185,6 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@test_truncate
# col_name data_type comment
a int
-b string
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out
index d0ba154e146..7dac902dc9a 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_copy_on_write_partitioned.q.out
@@ -287,14 +287,14 @@ POSTHOOK: query: insert into tbl_ice values (444, 'hola', 800), (555, 'schola',
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice
-Warning: Shuffle Join MERGEJOIN[284][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 23' is a cross product
-Warning: Shuffle Join MERGEJOIN[293][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 4' is a cross product
-Warning: Shuffle Join MERGEJOIN[287][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 14' is a cross product
-Warning: Shuffle Join MERGEJOIN[289][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 16' is a cross product
-Warning: Shuffle Join MERGEJOIN[281][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 25' is a cross product
-Warning: Shuffle Join MERGEJOIN[291][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 11' is a cross product
-Warning: Shuffle Join MERGEJOIN[282][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 24' is a cross product
-Warning: Shuffle Join MERGEJOIN[295][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 18' is a cross product
+Warning: Shuffle Join MERGEJOIN[279][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 22' is a cross product
+Warning: Shuffle Join MERGEJOIN[288][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 4' is a cross product
+Warning: Shuffle Join MERGEJOIN[282][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 14' is a cross product
+Warning: Shuffle Join MERGEJOIN[284][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 16' is a cross product
+Warning: Shuffle Join MERGEJOIN[276][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 24' is a cross product
+Warning: Shuffle Join MERGEJOIN[286][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 11' is a cross product
+Warning: Shuffle Join MERGEJOIN[277][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 23' is a cross product
+Warning: Shuffle Join MERGEJOIN[290][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 18' is a cross product
PREHOOK: query: explain update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
@@ -314,27 +314,27 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Reducer 2 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE)
+ Reducer 10 <- Reducer 2 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE)
Reducer 11 <- Reducer 10 (XPROD_EDGE), Reducer 27 (XPROD_EDGE)
Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE), Union 8 (CONTAINS)
Reducer 13 <- Map 26 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
- Reducer 14 <- Reducer 13 (XPROD_EDGE), Reducer 21 (XPROD_EDGE)
+ Reducer 14 <- Reducer 13 (XPROD_EDGE), Reducer 25 (XPROD_EDGE)
Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE)
Reducer 16 <- Reducer 15 (XPROD_EDGE), Reducer 29 (XPROD_EDGE)
- Reducer 17 <- Reducer 2 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE)
+ Reducer 17 <- Reducer 2 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE)
Reducer 18 <- Reducer 17 (XPROD_EDGE), Reducer 27 (XPROD_EDGE)
Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE)
Reducer 20 <- Reducer 19 (SIMPLE_EDGE), Union 8 (CONTAINS)
Reducer 21 <- Map 1 (CUSTOM_SIMPLE_EDGE)
- Reducer 22 <- Map 1 (CUSTOM_SIMPLE_EDGE)
- Reducer 23 <- Map 26 (XPROD_EDGE), Reducer 22 (XPROD_EDGE)
- Reducer 24 <- Map 26 (XPROD_EDGE), Reducer 22 (XPROD_EDGE)
- Reducer 25 <- Map 26 (XPROD_EDGE), Reducer 22 (XPROD_EDGE)
+ Reducer 22 <- Map 26 (XPROD_EDGE), Reducer 21 (XPROD_EDGE)
+ Reducer 23 <- Map 26 (XPROD_EDGE), Reducer 21 (XPROD_EDGE)
+ Reducer 24 <- Map 26 (XPROD_EDGE), Reducer 21 (XPROD_EDGE)
+ Reducer 25 <- Map 1 (CUSTOM_SIMPLE_EDGE)
Reducer 27 <- Map 26 (CUSTOM_SIMPLE_EDGE)
Reducer 28 <- Map 26 (SIMPLE_EDGE)
Reducer 29 <- Map 26 (CUSTOM_SIMPLE_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE)
Reducer 4 <- Reducer 27 (XPROD_EDGE), Reducer 3 (XPROD_EDGE)
Reducer 5 <- Reducer 28 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
@@ -351,33 +351,18 @@ STAGE PLANS:
Filter Operator
predicate: (a <= 5) (type: boolean)
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: a (type: int)
- outputColumnNames: a
+ Group By Operator
+ keys: a (type: int)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: a (type: int)
- minReductionHashAggr: 0.99
- mode: hash
- outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(), count(a)
- minReductionHashAggr: 0.99
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- null sort order:
- sort order:
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint), _col1 (type: bigint)
Select Operator
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -391,6 +376,17 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
+ Group By Operator
+ aggregations: count(), count(a)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
Execution mode: vectorized
Map 26
Map Operator Tree:
@@ -746,19 +742,6 @@ STAGE PLANS:
Statistics: Num rows: 19 Data size: 11116 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int)
Reducer 21
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0), count(VALUE._col1)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- null sort order:
- sort order:
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint), _col1 (type: bigint)
- Reducer 22
Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
@@ -781,7 +764,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
- Reducer 23
+ Reducer 22
Reduce Operator Tree:
Merge Join Operator
condition map:
@@ -798,7 +781,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 9 Data size: 1845 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: bigint)
- Reducer 24
+ Reducer 23
Reduce Operator Tree:
Merge Join Operator
condition map:
@@ -815,7 +798,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 9 Data size: 4428 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: string), _col7 (type: bigint)
- Reducer 25
+ Reducer 24
Reduce Operator Tree:
Merge Join Operator
condition map:
@@ -832,6 +815,19 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 9 Data size: 3681 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col7 (type: bigint)
+ Reducer 25
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
Reducer 27
Execution mode: vectorized
Reduce Operator Tree:
@@ -1061,14 +1057,14 @@ STAGE PLANS:
Stats Work
Basic Stats Work:
-Warning: Shuffle Join MERGEJOIN[284][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 23' is a cross product
-Warning: Shuffle Join MERGEJOIN[293][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 4' is a cross product
-Warning: Shuffle Join MERGEJOIN[287][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 14' is a cross product
-Warning: Shuffle Join MERGEJOIN[289][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 16' is a cross product
-Warning: Shuffle Join MERGEJOIN[281][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 25' is a cross product
-Warning: Shuffle Join MERGEJOIN[291][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 11' is a cross product
-Warning: Shuffle Join MERGEJOIN[282][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 24' is a cross product
-Warning: Shuffle Join MERGEJOIN[295][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 18' is a cross product
+Warning: Shuffle Join MERGEJOIN[279][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 22' is a cross product
+Warning: Shuffle Join MERGEJOIN[288][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 4' is a cross product
+Warning: Shuffle Join MERGEJOIN[282][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 14' is a cross product
+Warning: Shuffle Join MERGEJOIN[284][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 16' is a cross product
+Warning: Shuffle Join MERGEJOIN[276][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 24' is a cross product
+Warning: Shuffle Join MERGEJOIN[286][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 11' is a cross product
+Warning: Shuffle Join MERGEJOIN[277][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 23' is a cross product
+Warning: Shuffle Join MERGEJOIN[290][tables = [$hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 18' is a cross product
PREHOOK: query: update tbl_ice set b='Changed again' where a in (select a from tbl_ice where a <= 5) or c in (select c from tbl_ice where c > 800)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
@@ -1116,7 +1112,7 @@ POSTHOOK: query: insert into tbl_ice_other values (10, 'ten'), (333, 'hundred')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice_other
-Warning: Shuffle Join MERGEJOIN[269][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
+Warning: Shuffle Join MERGEJOIN[217][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 8' is a cross product
PREHOOK: query: explain update tbl_ice set b='Changed forever' where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
@@ -1138,21 +1134,41 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Map 1 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE)
- Reducer 11 <- Reducer 10 (SIMPLE_EDGE)
- Reducer 12 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE)
- Reducer 13 <- Reducer 12 (SIMPLE_EDGE)
- Reducer 14 <- Reducer 10 (CUSTOM_SIMPLE_EDGE)
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE)
- Reducer 3 <- Reducer 14 (XPROD_EDGE), Reducer 2 (XPROD_EDGE)
- Reducer 4 <- Reducer 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS)
- Reducer 6 <- Union 5 (SIMPLE_EDGE)
- Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Union 5 (CONTAINS)
- Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE)
- Reducer 9 <- Reducer 8 (SIMPLE_EDGE), Union 5 (CONTAINS)
+ Reducer 10 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+ Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+ Reducer 12 <- Reducer 11 (SIMPLE_EDGE)
+ Reducer 13 <- Map 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+ Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Union 4 (CONTAINS)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE)
+ Reducer 3 <- Map 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE), Union 4 (CONTAINS)
+ Reducer 5 <- Union 4 (SIMPLE_EDGE)
+ Reducer 6 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 7 <- Map 15 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
+ Reducer 8 <- Reducer 10 (XPROD_EDGE), Reducer 7 (XPROD_EDGE)
+ Reducer 9 <- Reducer 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE), Union 4 (CONTAINS)
#### A masked pattern was here ####
Vertices:
Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t2
+ filterExpr: a is not null (type: boolean)
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: a is not null (type: boolean)
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: a (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: vectorized
+ Map 15
Map Operator Tree:
TableScan
alias: tbl_ice
@@ -1221,97 +1237,20 @@ STAGE PLANS:
Statistics: Num rows: 9 Data size: 36 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
Execution mode: vectorized
- Map 15
- Map Operator Tree:
- TableScan
- alias: t2
- filterExpr: a is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: a is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: a (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Execution mode: vectorized
Reducer 10
- Reduce Operator Tree:
- Merge Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: _col0 (type: int)
- minReductionHashAggr: 0.99
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(), count(_col0)
- minReductionHashAggr: 0.99
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- null sort order:
- sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint), _col1 (type: bigint)
- Reducer 11
Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
- keys: KEY._col0 (type: int)
+ aggregations: count(VALUE._col0), count(VALUE._col1)
mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), true (type: boolean)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: boolean)
- Reducer 12
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Reducer 11
Reduce Operator Tree:
Merge Join Operator
condition map:
@@ -1327,7 +1266,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
- Reducer 13
+ Reducer 12
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
@@ -1373,20 +1312,182 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE
+ Reducer 13
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col5 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col5 (type: string)
+ Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string)
Reducer 14
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col5 ASC NULLS FIRST
+ partition by: _col5
+ raw input shape:
+ window functions:
+ window function definition
+ alias: row_number_window_0
+ name: row_number
+ window function: GenericUDAFRowNumberEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (row_number_window_0 = 1) (type: boolean)
+ Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string)
+ null sort order: zz
+ sort order: ++
+ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string)
+ Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int)
+ Reducer 2
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(), count(_col0)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Group By Operator
+ keys: _col0 (type: int)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 9 Data size: 39 Basic stats: COMPLETE Column stats: NONE
+ Reducer 3
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string)
+ null sort order: zz
+ sort order: ++
+ Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string)
+ Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int)
+ Reducer 5
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3)
+ File Output Operator
+ compressed: false
+ Dp Sort State: PARTITION_SORTED
+ Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.tbl_ice
+ Reducer 6
Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0), count(VALUE._col1)
+ keys: KEY._col0 (type: int)
mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- null sort order:
- sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint), _col1 (type: bigint)
- Reducer 2
+ outputColumnNames: _col0
+ Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), true (type: boolean)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: boolean)
+ Reducer 7
Reduce Operator Tree:
Merge Join Operator
condition map:
@@ -1401,7 +1502,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: string), _col6 (type: bigint), _col7 (type: string), _col9 (type: boolean)
- Reducer 3
+ Reducer 8
Reduce Operator Tree:
Merge Join Operator
condition map:
@@ -1429,7 +1530,7 @@ STAGE PLANS:
Map-reduce partition columns: _col5 (type: string)
Statistics: Num rows: 9 Data size: 2053 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: bigint), _col7 (type: string)
- Reducer 4
+ Reducer 9
Reduce Operator Tree:
Merge Join Operator
condition map:
@@ -1450,101 +1551,8 @@ STAGE PLANS:
Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string)
Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int)
- Reducer 6
- Execution mode: vectorized
- Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: int), VALUE._col6 (type: string), VALUE._col7 (type: int), KEY.iceberg_bucket(_col5, 16) (type: int), KEY.iceberg_truncate(_col6, 3) (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, iceberg_bucket(_col5, 16), iceberg_truncate(_col6, 3)
- File Output Operator
- compressed: false
- Dp Sort State: PARTITION_SORTED
- Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
- output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
- serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
- name: default.tbl_ice
- Reducer 7
- Reduce Operator Tree:
- Merge Join Operator
- condition map:
- Left Semi Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col2 (type: int), _col3 (type: bigint), _col4 (type: string), _col5 (type: bigint), _col6 (type: string), _col0 (type: int), 'Changed forever' (type: string), _col1 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 9 Data size: 79 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string)
- null sort order: zz
- sort order: ++
- Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string)
- Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int)
- Reducer 8
- Reduce Operator Tree:
- Merge Join Operator
- condition map:
- Left Semi Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col5 (type: string)
- null sort order: a
- sort order: +
- Map-reduce partition columns: _col5 (type: string)
- Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col6 (type: string)
- Reducer 9
- Execution mode: vectorized
- Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: bigint), KEY.reducesinkkey0 (type: string), VALUE._col5 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: _col5 ASC NULLS FIRST
- partition by: _col5
- raw input shape:
- window functions:
- window function definition
- alias: row_number_window_0
- name: row_number
- window function: GenericUDAFRowNumberEvaluator
- window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
- isPivotResult: true
- Statistics: Num rows: 9 Data size: 1900 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (row_number_window_0 = 1) (type: boolean)
- Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col3 (type: int), _col4 (type: bigint), _col5 (type: string), -1L (type: bigint), _col6 (type: string), _col0 (type: int), _col1 (type: string), _col2 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 4 Data size: 844 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string)
- null sort order: zz
- sort order: ++
- Map-reduce partition columns: iceberg_bucket(_col5, 16) (type: int), iceberg_truncate(_col6, 3) (type: string)
- Statistics: Num rows: 22 Data size: 3181 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int)
- Union 5
- Vertex: Union 5
+ Union 4
+ Vertex: Union 4
Stage: Stage-2
Dependency Collection
@@ -1563,7 +1571,7 @@ STAGE PLANS:
Stats Work
Basic Stats Work:
-Warning: Shuffle Join MERGEJOIN[269][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
+Warning: Shuffle Join MERGEJOIN[217][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 8' is a cross product
PREHOOK: query: update tbl_ice set b='Changed forever' where a in (select t1.a from tbl_ice t1 join tbl_ice_other t2 on t1.a = t2.a)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
@@ -1615,7 +1623,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_standard_other
POSTHOOK: Lineage: tbl_standard_other.a SCRIPT []
POSTHOOK: Lineage: tbl_standard_other.b SCRIPT []
-Warning: Shuffle Join MERGEJOIN[224][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 8' is a cross product
+Warning: Shuffle Join MERGEJOIN[189][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 8' is a cross product
PREHOOK: query: explain update tbl_ice set b='The last one' where a in (select t1.a from tbl_ice t1 join tbl_standard_other t2 on t1.a = t2.a)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
@@ -2074,7 +2082,7 @@ STAGE PLANS:
Stats Work
Basic Stats Work:
-Warning: Shuffle Join MERGEJOIN[224][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 8' is a cross product
+Warning: Shuffle Join MERGEJOIN[189][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 8' is a cross product
PREHOOK: query: update tbl_ice set b='The last one' where a in (select t1.a from tbl_ice t1 join tbl_standard_other t2 on t1.a = t2.a)
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_ice
diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out
index 1f837c4b586..ccf73acb866 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_avro.q.out
@@ -186,3 +186,46 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
333 Changed forever 56
444 The last one 800
555 Changed again 801
+PREHOOK: query: drop table if exists tbl_ice_all_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: drop table if exists tbl_ice_all_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: create external table tbl_ice_all_part(a int, b string) partitioned by spec (a, b) stored by iceberg stored as avro tblproperties ('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: query: create external table tbl_ice_all_part(a int, b string) partitioned by spec (a, b) stored by iceberg stored as avro tblproperties ('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tbl_ice_all_part
+PREHOOK: query: insert into tbl_ice_all_part values (1, 'one'), (2, 'two'), (3, 'three')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: query: insert into tbl_ice_all_part values (1, 'one'), (2, 'two'), (3, 'three')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tbl_ice_all_part
+PREHOOK: query: update tbl_ice_all_part set b='updated' where a = 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_all_part
+PREHOOK: Output: default@tbl_ice_all_part
+PREHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: query: update tbl_ice_all_part set b='updated' where a = 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_all_part
+POSTHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: Output: default@tbl_ice_all_part
+PREHOOK: query: select * from tbl_ice_all_part order by a, b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_all_part
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from tbl_ice_all_part order by a, b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_all_part
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 one
+2 updated
+3 three
diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out
index 84de34cb28b..66273d7216b 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_orc.q.out
@@ -186,3 +186,46 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
333 Changed forever 56
444 The last one 800
555 Changed again 801
+PREHOOK: query: drop table if exists tbl_ice_all_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: drop table if exists tbl_ice_all_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: create external table tbl_ice_all_part(a int, b string) partitioned by spec (a, b) stored by iceberg stored as orc tblproperties ('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: query: create external table tbl_ice_all_part(a int, b string) partitioned by spec (a, b) stored by iceberg stored as orc tblproperties ('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tbl_ice_all_part
+PREHOOK: query: insert into tbl_ice_all_part values (1, 'one'), (2, 'two'), (3, 'three')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: query: insert into tbl_ice_all_part values (1, 'one'), (2, 'two'), (3, 'three')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tbl_ice_all_part
+PREHOOK: query: update tbl_ice_all_part set b='updated' where a = 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_all_part
+PREHOOK: Output: default@tbl_ice_all_part
+PREHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: query: update tbl_ice_all_part set b='updated' where a = 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_all_part
+POSTHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: Output: default@tbl_ice_all_part
+PREHOOK: query: select * from tbl_ice_all_part order by a, b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_all_part
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from tbl_ice_all_part order by a, b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_all_part
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 one
+2 updated
+3 three
diff --git a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out
index ecc74a61c6c..3e6f66ad196 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/update_iceberg_partitioned_parquet.q.out
@@ -186,3 +186,46 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
333 Changed forever 56
444 The last one 800
555 Changed again 801
+PREHOOK: query: drop table if exists tbl_ice_all_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: drop table if exists tbl_ice_all_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: create external table tbl_ice_all_part(a int, b string) partitioned by spec (a, b) stored by iceberg tblproperties ('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: query: create external table tbl_ice_all_part(a int, b string) partitioned by spec (a, b) stored by iceberg tblproperties ('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tbl_ice_all_part
+PREHOOK: query: insert into tbl_ice_all_part values (1, 'one'), (2, 'two'), (3, 'three')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: query: insert into tbl_ice_all_part values (1, 'one'), (2, 'two'), (3, 'three')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tbl_ice_all_part
+PREHOOK: query: update tbl_ice_all_part set b='updated' where a = 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_all_part
+PREHOOK: Output: default@tbl_ice_all_part
+PREHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: query: update tbl_ice_all_part set b='updated' where a = 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_all_part
+POSTHOOK: Output: default@tbl_ice_all_part
+POSTHOOK: Output: default@tbl_ice_all_part
+PREHOOK: query: select * from tbl_ice_all_part order by a, b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice_all_part
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from tbl_ice_all_part order by a, b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice_all_part
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 one
+2 updated
+3 three
diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out
index 31676dd56a8..6965a3604c4 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out
@@ -899,66 +899,66 @@ Stage-6
Dependency Collection{}
Stage-4
Reducer 2 vectorized
- File Output Operator [FS_64]
+ File Output Operator [FS_59]
table:{"name:":"default.store_sales"}
- Select Operator [SEL_63]
+ Select Operator [SEL_58]
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col3","iceberg_bucket(_col2, 3)"]
<-Map 1 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_59]
+ SHUFFLE [RS_54]
PartitionCols:_col3, iceberg_bucket(_col2, 3)
- Select Operator [SEL_55] (rows=1 width=#Masked#)
+ Select Operator [SEL_50] (rows=1 width=#Masked#)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"]
- Filter Operator [FIL_51] (rows=1 width=#Masked#)
+ Filter Operator [FIL_46] (rows=1 width=#Masked#)
predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0) and _col33 is null)
- Select Operator [SEL_49] (rows=5 width=#Masked#)
+ Select Operator [SEL_44] (rows=5 width=#Masked#)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49"]
- Map Join Operator [MAPJOIN_48] (rows=5 width=#Masked#)
- BucketMapJoin:true,Conds:SEL_47._col2, _col1=RS_46._col8, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_c [...]
+ Map Join Operator [MAPJOIN_43] (rows=5 width=#Masked#)
+ BucketMapJoin:true,Conds:SEL_42._col2, _col1=RS_41._col8, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_c [...]
<-Map 5 [CUSTOM_EDGE] vectorized
- MULTICAST [RS_46]
+ MULTICAST [RS_41]
PartitionCols:_col7
- Select Operator [SEL_45] (rows=2 width=#Masked#)
+ Select Operator [SEL_40] (rows=2 width=#Masked#)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27"]
- Filter Operator [FIL_44] (rows=2 width=#Masked#)
+ Filter Operator [FIL_39] (rows=2 width=#Masked#)
predicate:((ss_sold_date_sk = 2451181) and ss_item_sk is not null and ss_customer_sk is not null)
TableScan [TS_2] (rows=2 width=#Masked#)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_sold_time_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"]
- <-Select Operator [SEL_47] (rows=5 width=#Masked#)
+ <-Select Operator [SEL_42] (rows=5 width=#Masked#)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"]
TableScan [TS_0] (rows=5 width=#Masked#)
default@ssv,s,Tbl:COMPLETE,Col:NONE,Grouping Num Buckets:3,Grouping Partition Columns:["ss_item_sk2"],Output:["ss_sold_time_sk","ss_item_sk2","ss_customer_sk2","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_in [...]
Reducer 3 vectorized
- File Output Operator [FS_66]
+ File Output Operator [FS_61]
table:{"name:":"default.store_sales"}
- Select Operator [SEL_65]
+ Select Operator [SEL_60]
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col3","iceberg_bucket(_col2, 3)"]
<-Map 1 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_60]
+ SHUFFLE [RS_55]
PartitionCols:_col3, iceberg_bucket(_col2, 3)
- Select Operator [SEL_56] (rows=1 width=#Masked#)
+ Select Operator [SEL_51] (rows=1 width=#Masked#)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"]
- Filter Operator [FIL_52] (rows=1 width=#Masked#)
+ Filter Operator [FIL_47] (rows=1 width=#Masked#)
predicate:(_col24 is null and _col47 is null and _col36 is null)
- Please refer to the previous Select Operator [SEL_49]
+ Please refer to the previous Select Operator [SEL_44]
Reducer 4 vectorized
- File Output Operator [FS_70]
+ File Output Operator [FS_65]
table:{"name:":"default.merge_tmp_table"}
- Select Operator [SEL_69] (rows=1 width=#Masked#)
+ Select Operator [SEL_64] (rows=1 width=#Masked#)
Output:["_col0"]
- Filter Operator [FIL_68] (rows=1 width=#Masked#)
+ Filter Operator [FIL_63] (rows=1 width=#Masked#)
predicate:(_col4 > 1L)
- Group By Operator [GBY_67] (rows=1 width=#Masked#)
+ Group By Operator [GBY_62] (rows=1 width=#Masked#)
Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3
<-Map 1 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_62]
+ SHUFFLE [RS_57]
PartitionCols:_col0, _col1, _col2, _col3
- Group By Operator [GBY_61] (rows=1 width=#Masked#)
+ Group By Operator [GBY_56] (rows=1 width=#Masked#)
Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["count()"],keys:_col4, _col40, _col6, _col45
- Select Operator [SEL_57] (rows=1 width=#Masked#)
+ Select Operator [SEL_52] (rows=1 width=#Masked#)
Output:["_col4","_col6","_col40","_col45"]
- Filter Operator [FIL_53] (rows=1 width=#Masked#)
+ Filter Operator [FIL_48] (rows=1 width=#Masked#)
predicate:((_col24 = _col34) and (_col47 = _col37) and (_col36 = 2451181) and (floor((_col34 / 1000)) * 1000) BETWEEN 1000 AND 2000 and (_col30 < 0))
- Please refer to the previous Select Operator [SEL_49]
+ Please refer to the previous Select Operator [SEL_44]
Stage-7
Stats Work{}
Stage-3
diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out
index e91e55d0d03..eeb9cbfe354 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out
@@ -757,10 +757,10 @@ PREHOOK: Input: default@tbl_ice_mixed_parted
POSTHOOK: query: describe tbl_ice_mixed_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_mixed_parted
-p1 string
b string
a int
-p2 string
+p1 string Transform: identity
+p2 string Transform: identity
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out
index fdf2679f5b2..0b7260d8420 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out
@@ -526,10 +526,10 @@ PREHOOK: Input: default@tbl_ice_orc_parted
POSTHOOK: query: describe tbl_ice_orc_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_orc_parted
-p1 string
b string
a int
-p2 string
+p1 string Transform: identity
+p2 string Transform: identity
# Partition Information
# col_name data_type comment
diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
index ba3f66f093c..b4a16f6690d 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
@@ -430,10 +430,10 @@ PREHOOK: Input: default@tbl_ice_parquet_parted
POSTHOOK: query: describe tbl_ice_parquet_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_parquet_parted
-p1 string
b string
a int
-p2 string
+p1 string Transform: identity
+p2 string Transform: identity
# Partition Information
# col_name data_type comment
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
index e74885e57a3..67e59155b67 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
@@ -31,7 +31,6 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
@@ -232,7 +231,7 @@ static VectorizedRowBatchCtx createFakeVrbCtx(MapWork mapWork) throws HiveExcept
if (paths.hasNext()) {
PartitionDesc partDesc = mapWork.getPathToPartitionInfo().get(paths.next());
if (partDesc != null) {
- LinkedHashMap<String, String> partSpec = partDesc.getPartSpec();
+ Map<String, String> partSpec = partDesc.getPartSpec();
if (partSpec != null && !partSpec.isEmpty()) {
partitionColumnCount = partSpec.size();
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractBaseAlterTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractBaseAlterTableAnalyzer.java
index 9e7688a5b29..4f34321ffcf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractBaseAlterTableAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractBaseAlterTableAnalyzer.java
@@ -96,7 +96,7 @@ protected void addInputsOutputsAlterTable(TableName tableName, Map<String, Strin
re.noLockNeeded();
inputs.add(re);
- if (AlterTableUtils.isFullPartitionSpec(table, partitionSpec)) {
+ if (AlterTableUtils.isFullPartitionSpec(table, partitionSpec) || table.hasNonNativePartitionSupport()) {
// Fully specified partition spec
Partition part = PartitionUtils.getPartition(db, table, partitionSpec, true);
outputs.add(new WriteEntity(part, writeType));
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java
index cd40dc91966..f04369fee34 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/show/ShowColumnsOperation.java
@@ -67,10 +67,7 @@ private List<FieldSchema> getColumnsByPattern() throws HiveException {
private List<FieldSchema> getCols() throws HiveException {
Table table = context.getDb().getTable(desc.getTableName());
- List<FieldSchema> allColumns = new ArrayList<>();
- allColumns.addAll(table.getCols());
- allColumns.addAll(table.getPartCols());
- return allColumns;
+ return table.getAllCols();
}
private Matcher getMatcher() {
@@ -94,13 +91,7 @@ private List<FieldSchema> filterColumns(List<FieldSchema> columns, Matcher match
}
if (desc.isSorted()) {
- result.sort(
- new Comparator<FieldSchema>() {
- @Override
- public int compare(FieldSchema f1, FieldSchema f2) {
- return f1.getName().compareTo(f2.getName());
- }
- });
+ result.sort(Comparator.comparing(FieldSchema::getName));
}
return result;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java
index b3d37fb9c40..8c005c8476f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableOperation.java
@@ -161,7 +161,8 @@ private void createTableNonReplaceMode(Table tbl) throws HiveException {
if (desc.isCTAS()) {
Table createdTable = context.getDb().getTable(tbl.getDbName(), tbl.getTableName());
DataContainer dc = new DataContainer(createdTable.getTTable());
- context.getQueryState().getLineageState().setLineage(createdTable.getPath(), dc, createdTable.getCols());
+ context.getQueryState().getLineageState().setLineage(
+ createdTable.getPath(), dc, createdTable);
// We did not create the table before moving the data files for a non-partitioned table i.e
// we used load file instead of load table (see SemanticAnalyzer#getFileSinkPlan() for
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
index 9086ad90c67..8d45892e17f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
@@ -200,7 +200,7 @@ private void getColumnDataColPathSpecified(Table table, Partition part, List<Fie
if (table.isPartitioned() && StatsUtils.checkCanProvideColumnStats(table)) {
Map<String, String> tableProps = table.getParameters() == null ?
new HashMap<>() : table.getParameters();
- if (partitionCol != null) {
+ if (partitionCol != null && !table.hasNonNativePartitionSupport()) {
addStatsForPartitionKeyColumn(table, colStats, colNames, tableProps, partitionCol);
} else {
addStatsForRegularColumn(table, colStats, colName, tableProps);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java
index 75f39291cd0..1ef6492057b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java
@@ -59,7 +59,6 @@
import java.io.DataOutputStream;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
@@ -70,7 +69,6 @@
import java.util.Set;
import java.util.TreeMap;
import java.util.Map.Entry;
-import java.util.stream.Collectors;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS;
import static org.apache.hadoop.hive.ql.ddl.ShowUtils.ALIGNMENT;
@@ -171,13 +169,7 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column
boolean isFormatted, boolean isOutputPadded) throws IOException {
String partitionData = "";
if (columnPath == null) {
- List<FieldSchema> partitionColumns = null;
- // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation
- if (table.isPartitioned()) {
- partitionColumns = table.hasNonNativePartitionSupport() ?
- table.getStorageHandler().getPartitionKeys(table) :
- table.getPartCols();
- }
+ List<FieldSchema> partitionColumns = table.getPartCols();
if (CollectionUtils.isNotEmpty(partitionColumns) &&
conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) {
TextMetaDataTable metaDataTable = new TextMetaDataTable();
@@ -204,13 +196,9 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column
}
private void addFormattedTableData(DataOutputStream out, Table table, Partition partition, boolean isOutputPadded)
- throws IOException, UnsupportedEncodingException {
- String formattedTableInfo = null;
- if (partition != null) {
- formattedTableInfo = getPartitionInformation(table, partition);
- } else {
- formattedTableInfo = getTableInformation(table, isOutputPadded);
- }
+ throws IOException {
+ String formattedTableInfo = (partition != null) ? getPartitionInformation(table, partition) :
+ getTableInformation(table, isOutputPadded);
if (table.getTableConstraintsInfo().isTableConstraintsInfoNotEmpty()) {
formattedTableInfo += getConstraintsInformation(table);
@@ -337,7 +325,7 @@ private void getStorageDescriptorInfo(StringBuilder tableInfo, Table table, Stor
List<String> skewedCoumnNames =
storageDesc.getSkewedInfo().getSkewedColNames().stream()
.sorted()
- .collect(Collectors.toList());
+ .toList();
formatOutput("Skewed Columns:", skewedCoumnNames.toString(), tableInfo);
}
@@ -345,16 +333,16 @@ private void getStorageDescriptorInfo(StringBuilder tableInfo, Table table, Stor
List<List<String>> skewedColumnValues =
storageDesc.getSkewedInfo().getSkewedColValues().stream()
.sorted(new VectorComparator<String>())
- .collect(Collectors.toList());
+ .toList();
formatOutput("Skewed Values:", skewedColumnValues.toString(), tableInfo);
}
- Map<List<String>, String> skewedColMap = new TreeMap<>(new VectorComparator<String>());
+ Map<List<String>, String> skewedColMap = new TreeMap<>(new VectorComparator<>());
skewedColMap.putAll(storageDesc.getSkewedInfo().getSkewedColValueLocationMaps());
if (MapUtils.isNotEmpty(skewedColMap)) {
formatOutput("Skewed Value to Path:", skewedColMap.toString(), tableInfo);
Map<List<String>, String> truncatedSkewedColMap =
- new TreeMap<List<String>, String>(new VectorComparator<String>());
+ new TreeMap<>(new VectorComparator<>());
// walk through existing map to truncate path so that test won't mask it then we can verify location is right
Set<Entry<List<String>, String>> entries = skewedColMap.entrySet();
for (Entry<List<String>, String> entry : entries) {
@@ -403,7 +391,7 @@ private void getPartitionMetaDataInformation(StringBuilder tableInfo, Partition
}
}
- private class VectorComparator<T extends Comparable<T>> implements Comparator<List<T>>{
+ private static final class VectorComparator<T extends Comparable<T>> implements Comparator<List<T>> {
@Override
public int compare(List<T> listA, List<T> listB) {
for (int i = 0; i < listA.size() && i < listB.size(); i++) {
@@ -438,7 +426,7 @@ private void displayAllParameters(Map<String, String> params, StringBuilder tabl
private void displayAllParameters(Map<String, String> params, StringBuilder tableInfo, boolean escapeUnicode,
boolean isOutputPadded) {
- List<String> keys = new ArrayList<String>(params.keySet());
+ List<String> keys = new ArrayList<>(params.keySet());
Collections.sort(keys);
for (String key : keys) {
String value = params.get(key);
@@ -626,7 +614,7 @@ private void addExtendedTableData(DataOutputStream out, Table table, Partition p
}
private void addExtendedConstraintData(DataOutputStream out, Table table)
- throws IOException, UnsupportedEncodingException {
+ throws IOException {
if (table.getTableConstraintsInfo().isTableConstraintsInfoNotEmpty()) {
out.write(("Constraints").getBytes(StandardCharsets.UTF_8));
out.write(Utilities.tabCode);
@@ -658,7 +646,7 @@ private void addExtendedConstraintData(DataOutputStream out, Table table)
}
private void addExtendedStorageData(DataOutputStream out, Table table)
- throws IOException, UnsupportedEncodingException {
+ throws IOException {
if (table.getStorageHandlerInfo() != null) {
out.write(("StorageHandlerInfo").getBytes(StandardCharsets.UTF_8));
out.write(Utilities.newLineCode);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java
index 7b6181b0415..1d7f0c52fa6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java
@@ -166,18 +166,17 @@ private void addTruncateTableOutputs(ASTNode root, Table table, Map<String, Stri
}
} else {
if (AlterTableUtils.isFullPartitionSpec(table, partitionSpec)) {
- if (table.hasNonNativePartitionSupport()) {
- table.getStorageHandler().validatePartSpec(table, partitionSpec);
- try {
- String partName = Warehouse.makePartName(partitionSpec, false);
- Partition partition = new DummyPartition(table, partName, partitionSpec);
- outputs.add(new WriteEntity(partition, writeType));
- } catch (MetaException e) {
- throw new SemanticException("Unable to construct name for dummy partition due to: ", e);
- }
- } else {
- Partition partition = PartitionUtils.getPartition(db, table, partitionSpec, true);
+ validatePartSpec(table, partitionSpec, (ASTNode) root.getChild(1), conf, true);
+ Partition partition = PartitionUtils.getPartition(db, table, partitionSpec, true);
+ outputs.add(new WriteEntity(partition, writeType));
+ } else if (table.hasNonNativePartitionSupport()) {
+ table.getStorageHandler().validatePartSpec(table, partitionSpec);
+ try {
+ String partName = Warehouse.makePartName(partitionSpec, false);
+ Partition partition = new DummyPartition(table, partName, partitionSpec);
outputs.add(new WriteEntity(partition, writeType));
+ } catch (MetaException e) {
+ throw new SemanticException("Unable to construct name for dummy partition due to: ", e);
}
} else {
validatePartSpec(table, partitionSpec, (ASTNode) root.getChild(1), conf, false);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java
index db7a5dfcd3d..5882e461650 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/PartitionUtils.java
@@ -23,7 +23,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.Map.Entry;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
index b5bb11bcea2..9ea98c922a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.ddl.table.storage.compact;
-import java.util.HashMap;
import java.util.Map;
import org.antlr.runtime.tree.Tree;
@@ -95,15 +94,13 @@ protected void analyzeCommand(TableName tableName, Map<String, String> partition
break;
case HiveParser.TOK_WHERE:
RowResolver rwsch = new RowResolver();
- Map<String, String> colTypes = new HashMap<>();
Table table;
try {
table = getDb().getTable(tableName);
- for (FieldSchema fs : table.getCols()) {
+ for (FieldSchema fs : table.getAllCols()) {
TypeInfo columnType = TypeInfoUtils.getTypeInfoFromTypeString(fs.getType());
- rwsch.put(tableName.getTable(), fs.getName(),
+ rwsch.put(tableName.getTable(), fs.getName(),
new ColumnInfo(fs.getName(), columnType, null, true));
- colTypes.put(fs.getName().toLowerCase(), fs.getType());
}
} catch (HiveException e) {
throw new SemanticException(e);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java
index ebe8f2f5277..e333ed85f43 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java
@@ -24,7 +24,6 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -254,7 +253,7 @@ public static String getPartialName(Partition p, int level) throws HiveException
* @throws HiveException
*/
public static String conflictingArchiveNameOrNull(Hive db, Table tbl,
- LinkedHashMap<String, String> partSpec)
+ Map<String, String> partSpec)
throws HiveException {
List<FieldSchema> partKeys = tbl.getPartitionKeys();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
index 6b338fbb330..b96936904ff 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
@@ -54,8 +54,6 @@ public class ColumnInfo implements Serializable {
* Indicates whether the column is a virtual column.
*/
private boolean isVirtualCol;
-
- private boolean isHiddenPartitionCol;
private ObjectInspector objectInspector;
@@ -129,7 +127,6 @@ public ColumnInfo(ColumnInfo columnInfo) {
this.isSkewedCol = columnInfo.isSkewedCol();
this.tabAlias = columnInfo.getTabAlias();
this.isVirtualCol = columnInfo.getIsVirtualCol();
- this.isHiddenPartitionCol = columnInfo.isHiddenPartitionCol();
this.isHiddenVirtualCol = columnInfo.isHiddenVirtualCol();
this.nullable = columnInfo.nullable;
this.setType(columnInfo.getType());
@@ -170,13 +167,9 @@ public String getTabAlias() {
}
public boolean getIsVirtualCol() {
- return isVirtualCol || isHiddenPartitionCol;
+ return isVirtualCol;
}
- public boolean isHiddenPartitionCol() {
- return isHiddenPartitionCol;
- }
-
public boolean isHiddenVirtualCol() {
return isHiddenVirtualCol;
}
@@ -209,10 +202,6 @@ public void setHiddenVirtualCol(boolean isHiddenVirtualCol) {
this.isHiddenVirtualCol = isHiddenVirtualCol;
}
- public void setHiddenPartitionCol(boolean isPartitionCol) {
- this.isHiddenPartitionCol = isPartitionCol;
- }
-
/**
* @return the isSkewedCol
*/
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
index b6d198264fa..d7e28bb603e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
@@ -39,7 +39,6 @@
import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
@@ -526,7 +525,7 @@ public List<String> getAlterTableStmtTableStatsColsAll(HiveConf conf, Table tbl)
List<String> accessedColumns = getTableColumnNames(tbl);
List<ColumnStatisticsObj> tableColumnStatistics = Hive.get(conf).getTableColumnStatistics(
tbl, accessedColumns, true);
-
+
ColumnStatisticsObj[] columnStatisticsObj = tableColumnStatistics.toArray(new ColumnStatisticsObj[0]);
for (ColumnStatisticsObj statisticsObj : columnStatisticsObj) {
alterTblStmt.add(getAlterTableStmtCol(
@@ -912,7 +911,8 @@ private String getExternal(Table table) {
private String getColumns(Table table) {
List<String> columnDescs = new ArrayList<>();
- for (FieldSchema column : table.getCols()) {
+ List<FieldSchema> colsToLookUp = table.hasNonNativePartitionSupport() ? table.getAllCols() : table.getCols();
+ for (FieldSchema column : colsToLookUp) {
String columnType = formatType(TypeInfoUtils.getTypeInfoFromTypeString(column.getType()));
String columnDesc = " " + unparseIdentifier(column.getName()) + " " + columnType;
if (column.getComment() != null) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 3eca5531f12..032181e3a69 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -31,7 +31,6 @@
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.HiveStatsUtils;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
@@ -555,20 +554,18 @@ public int execute() {
// match the number of columns in the file sink. For update there will be one too many
// (because of the ROW__ID), and in the case of the delete there will be just the
// ROW__ID, which we don't need to worry about from a lineage perspective.
- List<FieldSchema> tableCols = null;
switch (work.getLoadTableWork().getWriteType()) {
case DELETE:
case UPDATE:
// Pass an empty list as no columns will be written to the file.
// TODO I should be able to make this work for update
- tableCols = new ArrayList<>();
+ queryState.getLineageState().setLineage(tbd.getSourcePath(), dc, new ArrayList<>());
break;
default:
- tableCols = table.getCols();
+ queryState.getLineageState().setLineage(tbd.getSourcePath(), dc, table);
break;
}
- queryState.getLineageState().setLineage(tbd.getSourcePath(), dc, tableCols);
}
releaseLocks(tbd);
}
@@ -757,8 +754,7 @@ private DataContainer handleDynParts(Hive db, Table table, LoadTableDesc tbd,
// Don't set lineage on delete as we don't have all the columns
if (work.getLoadTableWork().getWriteType() != AcidUtils.Operation.DELETE &&
work.getLoadTableWork().getWriteType() != AcidUtils.Operation.UPDATE) {
- queryState.getLineageState().setLineage(tbd.getSourcePath(), dc,
- table.getCols());
+ queryState.getLineageState().setLineage(tbd.getSourcePath(), dc, table);
}
LOG.info("Loading partition " + entry.getKey());
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
index 787cadd256b..c37dbff5ac7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
@@ -547,8 +547,8 @@ public static Task<?> createViewTask(MetaData metaData, String dbNameToLoadIn, H
// texts using new DB name. Currently it refers to the source database name.
}
- CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getCols(), null, table.getParameters(),
- table.getPartColNames(), false, false, viewOriginalText, viewExpandedText,
+ CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getCols(), null, table.getParameters(),
+ table.getPartColNames(), false, false, viewOriginalText, viewExpandedText,
table.getPartCols(), null);
desc.setReplicationSpec(metaData.getReplicationSpec());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
index a0906cfb033..38c4dfb036b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
@@ -19,7 +19,6 @@
import java.io.IOException;
import java.util.Arrays;
-import java.util.LinkedHashMap;
import java.util.Map;
import java.util.stream.IntStream;
@@ -287,7 +286,7 @@ public static void getPartitionValues(VectorizedRowBatchCtx vrbCtx,
public static void getPartitionValues(VectorizedRowBatchCtx vrbCtx, PartitionDesc partDesc,
Object[] partitionValues) {
- LinkedHashMap<String, String> partSpec = partDesc.getPartSpec();
+ Map<String, String> partSpec = partDesc.getPartSpec();
for (int i = 0; i < vrbCtx.partitionColumnCount; i++) {
Object objectValue;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java
index c188eb09fdc..9f871d05feb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java
@@ -90,10 +90,7 @@ public List<String> getValues() {
Table table = this.getTable();
values = new ArrayList<>();
- // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation
- for (FieldSchema fs : table.hasNonNativePartitionSupport()
- ? table.getStorageHandler().getPartitionKeys(table)
- : table.getPartCols()) {
+ for (FieldSchema fs : table.getPartCols()) {
String val = partSpec.get(fs.getName());
values.add(val);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 31f695ed558..3a1b23e9f17 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -838,7 +838,7 @@ public void createTable(String tableName, List<String> columns,
* Create a table metadata and the directory for the table data
* @param tableName table name
* @param columns list of fields of the table
- * @param partCols partition keys of the table
+ * @param partColNames partition keys of the table
* @param fileInputFormat Class of the input format of the table data file
* @param fileOutputFormat Class of the output format of the table data file
* @param bucketCount number of buckets that each partition (or the table itself) should be
@@ -847,7 +847,7 @@ public void createTable(String tableName, List<String> columns,
* @param parameters Parameters for the table
* @throws HiveException
*/
- public void createTable(String tableName, List<String> columns, List<String> partCols,
+ public void createTable(String tableName, List<String> columns, List<String> partColNames,
Class<? extends InputFormat> fileInputFormat,
Class<?> fileOutputFormat, int bucketCount, List<String> bucketCols,
Map<String, String> parameters) throws HiveException {
@@ -864,13 +864,15 @@ public void createTable(String tableName, List<String> columns, List<String> par
tbl.getCols().add(field);
}
- if (partCols != null) {
- for (String partCol : partCols) {
+ if (partColNames != null) {
+ List<FieldSchema> partCols = new ArrayList<>();
+ for (String partCol : partColNames) {
FieldSchema part = new FieldSchema();
part.setName(partCol);
part.setType(STRING_TYPE_NAME); // default partition key
- tbl.getPartCols().add(part);
+ partCols.add(part);
}
+ tbl.setPartCols(partCols);
}
tbl.setSerializationLib(LazySimpleSerDe.class.getName());
tbl.setNumBuckets(bucketCount);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index 0cf02a95e39..ee55fde100f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -20,7 +20,6 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -430,10 +429,10 @@ private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable
// 1.2 Add column info corresponding to partition columns
ArrayList<ColumnInfo> partitionColumns = new ArrayList<ColumnInfo>();
- for (FieldSchema part_col : viewTable.getPartCols()) {
- colName = part_col.getName();
+ for (FieldSchema partCol : viewTable.getPartCols()) {
+ colName = partCol.getName();
colInfo = new ColumnInfo(colName,
- TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), null, true);
+ TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()), null, true);
rr.put(null, colName, colInfo);
cInfoLst.add(colInfo);
partitionColumns.add(colInfo);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
index 736e6e8c9f1..a17b7a51670 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
@@ -23,11 +23,12 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.hadoop.hive.common.StringInternUtils;
import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils;
@@ -124,7 +125,7 @@ public Partition(Table tbl, Map<String, String> partSpec, Path location) throws
public static org.apache.hadoop.hive.metastore.api.Partition createMetaPartitionObject(
Table tbl, Map<String, String> partSpec, Path location) throws HiveException {
- List<String> pvals = new ArrayList<String>();
+ List<String> pvals = new ArrayList<>();
for (FieldSchema field : tbl.getPartCols()) {
String val = partSpec.get(field.getName());
if (val == null || val.isEmpty()) {
@@ -173,7 +174,8 @@ protected void initialize(Table table,
// set default if location is not set and this is a physical
// table partition (not a view partition)
if (table.getDataLocation() != null) {
- Path partPath = new Path(table.getDataLocation(), Warehouse.makePartName(table.getPartCols(), tPartition.getValues()));
+ Path partPath = new Path(table.getDataLocation(),
+ Warehouse.makePartName(table.getPartCols(), tPartition.getValues()));
tPartition.getSd().setLocation(partPath.toString());
}
}
@@ -416,7 +418,7 @@ public Path[] getPath(Sample s) throws HiveException {
}
int scount = s.getSampleFraction();
- ArrayList<Path> ret = new ArrayList<Path>();
+ List<Path> ret = new ArrayList<>();
if (bcount == scount) {
ret.add(getBucketPath(s.getSampleNum() - 1));
@@ -428,7 +430,7 @@ public Path[] getPath(Sample s) throws HiveException {
}
// undersampling a bucket
ret.add(getBucketPath((s.getSampleNum() - 1) % bcount));
- } else if (bcount > scount) {
+ } else {
if ((bcount / scount) * scount != bcount) {
throw new HiveException("Sample Count" + scount
+ " is not a divisor of bucket count " + bcount + " for table "
@@ -439,11 +441,11 @@ public Path[] getPath(Sample s) throws HiveException {
ret.add(getBucketPath(i * scount + (s.getSampleNum() - 1)));
}
}
- return (ret.toArray(new Path[ret.size()]));
+ return (ret.toArray(new Path[0]));
}
}
- public LinkedHashMap<String, String> getSpec() {
+ public Map<String, String> getSpec() {
return table.createSpec(tPartition);
}
@@ -492,7 +494,14 @@ public Map<String, String> getParameters() {
}
public List<FieldSchema> getCols() {
- return getColsInternal(false);
+ List<FieldSchema> nonPartFields = new ArrayList<>();
+ Set<String> partFieldsName = table.getPartCols().stream().map(FieldSchema::getName).collect(Collectors.toSet());
+ for (FieldSchema field : getColsInternal(false)) {
+ if (!partFieldsName.contains(field.getName())) {
+ nonPartFields.add(field);
+ }
+ }
+ return nonPartFields;
}
public List<FieldSchema> getColsForMetastore() {
@@ -542,7 +551,7 @@ public void setLocation(String location) {
*/
public void setValues(Map<String, String> partSpec)
throws HiveException {
- List<String> pvals = new ArrayList<String>();
+ List<String> pvals = new ArrayList<>();
for (FieldSchema field : table.getPartCols()) {
String val = partSpec.get(field.getName());
if (val == null) {
@@ -582,12 +591,11 @@ public List<String> getSkewedColNames() {
return tPartition.getSd().getSkewedInfo().getSkewedColNames();
}
- public void setSkewedValueLocationMap(List<String> valList, String dirName)
- throws HiveException {
+ public void setSkewedValueLocationMap(List<String> valList, String dirName) {
Map<List<String>, String> mappings = tPartition.getSd().getSkewedInfo()
.getSkewedColValueLocationMaps();
if (null == mappings) {
- mappings = new HashMap<List<String>, String>();
+ mappings = new HashMap<>();
tPartition.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings);
}
@@ -612,8 +620,7 @@ public int hashCode() {
@Override
public boolean equals(Object obj) {
- if (obj instanceof Partition) {
- Partition o = (Partition) obj;
+ if (obj instanceof Partition o) {
return Objects.equals(tPartition, o.tPartition);
}
return false;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index 05faa38d7ca..ad16a2f22ce 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.metadata;
import java.io.IOException;
+import java.io.Serial;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
@@ -32,8 +33,10 @@
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
+
import java.util.stream.Stream;
+import com.google.common.collect.Maps;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -101,6 +104,7 @@
*/
public class Table implements Serializable {
+ @Serial
private static final long serialVersionUID = 1L;
static final private Logger LOG = LoggerFactory.getLogger("hive.ql.metadata.Table");
@@ -110,6 +114,10 @@ public class Table implements Serializable {
/**
* These fields are all cached fields. The information comes from tTable.
*/
+ private List<FieldSchema> tablePartCols;
+ private record TableColumn(int index, FieldSchema field) {}
+ private transient Map<String, TableColumn> columnsByName;
+ private transient List<FieldSchema> tableNonPartCols;
private transient Deserializer deserializer;
private Class<? extends OutputFormat> outputFormatClass;
private Class<? extends InputFormat> inputFormatClass;
@@ -194,6 +202,12 @@ public Table makeCopy() {
newTab.setMetaTable(this.getMetaTable());
newTab.setSnapshotRef(this.getSnapshotRef());
+ if (this.tablePartCols != null) {
+ newTab.tablePartCols = new ArrayList<>(this.tablePartCols.size());
+ for (FieldSchema fs : this.tablePartCols) {
+ newTab.tablePartCols.add(fs.deepCopy());
+ }
+ }
return newTab;
}
@@ -215,6 +229,7 @@ public org.apache.hadoop.hive.metastore.api.Table getTTable() {
*/
public void setTTable(org.apache.hadoop.hive.metastore.api.Table tTable) {
this.tTable = tTable;
+ resetColumnCache();
}
/**
@@ -226,11 +241,11 @@ public void setTTable(org.apache.hadoop.hive.metastore.api.Table tTable) {
{
sd.setSerdeInfo(new SerDeInfo());
sd.setNumBuckets(-1);
- sd.setBucketCols(new ArrayList<String>());
- sd.setCols(new ArrayList<FieldSchema>());
- sd.setParameters(new HashMap<String, String>());
- sd.setSortCols(new ArrayList<Order>());
- sd.getSerdeInfo().setParameters(new HashMap<String, String>());
+ sd.setBucketCols(new ArrayList<>());
+ sd.setCols(new ArrayList<>());
+ sd.setParameters(new HashMap<>());
+ sd.setSortCols(new ArrayList<>());
+ sd.getSerdeInfo().setParameters(new HashMap<>());
// We have to use MetadataTypedColumnsetSerDe because LazySimpleSerDe does
// not support a table with no columns.
sd.getSerdeInfo().setSerializationLib(MetadataTypedColumnsetSerDe.class.getName());
@@ -240,17 +255,17 @@ public void setTTable(org.apache.hadoop.hive.metastore.api.Table tTable) {
sd.setInputFormat(SequenceFileInputFormat.class.getName());
sd.setOutputFormat(HiveSequenceFileOutputFormat.class.getName());
SkewedInfo skewInfo = new SkewedInfo();
- skewInfo.setSkewedColNames(new ArrayList<String>());
- skewInfo.setSkewedColValues(new ArrayList<List<String>>());
- skewInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
+ skewInfo.setSkewedColNames(new ArrayList<>());
+ skewInfo.setSkewedColValues(new ArrayList<>());
+ skewInfo.setSkewedColValueLocationMaps(new HashMap<>());
sd.setSkewedInfo(skewInfo);
}
org.apache.hadoop.hive.metastore.api.Table t = new org.apache.hadoop.hive.metastore.api.Table();
{
t.setSd(sd);
- t.setPartitionKeys(new ArrayList<FieldSchema>());
- t.setParameters(new HashMap<String, String>());
+ t.setPartitionKeys(new ArrayList<>());
+ t.setParameters(new HashMap<>());
t.setTableType(TableType.MANAGED_TABLE.toString());
t.setDbName(databaseName);
t.setTableName(tableName);
@@ -403,7 +418,7 @@ public void setStorageHandlerInfo(StorageHandlerInfo storageHandlerInfo) {
this.storageHandlerInfo = storageHandlerInfo;
}
- final public Class<? extends InputFormat> getInputFormatClass() {
+ public final Class<? extends InputFormat> getInputFormatClass() {
if (inputFormatClass == null) {
try {
String className = tTable.getSd().getInputFormat();
@@ -423,7 +438,7 @@ final public Class<? extends InputFormat> getInputFormatClass() {
return inputFormatClass;
}
- final public Class<? extends OutputFormat> getOutputFormatClass() {
+ public final Class<? extends OutputFormat> getOutputFormatClass() {
if (outputFormatClass == null) {
try {
String className = tTable.getSd().getOutputFormat();
@@ -457,7 +472,7 @@ public void setMaterializedTable(boolean materializedTable) {
* Marker SemanticException, so that processing that allows for table validation failures
* and appropriately handles them can recover from these types of SemanticExceptions
*/
- public class ValidationFailureSemanticException extends SemanticException{
+ public static class ValidationFailureSemanticException extends SemanticException{
public ValidationFailureSemanticException(String s) {
super(s);
}
@@ -527,9 +542,9 @@ public TableType getTableType() {
return Enum.valueOf(TableType.class, tTable.getTableType());
}
- public ArrayList<StructField> getFields() {
+ public List<StructField> getFields() {
- ArrayList<StructField> fields = new ArrayList<StructField>();
+ List<StructField> fields = new ArrayList<>();
try {
Deserializer decoder = getDeserializer();
@@ -595,26 +610,41 @@ public boolean equals(Object obj) {
&& Objects.equals(snapshotRef, other.snapshotRef);
}
+
+ /**
+ * Returns partition columns for native tables, or from the
+ * storage handler when {@link #hasNonNativePartitionSupport()}.
+ */
public List<FieldSchema> getPartCols() {
- List<FieldSchema> partKeys = tTable.getPartitionKeys();
- if (partKeys == null) {
- partKeys = new ArrayList<>();
- tTable.setPartitionKeys(partKeys);
+ if (tablePartCols != null) {
+ return tablePartCols;
+ }
+ if (hasNonNativePartitionSupport()) {
+ List<FieldSchema> partCols = getStorageHandler().getPartitionKeys(this);
+ // Partition cols come from the handler, but user comments are stored in HMS sd.getCols().
+ // Copy HMS comments onto handler fields so DESCRIBE/SHOW match native table behavior; for
+ // imported non-native tables these comments may be invalid as the handler might not support them.
+ for (FieldSchema partCol : partCols) {
+ FieldSchema storageSchemaCol = getColumnByName(partCol.getName());
+ if (storageSchemaCol != null && storageSchemaCol.getComment() != null) {
+ partCol.setComment(storageSchemaCol.getComment());
+ }
+ }
+ tablePartCols = partCols;
+ } else {
+ tablePartCols = getPartitionKeys();
}
- return partKeys;
+ return tablePartCols;
}
public FieldSchema getPartColByName(String colName) {
return getPartCols().stream()
- .filter(key -> key.getName().toLowerCase().equals(colName))
- .findFirst().orElse(null);
+ .filter(key -> key.getName().toLowerCase().equals(colName))
+ .findFirst().orElse(null);
}
public List<String> getPartColNames() {
- List<FieldSchema> partCols = hasNonNativePartitionSupport() ?
- getStorageHandler().getPartitionKeys(this) : getPartCols();
- return partCols.stream().map(FieldSchema::getName)
- .collect(Collectors.toList());
+ return getPartCols().stream().map(FieldSchema::getName).toList();
}
public boolean hasNonNativePartitionSupport() {
@@ -672,7 +702,7 @@ public void setSkewedValueLocationMap(List<String> valList, String dirName) {
Map<List<String>, String> mappings = tTable.getSd().getSkewedInfo()
.getSkewedColValueLocationMaps();
if (null == mappings) {
- mappings = new HashMap<List<String>, String>();
+ mappings = new HashMap<>();
tTable.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings);
}
@@ -682,7 +712,7 @@ public void setSkewedValueLocationMap(List<String> valList, String dirName) {
public Map<List<String>, String> getSkewedColValueLocationMaps() {
return (tTable.getSd().getSkewedInfo() != null) ? tTable.getSd().getSkewedInfo()
- .getSkewedColValueLocationMaps() : new HashMap<List<String>, String>();
+ .getSkewedColValueLocationMaps() : Collections.emptyMap();
}
public void setSkewedColValues(List<List<String>> skewedValues) {
@@ -691,7 +721,7 @@ public void setSkewedColValues(List<List<String>> skewedValues) {
public List<List<String>> getSkewedColValues(){
return (tTable.getSd().getSkewedInfo() != null) ? tTable.getSd().getSkewedInfo()
- .getSkewedColValues() : new ArrayList<List<String>>();
+ .getSkewedColValues() : Collections.emptyList();
}
public void setSkewedColNames(List<String> skewedColNames) {
@@ -700,7 +730,7 @@ public void setSkewedColNames(List<String> skewedColNames) {
public List<String> getSkewedColNames() {
return (tTable.getSd().getSkewedInfo() != null) ? tTable.getSd().getSkewedInfo()
- .getSkewedColNames() : new ArrayList<String>();
+ .getSkewedColNames() : Collections.emptyList();
}
public SkewedInfo getSkewedInfo() {
@@ -728,8 +758,52 @@ private boolean isField(String col) {
return false;
}
+ public Integer getColumnIndexByName(String colName) {
+ TableColumn column = lazyColumnsByName().get(colName.toLowerCase());
+ return column != null ? column.index() : null;
+ }
+
+ public FieldSchema getColumnByName(String colName) {
+ TableColumn column = lazyColumnsByName().get(colName.toLowerCase());
+ return column != null ? column.field() : null;
+ }
+
+ private Map<String, TableColumn> lazyColumnsByName() {
+ if (columnsByName == null) {
+ this.columnsByName = indexColumnsByName();
+ }
+ return columnsByName;
+ }
+
+ private Map<String, TableColumn> indexColumnsByName() {
+ List<FieldSchema> fsList = new ArrayList<>(getColsInternal(false));
+ if (!hasNonNativePartitionSupport() || isView()) {
+ fsList.addAll(getPartitionKeys());
+ }
+ Map<String, TableColumn> colsByName = Maps.newHashMapWithExpectedSize(fsList.size());
+ for (int i = 0; i < fsList.size(); i++) {
+ colsByName.put(fsList.get(i).getName().toLowerCase(), new TableColumn(i, fsList.get(i)));
+ }
+ return colsByName;
+ }
+
public List<FieldSchema> getCols() {
- return getColsInternal(false);
+ if (tableNonPartCols != null) {
+ return tableNonPartCols;
+ }
+ if (!hasNonNativePartitionSupport()) {
+ tableNonPartCols = getColsInternal(false);
+ } else {
+ // For Iceberg, HMS sd.getCols() holds the full schema while partitionKeys is empty (HIVE-29633).
+ // Filter out handler partition columns to keep getCols() as data columns only.
+ Set<String> partColNames = getPartCols().stream()
+ .map(FieldSchema::getName)
+ .collect(Collectors.toSet());
+ tableNonPartCols = getColsInternal(false).stream()
+ .filter(field -> !partColNames.contains(field.getName()))
+ .toList();
+ }
+ return tableNonPartCols;
}
public List<FieldSchema> getColsForMetastore() {
@@ -762,14 +836,16 @@ private List<FieldSchema> getColsInternal(boolean forMs) {
* @return List<FieldSchema>
*/
public List<FieldSchema> getAllCols() {
- ArrayList<FieldSchema> f_list = new ArrayList<FieldSchema>();
- f_list.addAll(getCols());
- f_list.addAll(getPartCols());
- return f_list;
+ List<FieldSchema> fsList = new ArrayList<>(getColsInternal(false));
+ if (!hasNonNativePartitionSupport() || isView()) {
+ fsList.addAll(getPartitionKeys());
+ }
+ return fsList;
}
public void setPartCols(List<FieldSchema> partCols) {
tTable.setPartitionKeys(partCols);
+ resetColumnCache();
}
public String getCatName() {
@@ -813,12 +889,19 @@ public void setOutputFormatClass(String name) throws HiveException {
}
public boolean isPartitioned() {
- return hasNonNativePartitionSupport() ? getStorageHandler().isPartitioned(this) :
+ return hasNonNativePartitionSupport() ? getStorageHandler().isPartitioned(this) :
CollectionUtils.isNotEmpty(getPartCols());
}
public void setFields(List<FieldSchema> fields) {
tTable.getSd().setCols(fields);
+ resetColumnCache();
+ }
+
+ private void resetColumnCache() {
+ tableNonPartCols = null;
+ tablePartCols = null;
+ columnsByName = null;
}
public void setNumBuckets(int nb) {
@@ -916,6 +999,9 @@ public void setDbName(String databaseName) {
}
public List<FieldSchema> getPartitionKeys() {
+ if (tTable.getPartitionKeys() == null) {
+ tTable.setPartitionKeys(Collections.emptyList());
+ }
return tTable.getPartitionKeys();
}
@@ -1009,12 +1095,12 @@ public boolean isMaterializedView() {
* Use the information from this partition.
* @return Partition name to value mapping.
*/
- public LinkedHashMap<String, String> createSpec(
+ public Map<String, String> createSpec(
org.apache.hadoop.hive.metastore.api.Partition tp) {
List<FieldSchema> fsl = getPartCols();
List<String> tpl = tp.getValues();
- LinkedHashMap<String, String> spec = new LinkedHashMap<String, String>(fsl.size());
+ Map<String, String> spec = LinkedHashMap.newLinkedHashMap(fsl.size());
for (int i = 0; i < fsl.size(); i++) {
FieldSchema fs = fsl.get(i);
String value = tpl.get(i);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
index 3d3e4ce7663..6c358c98da7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
@@ -28,6 +28,7 @@
import java.util.Map;
import java.util.Set;
import java.util.Stack;
+import java.util.TreeSet;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.slf4j.Logger;
@@ -521,6 +522,8 @@ public static void setupNeededColumns(TableScanOperator scanOp, RowSchema inputR
cols.add(new FieldNode(VirtualColumn.RAWDATASIZE.getName()));
}
+ Set<String> nonNativePartitionCols = nonNativePartitionColNames(desc.getTableMetadata());
+
for (FieldNode fn : cols) {
String column = fn.getFieldName();
ColumnInfo colInfo = inputRS.getColumnInfo(column);
@@ -528,9 +531,9 @@ public static void setupNeededColumns(TableScanOperator scanOp, RowSchema inputR
continue;
}
referencedColumnNames.add(column);
- if (colInfo.getIsVirtualCol() && !colInfo.isHiddenPartitionCol()) {
+ if (colInfo.getIsVirtualCol() && !nonNativePartitionCols.contains(column)) {
// part is also a virtual column, but part col should not in this
- // list.
+ // list in case of non native tables like iceberg which have their partition support.
for (int j = 0; j < virtualCols.size(); j++) {
VirtualColumn vc = virtualCols.get(j);
if (vc.getName().equals(colInfo.getInternalName())) {
@@ -556,6 +559,15 @@ public static void setupNeededColumns(TableScanOperator scanOp, RowSchema inputR
scanOp.setReferencedColumns(referencedColumnNames);
}
+ private static Set<String> nonNativePartitionColNames(Table table) {
+ if (table == null || !table.hasNonNativePartitionSupport()) {
+ return Set.of();
+ }
+ Set<String> names = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ names.addAll(table.getPartColNames());
+ return names;
+ }
+
/**
* Factory method to get the ColumnPrunerDefaultProc class.
*
@@ -807,8 +819,7 @@ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
for (FieldNode col : cols) {
int index = originalOutputColumnNames.indexOf(col.getFieldName());
Table tab = cppCtx.getParseContext().getViewProjectToTableSchema().get(op);
- List<FieldSchema> fullFieldList = new ArrayList<FieldSchema>(tab.getCols());
- fullFieldList.addAll(tab.getPartCols());
+ List<FieldSchema> fullFieldList = tab.getAllCols();
cppCtx.getParseContext().getColumnAccessInfo()
.add(tab.getCompleteName(), fullFieldList.get(index).getName());
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
index dba737e382c..5aa52254b49 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
@@ -168,8 +168,15 @@ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Obje
Table table = ts.getConf().getTableMetadata();
- boolean nonEquiJoin = isNonEquiJoin(ctx.parent);
- if (table != null && table.isPartitionKey(column) && !nonEquiJoin) {
+ boolean prunable = table != null && !isNonEquiJoin(ctx.parent);
+ // Dynamic partition pruning only applies to equi-joins against an actual table.
+ if (prunable && table.isNonNative() &&
+ table.getStorageHandler().addDynamicSplitPruningEdge(table, ctx.parent)) {
+ // Non-native tables (e.g. Iceberg): the storage handler decides whether the column is prunable.
+ String columnType = table.getColumnByName(column).getType();
+ generateEventOperatorPlan(ctx, parseContext, ts, column, columnType, ctx.parent);
+ } else if (prunable && table.isPartitionKey(column) && !table.hasNonNativePartitionSupport()) {
+ // Native partitioned table.
String columnType = table.getPartColByName(column).getType();
String alias = ts.getConf().getAlias();
PrunedPartitionList plist = parseContext.getPrunedPartitions(alias, ts);
@@ -191,15 +198,10 @@ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Obje
// all partitions have been statically removed
LOG.debug("No partition pruning necessary.");
}
- } else if (table.isNonNative() &&
- table.getStorageHandler().addDynamicSplitPruningEdge(table, ctx.parent)) {
- generateEventOperatorPlan(ctx, parseContext, ts, column,
- table.getCols().stream().filter(e -> e.getName().equals(column)).
- map(e -> e.getType()).findFirst().get(), ctx.parent);
} else { // semijoin
LOG.debug("Column " + column + " is not a partition column");
if (semiJoin && !disableSemiJoinOptDueToExternalTable(parseContext.getConf(), ts, ctx)
- && ts.getConf().getFilterExpr() != null && !nonEquiJoin) {
+ && ts.getConf().getFilterExpr() != null && !isNonEquiJoin(ctx.parent)) {
LOG.debug("Initiate semijoin reduction for " + column + " ("
+ ts.getConf().getFilterExpr().getExprString());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 6007e488d39..8e041351ef3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -1768,7 +1768,7 @@ private ImmutablePair<Boolean, Boolean> validateInputFormatAndSchemaEvolution(Ma
* allColumnNameList and allTypeInfoList variables -- into the data and partition columns.
*/
- LinkedHashMap<String, String> partSpec = partDesc.getPartSpec();
+ Map<String, String> partSpec = partDesc.getPartSpec();
if (partSpec != null && partSpec.size() > 0) {
partitionColumnCount = partSpec.size();
dataColumnCount = dataAndPartColumnCount - partitionColumnCount;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java
index 11f02e295b8..3e8c83ec1a1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql.optimizer.ppr;
import java.util.ArrayList;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -53,7 +52,7 @@ public class PartExprEvalUtils {
* @throws HiveException
*/
static public Object evalExprWithPart(ExprNodeDesc expr, Partition p) throws HiveException {
- LinkedHashMap<String, String> partSpec = p.getSpec();
+ Map<String, String> partSpec = p.getSpec();
Properties partProps = p.getSchema();
String[] partKeyTypes;
@@ -61,8 +60,8 @@ static public Object evalExprWithPart(ExprNodeDesc expr, Partition p) throws Hiv
if (!partSpec.keySet().containsAll(expr.getCols())) {
return null;
}
- partKeyTypes = p.getTable().getStorageHandler().getPartitionKeys(p.getTable()).stream()
- .map(FieldSchema::getType).toArray(String[]::new);
+ partKeyTypes = p.getTable().getPartCols().stream().map(FieldSchema::getType)
+ .toArray(String[]::new);
} else {
String pcolTypes = partProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES);
partKeyTypes = pcolTypes.trim().split(":");
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
index 06912a1b322..05f3b85f271 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
@@ -175,7 +175,8 @@ private void analyzeAcidExport(ASTNode ast, Table exportTable, ASTNode tokRefOrN
//now generate insert statement
//insert into newTableName select * from ts <where partition spec>
StringBuilder rewrittenQueryStr = generateExportQuery(
- newTable.getPartCols(), tokRefOrNameExportTable, (ASTNode) tokRefOrNameExportTable.parent, newTableName);
+ newTable.getPartCols(),
+ tokRefOrNameExportTable, (ASTNode) tokRefOrNameExportTable.parent, newTableName);
ReparseResult rr = ParseUtils.parseRewrittenQuery(ctx, rewrittenQueryStr);
Context rewrittenCtx = rr.rewrittenCtx;
rewrittenCtx.setIsUpdateDeleteMerge(false); //it's set in parseRewrittenQuery()
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index ecc226d42a2..5e1640573db 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -3013,10 +3013,13 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc
ArrayList<ColumnInfo> partitionColumns = new ArrayList<ColumnInfo>();
// 3.2 Add column info corresponding to partition columns
- for (FieldSchema part_col : tabMetaData.getPartCols()) {
- colName = part_col.getName();
+ for (FieldSchema partCol : tabMetaData.getPartCols()) {
+ if (tabMetaData.hasNonNativePartitionSupport()) {
+ break;
+ }
+ colName = partCol.getName();
colInfo = new ColumnInfo(colName,
- TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()),
+ TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()),
isNullable(colName, nnc, pkc), tableAlias, true);
rr.put(tableAlias, colName, colInfo);
cInfoLst.add(colInfo);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java
index 9109f9cb608..10cb448746d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java
@@ -261,9 +261,10 @@ private void replaceSelectOperatorProcess(SelectOperator operator, Operator<? ex
for (int i = 0; i < selRSSig.size(); i++) {
columnNameToIndex.putIfAbsent(selRSSig.get(i).getAlias(), i);
}
- for (int i = 0; i < this.columns.size(); i++) {
- ColumnInfo col = columns.get(i);
- Integer selRSIdx = getSelRSColumnIndex(i, col, columnNameToIndex);
+ for (FieldSchema column : this.columns) {
+ int index = tbl.getColumnIndexByName(column.getName());
+ ColumnInfo col = columns.get(index);
+ Integer selRSIdx = getSelRSColumnIndex(column.getName(), col, columnNameToIndex);
if (selRSIdx == null) {
continue;
}
@@ -274,42 +275,66 @@ private void replaceSelectOperatorProcess(SelectOperator operator, Operator<? ex
columnExprMap.put(internalName, exprNodeDesc);
signature.add(selRSSig.get(selRSIdx));
}
- // if there is any partition column (in static partition or dynamic
- // partition or mixed case)
- int dynamicPartBegin = -1;
+ int dynPartsCount = 0;
+ if (partSpec != null) {
+ for (Map.Entry<String, String> entry : partSpec.entrySet()) {
+ if (entry.getValue() == null) {
+ dynPartsCount++;
+ }
+ }
+ }
+ // True when the input row already carries the static partition columns as real columns (e.g. Iceberg),
+ // i.e. it has more columns than just the data columns plus the dynamic partition columns.
+ boolean inputRRHasStaticParts = (this.columns.size() + dynPartsCount < columns.size());
+ // Column layout produced for the stats-gathering select:
+ // <-- non-partition cols -->|<-- static partition cols -->|<-- dynamic partition cols -->
+ // For non-native tables (e.g. Iceberg) partition columns are ordinary interleaved columns instead.
+ boolean dynamicPartSeen = false;
+ // Count of native static partition columns already emitted as constants. They are absent from the input
+ // row, so each one shifts the input position of the columns that follow it.
+ int staticPartShift = 0;
for (int i = 0; i < partitionColumns.size(); i++) {
- ExprNodeDesc exprNodeDesc;
- TypeInfo srcType;
String partColName = partitionColumns.get(i).getName();
+ int tableIndex = tbl.getColumnIndexByName(partColName);
+ boolean isStaticPartition = partSpec != null && partSpec.get(partColName) != null;
+
+ ExprNodeDesc exprNodeDesc;
// 2. deal with static partition columns
- if (partSpec != null && partSpec.containsKey(partColName)
- && partSpec.get(partColName) != null) {
- if (dynamicPartBegin > 0) {
- throw new SemanticException(
- "Dynamic partition columns should not come before static partition columns.");
+ if (isStaticPartition) {
+ if (inputRRHasStaticParts) {
+ // non-native (e.g. Iceberg): the partition column is a real column in the input row
+ exprNodeDesc = new ExprNodeColumnDesc(columns.get(tableIndex - staticPartShift));
+ } else {
+ if (dynamicPartSeen) {
+ throw new SemanticException(
+ "Dynamic partition columns should not come before static partition columns.");
+ }
+ // native: the static partition value is a constant, not present in the input row
+ exprNodeDesc = new ExprNodeConstantDesc(partSpec.get(partColName));
+ staticPartShift++;
}
- exprNodeDesc = new ExprNodeConstantDesc(partSpec.get(partColName));
- srcType = exprNodeDesc.getTypeInfo();
}
- // 3. dynamic partition columns
+ // 3. dynamic partition columns: always read from the input row
else {
- dynamicPartBegin++;
- ColumnInfo col = columns.get(this.columns.size() + dynamicPartBegin);
- exprNodeDesc = new ExprNodeColumnDesc(col);
- srcType = col.getType();
-
+ dynamicPartSeen = true;
+ exprNodeDesc = new ExprNodeColumnDesc(columns.get(tableIndex - staticPartShift));
}
- TypeInfo destType = selRSSig.get(this.columns.size() + i).getType();
+ TypeInfo srcType = exprNodeDesc.getTypeInfo();
+
+ // For non-native tables partition columns are interleaved, so find the select-RS position by name;
+ // for native tables it equals the table column index.
+ int selRSIndex = inputRRHasStaticParts ? columnNameToIndex.get(partColName) : tableIndex;
+ TypeInfo destType = selRSSig.get(selRSIndex).getType();
if (!srcType.equals(destType)) {
// This may be possible when srcType is string but destType is integer
exprNodeDesc = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor()
.createConversionCast(exprNodeDesc, (PrimitiveTypeInfo) destType);
}
colList.add(exprNodeDesc);
- String internalName = selRS.getColumnNames().get(this.columns.size() + i);
+ String internalName = selRS.getColumnNames().get(selRSIndex);
columnNames.add(internalName);
columnExprMap.put(internalName, exprNodeDesc);
- signature.add(selRSSig.get(this.columns.size() + i));
+ signature.add(selRSSig.get(selRSIndex));
}
operator.setConf(new SelectDesc(colList, columnNames));
operator.setColumnExprMap(columnExprMap);
@@ -317,7 +342,7 @@ private void replaceSelectOperatorProcess(SelectOperator operator, Operator<? ex
operator.setSchema(selRS);
}
- private Integer getSelRSColumnIndex(int i, ColumnInfo col, Map<String, Integer> columnNameToIndex) {
+ private Integer getSelRSColumnIndex(String columnName, ColumnInfo col, Map<String, Integer> columnNameToIndex) {
ObjectInspector objectInspector = col.getObjectInspector();
if (objectInspector == null) {
return null;
@@ -326,7 +351,7 @@ private Integer getSelRSColumnIndex(int i, ColumnInfo col, Map<String, Integer>
if (!columnSupported) {
return null;
}
- return columnNameToIndex.get(this.columns.get(i).getName());
+ return columnNameToIndex.get(columnName);
}
public String getCompleteName() {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
index e23e54aa523..88b4a4f0e9f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
@@ -107,7 +107,8 @@ private boolean shouldRewrite(ASTNode tree) {
*/
private static FieldSchemas getStatsEligibleFieldSchemas(Table tbl) {
List<FieldSchema> result = new ArrayList<>();
- for (FieldSchema col : tbl.getCols()) {
+ List<FieldSchema> colsToLookUp = tbl.hasNonNativePartitionSupport() ? tbl.getAllCols() : tbl.getCols();
+ for (FieldSchema col : colsToLookUp) {
String type = col.getType();
TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(type);
boolean isSupported = ColumnStatsAutoGatherContext.isColumnSupported(typeInfo.getCategory(), () -> typeInfo);
@@ -207,8 +208,7 @@ private static CharSequence genPartitionClause(Table tbl, List<TransformSpec> pa
private static String getColTypeOf(Table tbl, String partKey) {
- for (FieldSchema fs : tbl.hasNonNativePartitionSupport() ?
- tbl.getStorageHandler().getPartitionKeys(tbl) : tbl.getPartitionKeys()) {
+ for (FieldSchema fs : tbl.getPartCols()) {
if (partKey.equalsIgnoreCase(fs.getName())) {
return fs.getType().toLowerCase();
}
@@ -231,12 +231,13 @@ protected static List<FieldSchema> getFieldSchemasByColName(Table tbl, List<Stri
}
}
- for (FieldSchema col : tbl.getCols()) {
+ List<FieldSchema> colsToLookUp = tbl.hasNonNativePartitionSupport() ? tbl.getAllCols() : tbl.getCols();
+ for (FieldSchema col : colsToLookUp) {
specifiedColsMap.computeIfPresent(col.getName().toLowerCase(), (key, value) -> col);
}
List<FieldSchema> result = new ArrayList<>();
- List<String> tableColNames = new FieldSchemas(tbl.getCols()).getColName();
+ List<String> tableColNames = new FieldSchemas(colsToLookUp).getColName();
for (String colName : colNames) {
FieldSchema fs = specifiedColsMap.get(colName.toLowerCase());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 4d4956fbec1..dcf197a2c20 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -37,7 +37,6 @@
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.ReplChangeManager;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
-import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.ql.QueryState;
@@ -89,7 +88,6 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
/**
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index eb4a73f1e5e..9093570706f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -561,7 +561,7 @@ private void reparseAndSuperAnalyze(Table table, URI fromURI) throws SemanticExc
}
rewrittenQueryStr.append(getFullTableNameForSQL((ASTNode)(tableTree.getChild(0))));
- addPartitionColsToInsert(table.getPartCols(), inpPartSpec, rewrittenQueryStr);
+ addPartitionColsToInsert(parts, inpPartSpec, rewrittenQueryStr);
rewrittenQueryStr.append(" select * from ");
rewrittenQueryStr.append(tempTblName);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java
index 882840ffef5..4987384fbc0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MergeSemanticAnalyzer.java
@@ -29,7 +29,6 @@
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.rewrite.MergeStatement;
import org.apache.hadoop.hive.ql.parse.rewrite.RewriterFactory;
-import org.apache.hadoop.hive.ql.plan.HiveOperation;
import java.util.ArrayList;
import java.util.HashMap;
@@ -230,7 +229,7 @@ private MergeStatement.UpdateClause handleUpdate(ASTNode whenMatchedUpdateClause
String deleteExtraPredicate) throws SemanticException {
assert whenMatchedUpdateClause.getType() == HiveParser.TOK_MATCHED;
assert getWhenClauseOperation(whenMatchedUpdateClause).getType() == HiveParser.TOK_UPDATE;
- Map<String, String> newValuesMap = new HashMap<>(targetTable.getCols().size() + targetTable.getPartCols().size());
+ Map<String, String> newValuesMap = HashMap.newHashMap(targetTable.getAllCols().size());
ASTNode setClause = (ASTNode)getWhenClauseOperation(whenMatchedUpdateClause).getChild(0);
//columns being updated -> update expressions; "setRCols" (last param) is null because we use actual expressions
//before re-parsing, i.e. they are known to SemanticAnalyzer logic
@@ -240,7 +239,8 @@ private MergeStatement.UpdateClause handleUpdate(ASTNode whenMatchedUpdateClause
//insert into target (p1) select current_date(), 5, c3, p1 where ....
//since we take the RHS of set exactly as it was in Input, we don't need to deal with quoting/escaping column/table
//names
- List<FieldSchema> nonPartCols = targetTable.getCols();
+ List<FieldSchema> nonPartCols = targetTable.hasNonNativePartitionSupport() ? targetTable.getAllCols() :
+ targetTable.getCols();
Map<String, String> colNameToDefaultConstraint = getColNameToDefaultValueMap(targetTable);
for (FieldSchema fs : nonPartCols) {
String name = fs.getName();
@@ -303,7 +303,7 @@ private List<ASTNode> findWhenClauses(ASTNode tree, int start) throws SemanticEx
"Unexpected node type found: " + whenClause.getType() + addParseInfo(whenClause);
whenClauses.add(whenClause);
}
- if (whenClauses.size() <= 0) {
+ if (whenClauses.isEmpty()) {
//Futureproofing: the parser will actually not allow this
throw new SemanticException("Must have at least 1 WHEN clause in MERGE statement");
}
@@ -431,8 +431,7 @@ private static final class OnClauseAnalyzer {
OnClauseAnalyzer(ASTNode onClause, Table targetTable, String targetTableNameInSourceQuery,
HiveConf conf, String onClauseAsString) {
this.onClause = onClause;
- allTargetTableColumns.addAll(targetTable.getCols());
- allTargetTableColumns.addAll(targetTable.getPartCols());
+ allTargetTableColumns.addAll(targetTable.getAllCols());
this.targetTableNameInSourceQuery = unescapeIdentifier(targetTableNameInSourceQuery);
this.conf = conf;
this.onClauseAsString = onClauseAsString;
@@ -499,11 +498,7 @@ private void handleUnresolvedColumns() {
private void addColumn2Table(String tableName, String columnName) {
tableName = tableName.toLowerCase(); //normalize name for mapping
tableNamesFound.add(tableName);
- List<String> cols = table2column.get(tableName);
- if (cols == null) {
- cols = new ArrayList<>();
- table2column.put(tableName, cols);
- }
+ List<String> cols = table2column.computeIfAbsent(tableName, k -> new ArrayList<>());
//we want to preserve 'columnName' as it was in original input query so that rewrite
//looks as much as possible like original query
cols.add(columnName);
@@ -526,7 +521,7 @@ private String getPredicate() {
}
StringBuilder sb = new StringBuilder();
for (String col : targetCols) {
- if (sb.length() > 0) {
+ if (!sb.isEmpty()) {
sb.append(" AND ");
}
//but preserve table name in SQL
@@ -605,17 +600,15 @@ protected String getMatchedText(ASTNode n) {
}
protected boolean isAliased(ASTNode n) {
- switch (n.getType()) {
- case HiveParser.TOK_TABREF:
- return findTabRefIdxs(n)[0] != 0;
- case HiveParser.TOK_TABNAME:
- return false;
- case HiveParser.TOK_SUBQUERY:
+ return switch (n.getType()) {
+ case HiveParser.TOK_TABREF -> findTabRefIdxs(n)[0] != 0;
+ case HiveParser.TOK_TABNAME -> false;
+ case HiveParser.TOK_SUBQUERY -> {
assert n.getChildCount() > 1 : "Expected Derived Table to be aliased";
- return true;
- default:
- throw raiseWrongType("TOK_TABREF|TOK_TABNAME", n);
- }
+ yield true;
+ }
+ default -> throw raiseWrongType("TOK_TABREF|TOK_TABNAME", n);
+ };
}
/**
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
index 9964b936906..9edbe5b05e1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
@@ -123,15 +123,11 @@ public static ASTNode parse(
* @return boolean
*/
public static boolean isJoinToken(ASTNode node) {
- switch (node.getToken().getType()) {
- case HiveParser.TOK_JOIN:
- case HiveParser.TOK_LEFTOUTERJOIN:
- case HiveParser.TOK_RIGHTOUTERJOIN:
- case HiveParser.TOK_FULLOUTERJOIN:
- return true;
- default:
- return false;
- }
+ return switch (node.getToken().getType()) {
+ case HiveParser.TOK_JOIN, HiveParser.TOK_LEFTOUTERJOIN, HiveParser.TOK_RIGHTOUTERJOIN,
+ HiveParser.TOK_FULLOUTERJOIN -> true;
+ default -> false;
+ };
}
/**
@@ -163,12 +159,10 @@ public static List<String> validateColumnNameUniqueness(
// but it should not be a major bottleneck as the number of columns are
// anyway not so big
Iterator<FieldSchema> iterCols = fieldSchemas.iterator();
- List<String> colNames = new ArrayList<String>();
+ List<String> colNames = new ArrayList<>();
while (iterCols.hasNext()) {
String colName = iterCols.next().getName();
- Iterator<String> iter = colNames.iterator();
- while (iter.hasNext()) {
- String oldColName = iter.next();
+ for (String oldColName : colNames) {
if (colName.equalsIgnoreCase(oldColName)) {
throw new SemanticException(ErrorMsg.DUPLICATE_COLUMN_NAMES
.getMsg(oldColName));
@@ -286,7 +280,7 @@ public static Pair<Boolean, String> containsTokenOfType(ASTNode root, Integer ..
final Set<Integer> tokensToMatch = new HashSet<>(Arrays.asList(tokens));
final String[] matched = {null};
- boolean check = ParseUtils.containsTokenOfType(root, new PTFUtils.Predicate<ASTNode>() {
+ boolean check = ParseUtils.containsTokenOfType(root, new PTFUtils.Predicate<>() {
@Override
public boolean apply(ASTNode node) {
if (tokensToMatch.contains(node.getType())) {
@@ -302,7 +296,7 @@ public boolean apply(ASTNode node) {
}
public static boolean containsTokenOfType(ASTNode root, PTFUtils.Predicate<ASTNode> predicate) {
- Queue<ASTNode> queue = new ArrayDeque<ASTNode>();
+ Queue<ASTNode> queue = new ArrayDeque<>();
// BFS
queue.add(root);
@@ -535,7 +529,7 @@ public static String getKeywords(Set<String> excludes) {
if (excludes != null && excludes.contains(name)) {
continue;
}
- if (sb.length() > 0) {
+ if (!sb.isEmpty()) {
sb.append(",");
}
sb.append(name);
@@ -581,8 +575,7 @@ public static Map<Integer, List<ExprNodeGenericFuncDesc>> getFullPartitionSpecs(
CommonTree ast, Table table, Configuration conf, boolean canGroupExprs) throws SemanticException {
String defaultPartitionName = HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULT_PARTITION_NAME);
Map<String, String> colTypes = new HashMap<>();
- List<FieldSchema> partitionKeys = table.hasNonNativePartitionSupport() ?
- table.getStorageHandler().getPartitionKeys(table) : table.getPartitionKeys();
+ List<FieldSchema> partitionKeys = table.getPartCols();
for (FieldSchema fs : partitionKeys) {
colTypes.put(fs.getName().toLowerCase(), fs.getType());
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java
index 101f6b1fc3d..a181e26761d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java
@@ -117,19 +117,16 @@ protected void analyzeRewrittenTree(ASTNode rewrittenTree, Context rewrittenCtx)
*/
protected void checkValidSetClauseTarget(ASTNode colName, Table targetTable) throws SemanticException {
String columnName = normalizeColName(colName.getText());
-
- // Make sure this isn't one of the partitioning columns, that's not supported.
- for (FieldSchema fschema : targetTable.getPartCols()) {
- if (fschema.getName().equalsIgnoreCase(columnName)) {
- throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg());
- }
+ // Native partition columns cannot be updated; Iceberg partition columns are regular updatable columns.
+ if (!targetTable.hasNonNativePartitionSupport() && targetTable.isPartitionKey(columnName)) {
+ throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_PART_VALUE.getMsg());
}
//updating bucket column should move row from one file to another - not supported
if (targetTable.getBucketCols() != null && targetTable.getBucketCols().contains(columnName)) {
throw new SemanticException(ErrorMsg.UPDATE_CANNOT_UPDATE_BUCKET_VALUE, columnName);
}
boolean foundColumnInTargetTable = false;
- for (FieldSchema col : targetTable.getCols()) {
+ for (FieldSchema col : targetTable.getAllCols()) {
if (columnName.equalsIgnoreCase(col.getName())) {
foundColumnInTargetTable = true;
break;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 7781ded7bfa..ea981c98bb3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -5171,7 +5171,12 @@ RowResolver handleInsertStatementSpec(List<ExprNodeDesc> col_list, String dest,
}
List<ExprNodeDesc> newColList = new ArrayList<ExprNodeDesc>();
colListPos = 0;
- List<FieldSchema> targetTableCols = target != null ? target.getCols() : partition.getCols();
+ List<FieldSchema> targetTableCols;
+ if (target != null) {
+ targetTableCols = target.hasNonNativePartitionSupport() ? target.getAllCols() : target.getCols();
+ } else {
+ targetTableCols = partition.getCols();
+ }
List<String> targetTableColNames = new ArrayList<String>();
List<TypeInfo> targetTableColTypes = new ArrayList<TypeInfo>();
for(FieldSchema fs : targetTableCols) {
@@ -12008,47 +12013,44 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException {
// Determine row schema for TSOP.
// Include column names from SerDe, the partition and virtual columns.
rwsch = new RowResolver();
- try {
- // Including parameters passed in the query
- if (properties != null) {
- for (Entry<String, String> prop : properties.entrySet()) {
- if (tab.getSerdeParam(prop.getKey()) != null) {
- LOG.warn("SerDe property in input query overrides stored SerDe property");
- }
- tab.setSerdeParam(prop.getKey(), prop.getValue());
- }
- }
- // Obtain inspector for schema
- final Deserializer deserializer = tab.getDeserializer();
- StructObjectInspector rowObjectInspector = (StructObjectInspector) deserializer.getObjectInspector();
-
- deserializer.handleJobLevelConfiguration(conf);
- List<? extends StructField> fields = rowObjectInspector
- .getAllStructFieldRefs();
- Set<String> partCols = tab.hasNonNativePartitionSupport() ?
- Sets.newHashSet(tab.getPartColNames()) : Collections.emptySet();
- for (int i = 0; i < fields.size(); i++) {
- /**
- * if the column is a skewed column, use ColumnInfo accordingly
- */
- ColumnInfo colInfo = new ColumnInfo(fields.get(i).getFieldName(),
- TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i)
- .getFieldObjectInspector()), alias, false);
- if (partCols.contains(colInfo.getInternalName())) {
- colInfo.setHiddenPartitionCol(true);
+ // Including parameters passed in the query
+ if (properties != null) {
+ for (Entry<String, String> prop : properties.entrySet()) {
+ if (tab.getSerdeParam(prop.getKey()) != null) {
+ LOG.warn("SerDe property in input query overrides stored SerDe property");
}
- colInfo.setSkewedCol(isSkewedCol(alias, qb, fields.get(i).getFieldName()));
- rwsch.put(alias, fields.get(i).getFieldName(), colInfo);
+ tab.setSerdeParam(prop.getKey(), prop.getValue());
}
+ }
+ final Deserializer deserializer = tab.getDeserializer();
+ deserializer.handleJobLevelConfiguration(conf);
+ StructObjectInspector rowObjectInspector;
+ try {
+ rowObjectInspector = (StructObjectInspector) deserializer.getObjectInspector();
} catch (SerDeException e) {
throw new RuntimeException(e);
}
- // Hack!! - refactor once the metadata APIs with types are ready
- // Finally add the partitioning columns
- for (FieldSchema part_col : tab.getPartCols()) {
- LOG.trace("Adding partition col: " + part_col);
- rwsch.put(alias, part_col.getName(), new ColumnInfo(part_col.getName(),
- TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), alias, true));
+
+ int colCount = tab.getAllCols().size();
+ List<ColumnInfo> colInfoList = new ArrayList<>(Collections.nCopies(colCount, null));
+
+ // Build column info from the SerDe row schema (authoritative names/types), placed by table column index.
+ for (StructField field : rowObjectInspector.getAllStructFieldRefs()) {
+ String fieldName = field.getFieldName();
+ ColumnInfo colInfo = new ColumnInfo(fieldName,
+ TypeInfoUtils.getTypeInfoFromObjectInspector(field.getFieldObjectInspector()), alias, false);
+ colInfo.setSkewedCol(isSkewedCol(alias, qb, fieldName));
+ colInfoList.set(tab.getColumnIndexByName(fieldName), colInfo);
+ }
+ for (FieldSchema partCol : tab.getPartCols()) {
+ LOG.trace("Adding partition col: {} ", partCol);
+ ColumnInfo colInfo = new ColumnInfo(partCol.getName(),
+ TypeInfoFactory.getPrimitiveTypeInfo(partCol.getType()), alias, true);
+ colInfoList.set(tab.getColumnIndexByName(partCol.getName()), colInfo);
+ }
+
+ for (ColumnInfo colInfo : colInfoList) {
+ rwsch.put(alias, colInfo.getInternalName(), colInfo);
}
// put virtual columns into RowResolver.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java
index b72f2496d93..dbac6bffef2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteMergeRewriter.java
@@ -32,6 +32,8 @@
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.rewrite.sql.COWWithClauseBuilder;
import org.apache.hadoop.hive.ql.parse.rewrite.sql.MultiInsertSqlGenerator;
+import org.apache.hadoop.hive.ql.parse.rewrite.sql.PartitionSetValuesClause;
+import org.apache.hadoop.hive.ql.parse.rewrite.sql.SetValuesClauseBase;
import org.apache.hadoop.hive.ql.parse.rewrite.sql.SqlGeneratorFactory;
import java.util.ArrayList;
@@ -98,7 +100,8 @@ public ParseUtils.ReparseResult rewrite(Context ctx, MergeStatement mergeStateme
@Override
protected CopyOnWriteMergeWhenClauseSqlGenerator createMergeSqlGenerator(
MergeStatement mergeStatement, MultiInsertSqlGenerator sqlGenerator) {
- return new CopyOnWriteMergeWhenClauseSqlGenerator(conf, sqlGenerator, mergeStatement);
+ return new CopyOnWriteMergeWhenClauseSqlGenerator(conf, sqlGenerator, mergeStatement,
+ new PartitionSetValuesClause(conf, (newValue, alias) -> String.format("%s AS %s", newValue, alias)));
}
private void handleSource(MergeStatement mergeStatement, MultiInsertSqlGenerator sqlGenerator) {
@@ -140,8 +143,10 @@ static class CopyOnWriteMergeWhenClauseSqlGenerator extends MergeRewriter.MergeW
private int subQueryCount = 0;
CopyOnWriteMergeWhenClauseSqlGenerator(
- HiveConf conf, MultiInsertSqlGenerator sqlGenerator, MergeStatement mergeStatement) {
- super(conf, sqlGenerator, mergeStatement, shouldAddRowLineageColumnsForMerge(mergeStatement, conf));
+ HiveConf conf, MultiInsertSqlGenerator sqlGenerator,
+ MergeStatement mergeStatement, SetValuesClauseBase setvaluesClause) {
+ super(conf, sqlGenerator, mergeStatement,
+ shouldAddRowLineageColumnsForMerge(mergeStatement, conf), setvaluesClause);
this.cowWithClauseBuilder = new COWWithClauseBuilder();
}
@@ -202,9 +207,9 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau
sqlGenerator.append(hintStr);
hintStr = null;
}
- List<String> values = new ArrayList<>(targetTable.getCols().size() + targetTable.getPartCols().size());
+ List<String> values = new ArrayList<>(targetTable.getAllCols().size());
values.addAll(sqlGenerator.getDeleteValues(Context.Operation.MERGE));
- addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), values);
+ setValuesClause.addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), values);
addValuesForRowLineageForCopyOnMerge(isRowLineageSupported, values,
"NULL AS " + HiveUtils.unparseIdentifier(VirtualColumn.LAST_UPDATED_SEQUENCE_NUMBER.getName()), conf);
sqlGenerator.append(columnRefsFunc.apply(StringUtils.join(values, ",")));
@@ -215,11 +220,6 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau
columnRefsFunc);
sqlGenerator.append("\n");
}
-
- @Override
- protected String getRhsExpValue(String newValue, String alias) {
- return String.format("%s AS %s", newValue, alias);
- }
@Override
protected void handleWhenMatchedDelete(String onClauseAsString, String extraPredicate, String updateExtraPredicate,
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteUpdateRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteUpdateRewriter.java
index ae57198860c..2ac2062ce8b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteUpdateRewriter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/CopyOnWriteUpdateRewriter.java
@@ -22,6 +22,7 @@
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.RowLineageUtils;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -90,12 +91,14 @@ public ParseUtils.ReparseResult rewrite(Context context, UpdateStatement updateB
}
Map<Integer, ASTNode> setColExprs = new HashMap<>(updateBlock.getSetCols().size());
- List<FieldSchema> nonPartCols = updateBlock.getTargetTable().getCols();
- for (int i = 0; i < nonPartCols.size(); i++) {
+ Table targetTable = updateBlock.getTargetTable();
+ List<FieldSchema> colsToLookUp = targetTable.hasNonNativePartitionSupport() ? targetTable.getAllCols() :
+ targetTable.getCols();
+ for (int i = 0; i < colsToLookUp.size(); i++) {
if (columnOffset > 0 || i > 0) {
sqlGenerator.append(',');
}
- String name = nonPartCols.get(i).getName();
+ String name = colsToLookUp.get(i).getName();
ASTNode setCol = updateBlock.getSetCols().get(name);
String identifier = HiveUtils.unparseIdentifier(name, this.conf);
sqlGenerator.append(identifier);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java
index 3ec2e580f04..f3a17b5c8b2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/MergeRewriter.java
@@ -28,19 +28,19 @@
import org.apache.hadoop.hive.ql.Context.Operation;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.ParseUtils;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.StorageFormat;
import org.apache.hadoop.hive.ql.parse.rewrite.sql.MultiInsertSqlGenerator;
+import org.apache.hadoop.hive.ql.parse.rewrite.sql.SetValuesClause;
+import org.apache.hadoop.hive.ql.parse.rewrite.sql.SetValuesClauseBase;
import org.apache.hadoop.hive.ql.parse.rewrite.sql.SqlGeneratorFactory;
import org.apache.hadoop.hive.ql.session.SessionState;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import java.util.function.Function;
import java.util.function.UnaryOperator;
@@ -104,7 +104,8 @@ public ParseUtils.ReparseResult rewrite(Context ctx, MergeStatement mergeStateme
protected MergeWhenClauseSqlGenerator createMergeSqlGenerator(
MergeStatement mergeStatement, MultiInsertSqlGenerator sqlGenerator) {
- return new MergeWhenClauseSqlGenerator(conf, sqlGenerator, mergeStatement, isRowLineageSupported);
+ return new MergeWhenClauseSqlGenerator(
+ conf, sqlGenerator, mergeStatement, isRowLineageSupported, new SetValuesClause(conf));
}
private void handleSource(boolean hasWhenNotMatchedClause, String sourceAlias, String onClauseAsText,
@@ -177,14 +178,16 @@ protected static class MergeWhenClauseSqlGenerator implements MergeStatement.Mer
protected final MergeStatement mergeStatement;
protected String hintStr;
protected final boolean isRowLineageSupported;
+ protected final SetValuesClauseBase setValuesClause;
MergeWhenClauseSqlGenerator(HiveConf conf, MultiInsertSqlGenerator sqlGenerator, MergeStatement mergeStatement,
- boolean isRowLineageSupported) {
+ boolean isRowLineageSupported, SetValuesClauseBase setValuesClause) {
this.conf = conf;
this.sqlGenerator = sqlGenerator;
this.mergeStatement = mergeStatement;
this.hintStr = mergeStatement.getHintStr();
this.isRowLineageSupported = isRowLineageSupported;
+ this.setValuesClause = setValuesClause;
}
@Override
@@ -224,9 +227,9 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau
sqlGenerator.append(" -- update clause").append("\n");
List<String> valuesAndAcidSortKeys = new ArrayList<>(
- targetTable.getCols().size() + targetTable.getPartCols().size() + 1);
+ targetTable.getAllCols().size() + 1);
valuesAndAcidSortKeys.addAll(sqlGenerator.getSortKeys(Operation.MERGE));
- addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), valuesAndAcidSortKeys);
+ setValuesClause.addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), valuesAndAcidSortKeys);
sqlGenerator.appendInsertBranch(hintStr, valuesAndAcidSortKeys);
hintStr = null;
@@ -236,33 +239,11 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau
sqlGenerator.appendSortKeys();
}
- protected void addValues(Table targetTable, String targetAlias, Map<String, String> newValues,
- List<String> values) {
- UnaryOperator<String> formatter = name -> String.format("%s.%s", targetAlias,
- HiveUtils.unparseIdentifier(name, conf));
-
- for (FieldSchema fieldSchema : targetTable.getCols()) {
- if (newValues.containsKey(fieldSchema.getName())) {
- String rhsExp = newValues.get(fieldSchema.getName());
- values.add(getRhsExpValue(rhsExp, formatter.apply(fieldSchema.getName())));
- } else {
- values.add(formatter.apply(fieldSchema.getName()));
- }
- }
-
- targetTable.getPartCols().forEach(fieldSchema -> values.add(
- formatter.apply(fieldSchema.getName())));
- }
-
- protected String getRhsExpValue(String newValue, String alias) {
- return newValue;
- }
-
protected void addWhereClauseOfUpdate(String onClauseAsString, String extraPredicate, String deleteExtraPredicate,
MultiInsertSqlGenerator sqlGenerator) {
addWhereClauseOfUpdate(onClauseAsString, extraPredicate, deleteExtraPredicate, sqlGenerator, UnaryOperator.identity());
}
-
+
protected void addWhereClauseOfUpdate(String onClauseAsString, String extraPredicate, String deleteExtraPredicate,
MultiInsertSqlGenerator sqlGenerator, UnaryOperator<String> columnRefsFunc) {
StringBuilder whereClause = new StringBuilder(onClauseAsString);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/PartitionSplitUpdateRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/PartitionSplitUpdateRewriter.java
new file mode 100644
index 00000000000..797f36aa10e
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/PartitionSplitUpdateRewriter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.rewrite;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.rewrite.sql.MultiInsertSqlGenerator;
+import org.apache.hadoop.hive.ql.parse.rewrite.sql.SqlGeneratorFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class PartitionSplitUpdateRewriter extends SplitUpdateRewriter {
+
+ public PartitionSplitUpdateRewriter(HiveConf conf, SqlGeneratorFactory sqlGeneratorFactory) {
+ super(conf, sqlGeneratorFactory);
+ }
+
+ @Override
+ protected void appendPartitionColumns(UpdateStatement updateBlock, MultiInsertSqlGenerator sqlGenerator,
+ List<String> insertValues, Map<Integer, ASTNode> setColExprs,
+ int columnOffset) {
+ List<FieldSchema> partCols = updateBlock.getTargetTable().getPartCols();
+ boolean first = partCols.size() == updateBlock.getTargetTable().getAllCols().size();
+ for (int i = 0; i < partCols.size(); i++) {
+ if (first) {
+ first = false;
+ } else {
+ sqlGenerator.append(",");
+ }
+ appendUpdateColumn(updateBlock, sqlGenerator, insertValues, setColExprs,
+ partCols.get(i).getName(), updateBlock.getTargetTable().getCols().size() + i + columnOffset);
+ }
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java
index 84fcf186f6b..da60bc4ec9c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitMergeRewriter.java
@@ -22,6 +22,9 @@
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.rewrite.sql.MultiInsertSqlGenerator;
+import org.apache.hadoop.hive.ql.parse.rewrite.sql.PartitionSetValuesClause;
+import org.apache.hadoop.hive.ql.parse.rewrite.sql.SetValuesClause;
+import org.apache.hadoop.hive.ql.parse.rewrite.sql.SetValuesClauseBase;
import org.apache.hadoop.hive.ql.parse.rewrite.sql.SqlGeneratorFactory;
import java.util.ArrayList;
@@ -38,7 +41,9 @@ public SplitMergeRewriter(Hive db, HiveConf conf, SqlGeneratorFactory sqlGenerat
@Override
protected MergeWhenClauseSqlGenerator createMergeSqlGenerator(
MergeStatement mergeStatement, MultiInsertSqlGenerator sqlGenerator) {
- return new SplitMergeWhenClauseSqlGenerator(conf, sqlGenerator, mergeStatement, isRowLineageSupported);
+ return new SplitMergeWhenClauseSqlGenerator(conf, sqlGenerator, mergeStatement, isRowLineageSupported,
+ mergeStatement.getTargetTable().hasNonNativePartitionSupport() ?
+ new PartitionSetValuesClause(conf) : new SetValuesClause(conf));
}
static class SplitMergeWhenClauseSqlGenerator extends MergeWhenClauseSqlGenerator {
@@ -46,8 +51,8 @@ static class SplitMergeWhenClauseSqlGenerator extends MergeWhenClauseSqlGenerato
SplitMergeWhenClauseSqlGenerator(
HiveConf conf, MultiInsertSqlGenerator sqlGenerator, MergeStatement mergeStatement,
- boolean isRowLineageSupported) {
- super(conf, sqlGenerator, mergeStatement, isRowLineageSupported);
+ boolean isRowLineageSupported, SetValuesClauseBase setValuesClause) {
+ super(conf, sqlGenerator, mergeStatement, isRowLineageSupported, setValuesClause);
this.isRowLineageSupported = isRowLineageSupported;
}
@@ -58,8 +63,8 @@ public void appendWhenMatchedUpdateClause(MergeStatement.UpdateClause updateClau
String onClauseAsString = mergeStatement.getOnClauseAsText();
sqlGenerator.append(" -- update clause (insert part)\n");
- List<String> values = new ArrayList<>(targetTable.getCols().size() + targetTable.getPartCols().size());
- addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), values);
+ List<String> values = new ArrayList<>(targetTable.getAllCols().size());
+ setValuesClause.addValues(targetTable, targetAlias, updateClause.getNewValuesMap(), values);
addRowLineageColumnsForWhenMatchedUpdateClause(isRowLineageSupported, values, targetAlias, conf);
sqlGenerator.appendInsertBranch(hintStr, values);
hintStr = null;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java
index d14ddc7eb48..0f20669df94 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/SplitUpdateRewriter.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.CalcitePlanner;
import org.apache.hadoop.hive.ql.parse.HiveParser;
@@ -31,6 +32,7 @@
import org.apache.hadoop.hive.ql.parse.rewrite.sql.SqlGeneratorFactory;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -65,10 +67,11 @@ public ParseUtils.ReparseResult rewrite(Context context, UpdateStatement updateB
List<String> deleteValues = sqlGenerator.getDeleteValues(OPERATION);
int columnOffset = deleteValues.size();
- List<String> insertValues = new ArrayList<>(updateBlock.getTargetTable().getCols().size());
+ Table targetTable = updateBlock.getTargetTable();
+ List<String> insertValues = new ArrayList<>(Collections.nCopies(targetTable.getAllCols().size(), null));
boolean first = true;
- List<FieldSchema> nonPartCols = updateBlock.getTargetTable().getCols();
+ List<FieldSchema> nonPartCols = targetTable.getCols();
for (int i = 0; i < nonPartCols.size(); i++) {
if (first) {
first = false;
@@ -76,33 +79,12 @@ public ParseUtils.ReparseResult rewrite(Context context, UpdateStatement updateB
sqlGenerator.append(",");
}
- String name = nonPartCols.get(i).getName();
- ASTNode setCol = updateBlock.getSetCols().get(name);
- String identifier = HiveUtils.unparseIdentifier(name, this.conf);
-
- if (setCol != null) {
- if (setCol.getType() == HiveParser.TOK_TABLE_OR_COL &&
- setCol.getChildCount() == 1 && setCol.getChild(0).getType() == HiveParser.TOK_DEFAULT_VALUE) {
- sqlGenerator.append(updateBlock.getColNameToDefaultConstraint().get(name));
- } else {
- sqlGenerator.append(identifier);
- // This is one of the columns we're setting, record it's position so we can come back
- // later and patch it up. 0th is ROW_ID
- setColExprs.put(i + columnOffset, setCol);
- }
- } else {
- sqlGenerator.append(identifier);
- }
- sqlGenerator.append(" AS ");
- sqlGenerator.append(identifier);
-
- insertValues.add(sqlGenerator.qualify(identifier));
+ appendUpdateColumn(updateBlock, sqlGenerator, insertValues, setColExprs,
+ nonPartCols.get(i).getName(), i + columnOffset);
}
- if (updateBlock.getTargetTable().getPartCols() != null) {
- updateBlock.getTargetTable().getPartCols().forEach(
- fieldSchema -> insertValues.add(sqlGenerator.qualify(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf))));
- }
- addRowLineageColumnsForUpdate(updateBlock.getTargetTable(), sqlGenerator, insertValues, conf);
+
+ appendPartitionColumns(updateBlock, sqlGenerator, insertValues, setColExprs, columnOffset);
+ addRowLineageColumnsForUpdate(targetTable, sqlGenerator, insertValues, conf);
sqlGenerator.append(" FROM ").append(sqlGenerator.getTargetTableFullName()).append(") ");
sqlGenerator.appendSubQueryAlias().append("\n");
@@ -134,4 +116,40 @@ public ParseUtils.ReparseResult rewrite(Context context, UpdateStatement updateB
return rr;
}
+
+ protected void appendPartitionColumns(UpdateStatement updateBlock, MultiInsertSqlGenerator sqlGenerator,
+ List<String> insertValues, Map<Integer, ASTNode> setColExprs, int columnOffset) {
+ for (FieldSchema partCol : updateBlock.getTargetTable().getPartCols()) {
+ String identifier = HiveUtils.unparseIdentifier(partCol.getName(), conf);
+ insertValues.set(updateBlock.getTargetTable().getColumnIndexByName(partCol.getName()),
+ sqlGenerator.qualify(identifier));
+ }
+ }
+
+ protected void appendUpdateColumn(UpdateStatement updateBlock,
+ MultiInsertSqlGenerator sqlGenerator, List<String> insertValues,
+ Map<Integer, ASTNode> setColExprs, String columnName, int setColExprIndex) {
+ String identifier = HiveUtils.unparseIdentifier(columnName, conf);
+
+ // The insert value is placed for every column (data and partition).
+ int index = updateBlock.getTargetTable().getColumnIndexByName(columnName);
+ insertValues.set(index, sqlGenerator.qualify(identifier));
+
+ ASTNode setCol = updateBlock.getSetCols().get(columnName);
+ if (setCol != null) {
+ if (setCol.getType() == HiveParser.TOK_TABLE_OR_COL &&
+ setCol.getChildCount() == 1 && setCol.getChild(0).getType() == HiveParser.TOK_DEFAULT_VALUE) {
+ sqlGenerator.append(updateBlock.getColNameToDefaultConstraint().get(columnName));
+ } else {
+ sqlGenerator.append(identifier);
+ // This is one of the columns we're setting, record it's position so we can come back
+ // later and patch it up. 0th is ROW_ID
+ setColExprs.put(setColExprIndex, setCol);
+ }
+ } else {
+ sqlGenerator.append(identifier);
+ }
+ sqlGenerator.append(" AS ");
+ sqlGenerator.append(identifier);
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/UpdateRewriterFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/UpdateRewriterFactory.java
index 88949c9c6bd..fc677b45696 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/UpdateRewriterFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/UpdateRewriterFactory.java
@@ -51,7 +51,9 @@ public Rewriter<UpdateStatement> createRewriter(Table table, String targetTableF
if (copyOnWriteMode) {
return new CopyOnWriteUpdateRewriter(conf, sqlGeneratorFactory);
} else if (splitUpdate) {
- return new SplitUpdateRewriter(conf, sqlGeneratorFactory);
+ return table.hasNonNativePartitionSupport() ?
+ new PartitionSplitUpdateRewriter(conf, sqlGeneratorFactory) :
+ new SplitUpdateRewriter(conf, sqlGeneratorFactory);
} else {
if (AcidUtils.isNonNativeAcidTable(table)) {
throw new SemanticException(ErrorMsg.NON_NATIVE_ACID_UPDATE.getErrorCodedMsg());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java
index 7587daf1305..449449037a5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/MultiInsertSqlGenerator.java
@@ -77,9 +77,11 @@ public String qualify(String columnName) {
return String.format("%s.%s", subQueryAlias, columnName);
}
- public void appendInsertBranch(String hintStr, List<String> values) {
+ public void appendInsertBranchInsertClause() {
queryStr.append("INSERT INTO ").append(targetTableFullName);
- appendPartitionCols(targetTable);
+ }
+
+ public void appendInsertBranchSelectClause(String hintStr, List<String> values) {
queryStr.append("\n");
queryStr.append(INDENT);
@@ -92,6 +94,11 @@ public void appendInsertBranch(String hintStr, List<String> values) {
queryStr.append("\n");
}
+ public void appendInsertBranch(String hintStr, List<String> values) {
+ appendInsertBranchInsertClause();
+ appendInsertBranchSelectClause(hintStr, values);
+ }
+
public void appendDeleteBranch(String hintStr) {
List<String> deleteValues = getDeleteValues(Operation.DELETE);
appendInsertBranch(hintStr, deleteValues);
@@ -148,7 +155,7 @@ public void removeLastChar() {
}
public void appendPartColsOfTargetTableWithComma(String alias) {
- if (targetTable.getPartCols() == null || targetTable.getPartCols().isEmpty()) {
+ if (targetTable.getPartCols().isEmpty() || targetTable.hasNonNativePartitionSupport()) {
return;
}
queryStr.append(',');
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java
index 87e42680044..5bbc3f19a13 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/NativeAcidMultiInsertSqlGenerator.java
@@ -33,6 +33,13 @@ public NativeAcidMultiInsertSqlGenerator(Table table, String targetTableFullName
super(table, targetTableFullName, conf, subQueryAlias);
}
+ @Override
+ public void appendInsertBranch(String hintStr, List<String> values) {
+ appendInsertBranchInsertClause();
+ appendPartitionCols(targetTable);
+ appendInsertBranchSelectClause(hintStr, values);
+ }
+
@Override
public void appendAcidSelectColumns(Operation operation) {
queryStr.append("ROW__ID,");
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/PartitionSetValuesClause.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/PartitionSetValuesClause.java
new file mode 100644
index 00000000000..a7c5c27249d
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/PartitionSetValuesClause.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.rewrite.sql;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.BinaryOperator;
+
+public class PartitionSetValuesClause extends SetValuesClauseBase {
+
+ public PartitionSetValuesClause(HiveConf conf) {
+ super(conf);
+ }
+
+ public PartitionSetValuesClause(HiveConf conf, BinaryOperator<String> rhsExpValueFormatter) {
+ super(conf, rhsExpValueFormatter);
+ }
+
+ public void addValues(Table targetTable, String targetAlias, Map<String, String> newValues, List<String> values) {
+ for (FieldSchema fieldSchema : targetTable.getAllCols()) {
+ setColumnValue(targetAlias, newValues, values, fieldSchema);
+ }
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/SetValuesClause.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/SetValuesClause.java
new file mode 100644
index 00000000000..fdcf1cfd9e9
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/SetValuesClause.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.rewrite.sql;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+import java.util.List;
+import java.util.Map;
+
+public class SetValuesClause extends SetValuesClauseBase {
+
+ public SetValuesClause(HiveConf conf) {
+ super(conf);
+ }
+
+ public void addValues(Table targetTable, String targetAlias, Map<String, String> newValues, List<String> values) {
+ for (FieldSchema fieldSchema : targetTable.getCols()) {
+ setColumnValue(targetAlias, newValues, values, fieldSchema);
+ }
+ targetTable.getPartCols().forEach(fieldSchema -> values.add(
+ formatValue(targetAlias, fieldSchema.getName())));
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/SetValuesClauseBase.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/SetValuesClauseBase.java
new file mode 100644
index 00000000000..7dca12e5fbc
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/rewrite/sql/SetValuesClauseBase.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.rewrite.sql;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.BinaryOperator;
+
+public abstract class SetValuesClauseBase {
+
+ private final HiveConf conf;
+ private final BinaryOperator<String> rhsExpValueFormatter;
+
+ protected SetValuesClauseBase(HiveConf conf) {
+ this(conf, (newValue, alias) -> newValue);
+ }
+
+ protected SetValuesClauseBase(HiveConf conf, BinaryOperator<String> rhsExpValueFormatter) {
+ this.conf = conf;
+ this.rhsExpValueFormatter = rhsExpValueFormatter;
+ }
+
+ public abstract void addValues(
+ Table targetTable, String targetAlias, Map<String, String> newValues, List<String> values);
+
+ protected void setColumnValue(
+ String targetAlias, Map<String, String> newValues, List<String> values, FieldSchema fieldSchema) {
+ if (newValues.containsKey(fieldSchema.getName())) {
+ String rhsExp = newValues.get(fieldSchema.getName());
+ values.add(rhsExpValueFormatter.apply(rhsExp, formatValue(targetAlias, fieldSchema.getName())));
+ } else {
+ values.add(formatValue(targetAlias, fieldSchema.getName()));
+ }
+ }
+
+ protected String formatValue(String targetAlias, String name) {
+ return String.format("%s.%s", targetAlias, HiveUtils.unparseIdentifier(name, conf));
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
index 25f8afe6d6c..668fbfaacf8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
@@ -57,7 +57,7 @@ public class PartitionDesc implements Serializable, Cloneable {
private static final Interner<Class<?>> CLASS_INTERNER = Interners.newWeakInterner();
private TableDesc tableDesc;
- private LinkedHashMap<String, String> partSpec;
+ private Map<String, String> partSpec;
private Class<? extends InputFormat> inputFileFormatClass;
private Class<? extends OutputFormat> outputFileFormatClass;
private Properties properties;
@@ -73,7 +73,7 @@ public void setBaseFileName(String baseFileName) {
public PartitionDesc() {
}
- public PartitionDesc(final TableDesc table, final LinkedHashMap<String, String> partSpec) {
+ public PartitionDesc(final TableDesc table, final Map<String, String> partSpec) {
this.tableDesc = table;
setPartSpec(partSpec);
}
@@ -138,11 +138,11 @@ public void setTableDesc(TableDesc tableDesc) {
}
@Explain(displayName = "partition values", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
- public LinkedHashMap<String, String> getPartSpec() {
+ public Map<String, String> getPartSpec() {
return partSpec;
}
- public void setPartSpec(final LinkedHashMap<String, String> partSpec) {
+ public void setPartSpec(final Map<String, String> partSpec) {
StringInternUtils.internValuesInMap(partSpec);
this.partSpec = partSpec;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java
index 8229cd88dda..dc34a4dfa67 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.hooks.LineageInfo;
import org.apache.hadoop.hive.ql.hooks.LineageInfo.DataContainer;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.lineage.LineageCtx.Index;
/**
@@ -90,6 +91,19 @@ public synchronized void updateDirToOpMap(Path newPath, Path oldPath) {
}
}
+ /**
+ * Set the lineage information for the associated directory.
+ *
+ * @param dir The directory containing the query results.
+ * @param dc The associated data container.
+ * @param table table metadata used to resolve written columns for lineage mapping.
+ */
+ public synchronized void setLineage(Path dir, DataContainer dc, Table table) {
+ List<FieldSchema> cols = table.hasNonNativePartitionSupport() ?
+ table.getAllCols() : table.getCols();
+ setLineage(dir, dc, cols);
+ }
+
/**
* Set the lineage information for the associated directory.
*
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/CacheTag.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/CacheTag.java
index 66ef19e1d35..08a2c95468d 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/CacheTag.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/CacheTag.java
@@ -90,11 +90,11 @@ public static final CacheTag build(String catalogName, String dbAndTableName) {
return build(catalogName + "." + dbAndTableName);
}
- public static final CacheTag build(String catalogName, String dbAndTableName, LinkedHashMap<String, String> partDescMap) {
+ public static final CacheTag build(String catalogName, String dbAndTableName, Map<String, String> partDescMap) {
return build(catalogName + "." + dbAndTableName, partDescMap);
}
- public static final CacheTag build(String fullTableName, LinkedHashMap<String, String> partDescMap) {
+ public static final CacheTag build(String fullTableName, Map<String, String> partDescMap) {
if (StringUtils.isEmpty(fullTableName) || partDescMap == null || partDescMap.isEmpty()) {
throw new IllegalArgumentException();
}