You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kr...@apache.org on 2023/02/15 07:58:43 UTC
[hive] branch master updated: HIVE-27073: Apply SerDe properties when creating materialized view (Krisztian Kasa, reviewed by Laszlo Vegh, Steve Carlin)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6f480cd40b8 HIVE-27073: Apply SerDe properties when creating materialized view (Krisztian Kasa, reviewed by Laszlo Vegh, Steve Carlin)
6f480cd40b8 is described below
commit 6f480cd40b8ddf7383a48de878b96bffbdecea3b
Author: Krisztian Kasa <ka...@gmail.com>
AuthorDate: Wed Feb 15 08:58:36 2023 +0100
HIVE-27073: Apply SerDe properties when creating materialized view (Krisztian Kasa, reviewed by Laszlo Vegh, Steve Carlin)
---
.../src/test/queries/positive/mv_iceberg_partitioned_orc.q | 2 ++
.../src/test/queries/positive/mv_iceberg_partitioned_orc2.q | 2 ++
.../src/test/results/positive/mv_iceberg_partitioned_orc.q.out | 10 ++++++----
.../test/results/positive/mv_iceberg_partitioned_orc2.q.out | 10 ++++++----
.../hive/ql/ddl/view/create/CreateMaterializedViewDesc.java | 6 ++++++
5 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc.q b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc.q
index 88f03ba83e1..44a5561f926 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc.q
@@ -2,6 +2,8 @@
--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/
-- Mask random snapshot id
--! qt:replace:/(\s+current-snapshot-id\s+)\d+(\s*)/$1#SnapshotId#/
+-- Mask the totalSize value as it can change at file format library update
+--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
-- SORT_QUERY_RESULTS
drop materialized view if exists mat1;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc2.q b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc2.q
index d0be6ea68fe..3cb8c04f1d0 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc2.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_partitioned_orc2.q
@@ -1,6 +1,8 @@
-- MV data is stored by partitioned iceberg with partition spec
--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/
--! qt:replace:/(\s+current-snapshot-id\s+)\d+(\s*)/$1#SnapshotId#/
+-- Mask the totalSize value as it can change at file format library update
+--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
-- SORT_QUERY_RESULTS
drop materialized view if exists mat1;
diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out
index 5666d10d68d..8cda12100b4 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out
@@ -64,16 +64,17 @@ Table Parameters:
current-snapshot-id #SnapshotId#
engine.hive.enabled true
format-version 1
- iceberg.orc.files.only false
+ iceberg.orc.files.only true
metadata_location hdfs://### HDFS PATH ###
numFiles 2
numRows 2
previous_metadata_location hdfs://### HDFS PATH ###
storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 1282
+ totalSize #Masked#
#### A masked pattern was here ####
uuid #Masked#
+ write.format.default orc
# Storage Information
SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
@@ -143,17 +144,18 @@ Table Parameters:
current-snapshot-id #SnapshotId#
engine.hive.enabled true
format-version 2
- iceberg.orc.files.only false
+ iceberg.orc.files.only true
metadata_location hdfs://### HDFS PATH ###
numFiles 2
numRows 2
previous_metadata_location hdfs://### HDFS PATH ###
storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 1282
+ totalSize #Masked#
#### A masked pattern was here ####
uuid #Masked#
write.delete.mode merge-on-read
+ write.format.default orc
write.merge.mode merge-on-read
write.update.mode merge-on-read
diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out
index 169c00375ad..6a31b96ef77 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out
@@ -65,16 +65,17 @@ Table Parameters:
current-snapshot-id #SnapshotId#
engine.hive.enabled true
format-version 1
- iceberg.orc.files.only false
+ iceberg.orc.files.only true
metadata_location hdfs://### HDFS PATH ###
numFiles 2
numRows 2
previous_metadata_location hdfs://### HDFS PATH ###
storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 1282
+ totalSize #Masked#
#### A masked pattern was here ####
uuid #Masked#
+ write.format.default orc
# Storage Information
SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
@@ -145,17 +146,18 @@ Table Parameters:
current-snapshot-id #SnapshotId#
engine.hive.enabled true
format-version 2
- iceberg.orc.files.only false
+ iceberg.orc.files.only true
metadata_location hdfs://### HDFS PATH ###
numFiles 2
numRows 2
previous_metadata_location hdfs://### HDFS PATH ###
storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 1282
+ totalSize #Masked#
#### A masked pattern was here ####
uuid #Masked#
write.delete.mode merge-on-read
+ write.format.default orc
write.merge.mode merge-on-read
write.update.mode merge-on-read
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java
index 631a8868f10..452e861cefe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java
@@ -349,6 +349,12 @@ public class CreateMaterializedViewDesc implements DDLDesc, Serializable {
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE,
getStorageHandler());
}
+ if (getSerdeProps() != null) {
+ for (Map.Entry<String, String> entry : getSerdeProps().entrySet()) {
+ tbl.setSerdeParam(entry.getKey(), entry.getValue());
+ }
+ }
+
HiveStorageHandler storageHandler = tbl.getStorageHandler();
setColumnsAndStorePartitionTransformSpecOfTable(getSchema(), getPartCols(), conf, tbl);