You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2019/09/03 17:28:50 UTC
[hive] branch master updated: HIVE-22162: MVs are not using ACID
tables by default (Krisztian Kasa reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e56dfef HIVE-22162: MVs are not using ACID tables by default (Krisztian Kasa reviewed by Jesus Camacho Rodriguez)
e56dfef is described below
commit e56dfefebae39fdbabd6d9ced4bb0cdb643b4be4
Author: Krisztian Kasa <kk...@cloudera.com>
AuthorDate: Tue Sep 3 10:06:21 2019 -0700
HIVE-22162: MVs are not using ACID tables by default (Krisztian Kasa reviewed by Jesus Camacho Rodriguez)
---
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 18 ++++--
.../clientpositive/materialized_view_create_acid.q | 17 +++++
.../materialized_view_create_acid.q.out | 73 ++++++++++++++++++++++
3 files changed, 104 insertions(+), 4 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 826b23e..9644483 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -13417,10 +13417,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
boolean makeInsertOnly = !isTemporaryTable && HiveConf.getBoolVar(
conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY);
- boolean makeAcid = !isTemporaryTable &&
- MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID) &&
- HiveConf.getBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY) &&
- DbTxnManager.class.getCanonicalName().equals(HiveConf.getVar(conf, ConfVars.HIVE_TXN_MANAGER));
+ boolean makeAcid = !isTemporaryTable && makeAcid();
if ((makeInsertOnly || makeAcid || isTransactional)
&& !isExt && !isMaterialization && StringUtils.isBlank(storageFormat.getStorageHandler())
//don't overwrite user choice if transactional attribute is explicitly set
@@ -14065,6 +14062,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
unparseTranslator.enable();
if (isMaterialized) {
+ if (makeAcid()) {
+ if (tblProps == null) {
+ tblProps = new HashMap<>();
+ }
+ tblProps = convertToAcidByDefault(storageFormat, dbDotTable, null, tblProps);
+ }
+
createVwDesc = new CreateViewDesc(
dbDotTable, cols, comment, tblProps, partColNames, sortColNames, distributeColNames,
ifNotExists, isRebuild, rewriteEnabled, isAlterViewAs,
@@ -14087,6 +14091,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
return selectStmt;
}
+ private boolean makeAcid() {
+ return MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID) &&
+ HiveConf.getBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY) &&
+ DbTxnManager.class.getCanonicalName().equals(HiveConf.getVar(conf, ConfVars.HIVE_TXN_MANAGER));
+ }
+
// validate the (materialized) view statement
// check semantic conditions
private void validateCreateView()
diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_acid.q b/ql/src/test/queries/clientpositive/materialized_view_create_acid.q
new file mode 100644
index 0000000..35961af
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/materialized_view_create_acid.q
@@ -0,0 +1,17 @@
+SET hive.support.concurrency=true;
+SET hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+SET metastore.strict.managed.tables=true;
+SET hive.default.fileformat=textfile;
+SET hive.default.fileformat.managed=orc;
+
+SET metastore.create.as.acid=true;
+
+CREATE TABLE cmv_basetable_n4 (a int, b varchar(256), c decimal(10,2));
+
+INSERT INTO cmv_basetable_n4 VALUES (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8);
+
+CREATE MATERIALIZED VIEW cmv_mat_view_n4 disable rewrite
+AS SELECT a, b, c FROM cmv_basetable_n4;
+
+DESCRIBE FORMATTED cmv_mat_view_n4;
diff --git a/ql/src/test/results/clientpositive/materialized_view_create_acid.q.out b/ql/src/test/results/clientpositive/materialized_view_create_acid.q.out
new file mode 100644
index 0000000..52d9ae1
--- /dev/null
+++ b/ql/src/test/results/clientpositive/materialized_view_create_acid.q.out
@@ -0,0 +1,73 @@
+PREHOOK: query: CREATE TABLE cmv_basetable_n4 (a int, b varchar(256), c decimal(10,2))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cmv_basetable_n4
+POSTHOOK: query: CREATE TABLE cmv_basetable_n4 (a int, b varchar(256), c decimal(10,2))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cmv_basetable_n4
+PREHOOK: query: INSERT INTO cmv_basetable_n4 VALUES (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@cmv_basetable_n4
+POSTHOOK: query: INSERT INTO cmv_basetable_n4 VALUES (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@cmv_basetable_n4
+POSTHOOK: Lineage: cmv_basetable_n4.a SCRIPT []
+POSTHOOK: Lineage: cmv_basetable_n4.b SCRIPT []
+POSTHOOK: Lineage: cmv_basetable_n4.c SCRIPT []
+PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view_n4 disable rewrite
+AS SELECT a, b, c FROM cmv_basetable_n4
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@cmv_basetable_n4
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cmv_mat_view_n4
+POSTHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view_n4 disable rewrite
+AS SELECT a, b, c FROM cmv_basetable_n4
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@cmv_basetable_n4
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cmv_mat_view_n4
+PREHOOK: query: DESCRIBE FORMATTED cmv_mat_view_n4
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@cmv_mat_view_n4
+POSTHOOK: query: DESCRIBE FORMATTED cmv_mat_view_n4
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@cmv_mat_view_n4
+# col_name data_type comment
+a int
+b varchar(256)
+c decimal(10,2)
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MATERIALIZED_VIEW
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\"}}
+ bucketing_version 2
+ numFiles 1
+ numRows 5
+ rawDataSize 0
+ totalSize 924
+ transactional true
+ transactional_properties default
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# Materialized View Information
+Original Query: SELECT a, b, c FROM cmv_basetable_n4
+Expanded Query: SELECT `cmv_basetable_n4`.`a`, `cmv_basetable_n4`.`b`, `cmv_basetable_n4`.`c` FROM `default`.`cmv_basetable_n4`
+Rewrite Enabled: No
+Outdated for Rewriting: No