You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/08/14 20:15:23 UTC
hive git commit: HIVE-20343 : Hive 3: CTAS does not respect
transactional_properties (Sergey Shelukhin, reviewed by Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/master 93779cb10 -> 4ba12f2f4
HIVE-20343 : Hive 3: CTAS does not respect transactional_properties (Sergey Shelukhin, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4ba12f2f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4ba12f2f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4ba12f2f
Branch: refs/heads/master
Commit: 4ba12f2f47fb3318a139dc64efd7c0eabfca7a4c
Parents: 93779cb
Author: sergey <se...@apache.org>
Authored: Tue Aug 14 13:15:14 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Tue Aug 14 13:15:14 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 74 ++++++++++++--------
ql/src/test/queries/clientnegative/txnprops.q | 7 ++
.../test/results/clientnegative/txnprops.q.out | 5 ++
3 files changed, 56 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4ba12f2f/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 2ee562a..6312e74 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -12980,10 +12980,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* property map
* @return Modified table property map
*/
- private Map<String, String> addDefaultProperties(
+ private Map<String, String> validateAndAddDefaultProperties(
Map<String, String> tblProp, boolean isExt, StorageFormat storageFormat,
String qualifiedTableName, List<Order> sortCols, boolean isMaterialization,
- boolean isTemporaryTable) {
+ boolean isTemporaryTable) throws SemanticException {
Map<String, String> retValue;
if (tblProp == null) {
retValue = new HashMap<String, String>();
@@ -13002,6 +13002,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
}
+ if (!retValue.containsKey(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)
+ && retValue.containsKey(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES)) {
+ throw new SemanticException("Cannot specify "
+ + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES
+ + " without " + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+ }
if (isExt && HiveConf.getBoolVar(conf, ConfVars.HIVE_EXTERNALTABLE_PURGE_DEFAULT)) {
if (retValue.get(MetaStoreUtils.EXTERNAL_TABLE_PURGE) == null) {
@@ -13009,7 +13015,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
- boolean makeInsertOnly = !isTemporaryTable && HiveConf.getBoolVar(conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY);
+ boolean makeInsertOnly = !isTemporaryTable && HiveConf.getBoolVar(
+ conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY);
boolean makeAcid = !isTemporaryTable &&
MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID) &&
HiveConf.getBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY) &&
@@ -13024,35 +13031,42 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY);
}
if (makeAcid) {
- /*for CTAS, TransactionalValidationListener.makeAcid() runs to late to make table Acid
- so the initial write ends up running as non-acid...*/
- try {
- Class inputFormatClass = storageFormat.getInputFormat() == null ? null :
- Class.forName(storageFormat.getInputFormat());
- Class outputFormatClass = storageFormat.getOutputFormat() == null ? null :
- Class.forName(storageFormat.getOutputFormat());
- if (inputFormatClass == null || outputFormatClass == null ||
- !AcidInputFormat.class.isAssignableFrom(inputFormatClass) ||
- !AcidOutputFormat.class.isAssignableFrom(outputFormatClass)) {
- return retValue;
- }
- } catch (ClassNotFoundException e) {
- LOG.warn("Could not verify InputFormat=" + storageFormat.getInputFormat() + " or OutputFormat=" +
- storageFormat.getOutputFormat() + " for " + qualifiedTableName);
- return retValue;
- }
- if(sortCols != null && !sortCols.isEmpty()) {
- return retValue;
- }
- retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
- retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES,
- TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY);
- LOG.info("Automatically chose to make " + qualifiedTableName + " acid.");
+ retValue = convertToAcidByDefault(storageFormat, qualifiedTableName, sortCols, retValue);
}
}
return retValue;
}
+ private Map<String, String> convertToAcidByDefault(
+ StorageFormat storageFormat, String qualifiedTableName, List<Order> sortCols,
+ Map<String, String> retValue) {
+ /*for CTAS, TransactionalValidationListener.makeAcid() runs to late to make table Acid
+ so the initial write ends up running as non-acid...*/
+ try {
+ Class inputFormatClass = storageFormat.getInputFormat() == null ? null :
+ Class.forName(storageFormat.getInputFormat());
+ Class outputFormatClass = storageFormat.getOutputFormat() == null ? null :
+ Class.forName(storageFormat.getOutputFormat());
+ if (inputFormatClass == null || outputFormatClass == null ||
+ !AcidInputFormat.class.isAssignableFrom(inputFormatClass) ||
+ !AcidOutputFormat.class.isAssignableFrom(outputFormatClass)) {
+ return retValue;
+ }
+ } catch (ClassNotFoundException e) {
+ LOG.warn("Could not verify InputFormat=" + storageFormat.getInputFormat() + " or OutputFormat=" +
+ storageFormat.getOutputFormat() + " for " + qualifiedTableName);
+ return retValue;
+ }
+ if (sortCols != null && !sortCols.isEmpty()) {
+ return retValue;
+ }
+ retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
+ retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES,
+ TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY);
+ LOG.info("Automatically chose to make " + qualifiedTableName + " acid.");
+ return retValue;
+ }
+
/**
* Checks to see if given partition columns has DEFAULT or CHECK constraints (whether ENABLED or DISABLED)
* Or has NOT NULL constraints (only ENABLED)
@@ -13328,7 +13342,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException(
"Partition columns can only declared using their name and types in regular CREATE TABLE statements");
}
- tblProps = addDefaultProperties(
+ tblProps = validateAndAddDefaultProperties(
tblProps, isExt, storageFormat, dbDotTab, sortCols, isMaterialization, isTemporary);
addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, isTemporary, tblProps);
@@ -13352,7 +13366,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
break;
case CTLT: // create table like <tbl_name>
- tblProps = addDefaultProperties(
+ tblProps = validateAndAddDefaultProperties(
tblProps, isExt, storageFormat, dbDotTab, sortCols, isMaterialization, isTemporary);
addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, isTemporary, tblProps);
@@ -13436,7 +13450,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
"Partition columns can only declared using their names in CTAS statements");
}
- tblProps = addDefaultProperties(
+ tblProps = validateAndAddDefaultProperties(
tblProps, isExt, storageFormat, dbDotTab, sortCols, isMaterialization, isTemporary);
addDbAndTabToOutputs(qualifiedTabName, TableType.MANAGED_TABLE, isTemporary, tblProps);
tableDesc = new CreateTableDesc(qualifiedTabName[0], dbDotTab, isExt, isTemporary, cols,
http://git-wip-us.apache.org/repos/asf/hive/blob/4ba12f2f/ql/src/test/queries/clientnegative/txnprops.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/txnprops.q b/ql/src/test/queries/clientnegative/txnprops.q
new file mode 100644
index 0000000..3c2dcfb
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/txnprops.q
@@ -0,0 +1,7 @@
+--! qt:dataset:src
+
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+drop table test_props;
+create table test_props TBLPROPERTIES ("transactional_properties"="insert_only") as select * from src limit 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/4ba12f2f/ql/src/test/results/clientnegative/txnprops.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/txnprops.q.out b/ql/src/test/results/clientnegative/txnprops.q.out
new file mode 100644
index 0000000..24a59f3
--- /dev/null
+++ b/ql/src/test/results/clientnegative/txnprops.q.out
@@ -0,0 +1,5 @@
+PREHOOK: query: drop table test_props
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table test_props
+POSTHOOK: type: DROPTABLE
+FAILED: SemanticException Cannot specify transactional_properties without transactional