You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2018/01/02 18:13:22 UTC
hive git commit: HIVE-18294 - add switch to make acid table the
default (Eugene Koifman, reviewed by Alan Gates)
Repository: hive
Updated Branches:
refs/heads/master 4ecf2a755 -> 5b0d99383
HIVE-18294 - add switch to make acid table the default (Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5b0d9938
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5b0d9938
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5b0d9938
Branch: refs/heads/master
Commit: 5b0d99383d667572d649cafa97dc2b8a41bf8a7f
Parents: 4ecf2a7
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Jan 2 10:13:14 2018 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Jan 2 10:13:14 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/Driver.java | 3 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 63 ++++++++++++++------
.../TransactionalValidationListener.java | 55 +++++++++++++++++
.../hive/metastore/conf/MetastoreConf.java | 5 +-
4 files changed, 106 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5b0d9938/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index d632ee8..cfa38f4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1337,6 +1337,9 @@ public class Driver implements IDriver {
// If we've opened a transaction we need to commit or rollback rather than explicitly
// releasing the locks.
conf.unset(ValidTxnList.VALID_TXNS_KEY);
+ if(!checkConcurrency()) {
+ return;
+ }
if (txnMgr.isTxnOpen()) {
if (commit) {
if(conf.getBoolVar(ConfVars.HIVE_IN_TEST) && conf.getBoolVar(ConfVars.HIVETESTMODEROLLBACKTXN)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/5b0d9938/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 70fce32..10fe6fc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -77,6 +77,7 @@ import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -108,6 +109,7 @@ import org.apache.hadoop.hive.ql.exec.UnionOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.io.AcidInputFormat;
import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.AcidUtils.Operation;
@@ -119,6 +121,7 @@ import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
import org.apache.hadoop.hive.ql.metadata.DummyPartition;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -12202,14 +12205,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
validate(childTask, reworkMapredWork);
}
}
-
- /**
- * Get the row resolver given an operator.
- */
- public RowResolver getRowResolver(Operator opt) {
- return opParseCtx.get(opt).getRowResolver();
- }
-
/**
* Add default properties for table property. If a default parameter exists
* in the tblProp, the value in tblProp will be kept.
@@ -12219,7 +12214,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* @return Modified table property map
*/
private Map<String, String> addDefaultProperties(
- Map<String, String> tblProp, boolean isExt, StorageFormat storageFormat) {
+ Map<String, String> tblProp, boolean isExt, StorageFormat storageFormat,
+ String qualifiedTableName, List<Order> sortCols) {
Map<String, String> retValue;
if (tblProp == null) {
retValue = new HashMap<String, String>();
@@ -12238,16 +12234,45 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
}
- if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY)
+ boolean makeInsertOnly = HiveConf.getBoolVar(conf, ConfVars.HIVE_CREATE_TABLES_AS_INSERT_ONLY);
+ boolean makeAcid = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID) &&
+ HiveConf.getBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY) &&
+ DbTxnManager.class.getCanonicalName().equals(HiveConf.getVar(conf, ConfVars.HIVE_TXN_MANAGER));
+ if ((makeInsertOnly || makeAcid)
&& !isExt && StringUtils.isBlank(storageFormat.getStorageHandler())
+ //don't overwrite user choice if transactional attribute is explicitly set
&& !retValue.containsKey(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)) {
- retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
- String oldProps = retValue.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
- if (oldProps != null) {
- LOG.warn("Non-transactional table has transactional properties; overwriting " + oldProps);
+ if(makeInsertOnly) {
+ retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
+ retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES,
+ TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY);
+ }
+ if(makeAcid) {
+ /*for CTAS, TransactionalValidationListener.makeAcid() runs to late to make table Acid
+ so the initial write ends up running as non-acid...*/
+ try {
+ Class inputFormatClass = storageFormat.getInputFormat() == null ? null :
+ Class.forName(storageFormat.getInputFormat());
+ Class outputFormatClass = storageFormat.getOutputFormat() == null ? null :
+ Class.forName(storageFormat.getOutputFormat());
+ if (inputFormatClass == null || outputFormatClass == null ||
+ !AcidInputFormat.class.isAssignableFrom(inputFormatClass) ||
+ !AcidOutputFormat.class.isAssignableFrom(outputFormatClass)) {
+ return retValue;
+ }
+ } catch (ClassNotFoundException e) {
+ LOG.warn("Could not verify InputFormat=" + storageFormat.getInputFormat() + " or OutputFormat=" +
+ storageFormat.getOutputFormat() + " for " + qualifiedTableName);
+ return retValue;
+ }
+ if(sortCols != null && !sortCols.isEmpty()) {
+ return retValue;
+ }
+ retValue.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
+ retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES,
+ TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY);
+ LOG.info("Automatically chose to make " + qualifiedTableName + " acid.");
}
- retValue.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES,
- TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY);
}
return retValue;
}
@@ -12472,7 +12497,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
switch (command_type) {
case CREATE_TABLE: // REGULAR CREATE TABLE DDL
- tblProps = addDefaultProperties(tblProps, isExt, storageFormat);
+ tblProps = addDefaultProperties(tblProps, isExt, storageFormat, dbDotTab, sortCols);
CreateTableDesc crtTblDesc = new CreateTableDesc(dbDotTab, isExt, isTemporary, cols, partCols,
bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim,
@@ -12493,7 +12518,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
break;
case CTLT: // create table like <tbl_name>
- tblProps = addDefaultProperties(tblProps, isExt, storageFormat);
+ tblProps = addDefaultProperties(tblProps, isExt, storageFormat, dbDotTab, sortCols);
if (isTemporary) {
Table likeTable = getTable(likeTableName, false);
@@ -12570,7 +12595,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
- tblProps = addDefaultProperties(tblProps, isExt, storageFormat);
+ tblProps = addDefaultProperties(tblProps, isExt, storageFormat, dbDotTab, sortCols);
tableDesc = new CreateTableDesc(qualifiedTabName[0], dbDotTab, isExt, isTemporary, cols,
partCols, bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim,
rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim,
http://git-wip-us.apache.org/repos/asf/hive/blob/5b0d9938/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index df14752..f977f14 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.metastore;
import java.io.IOException;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -34,6 +35,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent;
import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent;
import org.apache.hadoop.hive.metastore.events.PreEventContext;
@@ -199,6 +201,57 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
}
/**
+ * Want to make a a newly create table Acid (unless it explicitly has transactional=false param)
+ * if table can support it. Also see SemanticAnalyzer.addDefaultProperties() which performs the
+ * same logic. This code path is more general since it is activated even if you create a table
+ * via Thrift, WebHCat etc but some operations like CTAS create the table (metastore object) as
+ * the last step (i.e. after the data is written) but write itself is has to be aware of the type
+ * of table so this Listener is too late.
+ */
+ private void makeAcid(Table newTable) throws MetaException {
+ if(newTable.getParameters() != null &&
+ newTable.getParameters().containsKey(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL)) {
+ LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: already has " +
+ hive_metastoreConstants.TABLE_IS_TRANSACTIONAL + "=" +
+ newTable.getParameters().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL));
+ return;
+ }
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ boolean makeAcid =
+ //no point making an acid table if these other props are not set since it will just throw
+ //exceptions when someone tries to use the table.
+ MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID) &&
+ MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.HIVE_SUPPORT_CONCURRENCY) &&
+ "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager".equals(
+ MetastoreConf.getVar(conf, MetastoreConf.ConfVars.HIVE_TXN_MANAGER)
+ );
+
+ if(makeAcid) {
+ if(!conformToAcid(newTable)) {
+ LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: wrong IO format");
+ return;
+ }
+ if(!TableType.MANAGED_TABLE.toString().equalsIgnoreCase(newTable.getTableType())) {
+ //todo should this check be in conformToAcid()?
+ LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: it's " +
+ newTable.getTableType());
+ return;
+ }
+ if(newTable.getSd().getSortColsSize() > 0) {
+ LOG.info("Could not make " + Warehouse.getQualifiedName(newTable) + " acid: it's sorted");
+ return;
+ }
+ //check if orc and not sorted
+ Map<String, String> parameters = newTable.getParameters();
+ if (parameters == null || parameters.isEmpty()) {
+ parameters = new HashMap<>();
+ }
+ parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
+ newTable.setParameters(parameters);
+ LOG.info("Automatically chose to make " + Warehouse.getQualifiedName(newTable) + " acid.");
+ }
+ }
+ /**
* Normalize case and make sure:
* 1. 'true' is the only value to be set for 'transactional' (if set at all)
* 2. If set to 'true', we should also enforce bucketing and ORC format
@@ -207,6 +260,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
Table newTable = context.getTable();
Map<String, String> parameters = newTable.getParameters();
if (parameters == null || parameters.isEmpty()) {
+ makeAcid(newTable);
return;
}
String transactional = null;
@@ -226,6 +280,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
}
if (transactional == null) {
+ makeAcid(newTable);
return;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5b0d9938/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index b46cc38..2e43dc8 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -53,7 +53,7 @@ import java.util.regex.Pattern;
/**
* A set of definitions of config values used by the Metastore. One of the key aims of this
* class is to provide backwards compatibility with existing Hive configuration keys while
- * allowing the metastore to have its own, Hive independant keys. For this reason access to the
+ * allowing the metastore to have its own, Hive independent keys. For this reason access to the
* underlying Configuration object should always be done via the static methods provided here
* rather than directly via {@link Configuration#get(String)} and
* {@link Configuration#set(String, String)}. All the methods of this class will handle checking
@@ -373,6 +373,9 @@ public class MetastoreConf {
CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName",
"javax.jdo.option.ConnectionUserName", "APP",
"Username to use against metastore database"),
+ CREATE_TABLES_AS_ACID("metastore.create.as.acid", "hive.create.as.acid", false,
+ "Whether the eligible tables should be created as full ACID by default. Does \n" +
+ "not apply to external tables, the ones using storage handlers, etc."),
COUNT_OPEN_TXNS_INTERVAL("metastore.count.open.txns.interval", "hive.count.open.txns.interval",
1, TimeUnit.SECONDS, "Time in seconds between checks to count open transactions."),
DATANUCLEUS_AUTOSTART("datanucleus.autoStartMechanismMode",