You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/11/15 22:42:25 UTC
svn commit: r1639928 - in /hive/trunk: metastore/if/
metastore/src/gen/thrift/gen-cpp/
metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/
metastore/src/gen/thrift/gen-php/metastore/
metastore/src/gen/thrift/gen-py/hive_metastor...
Author: hashutosh
Date: Sat Nov 15 21:42:25 2014
New Revision: 1639928
URL: http://svn.apache.org/r1639928
Log:
HIVE-8308 : Acid related table properties should be defined in one place and should be case insensitive (Alan Gates via Ashutosh Chauhan)
Modified:
hive/trunk/metastore/if/hive_metastore.thrift
hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php
hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java
Modified: hive/trunk/metastore/if/hive_metastore.thrift
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/if/hive_metastore.thrift?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/if/hive_metastore.thrift (original)
+++ hive/trunk/metastore/if/hive_metastore.thrift Sat Nov 15 21:42:25 2014
@@ -1139,6 +1139,7 @@ const string META_TABLE_PARTITION_COLUMN
const string FILE_INPUT_FORMAT = "file.inputformat",
const string FILE_OUTPUT_FORMAT = "file.outputformat",
const string META_TABLE_STORAGE = "storage_handler",
-
+const string TABLE_IS_TRANSACTIONAL = "transactional",
+const string TABLE_NO_AUTO_COMPACT = "no_auto_compaction",
Modified: hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp Sat Nov 15 21:42:25 2014
@@ -53,6 +53,10 @@ hive_metastoreConstants::hive_metastoreC
META_TABLE_STORAGE = "storage_handler";
+ TABLE_IS_TRANSACTIONAL = "transactional";
+
+ TABLE_NO_AUTO_COMPACT = "no_auto_compaction";
+
}
}}} // namespace
Modified: hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h Sat Nov 15 21:42:25 2014
@@ -36,6 +36,8 @@ class hive_metastoreConstants {
std::string FILE_INPUT_FORMAT;
std::string FILE_OUTPUT_FORMAT;
std::string META_TABLE_STORAGE;
+ std::string TABLE_IS_TRANSACTIONAL;
+ std::string TABLE_NO_AUTO_COMPACT;
};
extern const hive_metastoreConstants g_hive_metastore_constants;
Modified: hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java Sat Nov 15 21:42:25 2014
@@ -75,4 +75,8 @@ public class hive_metastoreConstants {
public static final String META_TABLE_STORAGE = "storage_handler";
+ public static final String TABLE_IS_TRANSACTIONAL = "transactional";
+
+ public static final String TABLE_NO_AUTO_COMPACT = "no_auto_compaction";
+
}
Modified: hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-php/metastore/Types.php Sat Nov 15 21:42:25 2014
@@ -12890,4 +12890,8 @@ $GLOBALS['hive_metastore_CONSTANTS']['FI
$GLOBALS['hive_metastore_CONSTANTS']['META_TABLE_STORAGE'] = "storage_handler";
+$GLOBALS['hive_metastore_CONSTANTS']['TABLE_IS_TRANSACTIONAL'] = "transactional";
+
+$GLOBALS['hive_metastore_CONSTANTS']['TABLE_NO_AUTO_COMPACT'] = "no_auto_compaction";
+
Modified: hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/constants.py Sat Nov 15 21:42:25 2014
@@ -30,3 +30,5 @@ META_TABLE_PARTITION_COLUMN_TYPES = "par
FILE_INPUT_FORMAT = "file.inputformat"
FILE_OUTPUT_FORMAT = "file.outputformat"
META_TABLE_STORAGE = "storage_handler"
+TABLE_IS_TRANSACTIONAL = "transactional"
+TABLE_NO_AUTO_COMPACT = "no_auto_compaction"
Modified: hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb (original)
+++ hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb Sat Nov 15 21:42:25 2014
@@ -49,3 +49,7 @@ FILE_OUTPUT_FORMAT = %q"file.outputforma
META_TABLE_STORAGE = %q"storage_handler"
+TABLE_IS_TRANSACTIONAL = %q"transactional"
+
+TABLE_NO_AUTO_COMPACT = %q"no_auto_compaction"
+
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Nov 15 21:42:25 2014
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
@@ -318,9 +319,6 @@ public class SemanticAnalyzer extends Ba
private static final String VALUES_TMP_TABLE_NAME_PREFIX = "Values__Tmp__Table__";
- @VisibleForTesting
- static final String ACID_TABLE_PROPERTY = "transactional";
-
private HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner;
private HashMap<TableScanOperator, PrunedPartitionList> opToPartList;
private HashMap<String, Operator<? extends OperatorDesc>> topOps;
@@ -12415,7 +12413,9 @@ public class SemanticAnalyzer extends Ba
private boolean isAcidTable(Table tab) {
if (tab == null) return false;
if (!SessionState.get().getTxnMgr().supportsAcid()) return false;
- return tab.getProperty(ACID_TABLE_PROPERTY) != null;
+ String tableIsTransactional =
+ tab.getProperty(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+ return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
}
private boolean isAcidOutputFormat(Class<? extends HiveOutputFormat> of) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java Sat Nov 15 21:42:25 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
import org.apache.hadoop.hive.metastore.txn.TxnHandler;
import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -53,8 +54,6 @@ public class Initiator extends Compactor
static final private String CLASS_NAME = Initiator.class.getName();
static final private Log LOG = LogFactory.getLog(CLASS_NAME);
- static final private String NO_COMPACTION = "NO_AUTO_COMPACTION";
-
private long checkInterval;
@Override
@@ -86,9 +85,8 @@ public class Initiator extends Compactor
try {
Table t = resolveTable(ci);
// check if no compaction set for this table
- if (t.getParameters().get(NO_COMPACTION) != null) {
- LOG.info("Table " + tableName(t) + " marked " + NO_COMPACTION +
- " so we will not compact it.");
+ if (noAutoCompactSet(t)) {
+ LOG.info("Table " + tableName(t) + " marked true so we will not compact it.");
continue;
}
@@ -279,4 +277,16 @@ public class Initiator extends Compactor
rqst.setRunas(runAs);
txnHandler.compact(rqst);
}
+
+ // Because TABLE_NO_AUTO_COMPACT was originally assumed to be NO_AUTO_COMPACT and then was moved
+ // to no_auto_compact, we need to check it in both cases.
+ private boolean noAutoCompactSet(Table t) {
+ String noAutoCompact =
+ t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT);
+ if (noAutoCompact == null) {
+ noAutoCompact =
+ t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT.toUpperCase());
+ }
+ return noAutoCompact != null && noAutoCompact.equalsIgnoreCase("true");
+ }
}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java Sat Nov 15 21:42:25 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FSDataInputS
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.exec.ExplainTask;
@@ -267,7 +268,7 @@ public class TestUpdateDeleteSemanticAna
// I have to create the tables here (rather than in setup()) because I need the Hive
// connection, which is conviently created by the semantic analyzer.
Map<String, String> params = new HashMap<String, String>(1);
- params.put(SemanticAnalyzer.ACID_TABLE_PROPERTY, "true");
+ params.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
db.createTable("T", Arrays.asList("a", "b"), null, OrcInputFormat.class,
OrcOutputFormat.class, 2, Arrays.asList("a"), params);
db.createTable("U", Arrays.asList("a", "b"), Arrays.asList("ds"), OrcInputFormat.class,
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java?rev=1639928&r1=1639927&r2=1639928&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java Sat Nov 15 21:42:25 2014
@@ -243,6 +243,32 @@ public class TestInitiator extends Compa
}
@Test
+ public void noCompactWhenNoCompactSetLowerCase() throws Exception {
+ Map<String, String> parameters = new HashMap<String, String>(1);
+ parameters.put("no_auto_compaction", "true");
+ Table t = newTable("default", "ncwncs", false, parameters);
+
+ HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10);
+
+ for (int i = 0; i < 11; i++) {
+ long txnid = openTxn();
+ LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default");
+ comp.setTablename("ncwncs");
+ List<LockComponent> components = new ArrayList<LockComponent>(1);
+ components.add(comp);
+ LockRequest req = new LockRequest(components, "me", "localhost");
+ req.setTxnid(txnid);
+ LockResponse res = txnHandler.lock(req);
+ txnHandler.abortTxn(new AbortTxnRequest(txnid));
+ }
+
+ startInitiator();
+
+ ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+ Assert.assertEquals(0, rsp.getCompactsSize());
+ }
+
+ @Test
public void noCompactWhenCompactAlreadyScheduled() throws Exception {
Table t = newTable("default", "ncwcas", false);