You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/06/26 04:17:15 UTC
[56/58] [abbrv] hive git commit: HIVE-19532 : fix tests for
master-txnstats branch (Sergey Shelukhin)
HIVE-19532 : fix tests for master-txnstats branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8f328324
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8f328324
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8f328324
Branch: refs/heads/master-txnstats
Commit: 8f328324287933f7005c46627e269b314d05e698
Parents: 4743c79
Author: sergey <se...@apache.org>
Authored: Mon Jun 25 21:01:23 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Mon Jun 25 21:01:23 2018 -0700
----------------------------------------------------------------------
.../hive/ql/parse/TestReplicationScenarios.java | 1 +
.../java/org/apache/hadoop/hive/ql/io/AcidUtils.java | 15 ++++++++++++---
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 6 ++++--
.../ql/parse/TestUpdateDeleteSemanticAnalyzer.java | 1 +
.../apache/hadoop/hive/metastore/HiveMetaStore.java | 3 ++-
5 files changed, 20 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8f328324/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index 689c859..a4e29c9 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -151,6 +151,7 @@ public class TestReplicationScenarios {
hconf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hconf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
hconf.set(HiveConf.ConfVars.HIVE_IN_TEST_REPL.varname, "true");
+ hconf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, true);
hconf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
hconf.set(HiveConf.ConfVars.HIVE_TXN_MANAGER.varname,
"org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager");
http://git-wip-us.apache.org/repos/asf/hive/blob/8f328324/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 7d35084..e67b579 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -1677,13 +1677,22 @@ public class AcidUtils {
}
String fullTableName = getFullTableName(tbl.getDbName(), tbl.getTableName());
if (txnId > 0 && isTransactionalTable(tbl)) {
- validWriteIdList =
- getTableValidWriteIdList(conf, fullTableName);
+ validWriteIdList = getTableValidWriteIdList(conf, fullTableName);
- if (validWriteIdList == null) {
+ // TODO: we shouldn't do this during normal Hive compilation, write IDs should be in conf.
+ // Can this still happen for DDLTask-based queries?
+ if (validWriteIdList == null && !HiveConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST)) {
+ LOG.warn("Obtaining write IDs from metastore for " + tbl.getTableName());
validWriteIdList = getTableValidWriteIdListWithTxnList(
conf, tbl.getDbName(), tbl.getTableName());
}
+ if (validWriteIdList == null) {
+ if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST)) {
+ throw new AssertionError("Cannot find valid write ID list for " + tbl.getTableName());
+ } else {
+ return null;
+ }
+ }
}
return new TableSnapshot(txnId,
validWriteIdList != null ? validWriteIdList.toString() : null);
http://git-wip-us.apache.org/repos/asf/hive/blob/8f328324/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index f9fab96..cc45195 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -24,8 +24,8 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import static org.apache.hadoop.hive.conf.Constants.MATERIALIZED_VIEW_REWRITING_TIME_WINDOW;
+import static org.apache.hadoop.hive.conf.Constants.MATERIALIZED_VIEW_REWRITING_TIME_WINDOW;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
import static org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer.makeBinaryPredicate;
@@ -62,6 +62,7 @@ import javax.annotation.Nullable;
import javax.jdo.JDODataStoreException;
import com.google.common.collect.ImmutableList;
+
import org.apache.calcite.plan.RelOptMaterialization;
import org.apache.calcite.plan.hep.HepPlanner;
import org.apache.calcite.plan.hep.HepProgramBuilder;
@@ -2512,7 +2513,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
: getMSC().add_partitions(partsToAdd, addPartitionDesc.isIfNotExists(), true)) {
out.add(new Partition(tbl, outPart));
}
- getMSC().alter_partitions(addPartitionDesc.getDbName(), addPartitionDesc.getTableName(), partsToAlter, null);
+ getMSC().alter_partitions(addPartitionDesc.getDbName(), addPartitionDesc.getTableName(),
+ partsToAlter, new EnvironmentContext());
for ( org.apache.hadoop.hive.metastore.api.Partition outPart :
getMSC().getPartitionsByNames(addPartitionDesc.getDbName(), addPartitionDesc.getTableName(),part_names)){
http://git-wip-us.apache.org/repos/asf/hive/blob/8f328324/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
index ffd0445..257a6ad 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
@@ -231,6 +231,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
conf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, true);
}
public void cleanupTables() throws HiveException {
http://git-wip-us.apache.org/repos/asf/hive/blob/8f328324/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 4296084..706d831 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -4877,7 +4877,8 @@ public class HiveMetaStore extends ThriftHiveMetastore {
throws TException {
alter_partitions_with_environment_context(
req.getDbName(), req.getTableName(), req.getPartitions(), req.getEnvironmentContext(),
- req.getTxnId(), req.getValidWriteIdList());
+ req.isSetTxnId() ? req.getTxnId() : -1,
+ req.isSetValidWriteIdList() ? req.getValidWriteIdList() : null);
return new AlterPartitionsResponse();
}