You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/10/13 23:54:49 UTC
hive git commit: HIVE-11923 : allow qtests to run via a single client
session for tez and llap (Siddharth Seth/Sergey Shelukhin,
reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master bb791bacb -> 07eaab396
HIVE-11923 : allow qtests to run via a single client session for tez and llap (Siddharth Seth/Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/07eaab39
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/07eaab39
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/07eaab39
Branch: refs/heads/master
Commit: 07eaab39607e66a397f4efe6824b2eaf98653327
Parents: bb791ba
Author: Sergey Shelukhin <se...@apache.org>
Authored: Tue Oct 13 14:47:17 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Oct 13 14:54:34 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/hbase/HBaseQTestUtil.java | 8 +--
.../org/apache/hadoop/hive/ql/QTestUtil.java | 61 ++++++++++++++------
.../test/queries/clientpositive/scriptfile1.q | 3 +
.../results/clientpositive/scriptfile1.q.out | 6 ++
.../clientpositive/spark/scriptfile1.q.out | 6 ++
.../clientpositive/tez/scriptfile1.q.out | 6 ++
6 files changed, 69 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
index 9c20f90..59fab2c 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
@@ -69,8 +69,8 @@ public class HBaseQTestUtil extends QTestUtil {
}
@Override
- public void createSources() throws Exception {
- super.createSources();
+ public void createSources(String tname) throws Exception {
+ super.createSources(tname);
conf.setBoolean("hive.test.init.phase", true);
@@ -96,8 +96,8 @@ public class HBaseQTestUtil extends QTestUtil {
}
@Override
- public void cleanUp() throws Exception {
- super.cleanUp();
+ public void cleanUp(String tname) throws Exception {
+ super.cleanUp(tname);
// drop in case leftover from unsuccessful run
db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, HBASE_SRC_NAME);
http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 174a988..309cdff 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -142,6 +142,7 @@ public class QTestUtil {
private final Set<String> qSortQuerySet;
private final Set<String> qHashQuerySet;
private final Set<String> qSortNHashQuerySet;
+ private final Set<String> qNoSessionReuseQuerySet;
private final Set<String> qJavaVersionSpecificOutput;
private static final String SORT_SUFFIX = ".sorted";
public static final HashSet<String> srcTables = new HashSet<String>();
@@ -401,6 +402,7 @@ public class QTestUtil {
qSortQuerySet = new HashSet<String>();
qHashQuerySet = new HashSet<String>();
qSortNHashQuerySet = new HashSet<String>();
+ qNoSessionReuseQuerySet = new HashSet<String>();
qJavaVersionSpecificOutput = new HashSet<String>();
QTestUtil.clusterType = clusterType;
@@ -556,12 +558,16 @@ public class QTestUtil {
} else if (matches(SORT_AND_HASH_QUERY_RESULTS, query)) {
qSortNHashQuerySet.add(qf.getName());
}
+ if (matches(NO_SESSION_REUSE, query)) {
+ qNoSessionReuseQuerySet.add(qf.getName());
+ }
}
private static final Pattern SORT_BEFORE_DIFF = Pattern.compile("-- SORT_BEFORE_DIFF");
private static final Pattern SORT_QUERY_RESULTS = Pattern.compile("-- SORT_QUERY_RESULTS");
private static final Pattern HASH_QUERY_RESULTS = Pattern.compile("-- HASH_QUERY_RESULTS");
private static final Pattern SORT_AND_HASH_QUERY_RESULTS = Pattern.compile("-- SORT_AND_HASH_QUERY_RESULTS");
+ private static final Pattern NO_SESSION_REUSE = Pattern.compile("-- NO_SESSION_REUSE");
private boolean matches(Pattern pattern, String query) {
Matcher matcher = pattern.matcher(query);
@@ -803,8 +809,13 @@ public class QTestUtil {
}
public void cleanUp() throws Exception {
+ cleanUp(null);
+ }
+
+ public void cleanUp(String tname) throws Exception {
+ boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
if(!isSessionStateStarted) {
- startSessionState();
+ startSessionState(canReuseSession);
}
if (System.getenv(QTEST_LEAVE_FILES) != null) {
return;
@@ -867,8 +878,13 @@ public class QTestUtil {
}
public void createSources() throws Exception {
+ createSources(null);
+ }
+
+ public void createSources(String tname) throws Exception {
+ boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
if(!isSessionStateStarted) {
- startSessionState();
+ startSessionState(canReuseSession);
}
if(cliDriver == null) {
@@ -908,8 +924,8 @@ public class QTestUtil {
}
public void init(String tname) throws Exception {
- cleanUp();
- createSources();
+ cleanUp(tname);
+ createSources(tname);
cliDriver.processCmd("set hive.cli.print.header=true;");
}
@@ -919,8 +935,8 @@ public class QTestUtil {
public String cliInit(String tname, boolean recreate) throws Exception {
if (recreate) {
- cleanUp();
- createSources();
+ cleanUp(tname);
+ createSources(tname);
}
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
@@ -955,11 +971,9 @@ public class QTestUtil {
ss.setIsSilent(true);
SessionState oldSs = SessionState.get();
- if (oldSs != null && (clusterType == MiniClusterType.llap
- || clusterType == MiniClusterType.spark || clusterType == MiniClusterType.miniSparkOnYarn)) {
- sparkSession = oldSs.getSparkSession();
- ss.setSparkSession(sparkSession);
- oldSs.setSparkSession(null);
+ boolean canReuseSession = !qNoSessionReuseQuerySet.contains(tname);
+ if (oldSs != null && canReuseSession
+ && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap)) {
// Copy the tezSessionState from the old CliSessionState.
tezSessionState = oldSs.getTezSession();
ss.setTezSession(tezSessionState);
@@ -967,6 +981,14 @@ public class QTestUtil {
oldSs.close();
}
+ if (oldSs != null && (clusterType == MiniClusterType.spark
+ || clusterType == MiniClusterType.miniSparkOnYarn)) {
+ sparkSession = oldSs.getSparkSession();
+ ss.setSparkSession(sparkSession);
+ oldSs.setSparkSession(null);
+ oldSs.close();
+ }
+
if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
oldSs.out.close();
}
@@ -1008,7 +1030,7 @@ public class QTestUtil {
};
}
- private CliSessionState startSessionState()
+ private CliSessionState startSessionState(boolean canReuseSession)
throws IOException {
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
@@ -1023,17 +1045,22 @@ public class QTestUtil {
ss.err = System.out;
SessionState oldSs = SessionState.get();
- if (oldSs != null && (clusterType == MiniClusterType.llap
- || clusterType == MiniClusterType.miniSparkOnYarn || clusterType == MiniClusterType.miniSparkOnYarn)) {
- sparkSession = oldSs.getSparkSession();
- ss.setSparkSession(sparkSession);
- oldSs.setSparkSession(null);
+ if (oldSs != null && canReuseSession
+ && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap)) {
// Copy the tezSessionState from the old CliSessionState.
tezSessionState = oldSs.getTezSession();
ss.setTezSession(tezSessionState);
oldSs.setTezSession(null);
oldSs.close();
}
+
+ if (oldSs != null && (clusterType == MiniClusterType.spark
+ || clusterType == MiniClusterType.miniSparkOnYarn)) {
+ sparkSession = oldSs.getSparkSession();
+ ss.setSparkSession(sparkSession);
+ oldSs.setSparkSession(null);
+ oldSs.close();
+ }
if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
oldSs.out.close();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/ql/src/test/queries/clientpositive/scriptfile1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/scriptfile1.q b/ql/src/test/queries/clientpositive/scriptfile1.q
index 49d2197..a075177 100644
--- a/ql/src/test/queries/clientpositive/scriptfile1.q
+++ b/ql/src/test/queries/clientpositive/scriptfile1.q
@@ -3,6 +3,9 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
-- SORT_QUERY_RESULTS
-- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
CREATE TABLE dest1(key INT, value STRING);
ADD FILE ../../ql/src/test/scripts/testgrep;
http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/ql/src/test/results/clientpositive/scriptfile1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/scriptfile1.q.out b/ql/src/test/results/clientpositive/scriptfile1.q.out
index d72ff5c..bf202f9 100644
--- a/ql/src/test/results/clientpositive/scriptfile1.q.out
+++ b/ql/src/test/results/clientpositive/scriptfile1.q.out
@@ -1,6 +1,9 @@
PREHOOK: query: -- SORT_QUERY_RESULTS
-- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
CREATE TABLE dest1(key INT, value STRING)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
@@ -8,6 +11,9 @@ PREHOOK: Output: default@dest1
POSTHOOK: query: -- SORT_QUERY_RESULTS
-- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
CREATE TABLE dest1(key INT, value STRING)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/ql/src/test/results/clientpositive/spark/scriptfile1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/scriptfile1.q.out b/ql/src/test/results/clientpositive/spark/scriptfile1.q.out
index d72ff5c..bf202f9 100644
--- a/ql/src/test/results/clientpositive/spark/scriptfile1.q.out
+++ b/ql/src/test/results/clientpositive/spark/scriptfile1.q.out
@@ -1,6 +1,9 @@
PREHOOK: query: -- SORT_QUERY_RESULTS
-- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
CREATE TABLE dest1(key INT, value STRING)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
@@ -8,6 +11,9 @@ PREHOOK: Output: default@dest1
POSTHOOK: query: -- SORT_QUERY_RESULTS
-- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
CREATE TABLE dest1(key INT, value STRING)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
http://git-wip-us.apache.org/repos/asf/hive/blob/07eaab39/ql/src/test/results/clientpositive/tez/scriptfile1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/scriptfile1.q.out b/ql/src/test/results/clientpositive/tez/scriptfile1.q.out
index d72ff5c..bf202f9 100644
--- a/ql/src/test/results/clientpositive/tez/scriptfile1.q.out
+++ b/ql/src/test/results/clientpositive/tez/scriptfile1.q.out
@@ -1,6 +1,9 @@
PREHOOK: query: -- SORT_QUERY_RESULTS
-- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
CREATE TABLE dest1(key INT, value STRING)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
@@ -8,6 +11,9 @@ PREHOOK: Output: default@dest1
POSTHOOK: query: -- SORT_QUERY_RESULTS
-- EXCLUDE_OS_WINDOWS
+
+-- NO_SESSION_REUSE
+
CREATE TABLE dest1(key INT, value STRING)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default