You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vg...@apache.org on 2018/05/15 22:40:53 UTC
[33/50] [abbrv] hive git commit: HIVE-19159:
TestMTQueries.testMTQueries1 failure (Laszlo Bodor via Zoltan Haindrich)
HIVE-19159: TestMTQueries.testMTQueries1 failure (Laszlo Bodor via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1db0521b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1db0521b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1db0521b
Branch: refs/heads/branch-3.0.0
Commit: 1db0521b593fd1e354c75fd719a98d231ded84d6
Parents: 685fc9f
Author: Laszlo Bodor <bo...@gmail.com>
Authored: Mon May 14 15:46:50 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Mon May 14 15:46:50 2018 +0200
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/TestMTQueries.java | 2 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 93 +++++++++++---------
2 files changed, 50 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1db0521b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
index 4838856..6ed872d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
@@ -42,7 +42,7 @@ public class TestMTQueries extends BaseTestQueries {
// derby fails creating multiple stats aggregator concurrently
util.getConf().setBoolean("hive.exec.submitviachild", true);
util.getConf().setBoolean("hive.exec.submit.local.task.via.child", true);
- util.getConf().setBoolean("hive.vectorized.execution.enabled", false);
+ util.getConf().setBoolean("hive.vectorized.execution.enabled", true);
util.getConf().set("hive.stats.dbclass", "fs");
util.getConf().set("hive.mapred.mode", "nonstrict");
util.getConf().set("hive.stats.column.autogather", "false");
http://git-wip-us.apache.org/repos/asf/hive/blob/1db0521b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 58680fe..f6729a9 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -36,6 +36,7 @@ import java.io.OutputStream;
import java.io.PrintStream;
import java.io.Serializable;
import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.sql.SQLException;
@@ -1200,11 +1201,13 @@ public class QTestUtil {
DatasetCollection datasets = parser.getDatasets();
for (String table : datasets.getTables()){
- initDataset(table);
+ synchronized (QTestUtil.class){
+ initDataset(table);
+ }
}
}
- private void initDataset(String table) {
+ protected void initDataset(String table) {
if (getSrcTables().contains(table)){
return;
}
@@ -1270,7 +1273,7 @@ public class QTestUtil {
initDataSetForTest(file);
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
- "org.apache.hadoop.hive.ql.security.DummyAuthenticator");
+ "org.apache.hadoop.hive.ql.security.DummyAuthenticator");
Utilities.clearWorkMap(conf);
CliSessionState ss = new CliSessionState(conf);
assert ss != null;
@@ -1287,6 +1290,30 @@ public class QTestUtil {
}
File outf = new File(logDir, stdoutName);
+
+ setSessionOutputs(fileName, ss, outf);
+
+ SessionState oldSs = SessionState.get();
+
+ boolean canReuseSession = !qNoSessionReuseQuerySet.contains(fileName);
+ restartSessions(canReuseSession, ss, oldSs);
+
+ closeSession(oldSs);
+
+ SessionState.start(ss);
+
+ cliDriver = new CliDriver();
+
+ if (fileName.equals("init_file.q")) {
+ ss.initFiles.add(AbstractCliConfig.HIVE_ROOT + "/data/scripts/test_init_file.sql");
+ }
+ cliDriver.processInitFiles(ss);
+
+ return outf.getAbsolutePath();
+ }
+
+ private void setSessionOutputs(String fileName, CliSessionState ss, File outf)
+ throws FileNotFoundException, Exception, UnsupportedEncodingException {
OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf));
if (qSortQuerySet.contains(fileName)) {
ss.out = new SortPrintStream(fo, "UTF-8");
@@ -1299,10 +1326,12 @@ public class QTestUtil {
}
ss.err = new CachingPrintStream(fo, true, "UTF-8");
ss.setIsSilent(true);
- SessionState oldSs = SessionState.get();
+ }
- boolean canReuseSession = !qNoSessionReuseQuerySet.contains(fileName);
- if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
+ private void restartSessions(boolean canReuseSession, CliSessionState ss, SessionState oldSs)
+ throws IOException {
+ if (oldSs != null && canReuseSession
+ && clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
// Copy the tezSessionState from the old CliSessionState.
TezSessionState tezSessionState = oldSs.getTezSession();
oldSs.setTezSession(null);
@@ -1316,27 +1345,9 @@ public class QTestUtil {
oldSs.setSparkSession(null);
oldSs.close();
}
-
- if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
- oldSs.out.close();
- }
- if (oldSs != null) {
- oldSs.close();
- }
- SessionState.start(ss);
-
- cliDriver = new CliDriver();
-
- if (fileName.equals("init_file.q")) {
- ss.initFiles.add(AbstractCliConfig.HIVE_ROOT + "/data/scripts/test_init_file.sql");
- }
- cliDriver.processInitFiles(ss);
-
- return outf.getAbsolutePath();
}
- private CliSessionState startSessionState(boolean canReuseSession)
- throws IOException {
+ private CliSessionState startSessionState(boolean canReuseSession) throws IOException {
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
@@ -1350,26 +1361,10 @@ public class QTestUtil {
ss.err = System.out;
SessionState oldSs = SessionState.get();
- if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
- // Copy the tezSessionState from the old CliSessionState.
- TezSessionState tezSessionState = oldSs.getTezSession();
- ss.setTezSession(tezSessionState);
- oldSs.setTezSession(null);
- oldSs.close();
- }
- if (oldSs != null && clusterType.getCoreClusterType() == CoreClusterType.SPARK) {
- sparkSession = oldSs.getSparkSession();
- ss.setSparkSession(sparkSession);
- oldSs.setSparkSession(null);
- oldSs.close();
- }
- if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
- oldSs.out.close();
- }
- if (oldSs != null) {
- oldSs.close();
- }
+ restartSessions(canReuseSession, ss, oldSs);
+
+ closeSession(oldSs);
SessionState.start(ss);
isSessionStateStarted = true;
@@ -1378,6 +1373,15 @@ public class QTestUtil {
return ss;
}
+ private void closeSession(SessionState oldSs) throws IOException {
+ if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
+ oldSs.out.close();
+ }
+ if (oldSs != null) {
+ oldSs.close();
+ }
+ }
+
public int executeAdhocCommand(String q) {
if (!q.contains(";")) {
return -1;
@@ -1986,6 +1990,7 @@ public class QTestUtil {
@Override
public void run() {
try {
+ qt.startSessionState(false);
// assumption is that environment has already been cleaned once globally
// hence each thread does not call cleanUp() and createSources() again
qt.cliInit(file, false);