You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/09/11 23:20:07 UTC
hive git commit: HIVE-11803: LLAP: MiniLlapCluster tests are slow
Repository: hive
Updated Branches:
refs/heads/llap 6e60a640c -> d6739a6b6
HIVE-11803: LLAP: MiniLlapCluster tests are slow
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d6739a6b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d6739a6b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d6739a6b
Branch: refs/heads/llap
Commit: d6739a6b64cd7e4bae48110cdba822424caa5952
Parents: 6e60a64
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Fri Sep 11 16:19:40 2015 -0500
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Fri Sep 11 16:19:40 2015 -0500
----------------------------------------------------------------------
data/conf/llap/hive-site.xml | 10 ----------
data/conf/tez/hive-site.xml | 5 -----
.../hadoop/hive/ql/TestLocationQueries.java | 2 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 21 +++++++-------------
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 10 +++++-----
5 files changed, 13 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d6739a6b/data/conf/llap/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml
index c002b1e..0e8c7c1 100644
--- a/data/conf/llap/hive-site.xml
+++ b/data/conf/llap/hive-site.xml
@@ -39,16 +39,6 @@
</property>
<property>
- <name>hive.execution.mode</name>
- <value>container</value>
-</property>
-
-<property>
- <name>mapred.tez.java.opts</name>
- <value>-Xmx128m</value>
-</property>
-
-<property>
<name>hive.tez.container.size</name>
<value>128</value>
<description></description>
http://git-wip-us.apache.org/repos/asf/hive/blob/d6739a6b/data/conf/tez/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/tez/hive-site.xml b/data/conf/tez/hive-site.xml
index b4abe90..8acf270 100644
--- a/data/conf/tez/hive-site.xml
+++ b/data/conf/tez/hive-site.xml
@@ -39,11 +39,6 @@
</property>
<property>
- <name>mapred.tez.java.opts</name>
- <value>-Xmx128m</value>
-</property>
-
-<property>
<name>hive.tez.container.size</name>
<value>128</value>
<description></description>
http://git-wip-us.apache.org/repos/asf/hive/blob/d6739a6b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
index 1fe4f48..6ee98cb 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
@@ -88,7 +88,7 @@ public class TestLocationQueries extends BaseTestQueries {
String hadoopVer, String locationSubdir)
throws Exception
{
- super(outDir, logDir, miniMr, null, hadoopVer, "", "", false);
+ super(outDir, logDir, miniMr, null, hadoopVer, "", "");
this.locationSubdir = locationSubdir;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d6739a6b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 68141ea..0d74d7f 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -347,15 +347,8 @@ public class QTestUtil {
}
public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
- String confDir, String hadoopVer, String initScript,
- String cleanupScript) throws Exception {
- this(outDir, logDir, clusterType, confDir,
- hadoopVer, initScript, cleanupScript, true);
- }
-
- public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
String confDir, String hadoopVer, String initScript,
- String cleanupScript, boolean withLlapIo) throws Exception {
+ String cleanupScript) throws Exception {
this.outDir = outDir;
this.logDir = logDir;
@@ -446,7 +439,7 @@ public class QTestUtil {
cleanUp();
}
- if (clusterType == MiniClusterType.tez) {
+ if (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap) {
SessionState.get().getTezSession().close(false);
}
setup.tearDown();
@@ -912,8 +905,8 @@ public class QTestUtil {
ss.setIsSilent(true);
SessionState oldSs = SessionState.get();
- if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.spark
- || clusterType == MiniClusterType.miniSparkOnYarn)) {
+ if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap
+ || clusterType == MiniClusterType.spark || clusterType == MiniClusterType.miniSparkOnYarn)) {
sparkSession = oldSs.getSparkSession();
ss.setSparkSession(sparkSession);
oldSs.setSparkSession(null);
@@ -976,8 +969,8 @@ public class QTestUtil {
ss.err = System.out;
SessionState oldSs = SessionState.get();
- if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.spark
- || clusterType == MiniClusterType.miniSparkOnYarn)) {
+ if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType == MiniClusterType.llap
+ || clusterType == MiniClusterType.miniSparkOnYarn || clusterType == MiniClusterType.miniSparkOnYarn)) {
sparkSession = oldSs.getSparkSession();
ss.setSparkSession(sparkSession);
oldSs.setSparkSession(null);
@@ -1838,7 +1831,7 @@ public class QTestUtil {
QTestUtil[] qt = new QTestUtil[qfiles.length];
for (int i = 0; i < qfiles.length; i++) {
qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20",
- defaultInitScript, defaultCleanupScript, false);
+ defaultInitScript, defaultCleanupScript);
qt[i].addFile(qfiles[i]);
qt[i].clearTestSideEffects();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d6739a6b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index e650cfd..876754d 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -390,11 +390,6 @@ public class Hadoop23Shims extends HadoopShimsSecure {
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode,
boolean isLlap) throws IOException {
- if (isLlap) {
- createAndLaunchLlapDaemon(conf);
- } else {
- miniLlapCluster = null;
- }
mr = new MiniTezCluster("hive", numberOfTaskTrackers);
conf.set("fs.defaultFS", nameNode);
conf.set("tez.am.log.level", "DEBUG");
@@ -402,6 +397,11 @@ public class Hadoop23Shims extends HadoopShimsSecure {
mr.init(conf);
mr.start();
this.conf = mr.getConfig();
+ if (isLlap) {
+ createAndLaunchLlapDaemon(this.conf);
+ } else {
+ miniLlapCluster = null;
+ }
}
private void createAndLaunchLlapDaemon(final Configuration conf) {