You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/09/29 02:40:22 UTC
hive git commit: HIVE-11923: allow qtests to run via a single client
session for tez and llap (Siddharth Seth reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/llap bf8acfb3b -> 7a3e74579
HIVE-11923: allow qtests to run via a single client session for tez and llap (Siddharth Seth reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7a3e7457
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7a3e7457
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7a3e7457
Branch: refs/heads/llap
Commit: 7a3e74579f33ef96abcdf7f45c507c6d47b912c7
Parents: bf8acfb
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Mon Sep 28 19:40:05 2015 -0500
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Mon Sep 28 19:40:05 2015 -0500
----------------------------------------------------------------------
data/conf/hive-log4j2.xml | 27 ++++++++++++++++++++
data/conf/llap/tez-site.xml | 6 +++++
data/conf/tez/tez-site.xml | 6 +++++
.../org/apache/hadoop/hive/ql/QTestUtil.java | 16 ++++++++++++
4 files changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/data/conf/hive-log4j2.xml
----------------------------------------------------------------------
diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
index 48a584f..860d756 100644
--- a/data/conf/hive-log4j2.xml
+++ b/data/conf/hive-log4j2.xml
@@ -72,6 +72,33 @@
<AppenderRef ref="EventCounter" />
</Root>
+ <Logger name="org.apache.hadoop.ipc" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop.security" level="INFO">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop.hdfs" level="INFO">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop.hdfs.server" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop.metrics2" level="INFO">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.mortbay" level="INFO">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop.yarn" level="INFO">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop.yarn.server" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.tez" level="INFO">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
<Logger name="org.apache.hadoop.conf.Configuration" level="ERROR">
<AppenderRef ref="${sys:hive.root.logger}"/>
</Logger>
http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/data/conf/llap/tez-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/llap/tez-site.xml b/data/conf/llap/tez-site.xml
new file mode 100644
index 0000000..940f390
--- /dev/null
+++ b/data/conf/llap/tez-site.xml
@@ -0,0 +1,6 @@
+<configuration>
+ <property>
+ <name>tez.am.dag.scheduler.class</name>
+ <value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/data/conf/tez/tez-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/tez/tez-site.xml b/data/conf/tez/tez-site.xml
new file mode 100644
index 0000000..940f390
--- /dev/null
+++ b/data/conf/tez/tez-site.xml
@@ -0,0 +1,6 @@
+<configuration>
+ <property>
+ <name>tez.am.dag.scheduler.class</name>
+ <value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/7a3e7457/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 4cab161..477a08d 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -81,6 +81,7 @@ import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
+import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -158,6 +159,7 @@ public class QTestUtil {
private final boolean miniMr = false;
private String hadoopVer = null;
private QTestSetup setup = null;
+ private TezSessionState tezSessionState = null;
private SparkSession sparkSession = null;
private boolean isSessionStateStarted = false;
private static final String javaVersion = getJavaVersion();
@@ -428,10 +430,16 @@ public class QTestUtil {
String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString());
if (clusterType == MiniClusterType.tez) {
+ if (confDir != null && !confDir.isEmpty()) {
+ conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+ + "/tez-site.xml"));
+ }
mr = shims.getMiniTezCluster(conf, 4, uriString, false);
} else if (clusterType == MiniClusterType.llap) {
if (confDir != null && !confDir.isEmpty()) {
conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+ + "/tez-site.xml"));
+ conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+ "/llap-daemon-site.xml"));
}
mr = shims.getMiniTezCluster(conf, 2, uriString, true);
@@ -952,6 +960,10 @@ public class QTestUtil {
sparkSession = oldSs.getSparkSession();
ss.setSparkSession(sparkSession);
oldSs.setSparkSession(null);
+ // Copy the tezSessionState from the old CliSessionState.
+ tezSessionState = oldSs.getTezSession();
+ ss.setTezSession(tezSessionState);
+ oldSs.setTezSession(null);
oldSs.close();
}
@@ -1016,6 +1028,10 @@ public class QTestUtil {
sparkSession = oldSs.getSparkSession();
ss.setSparkSession(sparkSession);
oldSs.setSparkSession(null);
+ // Copy the tezSessionState from the old CliSessionState.
+ tezSessionState = oldSs.getTezSession();
+ ss.setTezSession(tezSessionState);
+ oldSs.setTezSession(null);
oldSs.close();
}
if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {