You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/04/30 22:37:32 UTC
[1/3] hive git commit: HIVE-19309 : Add Arrow dependencies to
LlapServiceDriver (Eric Wohlstadter, reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/branch-3 9577a40b5 -> e621eeae7
refs/heads/master 577d32ef7 -> f8bc4868e
HIVE-19309 : Add Arrow dependencies to LlapServiceDriver (Eric Wohlstadter, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5616cbe4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5616cbe4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5616cbe4
Branch: refs/heads/master
Commit: 5616cbe40587c7cd218b15645c8563e6b4c6662e
Parents: 577d32e
Author: sergey <se...@apache.org>
Authored: Mon Apr 30 15:29:14 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Mon Apr 30 15:29:14 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 7 ++++++-
pom.xml | 3 +++
serde/pom.xml | 15 +++++++++++++++
3 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5616cbe4/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index 3eaaed7..b9845d4 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -400,7 +400,12 @@ public class LlapServiceDriver {
// log4j-1.2-API needed for NDC
org.apache.log4j.config.Log4j1ConfigurationFactory.class,
io.netty.util.NetUtil.class, // netty4
- org.jboss.netty.util.NetUtil.class //netty3
+ org.jboss.netty.util.NetUtil.class, //netty3
+ org.apache.arrow.vector.types.pojo.ArrowType.class, //arrow-vector
+ org.apache.arrow.memory.BaseAllocator.class, //arrow-memory
+ org.apache.arrow.flatbuf.Schema.class, //arrow-format
+ com.google.flatbuffers.Table.class, //flatbuffers
+ com.carrotsearch.hppc.ByteArrayDeque.class //hppc
};
for (Class<?> c : dependencies) {
http://git-wip-us.apache.org/repos/asf/hive/blob/5616cbe4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index afcf76e..92f1e2b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -119,6 +119,7 @@
<antlr.version>3.5.2</antlr.version>
<apache-directory-server.version>1.5.6</apache-directory-server.version>
<apache-directory-clientapi.version>0.1</apache-directory-clientapi.version>
+ <arrow.version>0.8.0</arrow.version>
<avatica.version>1.11.0</avatica.version>
<avro.version>1.7.7</avro.version>
<bonecp.version>0.8.0.RELEASE</bonecp.version>
@@ -142,6 +143,7 @@
<dropwizard.version>3.1.0</dropwizard.version>
<dropwizard-metrics-hadoop-metrics2-reporter.version>0.1.2</dropwizard-metrics-hadoop-metrics2-reporter.version>
<druid.version>0.12.0</druid.version>
+ <flatbuffers.version>1.2.0-3f79e055</flatbuffers.version>
<guava.version>19.0</guava.version>
<groovy.version>2.4.11</groovy.version>
<h2database.version>1.3.166</h2database.version>
@@ -149,6 +151,7 @@
<hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop</hadoop.bin.path>
<hamcrest.version>1.3</hamcrest.version>
<hbase.version>2.0.0-alpha4</hbase.version>
+ <hppc.version>0.7.2</hppc.version>
<!-- required for logging test to avoid including hbase which pulls disruptor transitively -->
<disruptor.version>3.3.7</disruptor.version>
<hikaricp.version>2.6.1</hikaricp.version>
http://git-wip-us.apache.org/repos/asf/hive/blob/5616cbe4/serde/pom.xml
----------------------------------------------------------------------
diff --git a/serde/pom.xml b/serde/pom.xml
index 3beeebd..e005585 100644
--- a/serde/pom.xml
+++ b/serde/pom.xml
@@ -66,6 +66,21 @@
<version>${commons-lang.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ <version>${arrow.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.carrotsearch</groupId>
+ <artifactId>hppc</artifactId>
+ <version>${hppc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.vlkan</groupId>
+ <artifactId>flatbuffers</artifactId>
+ <version>${flatbuffers.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
[3/3] hive git commit: HIVE-19309 : Add Arrow dependencies to
LlapServiceDriver (Eric Wohlstadter, reviewed by Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-19309 : Add Arrow dependencies to LlapServiceDriver (Eric Wohlstadter, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e621eeae
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e621eeae
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e621eeae
Branch: refs/heads/branch-3
Commit: e621eeae77857b50b64ec2f55a0fa0217144f68a
Parents: 9577a40
Author: sergey <se...@apache.org>
Authored: Mon Apr 30 15:29:14 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Mon Apr 30 15:30:55 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 7 ++++++-
pom.xml | 3 +++
serde/pom.xml | 15 +++++++++++++++
3 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e621eeae/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index 3eaaed7..b9845d4 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -400,7 +400,12 @@ public class LlapServiceDriver {
// log4j-1.2-API needed for NDC
org.apache.log4j.config.Log4j1ConfigurationFactory.class,
io.netty.util.NetUtil.class, // netty4
- org.jboss.netty.util.NetUtil.class //netty3
+ org.jboss.netty.util.NetUtil.class, //netty3
+ org.apache.arrow.vector.types.pojo.ArrowType.class, //arrow-vector
+ org.apache.arrow.memory.BaseAllocator.class, //arrow-memory
+ org.apache.arrow.flatbuf.Schema.class, //arrow-format
+ com.google.flatbuffers.Table.class, //flatbuffers
+ com.carrotsearch.hppc.ByteArrayDeque.class //hppc
};
for (Class<?> c : dependencies) {
http://git-wip-us.apache.org/repos/asf/hive/blob/e621eeae/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fe9aa82..9df8fb1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -119,6 +119,7 @@
<antlr.version>3.5.2</antlr.version>
<apache-directory-server.version>1.5.6</apache-directory-server.version>
<apache-directory-clientapi.version>0.1</apache-directory-clientapi.version>
+ <arrow.version>0.8.0</arrow.version>
<avatica.version>1.11.0</avatica.version>
<avro.version>1.7.7</avro.version>
<bonecp.version>0.8.0.RELEASE</bonecp.version>
@@ -142,6 +143,7 @@
<dropwizard.version>3.1.0</dropwizard.version>
<dropwizard-metrics-hadoop-metrics2-reporter.version>0.1.2</dropwizard-metrics-hadoop-metrics2-reporter.version>
<druid.version>0.12.0</druid.version>
+ <flatbuffers.version>1.2.0-3f79e055</flatbuffers.version>
<guava.version>19.0</guava.version>
<groovy.version>2.4.11</groovy.version>
<h2database.version>1.3.166</h2database.version>
@@ -149,6 +151,7 @@
<hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop</hadoop.bin.path>
<hamcrest.version>1.3</hamcrest.version>
<hbase.version>2.0.0-alpha4</hbase.version>
+ <hppc.version>0.7.2</hppc.version>
<!-- required for logging test to avoid including hbase which pulls disruptor transitively -->
<disruptor.version>3.3.7</disruptor.version>
<hikaricp.version>2.6.1</hikaricp.version>
http://git-wip-us.apache.org/repos/asf/hive/blob/e621eeae/serde/pom.xml
----------------------------------------------------------------------
diff --git a/serde/pom.xml b/serde/pom.xml
index 0247c32..fe5d2d6 100644
--- a/serde/pom.xml
+++ b/serde/pom.xml
@@ -66,6 +66,21 @@
<version>${commons-lang.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ <version>${arrow.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.carrotsearch</groupId>
+ <artifactId>hppc</artifactId>
+ <version>${hppc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.vlkan</groupId>
+ <artifactId>flatbuffers</artifactId>
+ <version>${flatbuffers.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
[2/3] hive git commit: HIVE-18037 : Migrate Slider LLAP package to
YARN Service framework for Hadoop 3.x (Gour Saha,
reviewed by Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-18037 : Migrate Slider LLAP package to YARN Service framework for Hadoop 3.x (Gour Saha, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f8bc4868
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f8bc4868
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f8bc4868
Branch: refs/heads/master
Commit: f8bc4868eced2ca83113579b626e279bbe6d5b13
Parents: 5616cbe
Author: sergey <se...@apache.org>
Authored: Mon Apr 30 15:30:33 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Mon Apr 30 15:30:33 2018 -0700
----------------------------------------------------------------------
bin/ext/llap.sh | 4 +-
binary-package-licenses/README | 1 -
.../org/apache/hadoop/hive/conf/HiveConf.java | 6 +-
jdbc/pom.xml | 1 -
.../hive/llap/registry/LlapServiceInstance.java | 2 +-
llap-server/bin/llapDaemon.sh | 2 +-
llap-server/changes_for_non_slider_install.txt | 2 +-
llap-server/pom.xml | 83 +----
.../hive/llap/cli/LlapOptionsProcessor.java | 50 ++-
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 11 +-
.../hadoop/hive/llap/cli/LlapSliderUtils.java | 211 ++++-------
.../hive/llap/cli/LlapStatusServiceDriver.java | 353 +++++--------------
.../hadoop/hive/llap/daemon/impl/QueryInfo.java | 2 +-
llap-server/src/main/resources/llap.py | 76 ----
llap-server/src/main/resources/package.py | 115 +++---
llap-server/src/main/resources/params.py | 40 ---
llap-server/src/main/resources/templates.py | 179 +++-------
packaging/src/main/assembly/bin.xml | 2 +-
pom.xml | 1 -
19 files changed, 302 insertions(+), 839 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/bin/ext/llap.sh
----------------------------------------------------------------------
diff --git a/bin/ext/llap.sh b/bin/ext/llap.sh
index 0462d26..91a54b3 100644
--- a/bin/ext/llap.sh
+++ b/bin/ext/llap.sh
@@ -17,7 +17,7 @@ THISSERVICE=llap
export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} "
llap () {
- TMPDIR=$(mktemp -d /tmp/staging-slider-XXXXXX)
+ TMPDIR=$(mktemp -d /tmp/staging-yarn-XXXXXX)
CLASS=org.apache.hadoop.hive.llap.cli.LlapServiceDriver;
if [ ! -f ${HIVE_LIB}/hive-cli-*.jar ]; then
echo "Missing Hive CLI Jar"
@@ -37,7 +37,7 @@ llap () {
# check for config files
test -f $TMPDIR/config.json
- python $HIVE_HOME/scripts/llap/slider/package.py --input $TMPDIR "$@"
+ python $HIVE_HOME/scripts/llap/yarn/package.py --input $TMPDIR "$@"
# remove temp files
rm -rf $TMPDIR
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/binary-package-licenses/README
----------------------------------------------------------------------
diff --git a/binary-package-licenses/README b/binary-package-licenses/README
index c801896..db6b7fe 100644
--- a/binary-package-licenses/README
+++ b/binary-package-licenses/README
@@ -74,7 +74,6 @@ pentaho-aggdesigner-algorithm
plexus-utils
regexp
server-metrics
-slider-core
snappy*
stax-api
super-csv
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 09bc572..f137d6c 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3893,15 +3893,15 @@ public class HiveConf extends Configuration {
"Number of RPC handlers for AM LLAP plugin endpoint."),
LLAP_DAEMON_WORK_DIRS("hive.llap.daemon.work.dirs", "",
"Working directories for the daemon. This should not be set if running as a YARN\n" +
- "application via Slider. It must be set when not running via Slider on YARN. If the value\n" +
- "is set when running as a Slider YARN application, the specified value will be used.",
+ "Service. It must be set when not running on YARN. If the value is set when\n" +
+ "running as a YARN Service, the specified value will be used.",
"llap.daemon.work.dirs"),
LLAP_DAEMON_YARN_SHUFFLE_PORT("hive.llap.daemon.yarn.shuffle.port", 15551,
"YARN shuffle port for LLAP-daemon-hosted shuffle.", "llap.daemon.yarn.shuffle.port"),
LLAP_DAEMON_YARN_CONTAINER_MB("hive.llap.daemon.yarn.container.mb", -1,
"llap server yarn container size in MB. Used in LlapServiceDriver and package.py", "llap.daemon.yarn.container.mb"),
LLAP_DAEMON_QUEUE_NAME("hive.llap.daemon.queue.name", null,
- "Queue name within which the llap slider application will run." +
+ "Queue name within which the llap application will run." +
" Used in LlapServiceDriver and package.py"),
// TODO Move the following 2 properties out of Configuration to a constant.
LLAP_DAEMON_CONTAINER_ID("hive.llap.daemon.container.id", null,
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 3c23a75..75129eb 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -255,7 +255,6 @@
<exclude>org.apache.directory.server:*</exclude>
<exclude>org.apache.geronimo.specs:*</exclude>
<exclude>org.apache.htrace:*</exclude>
- <exclude>org.apache.slider:*</exclude>
<exclude>org.codehaus.jackson:*</exclude>
<exclude>org.codehaus.jettison:*</exclude>
<exclude>org.datanucleus:*</exclude>
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java
index 30b1810..942f1d0 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java
@@ -51,7 +51,7 @@ public interface LlapServiceInstance extends ServiceInstance {
/**
* Memory and Executors available for the LLAP tasks
*
- * This does not include the size of the cache or the actual vCores allocated via Slider.
+ * This does not include the size of the cache or the actual vCores allocated via YARN.
*
* @return
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/bin/llapDaemon.sh
----------------------------------------------------------------------
diff --git a/llap-server/bin/llapDaemon.sh b/llap-server/bin/llapDaemon.sh
index 4945473..58188e2 100755
--- a/llap-server/bin/llapDaemon.sh
+++ b/llap-server/bin/llapDaemon.sh
@@ -113,7 +113,7 @@ case $startStop in
#rotate_log $logOut
echo starting llapdaemon, logging to $logLog and $logOut
export LLAP_DAEMON_LOGFILE=${LLAP_DAEMON_LOG_BASE}.log
- nohup nice -n $LLAP_DAEMON_NICENESS "$LLAP_DAEMON_BIN_HOME"/runLlapDaemon.sh run >> "$logOut" 2>&1 < /dev/null &
+ nohup nice -n $LLAP_DAEMON_NICENESS "$LLAP_DAEMON_BIN_HOME"/runLlapDaemon.sh run >> "$logOut" 2>&1 < /dev/null
echo $! > $pid
# capture the ulimit output
echo "ulimit -a" >> $logOut
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/changes_for_non_slider_install.txt
----------------------------------------------------------------------
diff --git a/llap-server/changes_for_non_slider_install.txt b/llap-server/changes_for_non_slider_install.txt
index ec20fe1..afcae31 100644
--- a/llap-server/changes_for_non_slider_install.txt
+++ b/llap-server/changes_for_non_slider_install.txt
@@ -1,4 +1,4 @@
--- Incomplete list of changes required to run LLAP as standalone daemons outside of YARN, instead of as a Slider App
+-- Incomplete list of changes required to run LLAP as standalone daemons outside of YARN, instead of as a YARN Service
hive.llap.daemon.work.dirs - Create dirs on each node, and set to this value.
hive.llap.daemon.service.hosts - Comma separated list of hosts on which llap will run.
ApplicationConstants.Environment.CONTAINER_ID should be available in the environment for secure clusters
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/pom.xml
----------------------------------------------------------------------
diff --git a/llap-server/pom.xml b/llap-server/pom.xml
index 6928f77..a57835e 100644
--- a/llap-server/pom.xml
+++ b/llap-server/pom.xml
@@ -122,6 +122,22 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-services-core</artifactId>
+ <version>${hadoop.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop.version}</version>
<optional>true</optional>
@@ -202,73 +218,6 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.slider</groupId>
- <artifactId>slider-core</artifactId>
- <version>${slider.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-web-proxy</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-digester</groupId>
- <artifactId>commons-digester</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-servlets</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.xml.bind</groupId>
- <artifactId>jaxb-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>jersey-guice</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-sslengine</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jettison</groupId>
- <artifactId>jettison</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
index c906a5d..2445075 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapOptionsProcessor.java
@@ -52,8 +52,8 @@ public class LlapOptionsProcessor {
public static final String OPTION_AUXHBASE = "auxhbase"; // used to localize jars
public static final String OPTION_JAVA_HOME = "javaHome"; // forward via config.json
public static final String OPTION_HIVECONF = "hiveconf"; // llap-daemon-site if relevant parameter
- public static final String OPTION_SLIDER_AM_CONTAINER_MB = "slider-am-container-mb"; // forward as arg
- public static final String OPTION_SLIDER_APPCONFIG_GLOBAL = "slider-appconfig-global"; // forward as arg
+ public static final String OPTION_SERVICE_AM_CONTAINER_MB = "service-am-container-mb"; // forward as arg
+ public static final String OPTION_SERVICE_APPCONFIG_GLOBAL = "service-appconfig-global"; // forward as arg
public static final String OPTION_LLAP_QUEUE = "queue"; // forward via config.json
public static final String OPTION_IO_THREADS = "iothreads"; // llap-daemon-site
@@ -61,12 +61,11 @@ public class LlapOptionsProcessor {
public static final String OPTION_ARGS = "args"; // forward as arg
public static final String OPTION_LOGLEVEL = "loglevel"; // forward as arg
public static final String OPTION_LOGGER = "logger"; // forward as arg
- public static final String OPTION_CHAOS_MONKEY = "chaosmonkey"; // forward as arg
- public static final String OPTION_SLIDER_KEYTAB_DIR = "slider-keytab-dir";
- public static final String OPTION_SLIDER_KEYTAB = "slider-keytab";
- public static final String OPTION_SLIDER_PRINCIPAL = "slider-principal";
- public static final String OPTION_SLIDER_PLACEMENT = "slider-placement";
- public static final String OPTION_SLIDER_DEFAULT_KEYTAB = "slider-default-keytab";
+ public static final String OPTION_SERVICE_KEYTAB_DIR = "service-keytab-dir";
+ public static final String OPTION_SERVICE_KEYTAB = "service-keytab";
+ public static final String OPTION_SERVICE_PRINCIPAL = "service-principal";
+ public static final String OPTION_SERVICE_PLACEMENT = "service-placement";
+ public static final String OPTION_SERVICE_DEFAULT_KEYTAB = "service-default-keytab";
public static final String OPTION_OUTPUT_DIR = "output";
public static final String OPTION_START = "startImmediately";
public static final String OPTION_HEALTH_PERCENT = "health-percent";
@@ -217,24 +216,21 @@ public class LlapOptionsProcessor {
LogHelpers.LLAP_LOGGER_NAME_QUERY_ROUTING + ", " + LogHelpers.LLAP_LOGGER_NAME_CONSOLE)
.create());
- options.addOption(OptionBuilder.hasArg().withArgName(OPTION_CHAOS_MONKEY).withLongOpt(OPTION_CHAOS_MONKEY)
- .withDescription("chaosmonkey interval").create('m'));
+ options.addOption(OptionBuilder.hasArg(false).withArgName(OPTION_SERVICE_DEFAULT_KEYTAB).withLongOpt(OPTION_SERVICE_DEFAULT_KEYTAB)
+ .withDescription("try to set default settings for Service AM keytab; mostly for dev testing").create());
- options.addOption(OptionBuilder.hasArg(false).withArgName(OPTION_SLIDER_DEFAULT_KEYTAB).withLongOpt(OPTION_SLIDER_DEFAULT_KEYTAB)
- .withDescription("try to set default settings for Slider AM keytab; mostly for dev testing").create());
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SERVICE_KEYTAB_DIR).withLongOpt(OPTION_SERVICE_KEYTAB_DIR)
+ .withDescription("Service AM keytab directory on HDFS (where the headless user keytab is stored by Service keytab installation, e.g. .yarn/keytabs/llap)").create());
- options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SLIDER_KEYTAB_DIR).withLongOpt(OPTION_SLIDER_KEYTAB_DIR)
- .withDescription("Slider AM keytab directory on HDFS (where the headless user keytab is stored by Slider keytab installation, e.g. .slider/keytabs/llap)").create());
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SERVICE_KEYTAB).withLongOpt(OPTION_SERVICE_KEYTAB)
+ .withDescription("Service AM keytab file name inside " + OPTION_SERVICE_KEYTAB_DIR).create());
- options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SLIDER_KEYTAB).withLongOpt(OPTION_SLIDER_KEYTAB)
- .withDescription("Slider AM keytab file name inside " + OPTION_SLIDER_KEYTAB_DIR).create());
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SERVICE_PRINCIPAL).withLongOpt(OPTION_SERVICE_PRINCIPAL)
+ .withDescription("Service AM principal; should be the user running the cluster, e.g. hive@EXAMPLE.COM").create());
- options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SLIDER_PRINCIPAL).withLongOpt(OPTION_SLIDER_PRINCIPAL)
- .withDescription("Slider AM principal; should be the user running the cluster, e.g. hive@EXAMPLE.COM").create());
-
- options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SLIDER_PLACEMENT).withLongOpt(OPTION_SLIDER_PLACEMENT)
- .withDescription("Slider placement policy; see slider documentation at https://slider.incubator.apache.org/docs/placement.html."
- + " 4 means anti-affinity (the default; unnecessary if LLAP is going to take more than half of the YARN capacity of a node), 0 is normal.").create());
+ options.addOption(OptionBuilder.hasArg().withArgName(OPTION_SERVICE_PLACEMENT).withLongOpt(OPTION_SERVICE_PLACEMENT)
+ .withDescription("Service placement policy; see YARN documentation at https://issues.apache.org/jira/browse/YARN-1042."
+ + " This is unnecessary if LLAP is going to take more than half of the YARN capacity of a node.").create());
options.addOption(OptionBuilder.hasArg().withArgName(OPTION_HEALTH_PERCENT).withLongOpt(OPTION_HEALTH_PERCENT)
.withDescription("Percentage of running containers after which LLAP application is considered healthy" +
@@ -291,12 +287,12 @@ public class LlapOptionsProcessor {
.create());
options.addOption(OptionBuilder.hasArg().withArgName("b")
- .withLongOpt(OPTION_SLIDER_AM_CONTAINER_MB)
- .withDescription("The size of the slider AppMaster container in MB").create('b'));
+ .withLongOpt(OPTION_SERVICE_AM_CONTAINER_MB)
+ .withDescription("The size of the service AppMaster container in MB").create('b'));
options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("property=value")
- .withLongOpt(OPTION_SLIDER_APPCONFIG_GLOBAL)
- .withDescription("Property (key=value) to be set in the global section of the Slider appConfig")
+ .withLongOpt(OPTION_SERVICE_APPCONFIG_GLOBAL)
+ .withDescription("Property (key=value) to be set in the global section of the Service appConfig")
.create());
options.addOption(OptionBuilder.hasArg().withArgName(OPTION_IO_THREADS)
@@ -366,7 +362,7 @@ public class LlapOptionsProcessor {
logger.equalsIgnoreCase(LogHelpers.LLAP_LOGGER_NAME_RFA));
}
- // loglevel, chaosmonkey & args are parsed by the python processor
+ // loglevel & args are parsed by the python processor
return new LlapOptions(name, instances, directory, executors, ioThreads, cache, size, xmx,
jars, isHbase, hiveconf, javaHome, queueName, logger, doStart, output, isHiveAux);
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index b9845d4..20fec7e 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -95,7 +95,7 @@ public class LlapServiceDriver {
private static final String HBASE_SERDE_CLASS = "org.apache.hadoop.hive.hbase.HBaseSerDe";
private static final String[] NEEDED_CONFIGS = LlapDaemonConfiguration.DAEMON_CONFIGS;
private static final String[] OPTIONAL_CONFIGS = LlapDaemonConfiguration.SSL_DAEMON_CONFIGS;
- private static final String OUTPUT_DIR_PREFIX = "llap-slider-";
+ private static final String OUTPUT_DIR_PREFIX = "llap-yarn-";
// This is not a config that users set in hive-site. It's only use is to share information
// between the java component of the service driver and the python component.
@@ -596,8 +596,9 @@ public class LlapServiceDriver {
}
rc = runPackagePy(args, tmpDir, scriptParent, version, outputDir);
if (rc == 0) {
- LlapSliderUtils.startCluster(conf, options.getName(), "llap-" + version + ".zip",
- packageDir, HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_QUEUE_NAME));
+ LlapSliderUtils.startCluster(conf, options.getName(),
+ "llap-" + version + ".tar.gz", packageDir,
+ HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_QUEUE_NAME));
}
} else {
rc = 0;
@@ -620,7 +621,7 @@ public class LlapServiceDriver {
private int runPackagePy(String[] args, Path tmpDir, Path scriptParent,
String version, String outputDir) throws IOException, InterruptedException {
- Path scriptPath = new Path(new Path(scriptParent, "slider"), "package.py");
+ Path scriptPath = new Path(new Path(scriptParent, "yarn"), "package.py");
List<String> scriptArgs = new ArrayList<>(args.length + 7);
scriptArgs.add("python");
scriptArgs.add(scriptPath.toString());
@@ -651,7 +652,7 @@ public class LlapServiceDriver {
private JSONObject createConfigJson(long containerSize, long cache, long xmx,
String java_home) throws JSONException {
- // extract configs for processing by the python fragments in Slider
+ // extract configs for processing by the python fragments in YARN Service
JSONObject configs = new JSONObject();
configs.put("java.home", java_home);
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java
index 8e5ae09..af47b26 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java
@@ -18,60 +18,53 @@
package org.apache.hadoop.hive.llap.cli;
-import com.google.common.io.Files;
import java.io.File;
import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.cli.status.LlapStatusHelpers.AppStatusBuilder;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
-import org.apache.slider.api.types.ApplicationDiagnostics;
-import org.apache.slider.client.SliderClient;
-import org.apache.slider.common.params.ActionCreateArgs;
-import org.apache.slider.common.params.ActionDestroyArgs;
-import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.slider.common.params.ActionInstallPackageArgs;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class LlapSliderUtils {
- private static final String SLIDER_GZ = "slider-agent.tar.gz";
- private static final Logger LOG = LoggerFactory.getLogger(LlapSliderUtils.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(LlapSliderUtils.class);
+ private static final String LLAP_PACKAGE_DIR = ".yarn/package/LLAP/";
- public static SliderClient createSliderClient(
+ public static ServiceClient createServiceClient(
Configuration conf) throws Exception {
- SliderClient sliderClient = new SliderClient() {
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- initHadoopBinding();
- }
- };
- Configuration sliderClientConf = new Configuration(conf);
- sliderClientConf = sliderClient.bindArgs(sliderClientConf,
- new String[]{"help"});
- sliderClient.init(sliderClientConf);
- sliderClient.start();
- return sliderClient;
+ ServiceClient serviceClient = new ServiceClient();
+ serviceClient.init(conf);
+ serviceClient.start();
+ return serviceClient;
}
- public static ApplicationReport getAppReport(String appName, SliderClient sliderClient,
+ public static ApplicationReport getAppReport(String appName, ServiceClient serviceClient,
long timeoutMs) throws
LlapStatusServiceDriver.LlapStatusCliException {
- Clock clock = new SystemClock();
+ Clock clock = SystemClock.getInstance();
long startTime = clock.getTime();
long timeoutTime = timeoutMs < 0 ? Long.MAX_VALUE : (startTime + timeoutMs);
ApplicationReport appReport = null;
+ ApplicationId appId;
+ try {
+ appId = serviceClient.getAppId(appName);
+ } catch (YarnException | IOException e) {
+ return null;
+ }
while (appReport == null) {
try {
- appReport = sliderClient.getYarnAppListClient().findInstance(appName);
+ appReport = serviceClient.getYarnClient().getApplicationReport(appId);
if (timeoutMs == 0) {
// break immediately if timeout is 0
break;
@@ -94,150 +87,78 @@ public class LlapSliderUtils {
return appReport;
}
- public static ApplicationDiagnostics getApplicationDiagnosticsFromYarnDiagnostics(
- ApplicationReport appReport, AppStatusBuilder appStatusBuilder, Logger LOG) {
- if (appReport == null) {
- return null;
- }
- String diagnostics = appReport.getDiagnostics();
- if (diagnostics == null || diagnostics.isEmpty()) {
- return null;
+ public static Service getService(Configuration conf, String name) {
+ LOG.info("Get service details for " + name);
+ ServiceClient sc;
+ try {
+ sc = createServiceClient(conf);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
+ Service service = null;
try {
- return ApplicationDiagnostics.fromJson(diagnostics);
- } catch (IOException e) {
- LOG.warn("Failed to parse application diagnostics from Yarn Diagnostics - {}", diagnostics);
- // Set the raw YARN diagnostics here - the caller won't know if they even exist.
- appStatusBuilder.setDiagnostics(diagnostics);
- return null;
+ service = sc.getStatus(name);
+ } catch (YarnException | IOException e) {
+ // Probably the app does not exist
+ LOG.info(e.getLocalizedMessage());
+ throw new RuntimeException(e);
+ } finally {
+ try {
+ sc.close();
+ } catch (IOException e) {
+ LOG.info("Failed to close service client", e);
+ }
}
+ return service;
}
public static void startCluster(Configuration conf, String name,
String packageName, Path packageDir, String queue) {
LOG.info("Starting cluster with " + name + ", "
+ packageName + ", " + queue + ", " + packageDir);
- SliderClient sc;
+ ServiceClient sc;
try {
- sc = createSliderClient(conf);
+ sc = createServiceClient(conf);
} catch (Exception e) {
throw new RuntimeException(e);
}
try {
- LOG.info("Executing the freeze command");
- ActionFreezeArgs freezeArgs = new ActionFreezeArgs();
- freezeArgs.force = true;
- freezeArgs.setWaittime(3600); // Wait forever (or at least for an hour).
try {
- sc.actionFreeze(name, freezeArgs);
- } catch (UnknownApplicationInstanceException ex) {
- LOG.info("There was no old application instance to freeze");
+ LOG.info("Executing the stop command");
+ sc.actionStop(name, true);
+ } catch (Exception ex) {
+ // Ignore exceptions from stop
+ LOG.info(ex.getLocalizedMessage());
}
- LOG.info("Executing the destroy command");
- ActionDestroyArgs destroyArg = new ActionDestroyArgs();
- destroyArg.force = true;
try {
- sc.actionDestroy(name, destroyArg);
- } catch (UnknownApplicationInstanceException ex) {
- LOG.info("There was no old application instance to destroy");
- }
- LOG.info("Executing the install command");
- ActionInstallPackageArgs installArgs = new ActionInstallPackageArgs();
- installArgs.name = "LLAP";
- installArgs.packageURI = new Path(packageDir, packageName).toString();
- installArgs.replacePkg = true;
- sc.actionInstallPkg(installArgs);
- LOG.info("Executing the create command");
- ActionCreateArgs createArgs = new ActionCreateArgs();
- createArgs.resources = new File(new Path(packageDir, "resources.json").toString());
- createArgs.template = new File(new Path(packageDir, "appConfig.json").toString());
- createArgs.setWaittime(3600);
- if (queue != null) {
- createArgs.queue = queue;
+ LOG.info("Executing the destroy command");
+ sc.actionDestroy(name);
+ } catch (Exception ex) {
+ // Ignore exceptions from destroy
+ LOG.info(ex.getLocalizedMessage());
}
- // See the comments in the method. SliderClient doesn't work in normal circumstances.
- File bogusSliderFile = startSetSliderLibDir();
+ LOG.info("Uploading the app tarball");
+ CoreFileSystem fs = new CoreFileSystem(conf);
+ fs.copyLocalFileToHdfs(new File(packageDir.toString(), packageName),
+ new Path(LLAP_PACKAGE_DIR), new FsPermission("755"));
+
+ LOG.info("Executing the launch command");
+ File yarnfile = new File(new Path(packageDir, "Yarnfile").toString());
+ Long lifetime = null; // unlimited lifetime
try {
- sc.actionCreate(name, createArgs);
+ sc.actionLaunch(yarnfile.getAbsolutePath(), name, lifetime, queue);
} finally {
- endSetSliderLibDir(bogusSliderFile);
}
- LOG.debug("Started the cluster via slider API");
+ LOG.debug("Started the cluster via service API");
} catch (YarnException | IOException e) {
throw new RuntimeException(e);
} finally {
try {
sc.close();
} catch (IOException e) {
- LOG.info("Failed to close slider client", e);
- }
- }
- }
-
- public static File startSetSliderLibDir() throws IOException {
- // TODO: this is currently required for the use of slider create API. Need SLIDER-1192.
- File sliderJarDir = SliderUtils.findContainingJar(SliderClient.class).getParentFile();
- File gz = new File(sliderJarDir, SLIDER_GZ);
- if (gz.exists()) {
- String path = sliderJarDir.getAbsolutePath();
- LOG.info("Setting slider.libdir based on jar file location: " + path);
- System.setProperty("slider.libdir", path);
- return null;
- }
-
- // There's no gz file next to slider jars. Due to the horror that is SliderClient, we'd have
- // to find it and copy it there. Let's try to find it. Also set slider.libdir.
- String path = System.getProperty("slider.libdir");
- gz = null;
- if (path != null && !path.isEmpty()) {
- LOG.info("slider.libdir was already set: " + path);
- gz = new File(path, SLIDER_GZ);
- if (!gz.exists()) {
- gz = null;
+ LOG.info("Failed to close service client", e);
}
}
- if (gz == null) {
- path = System.getenv("SLIDER_HOME");
- if (path != null && !path.isEmpty()) {
- gz = new File(new File(path, "lib"), SLIDER_GZ);
- if (gz.exists()) {
- path = gz.getParentFile().getAbsolutePath();
- LOG.info("Setting slider.libdir based on SLIDER_HOME: " + path);
- System.setProperty("slider.libdir", path);
- } else {
- gz = null;
- }
- }
- }
- if (gz == null) {
- // This is a terrible hack trying to find slider on a typical installation. Sigh...
- File rootDir = SliderUtils.findContainingJar(HiveConf.class)
- .getParentFile().getParentFile().getParentFile();
- File sliderJarDir2 = new File(new File(rootDir, "slider"), "lib");
- if (sliderJarDir2.exists()) {
- gz = new File(sliderJarDir2, SLIDER_GZ);
- if (gz.exists()) {
- path = sliderJarDir2.getAbsolutePath();
- LOG.info("Setting slider.libdir based on guesswork: " + path);
- System.setProperty("slider.libdir", path);
- } else {
- gz = null;
- }
- }
- }
- if (gz == null) {
- throw new IOException("Cannot find " + SLIDER_GZ + ". Please ensure SLIDER_HOME is set.");
- }
- File newGz = new File(sliderJarDir, SLIDER_GZ);
- LOG.info("Copying " + gz + " to " + newGz);
- Files.copy(gz, newGz);
- newGz.deleteOnExit();
- return newGz;
}
- public static void endSetSliderLibDir(File newGz) throws IOException {
- if (newGz == null || !newGz.exists()) return;
- LOG.info("Deleting " + newGz);
- newGz.delete();
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
index 65b4d81..e0ada45 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java
@@ -25,7 +25,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
-import java.net.URISyntaxException;
import java.text.DecimalFormat;
import java.util.Arrays;
import java.util.Collection;
@@ -52,17 +51,12 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
-import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.ClusterDescriptionKeys;
-import org.apache.slider.api.StateValues;
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.api.types.ApplicationDiagnostics;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.client.SliderClient;
-import org.apache.slider.common.params.ActionDiagnosticArgs;
-import org.apache.slider.core.exceptions.SliderException;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.SerializationConfig;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -71,7 +65,7 @@ import org.slf4j.LoggerFactory;
public class LlapStatusServiceDriver {
- private static final EnumSet<State> NO_SLIDER_INFO_STATES = EnumSet.of(
+ private static final EnumSet<State> NO_YARN_SERVICE_INFO_STATES = EnumSet.of(
State.APP_NOT_FOUND, State.COMPLETE, State.LAUNCHING);
private static final EnumSet<State> LAUNCHING_STATES = EnumSet.of(
State.LAUNCHING, State.RUNNING_PARTIAL, State.RUNNING_ALL);
@@ -119,11 +113,11 @@ public class LlapStatusServiceDriver {
private static final long LOG_SUMMARY_INTERVAL = 15000L; // Log summary every ~15 seconds.
- private static final String LLAP_KEY = "LLAP";
+ private static final String LLAP_KEY = "llap";
private final Configuration conf;
private final Clock clock = new SystemClock();
private String appName = null;
- private SliderClient sliderClient = null;
+ private ServiceClient serviceClient = null;
private Configuration llapRegistryConf = null;
private LlapRegistryService llapRegistry = null;
@@ -203,7 +197,7 @@ public class LlapStatusServiceDriver {
if (StringUtils.isEmpty(appName)) {
appName = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS);
if (appName.startsWith("@") && appName.length() > 1) {
- // This is a valid slider app name. Parse it out.
+ // This is a valid YARN Service name. Parse it out.
appName = appName.substring(1);
} else {
// Invalid app name. Checked later.
@@ -213,7 +207,7 @@ public class LlapStatusServiceDriver {
if (StringUtils.isEmpty(appName)) {
String message =
"Invalid app name. This must be setup via config or passed in as a parameter." +
- " This tool works with clusters deployed by Slider/YARN";
+ " This tool works with clusters deployed by YARN Service";
LOG.info(message);
return ExitCode.INCORRECT_USAGE.getInt();
}
@@ -225,13 +219,13 @@ public class LlapStatusServiceDriver {
}
try {
- if (sliderClient == null) {
- sliderClient = LlapSliderUtils.createSliderClient(conf);
+ if (serviceClient == null) {
+ serviceClient = LlapSliderUtils.createServiceClient(conf);
}
} catch (Exception e) {
LlapStatusCliException le = new LlapStatusCliException(
- LlapStatusServiceDriver.ExitCode.SLIDER_CLIENT_ERROR_CREATE_FAILED,
- "Failed to create slider client", e);
+ LlapStatusServiceDriver.ExitCode.SERVICE_CLIENT_ERROR_CREATE_FAILED,
+ "Failed to create service client", e);
logError(le);
return le.getExitCode().getInt();
}
@@ -239,13 +233,14 @@ public class LlapStatusServiceDriver {
// Get the App report from YARN
ApplicationReport appReport;
try {
- appReport = LlapSliderUtils.getAppReport(appName, sliderClient, options.getFindAppTimeoutMs());
+ appReport = LlapSliderUtils.getAppReport(appName, serviceClient,
+ options.getFindAppTimeoutMs());
} catch (LlapStatusCliException e) {
logError(e);
return e.getExitCode().getInt();
}
- // Process the report to decide whether to go to slider.
+ // Process the report
ExitCode ret;
try {
ret = processAppReport(appReport, appStatusBuilder);
@@ -256,26 +251,16 @@ public class LlapStatusServiceDriver {
if (ret != ExitCode.SUCCESS) {
return ret.getInt();
- } else if (NO_SLIDER_INFO_STATES.contains(appStatusBuilder.getState())) {
+ } else if (NO_YARN_SERVICE_INFO_STATES.contains(appStatusBuilder.getState())) {
return ExitCode.SUCCESS.getInt();
} else {
- // Get information from slider.
+ // Get information from YARN Service
try {
- ret = populateAppStatusFromSliderStatus(appName, sliderClient, appStatusBuilder);
- } catch (LlapStatusCliException e) {
- // In case of failure, send back whatever is constructed sop far - which wouldbe from the AppReport
- logError(e);
- return e.getExitCode().getInt();
- }
- }
-
-
- if (ret != ExitCode.SUCCESS) {
- return ret.getInt();
- } else {
- try {
- ret = populateAppStatusFromSliderDiagnostics(appName, sliderClient, appStatusBuilder);
+ ret = populateAppStatusFromServiceStatus(appName, serviceClient,
+ appStatusBuilder);
} catch (LlapStatusCliException e) {
+ // In case of failure, send back whatever is constructed so far -
+ // which would be from the AppReport
logError(e);
return e.getExitCode().getInt();
}
@@ -315,57 +300,6 @@ public class LlapStatusServiceDriver {
}
}
- private SliderClient createSliderClient() throws LlapStatusCliException {
- if (sliderClient != null) {
- return sliderClient;
- }
-
- try {
- sliderClient = LlapSliderUtils.createSliderClient(conf);
- } catch (Exception e) {
- throw new LlapStatusCliException(ExitCode.SLIDER_CLIENT_ERROR_CREATE_FAILED,
- "Failed to create slider client", e);
- }
- return sliderClient;
- }
-
- private ApplicationReport getAppReport(String appName, SliderClient sliderClient,
- long timeoutMs) throws LlapStatusCliException {
-
- long startTime = clock.getTime();
- long timeoutTime = timeoutMs < 0 ? Long.MAX_VALUE : (startTime + timeoutMs);
- ApplicationReport appReport = null;
-
- // TODO HIVE-13454 Maybe add an option to wait for a certain amount of time for the app to
- // move to running state. Potentially even wait for the containers to be launched.
-
-// while (clock.getTime() < timeoutTime && appReport == null) {
-
- while (appReport == null) {
- try {
- appReport = sliderClient.getYarnAppListClient().findInstance(appName);
- if (timeoutMs == 0) {
- // break immediately if timeout is 0
- break;
- }
- // Otherwise sleep, and try again.
- if (appReport == null) {
- long remainingTime = Math.min(timeoutTime - clock.getTime(), 500l);
- if (remainingTime > 0) {
- Thread.sleep(remainingTime);
- } else {
- break;
- }
- }
- } catch (Exception e) { // No point separating IOException vs YarnException vs others
- throw new LlapStatusCliException(ExitCode.YARN_ERROR,
- "Failed to get Yarn AppReport", e);
- }
- }
- return appReport;
- }
-
-
/**
* Populates parts of the AppStatus
*
@@ -399,7 +333,7 @@ public class LlapStatusServiceDriver {
return ExitCode.SUCCESS;
case RUNNING:
appStatusBuilder.maybeCreateAndGetAmInfo().setAppId(appReport.getApplicationId().toString());
- // If the app state is running, get additional information from Slider itself.
+ // If the app state is running, get additional information from YARN Service
return ExitCode.SUCCESS;
case FINISHED:
case FAILED:
@@ -407,14 +341,7 @@ public class LlapStatusServiceDriver {
appStatusBuilder.maybeCreateAndGetAmInfo().setAppId(appReport.getApplicationId().toString());
appStatusBuilder.setAppFinishTime(appReport.getFinishTime());
appStatusBuilder.setState(State.COMPLETE);
- ApplicationDiagnostics appDiagnostics =
- LlapSliderUtils.getApplicationDiagnosticsFromYarnDiagnostics(
- appReport, appStatusBuilder, LOG);
- if (appDiagnostics == null) {
- LOG.warn("AppDiagnostics not available for YARN application report");
- } else {
- processAppDiagnostics(appStatusBuilder, appDiagnostics, true);
- }
+ // add log links and other diagnostics from YARN Service
return ExitCode.SUCCESS;
default:
throw new LlapStatusCliException(ExitCode.INTERNAL_ERROR,
@@ -422,135 +349,57 @@ public class LlapStatusServiceDriver {
}
}
-
-
-
-
/**
- * Populates information from SliderStatus.
+ * Populates information from YARN Service Status.
*
* @param appName
- * @param sliderClient
+ * @param serviceClient
* @param appStatusBuilder
- * @return an ExitCode. An ExitCode other than ExitCode.SUCCESS implies future progress not possible
+ * @return an ExitCode. An ExitCode other than ExitCode.SUCCESS implies future
+ * progress not possible
* @throws LlapStatusCliException
*/
- private ExitCode populateAppStatusFromSliderStatus(String appName, SliderClient sliderClient, AppStatusBuilder appStatusBuilder) throws
- LlapStatusCliException {
-
- ClusterDescription clusterDescription;
+ private ExitCode populateAppStatusFromServiceStatus(String appName,
+ ServiceClient serviceClient, AppStatusBuilder appStatusBuilder)
+ throws LlapStatusCliException {
+ ExitCode exitCode = ExitCode.YARN_ERROR;
try {
- clusterDescription = sliderClient.getClusterDescription(appName);
- } catch (SliderException e) {
- throw new LlapStatusCliException(ExitCode.SLIDER_CLIENT_ERROR_OTHER,
- "Failed to get cluster description from slider. SliderErrorCode=" + (e).getExitCode(), e);
- } catch (Exception e) {
- throw new LlapStatusCliException(ExitCode.SLIDER_CLIENT_ERROR_OTHER,
- "Failed to get cluster description from slider", e);
- }
-
- if (clusterDescription == null) {
- LOG.info("Slider ClusterDescription not available");
- return ExitCode.SLIDER_CLIENT_ERROR_OTHER; // ClusterDescription should always be present.
- } else {
- // Process the Cluster Status returned by slider.
- appStatusBuilder.setOriginalConfigurationPath(clusterDescription.originConfigurationPath);
- appStatusBuilder.setGeneratedConfigurationPath(clusterDescription.generatedConfigurationPath);
- appStatusBuilder.setAppStartTime(clusterDescription.createTime);
-
- // Finish populating AMInfo
- appStatusBuilder.maybeCreateAndGetAmInfo().setAmWebUrl(clusterDescription.getInfo(StatusKeys.INFO_AM_WEB_URL));
- appStatusBuilder.maybeCreateAndGetAmInfo().setHostname(clusterDescription.getInfo(StatusKeys.INFO_AM_HOSTNAME));
- appStatusBuilder.maybeCreateAndGetAmInfo().setContainerId(clusterDescription.getInfo(StatusKeys.INFO_AM_CONTAINER_ID));
-
-
- if (clusterDescription.statistics != null) {
- Map<String, Integer> llapStats = clusterDescription.statistics.get(LLAP_KEY);
- if (llapStats != null) {
- int desiredContainers = llapStats.get(StatusKeys.STATISTICS_CONTAINERS_DESIRED);
- int liveContainers = llapStats.get(StatusKeys.STATISTICS_CONTAINERS_LIVE);
- appStatusBuilder.setDesiredInstances(desiredContainers);
- appStatusBuilder.setLiveInstances(liveContainers);
- } else {
- throw new LlapStatusCliException(ExitCode.SLIDER_CLIENT_ERROR_OTHER,
- "Failed to get statistics for LLAP"); // Error since LLAP should always exist.
+ Service service = serviceClient.getStatus(appName);
+ if (service != null) {
+ // How to get config paths and AmInfo
+ ServiceState state = service.getState();
+ appStatusBuilder.setAppStartTime(service.getLaunchTime() == null ? 0
+ : service.getLaunchTime().getTime());
+ appStatusBuilder.setDesiredInstances(
+ service.getComponent(LLAP_KEY).getNumberOfContainers() == null ? 0
+ : service.getComponent(LLAP_KEY).getNumberOfContainers()
+ .intValue());
+ appStatusBuilder.setLiveInstances(
+ service.getComponent(LLAP_KEY).getContainers().size());
+ for (Container cont : service.getComponent(LLAP_KEY).getContainers()) {
+ LlapInstance llapInstance = new LlapInstance(cont.getHostname(),
+ cont.getId());
+ appStatusBuilder.addNewRunningLlapInstance(llapInstance);
}
- // TODO HIVE-13454 Use some information from here such as containers.start.failed
- // and containers.failed.recently to provide an estimate of whether this app is healthy or not.
- } else {
- throw new LlapStatusCliException(ExitCode.SLIDER_CLIENT_ERROR_OTHER,
- "Failed to get statistics"); // Error since statistics should always exist.
- }
-
- // Code to locate container status via slider. Not using this at the moment.
- if (clusterDescription.status != null) {
- Object liveObject = clusterDescription.status.get(ClusterDescriptionKeys.KEY_CLUSTER_LIVE);
- if (liveObject != null) {
- Map<String, Map<String, Map<String, Object>>> liveEntity =
- (Map<String, Map<String, Map<String, Object>>>) liveObject;
- Map<String, Map<String, Object>> llapEntity = liveEntity.get(LLAP_KEY);
-
- if (llapEntity != null) { // Not a problem. Nothing has come up yet.
- for (Map.Entry<String, Map<String, Object>> containerEntry : llapEntity.entrySet()) {
- String containerIdString = containerEntry.getKey();
- Map<String, Object> containerParams = containerEntry.getValue();
-
- String host = (String) containerParams.get("host");
-
- LlapInstance
- llapInstance = new LlapInstance(host, containerIdString);
-
- appStatusBuilder.addNewRunningLlapInstance(llapInstance);
- }
- }
-
+ if (state == ServiceState.STABLE) {
+ exitCode = ExitCode.SUCCESS;
}
+ } else {
+ exitCode = ExitCode.SERVICE_CLIENT_ERROR_OTHER;
}
-
- return ExitCode.SUCCESS;
-
- }
- }
-
- /**
- * Populates information based on the slider diagnostics call. Must be invoked
- * after populating status from slider status.
- * @param appName
- * @param sliderClient
- * @param appStatusBuilder
- * @return
- * @throws LlapStatusCliException
- */
- private ExitCode populateAppStatusFromSliderDiagnostics(String appName,
- SliderClient sliderClient,
- AppStatusBuilder appStatusBuilder) throws
- LlapStatusCliException {
-
- ApplicationDiagnostics appDiagnostics;
- try {
- ActionDiagnosticArgs args = new ActionDiagnosticArgs();
- args.containers = true;
- args.name = appName;
- appDiagnostics =
- sliderClient.actionDiagnosticContainers(args);
- } catch (YarnException | IOException | URISyntaxException e) {
- throw new LlapStatusCliException(
- ExitCode.SLIDER_CLIENT_ERROR_OTHER,
- "Failed to get container diagnostics from slider", e);
- }
- if (appDiagnostics == null) {
- LOG.info("Slider container diagnostics not available");
- return ExitCode.SLIDER_CLIENT_ERROR_OTHER;
- }
-
- processAppDiagnostics(appStatusBuilder, appDiagnostics, false);
-
- return ExitCode.SUCCESS;
+ } catch (IOException | YarnException e) {
+ LlapStatusCliException le = new LlapStatusCliException(
+ LlapStatusServiceDriver.ExitCode.SERVICE_CLIENT_ERROR_OTHER,
+ "Failed to get service status", e);
+ logError(le);
+ exitCode = le.getExitCode();
+ }
+ return exitCode;
}
/**
* Populate additional information for containers from the LLAP registry. Must be invoked
- * after Slider status. Also after slider-diagnostics.
+ * after YARN Service status and diagnostics.
* @param appStatusBuilder
* @return an ExitCode. An ExitCode other than ExitCode.SUCCESS implies future progress not possible
* @throws LlapStatusCliException
@@ -584,7 +433,7 @@ public class LlapStatusServiceDriver {
appStatusBuilder.clearRunningLlapInstances();
return ExitCode.SUCCESS;
} else {
- // Tracks instances known by both slider and llap.
+ // Tracks instances known by both YARN Service and llap.
List<LlapInstance> validatedInstances = new LinkedList<>();
List<String> llapExtraInstances = new LinkedList<>();
@@ -612,7 +461,8 @@ public class LlapStatusServiceDriver {
appStatusBuilder.setLiveInstances(validatedInstances.size());
appStatusBuilder.setLaunchingInstances(llapExtraInstances.size());
- if (validatedInstances.size() >= appStatusBuilder.getDesiredInstances()) {
+ if (appStatusBuilder.getDesiredInstances() != null && validatedInstances
+ .size() >= appStatusBuilder.getDesiredInstances()) {
appStatusBuilder.setState(State.RUNNING_ALL);
if (validatedInstances.size() > appStatusBuilder.getDesiredInstances()) {
LOG.warn("Found more entries in LLAP registry, as compared to desired entries");
@@ -633,7 +483,7 @@ public class LlapStatusServiceDriver {
}
if (llapExtraInstances.size() > 0) {
// Old containers which are likely shutting down, or new containers which
- // launched between slider-status/slider-diagnostics. Skip for this iteration.
+ // launched between YARN Service status/diagnostics. Skip for this iteration.
LOG.debug("Instances likely to shutdown soon: {}", llapExtraInstances);
}
@@ -643,52 +493,6 @@ public class LlapStatusServiceDriver {
return ExitCode.SUCCESS;
}
-
- private static void processAppDiagnostics(AppStatusBuilder appStatusBuilder,
- ApplicationDiagnostics appDiagnostics, boolean appComplete) {
- // For a running app this should be empty.
- String finalMessage = appDiagnostics.getFinalMessage();
- Collection<ContainerInformation> containerInfos =
- appDiagnostics.getContainers();
- appStatusBuilder.setDiagnostics(finalMessage);
- if (containerInfos != null) {
- for (ContainerInformation containerInformation : containerInfos) {
- if (containerInformation.getState() == StateValues.STATE_LIVE && !appComplete) {
- LlapInstance instance = appStatusBuilder
- .removeAndGetCompletedLlapInstanceForContainer(
- containerInformation.getContainerId());
- if (instance ==
- null) { // New launch. Not available during slider status, but available now.
- instance = new LlapInstance(containerInformation.getHost(),
- containerInformation.getContainerId());
- }
- instance.setLogUrl(containerInformation.getLogLink());
- appStatusBuilder.addNewRunningLlapInstance(instance);
- } else if (containerInformation.getState() ==
- StateValues.STATE_STOPPED || appComplete) {
- LlapInstance instance =
- new LlapInstance(containerInformation.getHost(),
- containerInformation.getContainerId());
- instance.setLogUrl(containerInformation.getLogLink());
- if (appComplete && containerInformation.getExitCode() !=
- ContainerExitStatus.INVALID) {
- instance
- .setYarnContainerExitStatus(containerInformation.getExitCode());
- }
- instance.setDiagnostics(containerInformation.getDiagnostics());
- appStatusBuilder.addNewCompleteLlapInstance(instance);
- } else {
- LOG.warn("Unexpected containerstate={}, for container={}",
- containerInformation.getState(), containerInformation);
- }
- }
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("ContainerInfos is null");
- }
- }
- }
-
private static String constructCompletedContainerDiagnostics(List<LlapInstance> completedInstances) {
StringBuilder sb = new StringBuilder();
if (completedInstances == null || completedInstances.size() == 0) {
@@ -709,7 +513,7 @@ public class LlapStatusServiceDriver {
ContainerExitStatus.KILLED_EXCEEDED_VMEM) {
sb.append("\tKILLED container (by YARN for exceeding memory limits): ");
} else {
- // TODO HIVE-15865 Handle additional reasons like OS launch failed (Slider needs to give this info)
+ // TODO HIVE-15865 Handle additional reasons like OS launch failed
sb.append("\tFAILED container: ");
}
sb.append(" ").append(instance.getContainerId());
@@ -763,8 +567,8 @@ public class LlapStatusServiceDriver {
sb.append("\n").append(containerDiagnostics);
}
- // TODO HIVE-15865: Include information about pending requests, and last allocation time
- // once Slider provides this information.
+ // TODO HIVE-15865: Include information about pending requests, and last
+ // allocation time once YARN Service provides this information.
break;
case RUNNING_ALL:
sb.append("LLAP Application running with ApplicationId=")
@@ -796,8 +600,8 @@ public class LlapStatusServiceDriver {
SUCCESS(0),
INCORRECT_USAGE(10),
YARN_ERROR(20),
- SLIDER_CLIENT_ERROR_CREATE_FAILED(30),
- SLIDER_CLIENT_ERROR_OTHER(31),
+ SERVICE_CLIENT_ERROR_CREATE_FAILED(30),
+ SERVICE_CLIENT_ERROR_OTHER(31),
LLAP_REGISTRY_ERROR(40),
LLAP_JSON_GENERATION_ERROR(50),
// Error in the script itself - likely caused by an incompatible change, or new functionality / states added.
@@ -844,7 +648,7 @@ public class LlapStatusServiceDriver {
public static void main(String[] args) {
LOG.info("LLAP status invoked with arguments = {}", Arrays.toString(args));
int ret = ExitCode.SUCCESS.getInt();
- Clock clock = new SystemClock();
+ Clock clock = SystemClock.getInstance();
long startTime = clock.getTime();
long lastSummaryLogTime = -1;
@@ -918,7 +722,8 @@ public class LlapStatusServiceDriver {
if (ret == ExitCode.SUCCESS.getInt()) {
if (watchMode) {
- // slider has started llap application, now if for some reason state changes to COMPLETE then fail fast
+ // YARN Service has started llap application, now if for some reason
+ // state changes to COMPLETE then fail fast
if (launchingState == null && LAUNCHING_STATES.contains(currentState)) {
launchingState = currentState;
}
@@ -984,12 +789,12 @@ public class LlapStatusServiceDriver {
LOG.warn("Watch mode enabled and got YARN error. Retrying..");
numAttempts--;
continue;
- } else if (ret == ExitCode.SLIDER_CLIENT_ERROR_CREATE_FAILED.getInt() && watchMode) {
- LOG.warn("Watch mode enabled and slider client creation failed. Retrying..");
+ } else if (ret == ExitCode.SERVICE_CLIENT_ERROR_CREATE_FAILED.getInt() && watchMode) {
+ LOG.warn("Watch mode enabled and YARN Service client creation failed. Retrying..");
numAttempts--;
continue;
- } else if (ret == ExitCode.SLIDER_CLIENT_ERROR_OTHER.getInt() && watchMode) {
- LOG.warn("Watch mode enabled and got slider client error. Retrying..");
+ } else if (ret == ExitCode.SERVICE_CLIENT_ERROR_OTHER.getInt() && watchMode) {
+ LOG.warn("Watch mode enabled and got YARN Service client error. Retrying..");
numAttempts--;
continue;
} else if (ret == ExitCode.LLAP_REGISTRY_ERROR.getInt() && watchMode) {
@@ -1066,8 +871,8 @@ public class LlapStatusServiceDriver {
}
private void close() {
- if (sliderClient != null) {
- sliderClient.stop();
+ if (serviceClient != null) {
+ serviceClient.stop();
}
if (llapRegistry != null) {
llapRegistry.stop();
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
index d2e9396..00fed15 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
@@ -184,7 +184,7 @@ public class QueryInfo {
private static String createAppSpecificLocalDir(String baseDir, String applicationIdString,
String user, int dagIdentifier) {
// TODO This is broken for secure clusters. The app will not have permission to create these directories.
- // May work via Slider - since the directory would already exist. Otherwise may need a custom shuffle handler.
+ // May work via YARN Service - since the directory would already exist. Otherwise may need a custom shuffle handler.
// TODO This should be the process user - and not the user on behalf of whom the query is being submitted.
return baseDir + File.separator + "usercache" + File.separator + user + File.separator +
"appcache" + File.separator + applicationIdString + File.separator + dagIdentifier;
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/src/main/resources/llap.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/llap.py b/llap-server/src/main/resources/llap.py
deleted file mode 100644
index 26756ce..0000000
--- a/llap-server/src/main/resources/llap.py
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import sys
-import os
-import subprocess
-from resource_management import *
-from os.path import dirname
-from os.path import join as join_path
-
-class Llap(Script):
- def install(self, env):
- self.install_packages(env)
- pass
-
- def configure(self, env):
- import params
- env.set_params(params)
-
- def start(self, env):
- import params
- env.set_params(params)
- os.environ['JAVA_HOME'] = format('{java64_home}')
- # this is the same as TEZ_PREFIX
- os.environ['LLAP_DAEMON_HOME'] = format('{app_root}')
- os.environ['LLAP_DAEMON_TMP_DIR'] = format('{app_tmp_dir}')
- # this is the location where we have the llap server components (shell scripts)
- os.environ['LLAP_DAEMON_BIN_HOME'] = format('{app_root}/bin')
- # location containing llap-daemon-site.xml, tez and yarn configuration xmls as well.
- os.environ['LLAP_DAEMON_CONF_DIR'] = format("{app_root}/conf/")
- os.environ['LLAP_DAEMON_LOG_DIR'] = format("{app_log_dir}/")
- os.environ['LLAP_DAEMON_LOGGER'] = format("{app_logger}")
- os.environ['LLAP_DAEMON_LOG_LEVEL'] = format("{app_log_level}")
- os.environ['LLAP_DAEMON_HEAPSIZE'] = format("{memory_val}")
- os.environ['LLAP_DAEMON_PID_DIR'] = dirname(format("{pid_file}"))
- os.environ['LLAP_DAEMON_LD_PATH'] = format('{library_path}')
- os.environ['LLAP_DAEMON_OPTS'] = format('{daemon_args}')
- print "Debug from LLAP python script"
- print os.environ['LLAP_DAEMON_CONF_DIR']
- self.configure(env)
- location = "bash -x {app_root}/bin/llapDaemon.sh start &> {app_log_dir}/shell.out"
- process_cmd = format(location)
-
- subprocess.call(process_cmd, shell=True
- )
-
- def stop(self, env):
- import params
- env.set_params(params)
-
- def status(self, env):
- import params
- env.set_params(params)
- check_process_status(params.pid_file)
-
-if __name__ == "__main__":
- Llap().execute()
-
-# vim: tabstop=24 expandtab shiftwidth=4 softtabstop=4
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/src/main/resources/package.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/package.py b/llap-server/src/main/resources/package.py
index 21c34e9..9eb3fd7 100644
--- a/llap-server/src/main/resources/package.py
+++ b/llap-server/src/main/resources/package.py
@@ -10,7 +10,7 @@ import shutil
import tarfile
import zipfile
-from templates import metainfo, appConfig, resources, runner
+from templates import yarnfile, runner
class LlapResource(object):
def __init__(self, config):
@@ -40,8 +40,7 @@ class LlapResource(object):
if (not config.get("hive.llap.daemon.queue.name","")):
self.queueString = ""
else:
- self.queueString = "--queue "
- self.queueString += config["hive.llap.daemon.queue.name"]
+ self.queueString = config["hive.llap.daemon.queue.name"]
if (not config.get("private.hive.llap.servicedriver.cluster.name")):
self.clusterName="llap0"
@@ -58,13 +57,13 @@ def zipdir(path, zip, prefix="."):
dst = src.replace(path, prefix)
zip.write(src, dst)
-def slider_appconfig_global_property(arg):
+def service_appconfig_global_property(arg):
kv = arg.split("=")
if len(kv) != 2:
raise argparse.ArgumentTypeError("Value must be split into two parts separated by =")
return tuple(kv)
-def construct_slider_site_global_string(kvs):
+def construct_service_site_global_string(kvs):
if not kvs:
return ""
kvs = map(lambda a : a[0], kvs)
@@ -76,7 +75,7 @@ def main(args):
if not version:
version = strftime("%d%b%Y", gmtime())
home = os.getenv("HIVE_HOME")
- output = "llap-slider-%(version)s" % ({"version": version})
+ output = "llap-yarn-%(version)s" % ({"version": version})
parser = argparse.ArgumentParser()
parser.add_argument("--instances", type=int, default=1)
parser.add_argument("--output", default=output)
@@ -85,18 +84,17 @@ def main(args):
parser.add_argument("--name", default="llap0")
parser.add_argument("--loglevel", default="INFO")
parser.add_argument("--logger", default="query-routing")
- parser.add_argument("--chaosmonkey", type=int, default=0)
- parser.add_argument("--slider-am-container-mb", type=int, default=1024)
- parser.add_argument("--slider-appconfig-global", nargs='*', type=slider_appconfig_global_property, action='append')
- parser.add_argument("--slider-keytab-dir", default="")
- parser.add_argument("--slider-keytab", default="")
- parser.add_argument("--slider-principal", default="")
- parser.add_argument("--slider-default-keytab", dest='slider_default_keytab', action='store_true')
- parser.add_argument("--slider-placement", type=int, default=4)
+ parser.add_argument("--service-am-container-mb", type=int, default=1024)
+ parser.add_argument("--service-appconfig-global", nargs='*', type=service_appconfig_global_property, action='append')
+ parser.add_argument("--service-keytab-dir", default="")
+ parser.add_argument("--service-keytab", default="")
+ parser.add_argument("--service-principal", default="")
+ parser.add_argument("--service-default-keytab", dest='service_default_keytab', action='store_true')
+ parser.add_argument("--service-placement", type=int, default=4)
parser.add_argument("--health-percent", type=int, default=80)
parser.add_argument("--health-time-window-secs", type=int, default=300)
parser.add_argument("--health-init-delay-secs", type=int, default=400)
- parser.set_defaults(slider_default_keytab=False)
+ parser.set_defaults(service_default_keytab=False)
parser.add_argument("--startImmediately", dest='start_immediately', action='store_true')
parser.add_argument("--javaChild", dest='java_child', action='store_true')
parser.set_defaults(start_immediately=False)
@@ -114,18 +112,27 @@ def main(args):
input = args.input
output = args.output
- slider_am_jvm_heapsize = max(args.slider_am_container_mb * 0.8, args.slider_am_container_mb - 1024)
- slider_keytab_dir = args.slider_keytab_dir
- slider_keytab = args.slider_keytab
- slider_principal = args.slider_principal
+ service_am_jvm_heapsize = max(args.service_am_container_mb * 0.8, args.service_am_container_mb - 1024)
+ service_keytab_dir = args.service_keytab_dir
+ service_keytab = args.service_keytab
+ service_principal = args.service_principal
# set the defaults only if the defaults are enabled
- if args.slider_default_keytab:
- if not slider_keytab_dir:
- slider_keytab_dir = ".slider/keytabs/llap"
- if not slider_keytab:
- slider_keytab = "llap.keytab"
- if not slider_principal:
- slider_principal = "llap@EXAMPLE.COM"
+ if args.service_default_keytab:
+ if not service_keytab_dir:
+ service_keytab_dir = ".yarn/keytabs/llap"
+ if not service_keytab:
+ service_keytab = "llap.keytab"
+ if not service_principal:
+ service_principal = "llap@EXAMPLE.COM"
+ service_keytab_path = service_keytab_dir
+ if service_keytab_path:
+ if service_keytab:
+ service_keytab_path += "/" + service_keytab
+ else:
+ service_keytab_path = service_keytab
+ if service_keytab_path:
+ service_keytab_path = "hdfs:///user/hive/" + service_keytab_path
+
if not input:
print "Cannot find input files"
sys.exit(1)
@@ -140,8 +147,6 @@ def main(args):
if long(max_direct_memory) > 0:
daemon_args = " -XX:MaxDirectMemorySize=%s %s" % (max_direct_memory, daemon_args)
daemon_args = " -Dhttp.maxConnections=%s %s" % ((max(args.instances, resource.executors) + 1), daemon_args)
- # 5% container failure every monkey_interval seconds
- monkey_percentage = 5 # 5%
vars = {
"home" : home,
"version" : version,
@@ -156,19 +161,15 @@ def main(args):
"daemon_loglevel" : args.loglevel,
"daemon_logger" : args.logger,
"queue.string" : resource.queueString,
- "monkey_interval" : args.chaosmonkey,
- "monkey_percentage" : monkey_percentage,
- "monkey_enabled" : args.chaosmonkey > 0,
- "slider.am.container.mb" : args.slider_am_container_mb,
- "slider_appconfig_global_append": construct_slider_site_global_string(args.slider_appconfig_global),
- "slider_am_jvm_heapsize" : slider_am_jvm_heapsize,
- "slider_keytab_dir" : slider_keytab_dir,
- "slider_keytab" : slider_keytab,
- "slider_principal" : slider_principal,
- "placement" : args.slider_placement,
- "health_percent": args.health_percent,
- "health_time_window": args.health_time_window_secs,
- "health_init_delay": args.health_init_delay_secs
+ "service.am.container.mb" : args.service_am_container_mb,
+ "service_appconfig_global_append": construct_service_site_global_string(args.service_appconfig_global),
+ "service_am_jvm_heapsize" : service_am_jvm_heapsize,
+ "service_keytab_path" : service_keytab_path,
+ "service_principal" : service_principal,
+ "placement" : args.service_placement,
+ "health_percent": args.health_percent,
+ "health_time_window": args.health_time_window_secs,
+ "health_init_delay": args.health_init_delay_secs
}
if not exists(output):
@@ -180,47 +181,25 @@ def main(args):
shutil.rmtree(dst)
shutil.copytree(src, dst)
- # Make the zip package
- tmp = join(output, "tmp")
- pkg = join(tmp, "package")
-
- src = join(home, "scripts", "llap", "slider")
- dst = join(pkg, "scripts")
- if exists(dst):
- shutil.rmtree(dst)
- shutil.copytree(src, dst)
-
- with open(join(tmp, "metainfo.xml"),"w") as f:
- f.write(metainfo % vars)
-
- os.mkdir(join(pkg, "files"))
+ # Make the llap tarball
print "%s Prepared the files" % (strftime("%H:%M:%S", gmtime()))
- tarball = tarfile.open(join(pkg, "files", "llap-%s.tar.gz" % version), "w:gz")
+ tarball = tarfile.open(join(output, "llap-%s.tar.gz" % version), "w:gz")
# recursive add + -C chdir inside
tarball.add(input, "")
tarball.close()
- zipped = zipfile.ZipFile(join(output, "llap-%s.zip" % version), "w")
- zipdir(tmp, zipped)
- zipped.close()
print "%s Packaged the files" % (strftime("%H:%M:%S", gmtime()))
- # cleanup after making zip pkg
- shutil.rmtree(tmp)
-
- with open(join(output, "appConfig.json"), "w") as f:
- f.write(appConfig % vars)
-
- with open(join(output, "resources.json"), "w") as f:
- f.write(resources % vars)
+ with open(join(output, "Yarnfile"), "w") as f:
+ f.write(yarnfile % vars)
with open(join(output, "run.sh"), "w") as f:
f.write(runner % vars)
os.chmod(join(output, "run.sh"), 0700)
if not args.java_child:
- print "%s Prepared %s/run.sh for running LLAP on Slider" % (strftime("%H:%M:%S", gmtime()), output)
+ print "%s Prepared %s/run.sh for running LLAP on YARN" % (strftime("%H:%M:%S", gmtime()), output)
if __name__ == "__main__":
main(sys.argv[1:])
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/src/main/resources/params.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/params.py b/llap-server/src/main/resources/params.py
deleted file mode 100644
index 8972ba1..0000000
--- a/llap-server/src/main/resources/params.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-
-# server configurations
-config = Script.get_config()
-
-app_root = config['configurations']['global']['app_root']
-java64_home = config['hostLevelParams']['java_home']
-pid_file = config['configurations']['global']['pid_file']
-
-additional_cp = config['configurations']['global']['additional_cp']
-app_log_dir = config['configurations']['global']['app_log_dir']
-app_tmp_dir = config['configurations']['global']['app_tmp_dir']
-app_logger = config['configurations']['global']['app_logger']
-app_log_level = config['configurations']['global']['app_log_level']
-daemon_args = config['configurations']['global']['daemon_args']
-
-port = config['configurations']['global']['listen_port']
-memory_val = config['configurations']['global']['memory_val']
-library_path = config['configurations']['global']['library_path']
-
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/llap-server/src/main/resources/templates.py
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/templates.py b/llap-server/src/main/resources/templates.py
index 3d747a2..00a84cf 100644
--- a/llap-server/src/main/resources/templates.py
+++ b/llap-server/src/main/resources/templates.py
@@ -1,138 +1,69 @@
-metainfo = """<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <application>
- <name>LLAP</name>
- <comment>LLAP is a daemon service that works with a cache and works on SQL constructs.</comment>
- <version>%(version)s</version>
- <exportedConfigs>None</exportedConfigs>
- <exportGroups>
- <exportGroup>
- <name>Servers</name>
- <exports>
- <export>
- <name>instances</name>
- <value>${LLAP_HOST}:${site.global.listen_port}</value>
- </export>
- </exports>
- </exportGroup>
- </exportGroups>
-
- <components>
- <component>
- <name>LLAP</name>
- <category>MASTER</category>
- <compExports>Servers-instances</compExports>
- <commandScript>
- <script>scripts/llap.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </component>
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osType>any</osType>
- <packages>
- <package>
- <type>tarball</type>
- <name>files/llap-%(version)s.tar.gz</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- </application>
-</metainfo>
-"""
-
-appConfig = """
+yarnfile = """
{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": ".slider/package/LLAP/llap-%(version)s.zip",
- "java_home": "%(java_home)s",
- "site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/",
- "site.global.app_tmp_dir": "${AGENT_WORK_ROOT}/tmp/",
- "site.global.app_logger": "%(daemon_logger)s",
- "site.global.app_log_level": "%(daemon_loglevel)s",
- "site.global.additional_cp": "%(hadoop_home)s",
- "site.global.daemon_args": "%(daemon_args)s",
- "site.global.library_path": "%(hadoop_home)s/lib/native",
- "site.global.memory_val": "%(heap)d",
- "site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/llap-daemon.pid",
- "internal.chaos.monkey.probability.amlaunchfailure": "0",
- "internal.chaos.monkey.probability.containerfailure": "%(monkey_percentage)d",
- "internal.chaos.monkey.interval.seconds": "%(monkey_interval)d",
- "internal.chaos.monkey.enabled": "%(monkey_enabled)s"%(slider_appconfig_global_append)s
-
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "%(slider_am_jvm_heapsize)dM",
- "slider.hdfs.keytab.dir": "%(slider_keytab_dir)s",
- "slider.am.login.keytab.name": "%(slider_keytab)s",
- "slider.keytab.principal.name": "%(slider_principal)s"
- }
- }
-}
-"""
-
-resources = """
-{
- "schema" : "http://example.org/specification/v2.0.0",
- "metadata" : {
- },
- "global" : {
- "yarn.log.include.patterns": ".*\\\\.done"
- },
- "components": {
- "slider-appmaster": {
- "yarn.memory": "%(slider.am.container.mb)d",
- "yarn.component.instances": "1"
- },
- "LLAP": {
- "yarn.role.priority": "1",
- "yarn.component.instances": "%(instances)d",
- "yarn.resource.normalization.enabled": "false",
- "yarn.memory": "%(container.mb)d",
+ "name": "%(name)s",
+ "version": "1.0.0",
+ "queue": "%(queue.string)s",
+ "configuration": {
+ "properties": {
+ "yarn.service.rolling-log.include-pattern": ".*\\\\.done",
"yarn.component.placement.policy" : "%(placement)d",
"yarn.container.health.threshold.percent": "%(health_percent)d",
"yarn.container.health.threshold.window.secs": "%(health_time_window)d",
- "yarn.container.health.threshold.init.delay.secs": "%(health_init_delay)d"
+ "yarn.container.health.threshold.init.delay.secs": "%(health_init_delay)d"%(service_appconfig_global_append)s
}
+ },
+ "components": [
+ {
+ "name": "llap",
+ "number_of_containers": %(instances)d,
+ "launch_command": "$LLAP_DAEMON_BIN_HOME/llapDaemon.sh start &> $LLAP_DAEMON_TMP_DIR/shell.out",
+ "artifact": {
+ "id": ".yarn/package/LLAP/llap-%(version)s.tar.gz",
+ "type": "TARBALL"
+ },
+ "resource": {
+ "cpus": 1,
+ "memory": "%(container.mb)d"
+ },
+ "configuration": {
+ "env": {
+ "JAVA_HOME": "%(java_home)s",
+ "LLAP_DAEMON_HOME": "$PWD/lib/",
+ "LLAP_DAEMON_TMP_DIR": "$PWD/tmp/",
+ "LLAP_DAEMON_BIN_HOME": "$PWD/lib/bin/",
+ "LLAP_DAEMON_CONF_DIR": "$PWD/lib/conf/",
+ "LLAP_DAEMON_LOG_DIR": "<LOG_DIR>",
+ "LLAP_DAEMON_LOGGER": "%(daemon_logger)s",
+ "LLAP_DAEMON_LOG_LEVEL": "%(daemon_loglevel)s",
+ "LLAP_DAEMON_HEAPSIZE": "%(heap)d",
+ "LLAP_DAEMON_PID_DIR": "$PWD/lib/app/run/",
+ "LLAP_DAEMON_LD_PATH": "%(hadoop_home)s/lib/native",
+ "LLAP_DAEMON_OPTS": "%(daemon_args)s",
+
+ "APP_ROOT": "<WORK_DIR>/app/install/",
+ "APP_TMP_DIR": "<WORK_DIR>/tmp/"
+ }
+ }
+ }
+ ],
+ "kerberos_principal" : {
+ "principal_name" : "%(service_principal)s",
+ "keytab" : "%(service_keytab_path)s"
+ },
+ "quicklinks": {
+ "LLAP Daemon JMX Endpoint": "http://llap-0.${SERVICE_NAME}.${USER}.${DOMAIN}:15002/jmx"
}
}
"""
-# placement policy "4" is a bit-mask
-# only bit set is Slider PlacementPolicy.ANTI_AFFINITY_REQUIRED(4)
+# Placement policy feature like ANTI AFFINITY is not yet merged to trunk in YARN
runner = """
#!/bin/bash -e
BASEDIR=$(dirname $0)
-slider stop %(name)s --wait 10 || slider stop %(name)s --force --wait 30
-slider destroy %(name)s --force || slider destroy %(name)s
-slider package --install --name LLAP --package $BASEDIR/llap-%(version)s.zip --replacepkg
-slider create %(name)s --resources $BASEDIR/resources.json --template $BASEDIR/appConfig.json %(queue.string)s
+yarn app -stop %(name)s
+yarn app -destroy %(name)s
+hdfs dfs -mkdir -p .yarn/package/LLAP
+hdfs dfs -copyFromLocal -f $BASEDIR/llap-%(version)s.tar.gz .yarn/package/LLAP
+yarn app -launch %(name)s $BASEDIR/Yarnfile
"""
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/packaging/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml
index 5d934ac..6b2d678 100644
--- a/packaging/src/main/assembly/bin.xml
+++ b/packaging/src/main/assembly/bin.xml
@@ -206,7 +206,7 @@
<include>**/*.py</include>
<include>**/*.xml</include>
</includes>
- <outputDirectory>scripts/llap/slider</outputDirectory>
+ <outputDirectory>scripts/llap/yarn</outputDirectory>
</fileSet>
<fileSet>
http://git-wip-us.apache.org/repos/asf/hive/blob/f8bc4868/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 92f1e2b..e113399 100644
--- a/pom.xml
+++ b/pom.xml
@@ -198,7 +198,6 @@
<ST4.version>4.0.4</ST4.version>
<storage-api.version>2.7.0-SNAPSHOT</storage-api.version>
<tez.version>0.9.1</tez.version>
- <slider.version>0.92.0-incubating</slider.version>
<super-csv.version>2.2.0</super-csv.version>
<spark.version>2.3.0</spark.version>
<scala.binary.version>2.11</scala.binary.version>