You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2015/01/29 21:40:35 UTC
svn commit: r1655848 [1/4] - in /hive/branches/llap/llap-server: ./ bin/
src/gen/ src/gen/protobuf/ src/gen/protobuf/gen-java/
src/gen/protobuf/gen-java/org/ src/gen/protobuf/gen-java/org/apache/
src/gen/protobuf/gen-java/org/apache/hadoop/ src/gen/pro...
Author: gunther
Date: Thu Jan 29 20:40:34 2015
New Revision: 1655848
URL: http://svn.apache.org/r1655848
Log:
HIVE-9506: LLAP: Add an execution daemon (Siddharth Seth via Gunther Hagleitner)
Added:
hive/branches/llap/llap-server/bin/
hive/branches/llap/llap-server/bin/llap-daemon-env.sh
hive/branches/llap/llap-server/bin/llapDaemon.sh
hive/branches/llap/llap-server/bin/runLlapDaemon.sh
hive/branches/llap/llap-server/src/gen/
hive/branches/llap/llap-server/src/gen/protobuf/
hive/branches/llap/llap-server/src/gen/protobuf/gen-java/
hive/branches/llap/llap-server/src/gen/protobuf/gen-java/org/
hive/branches/llap/llap-server/src/gen/protobuf/gen-java/org/apache/
hive/branches/llap/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/
hive/branches/llap/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/
hive/branches/llap/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/
hive/branches/llap/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/
hive/branches/llap/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/
hive/branches/llap/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/ContainerRunner.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonConfiguration.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonProtocolBlockingPB.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolClientImpl.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/FadvisedChunkedFile.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/FadvisedFileRegion.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/IndexCache.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/shufflehandler/ShuffleHandler.java
hive/branches/llap/llap-server/src/java/org/apache/tez/
hive/branches/llap/llap-server/src/java/org/apache/tez/dag/
hive/branches/llap/llap-server/src/java/org/apache/tez/dag/app/
hive/branches/llap/llap-server/src/java/org/apache/tez/dag/app/launcher/
hive/branches/llap/llap-server/src/java/org/apache/tez/dag/app/launcher/DaemonContainerLauncher.java
hive/branches/llap/llap-server/src/java/org/apache/tez/dag/app/rm/
hive/branches/llap/llap-server/src/java/org/apache/tez/dag/app/rm/DaemonTaskSchedulerService.java
hive/branches/llap/llap-server/src/protobuf/
hive/branches/llap/llap-server/src/protobuf/LlapDaemonProtocol.proto
hive/branches/llap/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/
hive/branches/llap/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/
hive/branches/llap/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
hive/branches/llap/llap-server/src/test/resources/
hive/branches/llap/llap-server/src/test/resources/llap-daemon-log4j.properties
hive/branches/llap/llap-server/src/test/resources/llap-daemon-site.xml
hive/branches/llap/llap-server/src/test/resources/log4j.properties
Modified:
hive/branches/llap/llap-server/pom.xml
Added: hive/branches/llap/llap-server/bin/llap-daemon-env.sh
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/bin/llap-daemon-env.sh?rev=1655848&view=auto
==============================================================================
--- hive/branches/llap/llap-server/bin/llap-daemon-env.sh (added)
+++ hive/branches/llap/llap-server/bin/llap-daemon-env.sh Thu Jan 29 20:40:34 2015
@@ -0,0 +1,11 @@
+# Required
+#export LLAP_DAEMON_HOME=
+# TODO Remove this.
+#export LLAP_DAEMON_BIN_HOME=
+
+# Optional
+#export LLAP_DAEMON_LOG_DIR=
+#export LLAP_DAEMON_PID_DIR=
+#export LLAP_DAEMON_USER_CLASSPATH=
+#export LLAP_DAEMON_OPTS=
+#export LLAP_DAEMON_LOGGER=INFO,console
Added: hive/branches/llap/llap-server/bin/llapDaemon.sh
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/bin/llapDaemon.sh?rev=1655848&view=auto
==============================================================================
--- hive/branches/llap/llap-server/bin/llapDaemon.sh (added)
+++ hive/branches/llap/llap-server/bin/llapDaemon.sh Thu Jan 29 20:40:34 2015
@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Runs a yarn command as a daemon.
+#
+# Environment Variables
+#
+# LLAP_DAEMON_HOME - Directory with jars
+# LLAP_DAEMON_BIN_HOME - Directory with binaries
+# LLAP_DAEMON_CONF_DIR Conf dir for llap-daemon-site.xml
+# LLAP_DAEMON_LOG_DIR - defaults to /tmp
+# LLAP_DAEMON_PID_DIR The pid files are stored. /tmp by default.
+# LLAP_DAEMON_NICENESS The scheduling priority for daemons. Defaults to 0.
+##
+
+#set -x
+
+usage="Usage: llap-daemon.sh (start|stop) "
+
+# if no args specified, show usage
+if [ $# -le 0 ]; then
+ echo $usage
+ exit 1
+fi
+
+# get arguments
+startStop=$1
+shift
+
+
+rotate_log ()
+{
+ log=$1;
+ num=5;
+ if [ -n "$2" ]; then
+ num=$2
+ fi
+ if [ -f "$log" ]; then # rotate logs
+ while [ $num -gt 1 ]; do
+ prev=`expr $num - 1`
+ [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
+ num=$prev
+ done
+ mv "$log" "$log.$num";
+ fi
+}
+
+if [ "${LLAP_DAEMON_CONF_DIR}" = "" ] ; then
+ echo "LLAP_DAEMON_CONF_DIR must be specified"
+ exit 1
+fi
+
+if [ -f "${LLAP_DAEMON_CONF_DIR}/llap-daemon-env.sh" ] ; then
+ . "${LLAP_DAEMON_CONF_DIR}/llap-daemon-env.sh"
+fi
+
+# get log directory
+if [ "$LLAP_DAEMON_LOG_DIR" = "" ]; then
+ export LLAP_DAEMON_LOG_DIR="/tmp/llapDaemonLogs"
+fi
+
+if [ ! -w "$LLAP_DAEMON_LOG_DIR" ] ; then
+ mkdir -p "$LLAP_DAEMON_LOG_DIR"
+ chown $USER $LLAP_DAEMON_LOG_DIR
+fi
+
+if [ "$LLAP_DAEMON_PID_DIR" = "" ]; then
+ LLAP_DAEMON_PID_DIR=/tmp
+fi
+
+# some variables
+LLAP_DAEMON_LOG_BASE=llap-daemon-$USER-$HOSTNAME
+export LLAP_DAEMON_LOG_FILE=$LLAP_DAEMON_LOG_BASE.log
+if [ ! -n "${LLAP_DAEMON_LOGGER}" ]; then
+ echo "LLAP_DAEMON_LOGGER not defined... using defaults"
+ LLAP_DAEMON_LOGGER=${LOG_LEVEL_DEFAULT}
+fi
+logLog=$LLAP_DAEMON_LOG_DIR/$LLAP_DAEMON_LOG_BASE.log
+logOut=$LLAP_DAEMON_LOG_DIR/$LLAP_DAEMON_LOG_BASE.out
+pid=$LLAP_DAEMON_PID_DIR/llap-daemon-$USER.pid
+LLAP_DAEMON_STOP_TIMEOUT=${LLAP_DAEMON_STOP_TIMEOUT:-2}
+
+# Set default scheduling priority
+if [ "$LLAP_DAEMON_NICENESS" = "" ]; then
+ export LLAP_DAEMON_NICENESS=0
+fi
+
+case $startStop in
+
+ (start)
+
+ [ -w "$LLAP_DAEMON_PID_DIR" ] || mkdir -p "$LLAP_DAEMON_PID_DIR"
+
+ if [ -f $pid ]; then
+ if kill -0 `cat $pid` > /dev/null 2>&1; then
+ echo llapdaemon running as process `cat $pid`. Stop it first.
+ exit 1
+ fi
+ fi
+
+ #rotate_log $logLog
+ #rotate_log $logOut
+ echo starting llapdaemon, logging to $logLog and $logOut
+ nohup nice -n $LLAP_DAEMON_NICENESS "$LLAP_DAEMON_BIN_HOME"/bin/runLlapDaemon.sh run > "$logOut" 2>&1 < /dev/null &
+ echo $! > $pid
+ ;;
+
+ (stop)
+
+ if [ -f $pid ]; then
+ TARGET_PID=`cat $pid`
+ if kill -0 $TARGET_PID > /dev/null 2>&1; then
+ echo stopping llapDaemon
+ kill $TARGET_PID
+ sleep $LLAP_DAEMON_STOP_TIMEOUT
+ if kill -0 $TARGET_PID > /dev/null 2>&1; then
+ echo "llapDaemon did not stop gracefully after $LLAP_DAEMON_STOP_TIMEOUT seconds: killing with kill -9"
+ kill -9 $TARGET_PID
+ fi
+ else
+ echo no llapDaemon to stop
+ fi
+ rm -f $pid
+ else
+ echo no llapDaemon to stop
+ fi
+ ;;
+
+ (*)
+ echo $usage
+ exit 1
+ ;;
+
+esac
+
+
Added: hive/branches/llap/llap-server/bin/runLlapDaemon.sh
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/bin/runLlapDaemon.sh?rev=1655848&view=auto
==============================================================================
--- hive/branches/llap/llap-server/bin/runLlapDaemon.sh (added)
+++ hive/branches/llap/llap-server/bin/runLlapDaemon.sh Thu Jan 29 20:40:34 2015
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Environment Variables
+# LLAP_DAEMON_HOME
+# LLAP_DAEMON_USER_CLASSPATH
+# LLAP_DAEMON_HEAPSIZE - MB
+# LLAP_DAEMON_OPTS - additional options
+# LLAP_DAEMON_LOGGER - default is INFO,console
+# LLAP_DAEMON_LOG_DIR - defaults to /tmp
+# LLAP_DAEMON_LOG_FILE -
+# LLAP_DAEMON_CONF_DIR
+
+function print_usage() {
+ echo "Usage: llap-daemon.sh [COMMAND]"
+ echo "Commands: "
+ echo " classpath print classpath"
+ echo " run run the daemon"
+}
+
+# if no args specified, show usage
+if [ $# = 0 ]; then
+ print_usage
+ exit 1
+fi
+
+# get arguments
+COMMAND=$1
+shift
+
+
+JAVA=$JAVA_HOME/bin/java
+LOG_LEVEL_DEFAULT="INFO,console"
+JAVA_OPTS_BASE="-server -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps"
+
+# CLASSPATH initially contains $HADOOP_CONF_DIR & $YARN_CONF_DIR
+if [ ! -d "$HADOOP_CONF_DIR" ]; then
+ echo No HADOOP_CONF_DIR set, or is not a directory.
+ echo Please specify it in the environment.
+ exit 1
+fi
+
+if [ ! -d "${LLAP_DAEMON_HOME}" ]; then
+ echo No LLAP_DAEMON_HOME set, or is not a directory.
+ echo Please specify it in the environment.
+ exit 1
+fi
+
+if [ ! -d "${LLAP_DAEMON_CONF_DIR}" ]; then
+ echo No LLAP_DAEMON_CONF_DIR set, or is not a directory.
+ echo Please specify it in the environment.
+ exit 1
+fi
+
+if [ ! -n "${LLAP_DAEMON_LOGGER}" ]; then
+ echo "LLAP_DAEMON_LOGGER not defined... using defaults"
+ LLAP_DAEMON_LOGGER=${LOG_LEVEL_DEFAULT}
+fi
+
+CLASSPATH=${LLAP_DAEMON_CONF_DIR}:${LLAP_DAEMON_HOME}/*:${LLAP_DAEMON_HOME}/lib/*:`${HADOOP_PREFIX}/bin/hadoop classpath`:.
+
+if [ -n "LLAP_DAEMON_USER_CLASSPATH" ]; then
+ CLASSPATH=${CLASSPATH}:${LLAP_DAEMON_USER_CLASSPATH}
+fi
+
+if [ ! -n "${LLAP_DAEMON_LOG_DIR}" ]; then
+ echo "LLAP_DAEMON_LOG_DIR not defined. Using default"
+ LLAP_DAEMON_LOG_DIR="/tmp/llapDaemonLogs"
+fi
+
+if [ "$LLAP_DAEMON_LOGFILE" = "" ]; then
+ LLAP_DAEMON_LOG_FILE='llapdaemon.log'
+fi
+
+if [ "$LLAP_DAEMON_HEAPSIZE" = "" ]; then
+ LLAP_DAEMON_HEAPSIZE=4096
+fi
+
+# Figure out classes based on the command
+
+if [ "$COMMAND" = "classpath" ] ; then
+ echo $CLASSPATH
+ exit
+elif [ "$COMMAND" = "run" ] ; then
+ CLASS='org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon'
+fi
+
+LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} ${JAVA_OPTS_BASE}"
+LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} -Dlog4j.configuration=llap-daemon-log4j.properties"
+LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} -Dllap.daemon.log.dir=${LLAP_DAEMON_LOG_DIR}"
+LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} -Dllap.daemon.log.file=${LLAP_DAEMON_LOG_FILE}"
+LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} -Dllap.daemon.root.logger=${LLAP_DAEMON_LOGGER}"
+
+exec "$JAVA" -Dproc_llapdaemon -Xms${LLAP_DAEMON_HEAPSIZE}m -Xmx${LLAP_DAEMON_HEAPSIZE}m ${LLAP_DAEMON_OPTS} -classpath "$CLASSPATH" $CLASS "$@"
+
+
Modified: hive/branches/llap/llap-server/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/pom.xml?rev=1655848&r1=1655847&r2=1655848&view=diff
==============================================================================
--- hive/branches/llap/llap-server/pom.xml (original)
+++ hive/branches/llap/llap-server/pom.xml Thu Jan 29 20:40:34 2015
@@ -159,8 +159,59 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-internals</artifactId>
+ <version>${tez.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-library</artifactId>
+ <version>${tez.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-dag</artifactId>
+ <version>${tez.version}</version>
+ <optional>true</optional>
+ </dependency>
</dependencies>
</profile>
+ <profile>
+ <id>protobuf</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>generate-protobuf-sources</id>
+ <phase>generate-sources</phase>
+ <configuration>
+ <target>
+ <property name="protobuf.src.dir" location="${basedir}/src/protobuf"/>
+ <property name="protobuf.build.dir" location="${basedir}/src/gen/protobuf/gen-java"/>
+ <echo>Building LLAP-Server Protobuf</echo>
+ <mkdir dir="${protobuf.build.dir}"/>
+ <exec executable="protoc" failonerror="true">
+ <arg value="--java_out=${protobuf.build.dir}"/>
+ <arg value="-I=${protobuf.src.dir}"/>
+ <arg value="${protobuf.src.dir}/LlapDaemonProtocol.proto"/>
+ </exec>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>