samza git commit: SAMZA-1817: Adding pathing jar support for Long Classpath

[bo...@apache.org]

Repository: samza
Updated Branches:
  refs/heads/master 69895c34a -> a60da1908


SAMZA-1817: Adding pathing jar support for Long Classpath

- To support long classpath with deterministic jar loading we use a concept called pathing jar
- A new archive called pathing.jar is created with a custom manifest
- This custom manifest file contains the deterministic classpath that we construct

Readings:
- [Adding Classes to the JAR File's Classpath
](https://docs.oracle.com/javase/tutorial/deployment/jar/downman.html)
- [How to avoid argument line too long with manifest files
](https://stackoverflow.com/questions/3057841/too-long-line-in-manifest-file-while-trying-to-create-jar)
- [Pathing Jar information: (from tools)
](https://stackoverflow.com/questions/201816/how-to-set-a-long-java-classpath-in-windows)

Author: Sanil15 <sa...@gmail.com>

Reviewers: Boris Shkolnik <bo...@apache.org>, svenkataraman@linkedin.com

Closes #824 from Sanil15/SAMZA-1817-pathing


Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/a60da190
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/a60da190
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/a60da190

Branch: refs/heads/master
Commit: a60da1908c280e31a2403cf7ef9afdfd8b605163
Parents: 69895c3
Author: Sanil15 <sa...@gmail.com>
Authored: Mon Jan 7 09:55:23 2019 -0800
Committer: Boris S <bs...@linkedin.com>
Committed: Mon Jan 7 09:55:23 2019 -0800

----------------------------------------------------------------------
 samza-shell/src/main/bash/run-class.sh | 33 +++++++++++++++++++----------
 1 file changed, 22 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/samza/blob/a60da190/samza-shell/src/main/bash/run-class.sh
----------------------------------------------------------------------
diff --git a/samza-shell/src/main/bash/run-class.sh b/samza-shell/src/main/bash/run-class.sh
index 440dbdc..e181121 100755
--- a/samza-shell/src/main/bash/run-class.sh
+++ b/samza-shell/src/main/bash/run-class.sh
@@ -38,7 +38,6 @@ fi
 
 HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HOME/.samza}"
 HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
-CLASSPATH=$HADOOP_CONF_DIR
 GC_LOG_ROTATION_OPTS="-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10241024"
 DEFAULT_LOG4J_FILE=$base_dir/lib/log4j.xml
 DEFAULT_LOG4J2_FILE=$base_dir/lib/log4j2.xml
@@ -51,6 +50,7 @@ export JOB_LIB_DIR=$JOB_LIB_DIR
 
 echo JOB_LIB_DIR=$JOB_LIB_DIR
 echo BASE_LIB_DIR=$BASE_LIB_DIR
+CLASSPATH=""
 if [ -d "$JOB_LIB_DIR" ] && [ "$JOB_LIB_DIR" != "$BASE_LIB_DIR" ]; then
   # build a common classpath
   # this class path will contain all the jars from the framework and the job's libs.
@@ -62,8 +62,9 @@ if [ -d "$JOB_LIB_DIR" ] && [ "$JOB_LIB_DIR" != "$BASE_LIB_DIR" ]; then
   job_jars=`for file in $JOB_LIB_DIR/*.[jw]ar; do name=\`basename $file\`; if [[ $base_jars != *"$name"* ]]; then echo "$file"; fi; done`
   # get all lib jars and reverse sort it by versions
   all_jars=`for file in $base_jars $job_jars; do echo \`basename $file|sed 's/.*[-]\([0-9]\+\..*\)[jw]ar$/\1/'\` $file; done|sort -t. -k 1,1nr -k 2,2nr -k 3,3nr -k 4,4nr|awk '{print $2}'`
-  # generate the class path based on the sorted result
-  for jar in $all_jars; do CLASSPATH=$CLASSPATH:$jar; done
+  # generate the class path based on the sorted result, all the jars need to be appended on newlines
+  # to ensure java argument length of 72 bytes is not violated
+  for jar in $all_jars; do CLASSPATH=$CLASSPATH" $jar \n"; done
 
   # for debug only
   echo base_jars=$base_jars
@@ -71,18 +72,27 @@ if [ -d "$JOB_LIB_DIR" ] && [ "$JOB_LIB_DIR" != "$BASE_LIB_DIR" ]; then
   echo all_jars=$all_jars
   echo generated combined CLASSPATH=$CLASSPATH
 else
-  # default behaviour
-  # Wildcarding only includes *.jar and *.JAR files in classpath
-  CLASSPATH=$CLASSPATH:"$BASE_LIB_DIR/*";
-  # We handle .war separately
-  for file in $BASE_LIB_DIR/*.war;
+  # default behavior, all the jars need to be appended on newlines
+  # to ensure line argument length of 72 bytes is not violated
+  for file in $BASE_LIB_DIR/*.[jw]ar;
   do
-    CLASSPATH=$CLASSPATH:$file
+    CLASSPATH=$CLASSPATH" $file \n"
   done
   echo generated from BASE_LIB_DIR CLASSPATH=$CLASSPATH
 fi
 
 if [ -z "$JAVA_HOME" ]; then
+  JAR="jar"
+else
+  JAR="$JAVA_HOME/bin/jar"
+fi
+
+# Newlines and spaces are intended to ensure proper parsing of manifest in pathing jar
+printf "Class-Path: \n $CLASSPATH \n" > manifest.txt
+# Creates a new archive and adds custom manifest information to pathing.jar
+eval "$JAR -cvmf manifest.txt pathing.jar"
+
+if [ -z "$JAVA_HOME" ]; then
   JAVA="java"
 else
   JAVA="$JAVA_HOME/bin/java"
@@ -142,5 +152,6 @@ fi
 # Check if 64 bit is set. If not - try and set it if it's supported
 [[ $JAVA_OPTS != *-d64* ]] && check_and_enable_64_bit_mode
 
-echo $JAVA $JAVA_OPTS -cp $CLASSPATH "$@"
-exec $JAVA $JAVA_OPTS -cp $CLASSPATH "$@"
+# HADOOP_CONF_DIR should be supplied to classpath explicitly for Yarn to parse configs
+echo $JAVA $JAVA_OPTS -cp $HADOOP_CONF_DIR:pathing.jar "$@"
+exec $JAVA $JAVA_OPTS -cp $HADOOP_CONF_DIR:pathing.jar "$@"
\ No newline at end of file