[samza] branch master updated: SAMZA-2364: Include the localized resource lib directory in the classpath of SamzaContainer (#1207)

[sh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

shanthoosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git


The following commit(s) were added to refs/heads/master by this push:
     new de215f1  SAMZA-2364: Include the localized resource lib directory in the classpath of SamzaContainer (#1207)
de215f1 is described below

commit de215f12e0904da625a5beba9fc7c126b611162a
Author: shanthoosh <sp...@usc.edu>
AuthorDate: Tue Nov 5 12:03:07 2019 -0800

    SAMZA-2364: Include the localized resource lib directory in the classpath of SamzaContainer (#1207)
    
    * SAMZA-2364: Include the localized resource lib directory in the classpath of SamzaContainer.
    
    * Fix checkstyle errors.
    
    * Address review comment.
---
 .../org/apache/samza/config/ShellCommandConfig.scala     | 16 ++++++++++++++++
 .../scala/org/apache/samza/job/ShellCommandBuilder.scala |  3 ++-
 samza-shell/src/main/bash/run-class.sh                   |  8 +++++++-
 3 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala b/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala
index 9722608..702445f 100644
--- a/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala
+++ b/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala
@@ -87,10 +87,24 @@ object ShellCommandConfig {
    */
   val EXECUTION_PLAN_DIR = "EXECUTION_PLAN_DIR"
 
+  /**
+   * Points to the lib directory of the localized resources(other than the framework dependencies).
+   */
+  val ENV_ADDITIONAL_CLASSPATH_DIR = "ADDITIONAL_CLASSPATH_DIR"
+
   val COMMAND_SHELL_EXECUTE = "task.execute"
   val TASK_JVM_OPTS = "task.opts"
   val TASK_JAVA_HOME = "task.java.home"
 
+  /**
+   * SamzaContainer uses JARs from the lib directory of the framework in it classpath. In some cases, it is necessary to include
+   * the jars from lib directories of the resources that are localized along with the framework dependencies. These resources are logically
+   * independent of the framework and cannot be bundled with the framework dependencies. The URI of these resources are set dynamically at
+   * run-time before launching the SamzaContainer. This environment variable can be set to a lib directory of the localized resource and
+   * it will be included in the java classpath of the SamzaContainer.
+   */
+  val ADDITIONAL_CLASSPATH_DIR = "additional.classpath.dir"
+
   implicit def Config2ShellCommand(config: Config) = new ShellCommandConfig(config)
 }
 
@@ -100,4 +114,6 @@ class ShellCommandConfig(config: Config) extends ScalaMapConfig(config) {
   def getTaskOpts = getOption(ShellCommandConfig.TASK_JVM_OPTS)
 
   def getJavaHome = getOption(ShellCommandConfig.TASK_JAVA_HOME)
+
+  def getAdditionalClasspathDir(): Option[String] = getOption(ShellCommandConfig.ADDITIONAL_CLASSPATH_DIR)
 }
diff --git a/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala b/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala
index 2dd21c6..42c50e1 100644
--- a/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala
+++ b/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala
@@ -38,7 +38,8 @@ class ShellCommandBuilder extends CommandBuilder {
     val envMap = Map(
       ShellCommandConfig.ENV_CONTAINER_ID -> id.toString,
       ShellCommandConfig.ENV_COORDINATOR_URL -> url.toString,
-      ShellCommandConfig.ENV_JAVA_OPTS -> config.getTaskOpts.getOrElse(""))
+      ShellCommandConfig.ENV_JAVA_OPTS -> config.getTaskOpts.getOrElse(""),
+      ShellCommandConfig.ENV_ADDITIONAL_CLASSPATH_DIR -> config.getAdditionalClasspathDir.getOrElse(""))
 
     val envMapWithJavaHome = config.getJavaHome match {
       case Some(javaHome) => envMap + (ShellCommandConfig.ENV_JAVA_HOME -> javaHome)
diff --git a/samza-shell/src/main/bash/run-class.sh b/samza-shell/src/main/bash/run-class.sh
index 5b3b7d1..2b7d22e 100755
--- a/samza-shell/src/main/bash/run-class.sh
+++ b/samza-shell/src/main/bash/run-class.sh
@@ -127,4 +127,10 @@ fi
 
 # HADOOP_CONF_DIR should be supplied to classpath explicitly for Yarn to parse configs
 echo $JAVA $JAVA_OPTS -cp $HADOOP_CONF_DIR:pathing.jar "$@"
-exec $JAVA $JAVA_OPTS -cp $HADOOP_CONF_DIR:pathing.jar "$@"
\ No newline at end of file
+
+## If localized resource lib directory is defined, then include it in the classpath.
+if [[ -z "${ADDITIONAL_CLASSPATH_DIR}" ]]; then
+   exec $JAVA $JAVA_OPTS -cp $HADOOP_CONF_DIR:pathing.jar "$@"
+else
+  exec $JAVA $JAVA_OPTS -cp $HADOOP_CONF_DIR:pathing.jar:$ADDITIONAL_CLASSPATH_DIR "$@"
+fi
\ No newline at end of file