You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2016/10/11 04:04:50 UTC
oozie git commit: OOZIE-2658 --driver-class-path can overwrite the
classpath in SparkMain (gezapeti via rkanter)
Repository: oozie
Updated Branches:
refs/heads/master 962d650bd -> e0016c93a
OOZIE-2658 --driver-class-path can overwrite the classpath in SparkMain (gezapeti via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e0016c93
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e0016c93
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e0016c93
Branch: refs/heads/master
Commit: e0016c93ad903bdee07fa63b9265382f1c6e3a62
Parents: 962d650
Author: Robert Kanter <rk...@cloudera.com>
Authored: Mon Oct 10 21:02:47 2016 -0700
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Mon Oct 10 21:02:47 2016 -0700
----------------------------------------------------------------------
release-log.txt | 1 +
.../apache/oozie/action/hadoop/SparkMain.java | 64 +++++++++++---------
2 files changed, 37 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/e0016c93/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index e855a66..a34525e 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -5,6 +5,7 @@ OOZIE-2634 Queue dump command message is confusing when the queue is empty (andr
-- Oozie 4.3.0 release
+OOZIE-2658 --driver-class-path can overwrite the classpath in SparkMain (gezapeti via rkanter)
OOZIE-1814 Oozie should mask any passwords in logs and REST interfaces (andras.piros via rkanter)
OOZIE-2622 ExtJS 2.2 is no longer available (rkanter)
OOZIE-2606 Set spark.yarn.jars to fix Spark 2.0 with Oozie (satishsaley via rohini)
http://git-wip-us.apache.org/repos/asf/oozie/blob/e0016c93/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 539fb5c..19a39a9 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -49,6 +49,7 @@ public class SparkMain extends LauncherMain {
private static final String JOB_NAME_OPTION = "--name";
private static final String CLASS_NAME_OPTION = "--class";
private static final String VERBOSE_OPTION = "--verbose";
+ private static final String DRIVER_CLASSPATH_OPTION = "--driver-class-path";
private static final String EXECUTOR_CLASSPATH = "spark.executor.extraClassPath=";
private static final String DRIVER_CLASSPATH = "spark.driver.extraClassPath=";
private static final String HIVE_SECURITY_TOKEN = "spark.yarn.security.tokens.hive.enabled";
@@ -116,29 +117,31 @@ public class SparkMain extends LauncherMain {
if(jarPath!=null && jarPath.endsWith(".py")){
isPyspark = true;
}
- boolean addedExecutorClasspath = false;
- boolean addedDriverClasspath = false;
boolean addedHiveSecurityToken = false;
boolean addedHBaseSecurityToken = false;
+ StringBuilder driverClassPath = new StringBuilder();
+ StringBuilder executorClassPath = new StringBuilder();
String sparkOpts = actionConf.get(SparkActionExecutor.SPARK_OPTS);
if (StringUtils.isNotEmpty(sparkOpts)) {
List<String> sparkOptions = splitSparkOpts(sparkOpts);
for (int i = 0; i < sparkOptions.size(); i++) {
String opt = sparkOptions.get(i);
+ boolean addToSparkArgs = true;
if (yarnClusterMode || yarnClientMode) {
if (opt.startsWith(EXECUTOR_CLASSPATH)) {
- // Include the current working directory (of executor
- // container) in executor classpath, because it will contain
- // localized files
- opt = opt + File.pathSeparator + PWD;
- addedExecutorClasspath = true;
+ appendWithPathSeparator(opt.substring(EXECUTOR_CLASSPATH.length()), executorClassPath);
+ addToSparkArgs = false;
}
if (opt.startsWith(DRIVER_CLASSPATH)) {
- // Include the current working directory (of driver
- // container) in executor classpath, because it will contain
- // localized files
- opt = opt + File.pathSeparator + PWD;
- addedDriverClasspath = true;
+ appendWithPathSeparator(opt.substring(DRIVER_CLASSPATH.length()), driverClassPath);
+ addToSparkArgs = false;
+ }
+ if (opt.equals(DRIVER_CLASSPATH_OPTION)) {
+ // we need the next element after this option
+ appendWithPathSeparator(sparkOptions.get(i + 1), driverClassPath);
+ // increase i to skip the next element.
+ i++;
+ addToSparkArgs = false;
}
}
if (opt.startsWith(HIVE_SECURITY_TOKEN)) {
@@ -147,25 +150,24 @@ public class SparkMain extends LauncherMain {
if (opt.startsWith(HBASE_SECURITY_TOKEN)) {
addedHBaseSecurityToken = true;
}
- sparkArgs.add(opt);
+ if(addToSparkArgs) {
+ sparkArgs.add(opt);
+ }
}
}
if ((yarnClusterMode || yarnClientMode)) {
- if (!addedExecutorClasspath) {
- // Include the current working directory (of executor container)
- // in executor classpath, because it will contain localized
- // files
- sparkArgs.add("--conf");
- sparkArgs.add(EXECUTOR_CLASSPATH + PWD);
- }
- if (!addedDriverClasspath) {
- // Include the current working directory (of driver container)
- // in executor classpath, because it will contain localized
- // files
- sparkArgs.add("--conf");
- sparkArgs.add(DRIVER_CLASSPATH + PWD);
- }
+ // Include the current working directory (of executor container)
+ // in executor classpath, because it will contain localized
+ // files
+ appendWithPathSeparator(PWD, executorClassPath);
+ appendWithPathSeparator(PWD, driverClassPath);
+
+ sparkArgs.add("--conf");
+ sparkArgs.add(EXECUTOR_CLASSPATH + executorClassPath.toString());
+
+ sparkArgs.add("--conf");
+ sparkArgs.add(DRIVER_CLASSPATH + driverClassPath.toString());
}
sparkArgs.add("--conf");
sparkArgs.add("spark.executor.extraJavaOptions=-Dlog4j.configuration=" + SPARK_LOG4J_PROPS);
@@ -492,4 +494,10 @@ public class SparkMain extends LauncherMain {
return manifest.getMainAttributes().getValue("Specification-Version");
}
-}
\ No newline at end of file
+ private void appendWithPathSeparator(String what, StringBuilder to){
+ if(to.length() > 0){
+ to.append(File.pathSeparator);
+ }
+ to.append(what);
+ }
+}