You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2016/05/19 06:37:27 UTC
oozie git commit: OOZIE-2531 Prevent Spark trying for token which is
already available (satishsaley via rkanter)
Repository: oozie
Updated Branches:
refs/heads/master ba7a7b85e -> ec70027da
OOZIE-2531 Prevent Spark trying for token which is already available (satishsaley via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ec70027d
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ec70027d
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ec70027d
Branch: refs/heads/master
Commit: ec70027da0012aa8a7dd0001a05e524c76a62b1c
Parents: ba7a7b8
Author: Robert Kanter <rk...@rkanter-MBP.local>
Authored: Wed May 18 23:37:16 2016 -0700
Committer: Robert Kanter <rk...@rkanter-MBP.local>
Committed: Wed May 18 23:37:16 2016 -0700
----------------------------------------------------------------------
release-log.txt | 1 +
.../apache/oozie/action/hadoop/SparkMain.java | 19 ++++++++++++++++++-
2 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/ec70027d/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 8ea3c4a..042a9f6 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2531 Prevent Spark trying for token which is already available (satishsaley via rkanter)
OOZIE-2509 SLA job status can stuck in running state (puru)
OOZIE-2529 Support adding secret keys to Credentials of Launcher (satishsaley via rohini)
OOZIE-1402 Increase retry interval for non-progressing coordinator action with fix value (satishsaley via puru)
http://git-wip-us.apache.org/repos/asf/oozie/blob/ec70027d/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 4587ab4..a09ecfb 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -37,6 +37,8 @@ public class SparkMain extends LauncherMain {
private static final String DRIVER_CLASSPATH = "spark.driver.extraClassPath=";
private static final String DIST_FILES = "spark.yarn.dist.files=";
private static final String JARS_OPTION = "--jars";
+ private static final String HIVE_SECURITY_TOKEN = "spark.yarn.security.tokens.hive.enabled";
+ private static final String HBASE_SECURITY_TOKEN = "spark.yarn.security.tokens.hbase.enabled";
private String sparkJars = null;
private String sparkClasspath = null;
@@ -104,6 +106,8 @@ public class SparkMain extends LauncherMain {
boolean addedDriverClasspath = false;
boolean addedDistFiles = false;
boolean addedJars = false;
+ boolean addedHiveSecurityToken = false;
+ boolean addedHBaseSecurityToken = false;
String sparkOpts = actionConf.get(SparkActionExecutor.SPARK_OPTS);
if (StringUtils.isNotEmpty(sparkOpts)) {
List<String> sparkOptions = splitSparkOpts(sparkOpts);
@@ -135,6 +139,12 @@ public class SparkMain extends LauncherMain {
addedDriverClasspath = true;
}
}
+ if (opt.startsWith(HIVE_SECURITY_TOKEN)) {
+ addedHiveSecurityToken = true;
+ }
+ if (opt.startsWith(HBASE_SECURITY_TOKEN)) {
+ addedHBaseSecurityToken = true;
+ }
sparkArgs.add(opt);
}
}
@@ -156,7 +166,14 @@ public class SparkMain extends LauncherMain {
sparkArgs.add("--conf");
sparkArgs.add(DIST_FILES + sparkJars);
}
-
+ if (!addedHiveSecurityToken) {
+ sparkArgs.add("--conf");
+ sparkArgs.add(HIVE_SECURITY_TOKEN + "=false");
+ }
+ if (!addedHBaseSecurityToken) {
+ sparkArgs.add("--conf");
+ sparkArgs.add(HBASE_SECURITY_TOKEN + "=false");
+ }
if (!sparkArgs.contains(VERBOSE_OPTION)) {
sparkArgs.add(VERBOSE_OPTION);
}