You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ro...@apache.org on 2016/09/27 23:22:46 UTC
oozie git commit: OOZIE-2194 oozie job -kill doesn't work with spark
action (abhishekbafna via rohini)
Repository: oozie
Updated Branches:
refs/heads/master d330d4066 -> 376cdb3d8
OOZIE-2194 oozie job -kill doesn't work with spark action (abhishekbafna via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/376cdb3d
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/376cdb3d
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/376cdb3d
Branch: refs/heads/master
Commit: 376cdb3d8cd4f3a5c2c3f3e771df19c50e57b8f4
Parents: d330d40
Author: Rohini Palaniswamy <ro...@apache.org>
Authored: Tue Sep 27 16:22:39 2016 -0700
Committer: Rohini Palaniswamy <ro...@apache.org>
Committed: Tue Sep 27 16:22:39 2016 -0700
----------------------------------------------------------------------
release-log.txt | 1 +
.../src/main/java/org/apache/oozie/action/hadoop/JavaMain.java | 1 +
.../main/java/org/apache/oozie/action/hadoop/LauncherMain.java | 2 +-
.../src/main/java/org/apache/oozie/action/hadoop/ShellMain.java | 1 +
.../src/main/java/org/apache/oozie/action/hadoop/SparkMain.java | 5 +++++
5 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index b03a61a..1ca7e2e 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -3,6 +3,7 @@
-- Oozie 4.3.0 release
+OOZIE-2194 oozie job -kill doesn't work with spark action (abhishekbafna via rohini)
OOZIE-2501 ZK reentrant lock doesn't work for few cases (puru)
OOZIE-2582 Populating external child Ids for action failures (abhishekbafna via rohini)
OOZIE-2678 Oozie job -kill doesn't work with tez jobs (abhishekbafna via rohini)
http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
index 16bd4e6..30d68e2 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
@@ -42,6 +42,7 @@ public class JavaMain extends LauncherMain {
setYarnTag(actionConf);
setApplicationTags(actionConf, TEZ_APPLICATION_TAGS);
+ setApplicationTags(actionConf, SPARK_YARN_TAGS);
LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
index 815f60b..785ca5e 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
@@ -31,7 +31,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.Map;
-import java.util.Properties;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -52,6 +51,7 @@ public abstract class LauncherMain {
public static final String HADOOP_JOBS = "hadoopJobs";
public static final String MAPREDUCE_JOB_TAGS = "mapreduce.job.tags";
public static final String TEZ_APPLICATION_TAGS = "tez.application.tags";
+ public static final String SPARK_YARN_TAGS = "spark.yarn.tags";
protected static String[] HADOOP_SITE_FILES = new String[]
{"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"};
http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
index 1e9d8af..f109318 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
@@ -65,6 +65,7 @@ public class ShellMain extends LauncherMain {
Configuration actionConf = loadActionConf();
setYarnTag(actionConf);
setApplicationTags(actionConf, TEZ_APPLICATION_TAGS);
+ setApplicationTags(actionConf, SPARK_YARN_TAGS);
int exitCode = execute(actionConf);
if (exitCode != 0) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/376cdb3d/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index bf2869b..407ba4b 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -159,6 +159,11 @@ public class SparkMain extends LauncherMain {
sparkArgs.add("--conf");
sparkArgs.add("spark.driver.extraJavaOptions=-Dlog4j.configuration=" + SPARK_LOG4J_PROPS);
+ if (actionConf.get(MAPREDUCE_JOB_TAGS) != null) {
+ sparkArgs.add("--conf");
+ sparkArgs.add("spark.yarn.tags=" + actionConf.get(MAPREDUCE_JOB_TAGS));
+ }
+
if (!addedHiveSecurityToken) {
sparkArgs.add("--conf");
sparkArgs.add(HIVE_SECURITY_TOKEN + "=false");