You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2014/03/06 07:13:10 UTC
git commit: OOZIE-1722 When an ApplicationMaster restarts,
it restarts the launcher job (rkanter)
Repository: oozie
Updated Branches:
refs/heads/master 7739e93c0 -> 2545262f0
OOZIE-1722 When an ApplicationMaster restarts, it restarts the launcher job (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2545262f
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2545262f
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2545262f
Branch: refs/heads/master
Commit: 2545262f037554849e3dd0069290d0b6aed37cb6
Parents: 7739e93
Author: Robert Kanter <rk...@cloudera.com>
Authored: Wed Mar 5 22:11:41 2014 -0800
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Wed Mar 5 22:11:41 2014 -0800
----------------------------------------------------------------------
.../action/hadoop/DistcpActionExecutor.java | 14 +--
.../oozie/action/hadoop/JavaActionExecutor.java | 25 ++++-
.../action/hadoop/LauncherMapperHelper.java | 23 +++-
.../action/hadoop/TestJavaActionExecutor.java | 9 +-
.../oozie/action/hadoop/TestLauncher.java | 1 +
hadooplibs/hadoop-utils-0.23/pom.xml | 43 ++++++++
.../action/hadoop/LauncherMainHadoopUtils.java | 35 ++++++
hadooplibs/hadoop-utils-1/pom.xml | 43 ++++++++
.../action/hadoop/LauncherMainHadoopUtils.java | 35 ++++++
hadooplibs/hadoop-utils-2/pom.xml | 43 ++++++++
.../action/hadoop/LauncherMainHadoopUtils.java | 35 ++++++
hadooplibs/hadoop-utils-3/pom.xml | 43 ++++++++
.../action/hadoop/LauncherMainHadoopUtils.java | 108 +++++++++++++++++++
hadooplibs/pom.xml | 4 +
pom.xml | 6 ++
release-log.txt | 1 +
.../action/hadoop/TestDistCpActionExecutor.java | 7 ++
sharelib/hive/pom.xml | 5 +
.../apache/oozie/action/hadoop/HiveMain.java | 2 +
sharelib/oozie/pom.xml | 5 +
.../apache/oozie/action/hadoop/JavaMain.java | 84 +++++++++++++++
.../oozie/action/hadoop/LauncherMapper.java | 22 +++-
.../oozie/action/hadoop/MapReduceMain.java | 32 ++++--
sharelib/pig/pom.xml | 5 +
.../org/apache/oozie/action/hadoop/PigMain.java | 2 +
sharelib/sqoop/pom.xml | 5 +
.../apache/oozie/action/hadoop/SqoopMain.java | 2 +
27 files changed, 613 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
index fcf69e7..fe31d7b 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
@@ -20,6 +20,8 @@ package org.apache.oozie.action.hadoop;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.service.Services;
import org.apache.oozie.util.XLog;
import org.jdom.Element;
@@ -35,22 +37,22 @@ public class DistcpActionExecutor extends JavaActionExecutor{
super("distcp");
}
- /* (non-Javadoc)
- * @see org.apache.oozie.action.hadoop.JavaActionExecutor#getLauncherMain(org.apache.hadoop.conf.Configuration, org.jdom.Element)
- */
@Override
- protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
+ Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
+ throws ActionExecutorException {
+ actionConf = super.setupActionConf(actionConf, context, actionXml, appPath);
String classNameDistcp = CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS;
String name = getClassNamebyType(DISTCP_TYPE);
if(name != null){
classNameDistcp = name;
}
- return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, classNameDistcp);
+ actionConf.set(JavaMain.JAVA_MAIN_CLASS, classNameDistcp);
+ return actionConf;
}
@Override
public List<Class> getLauncherClasses() {
- return null;
+ return super.getLauncherClasses();
}
/**
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 68d77a8..860eb51 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -101,6 +101,7 @@ public class JavaActionExecutor extends ActionExecutor {
public static final String YARN_AM_RESOURCE_MB = "yarn.app.mapreduce.am.resource.mb";
public static final String YARN_AM_COMMAND_OPTS = "yarn.app.mapreduce.am.command-opts";
public static final String YARN_AM_ENV = "yarn.app.mapreduce.am.env";
+ private static final String JAVA_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.JavaMain";
public static final int YARN_MEMORY_MB_MIN = 512;
private static int maxActionOutputLen;
private static int maxExternalStatsSize;
@@ -138,7 +139,14 @@ public class JavaActionExecutor extends ActionExecutor {
}
public List<Class> getLauncherClasses() {
- return null;
+ List<Class> classes = new ArrayList<Class>();
+ try {
+ classes.add(Class.forName(JAVA_MAIN_CLASS_NAME));
+ }
+ catch (ClassNotFoundException e) {
+ throw new RuntimeException("Class not found", e);
+ }
+ return classes;
}
@Override
@@ -386,6 +394,9 @@ public class JavaActionExecutor extends ActionExecutor {
has.checkSupportedFilesystem(appPath.toUri());
+ // Set the Java Main Class for the Java action to give to the Java launcher
+ setJavaMain(actionConf, actionXml);
+
parseJobXmlAndConfiguration(context, actionXml, appPath, actionConf);
return actionConf;
}
@@ -657,9 +668,15 @@ public class JavaActionExecutor extends ActionExecutor {
protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
+ return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, JavaMain.class.getName());
+ }
+
+ private void setJavaMain(Configuration actionConf, Element actionXml) {
Namespace ns = actionXml.getNamespace();
Element e = actionXml.getChild("main-class", ns);
- return e.getTextTrim();
+ if (e != null) {
+ actionConf.set(JavaMain.JAVA_MAIN_CLASS, e.getTextTrim());
+ }
}
private static final String QUEUE_NAME = "mapred.job.queue.name";
@@ -687,6 +704,9 @@ public class JavaActionExecutor extends ActionExecutor {
JobConf launcherJobConf = createBaseHadoopConf(context, actionXml);
setupLauncherConf(launcherJobConf, actionXml, appPathRoot, context);
+ // Properties for when a launcher job's AM gets restarted
+ LauncherMapperHelper.setupYarnRestartHandling(launcherJobConf, actionConf, action.getId());
+
String actionShareLibProperty = actionConf.get(ACTION_SHARELIB_FOR + getType());
if (actionShareLibProperty != null) {
launcherJobConf.set(ACTION_SHARELIB_FOR + getType(), actionShareLibProperty);
@@ -719,6 +739,7 @@ public class JavaActionExecutor extends ActionExecutor {
LauncherMapperHelper.setupLauncherInfo(launcherJobConf, jobId, actionId, actionDir, recoveryId, actionConf,
prepareXML);
+ // Set the launcher Main Class
LauncherMapperHelper.setupMainClass(launcherJobConf, getLauncherMain(launcherJobConf, actionXml));
LauncherMapperHelper.setupLauncherURIHandlerConf(launcherJobConf);
LauncherMapperHelper.setupMaxOutputData(launcherJobConf, maxActionOutputLen);
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
index 07de214..eeda715 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
@@ -24,6 +24,9 @@ import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.math.BigInteger;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collection;
@@ -42,6 +45,7 @@ import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.HadoopAccessorException;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.Services;
@@ -70,8 +74,8 @@ public class LauncherMapperHelper {
}
public static void setupMainClass(Configuration launcherConf, String javaMainClass) {
- // Only set the javaMainClass if its not null or empty string (should be the case except for java action), this way the user
- // can override the action's main class via <configuration> property
+ // Only set the javaMainClass if its not null or empty string, this way the user can override the action's main class via
+ // <configuration> property
if (javaMainClass != null && !javaMainClass.equals("")) {
launcherConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass);
}
@@ -162,6 +166,21 @@ public class LauncherMapperHelper {
launcherConf.set("mapred.output.dir", new Path(actionDir, "output").toString());
}
+ public static void setupYarnRestartHandling(JobConf launcherJobConf, Configuration actionConf, String actionId)
+ throws NoSuchAlgorithmException {
+ launcherJobConf.setLong("oozie.job.launch.time", System.currentTimeMillis());
+ // Tags are limited to 100 chars so we need to hash them to make sure (the actionId otherwise doesn't have a max length)
+ String tag = getTag(actionId);
+ actionConf.set("mapreduce.job.tags", tag);
+ }
+
+ private static String getTag(String actionId) throws NoSuchAlgorithmException {
+ MessageDigest digest = MessageDigest.getInstance("MD5");
+ digest.update(actionId.getBytes(), 0, actionId.length());
+ String md5 = "oozie-" + new BigInteger(1, digest.digest()).toString(16);
+ return md5;
+ }
+
public static boolean isMainDone(RunningJob runningJob) throws IOException {
return runningJob.isComplete();
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 7841076..c8b8caf 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -25,11 +25,9 @@ import java.io.OutputStream;
import java.io.StringReader;
import java.net.URI;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
-import java.util.List;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
@@ -57,7 +55,6 @@ import org.apache.oozie.service.ShareLibService;
import org.apache.oozie.service.UUIDService;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.service.WorkflowStoreService;
-import org.apache.oozie.util.HCatURI;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
@@ -88,9 +85,10 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
}
+ @SuppressWarnings("unchecked")
public void testSetupMethods() throws Exception {
JavaActionExecutor ae = new JavaActionExecutor();
- assertEquals(null, ae.getLauncherClasses());
+ assertEquals(Arrays.asList(JavaMain.class), ae.getLauncherClasses());
Configuration conf = new XConfiguration();
conf.set("user.name", "a");
try {
@@ -241,7 +239,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
conf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf);
ae.setupLauncherConf(conf, actionXml, getFsTestCaseDir(), context);
- assertEquals("MAIN-CLASS", ae.getLauncherMain(conf, actionXml));
+ assertEquals("MAIN-CLASS", actionConf.get("oozie.action.java.main", "null"));
+ assertEquals("org.apache.oozie.action.hadoop.JavaMain", ae.getLauncherMain(conf, actionXml));
assertTrue(conf.get("mapred.child.java.opts").contains("JAVA-OPTS"));
assertEquals(Arrays.asList("A1", "A2"), Arrays.asList(LauncherMapper.getMainArguments(conf)));
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
index b55b9d7..67eda7e 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
@@ -78,6 +78,7 @@ public class TestLauncher extends XFsTestCase {
Configuration actionConf = new XConfiguration();
LauncherMapperHelper.setupLauncherInfo(jobConf, "1", "1@a", actionDir, "1@a-0", actionConf, "");
+ LauncherMapperHelper.setupYarnRestartHandling(jobConf, jobConf, "1@a");
assertEquals("1", actionConf.get("oozie.job.id"));
assertEquals("1@a", actionConf.get("oozie.action.id"));
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-0.23/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-0.23/pom.xml b/hadooplibs/hadoop-utils-0.23/pom.xml
new file mode 100644
index 0000000..13e736f
--- /dev/null
+++ b/hadooplibs/hadoop-utils-0.23/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-main</artifactId>
+ <version>4.1.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-utils</artifactId>
+ <version>0.23.5.oozie-4.1.0-SNAPSHOT</version>
+ <description>Apache Oozie Hadoop Utils ${project.version}</description>
+ <name>Apache Oozie Hadoop Utils ${project.version}</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
new file mode 100644
index 0000000..07a3c29
--- /dev/null
+++ b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+
+
+public class LauncherMainHadoopUtils {
+
+ private LauncherMainHadoopUtils() {
+ }
+
+ public static String getYarnJobForMapReduceAction(Configuration actionConf) {
+ return null;
+ }
+
+ public static void killChildYarnJobs(Configuration actionConf) {
+ // no-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-1/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-1/pom.xml b/hadooplibs/hadoop-utils-1/pom.xml
new file mode 100644
index 0000000..0d10e7a
--- /dev/null
+++ b/hadooplibs/hadoop-utils-1/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-main</artifactId>
+ <version>4.1.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-utils</artifactId>
+ <version>1.1.1.oozie-4.1.0-SNAPSHOT</version>
+ <description>Apache Oozie Hadoop Utils ${project.version}</description>
+ <name>Apache Oozie Hadoop Utils ${project.version}</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
new file mode 100644
index 0000000..07a3c29
--- /dev/null
+++ b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+
+
+public class LauncherMainHadoopUtils {
+
+ private LauncherMainHadoopUtils() {
+ }
+
+ public static String getYarnJobForMapReduceAction(Configuration actionConf) {
+ return null;
+ }
+
+ public static void killChildYarnJobs(Configuration actionConf) {
+ // no-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-2/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-2/pom.xml b/hadooplibs/hadoop-utils-2/pom.xml
new file mode 100644
index 0000000..3f48066
--- /dev/null
+++ b/hadooplibs/hadoop-utils-2/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-main</artifactId>
+ <version>4.1.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-utils</artifactId>
+ <version>2.2.0.oozie-4.1.0-SNAPSHOT</version>
+ <description>Apache Oozie Hadoop Utils ${project.version}</description>
+ <name>Apache Oozie Hadoop Utils ${project.version}</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
new file mode 100644
index 0000000..07a3c29
--- /dev/null
+++ b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+
+
+public class LauncherMainHadoopUtils {
+
+ private LauncherMainHadoopUtils() {
+ }
+
+ public static String getYarnJobForMapReduceAction(Configuration actionConf) {
+ return null;
+ }
+
+ public static void killChildYarnJobs(Configuration actionConf) {
+ // no-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-3/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-3/pom.xml b/hadooplibs/hadoop-utils-3/pom.xml
new file mode 100644
index 0000000..07afb2d
--- /dev/null
+++ b/hadooplibs/hadoop-utils-3/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-main</artifactId>
+ <version>4.1.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-utils</artifactId>
+ <version>3.0.0-SNAPSHOT.oozie-4.1.0-SNAPSHOT</version>
+ <description>Apache Oozie Hadoop Utils ${project.version}</description>
+ <name>Apache Oozie Hadoop Utils ${project.version}</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
new file mode 100644
index 0000000..db827e5
--- /dev/null
+++ b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.mapreduce.TypeConverter;
+
+public class LauncherMainHadoopUtils {
+
+ private LauncherMainHadoopUtils() {
+ }
+
+ private static Set<ApplicationId> getChildYarnJobs(Configuration actionConf) {
+ Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>();
+ long startTime = 0L;
+ try {
+ startTime = Long.parseLong((System.getProperty("oozie.job.launch.time")));
+ } catch(NumberFormatException nfe) {
+ throw new RuntimeException("Could not find Oozie job launch time", nfe);
+ }
+ String tag = actionConf.get("mapreduce.job.tags");
+ if (tag == null) {
+ throw new RuntimeException("Could not find Yarn tags property (mapreduce.job.tags)");
+ }
+ GetApplicationsRequest gar = GetApplicationsRequest.newInstance();
+ gar.setScope(ApplicationsRequestScope.OWN);
+ gar.setStartRange(startTime, System.currentTimeMillis());
+ gar.setApplicationTags(Collections.singleton(tag));
+ try {
+ ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(actionConf, ApplicationClientProtocol.class);
+ GetApplicationsResponse apps = proxy.getApplications(gar);
+ List<ApplicationReport> appsList = apps.getApplicationList();
+ for(ApplicationReport appReport : appsList) {
+ childYarnJobs.add(appReport.getApplicationId());
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException("Exception occurred while finding child jobs", ioe);
+ } catch (YarnException ye) {
+ throw new RuntimeException("Exception occurred while finding child jobs", ye);
+ }
+ return childYarnJobs;
+ }
+
+ public static String getYarnJobForMapReduceAction(Configuration actionConf) {
+ Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf);
+ String childJobId = null;
+ if (!childYarnJobs.isEmpty()) {
+ ApplicationId childJobYarnId = childYarnJobs.iterator().next();
+ System.out.println("Found Map-Reduce job [" + childJobYarnId + "] already running");
+ // Need the JobID version for Oozie
+ childJobId = TypeConverter.fromYarn(childJobYarnId).toString();
+ }
+ return childJobId;
+ }
+
+ public static void killChildYarnJobs(Configuration actionConf) {
+ try {
+ Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf);
+ if (!childYarnJobs.isEmpty()) {
+ System.out.println();
+ System.out.println("Found [" + childYarnJobs.size() + "] Map-Reduce jobs from this launcher");
+ System.out.println("Killing existing jobs and starting over:");
+ YarnClient yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(actionConf);
+ yarnClient.start();
+ for (ApplicationId app : childYarnJobs) {
+ System.out.print("Killing job [" + app + "] ... ");
+ yarnClient.killApplication(app);
+ System.out.println("Done");
+ }
+ System.out.println();
+ }
+ } catch (YarnException ye) {
+ throw new RuntimeException("Exception occurred while killing child job(s)", ye);
+ } catch (IOException ioe) {
+ throw new RuntimeException("Exception occurred while killing child job(s)", ioe);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/hadooplibs/pom.xml
----------------------------------------------------------------------
diff --git a/hadooplibs/pom.xml b/hadooplibs/pom.xml
index 35875c8..788788c 100644
--- a/hadooplibs/pom.xml
+++ b/hadooplibs/pom.xml
@@ -35,12 +35,15 @@
<module>hadoop-1</module>
<module>hadoop-distcp-1</module>
<module>hadoop-test-1</module>
+ <module>hadoop-utils-1</module>
<module>hadoop-2</module>
<module>hadoop-test-2</module>
<module>hadoop-distcp-2</module>
+ <module>hadoop-utils-2</module>
<module>hadoop-0.23</module>
<module>hadoop-test-0.23</module>
<module>hadoop-distcp-0.23</module>
+ <module>hadoop-utils-0.23</module>
</modules>
<build>
@@ -75,6 +78,7 @@
<module>hadoop-3</module>
<module>hadoop-test-3</module>
<module>hadoop-distcp-3</module>
+ <module>hadoop-utils-3</module>
</modules>
</profile>
</profiles>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f960117..99fb083 100644
--- a/pom.xml
+++ b/pom.xml
@@ -780,6 +780,12 @@
<version>4.01</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-utils</artifactId>
+ <version>${hadooplib.version}</version>
+ </dependency>
+
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index adf1fef..51d8ada 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1722 When an ApplicationMaster restarts, it restarts the launcher job (rkanter)
OOZIE-1727 Upgrade hadoop-minikdc (puru via rkanter)
OOZIE-1711 TestLiteWorkflowAppParser fails against Hadoop 2 (rkanter)
OOZIE-1714 Update Derby driver version to latest (rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/distcp/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java
----------------------------------------------------------------------
diff --git a/sharelib/distcp/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java b/sharelib/distcp/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java
index 25b1a23..b075957 100644
--- a/sharelib/distcp/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java
+++ b/sharelib/distcp/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.Arrays;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobClient;
@@ -44,6 +45,12 @@ public class TestDistCpActionExecutor extends ActionExecutorTestCase{
setSystemProperty("oozie.service.ActionService.executor.classes", DistcpActionExecutor.class.getName());
}
+ @SuppressWarnings("unchecked")
+ public void testSetupMethods() throws Exception {
+ DistcpActionExecutor ae = new DistcpActionExecutor();
+ assertEquals(Arrays.asList(JavaMain.class), ae.getLauncherClasses());
+ }
+
public void testDistCpFile() throws Exception {
Path inputPath = new Path(getFsTestCaseDir(), "input.txt");
final Path outputPath = new Path(getFsTestCaseDir(), "output.txt");
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/hive/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/hive/pom.xml b/sharelib/hive/pom.xml
index e355d97..5ca5cb8 100644
--- a/sharelib/hive/pom.xml
+++ b/sharelib/hive/pom.xml
@@ -152,6 +152,11 @@
<artifactId>oozie-sharelib-oozie</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-utils</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
index 8fa5ec8..348c606 100644
--- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
+++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
@@ -264,6 +264,8 @@ public class HiveMain extends LauncherMain {
}
System.out.println();
+ LauncherMainHadoopUtils.killChildYarnJobs(hiveConf);
+
System.out.println("=================================================================");
System.out.println();
System.out.println(">>> Invoking Hive command line now >>>");
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/oozie/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/oozie/pom.xml b/sharelib/oozie/pom.xml
index c70582d..22a01b6 100644
--- a/sharelib/oozie/pom.xml
+++ b/sharelib/oozie/pom.xml
@@ -58,6 +58,11 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-utils</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
new file mode 100644
index 0000000..d7eb17b
--- /dev/null
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+public class JavaMain extends LauncherMain {
+ public static final String JAVA_MAIN_CLASS = "oozie.action.java.main";
+
+ /**
+ * @param args Invoked from LauncherMapper:map()
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ run(JavaMain.class, args);
+ }
+
+ @Override
+ protected void run(String[] args) throws Exception {
+
+ Configuration actionConf = loadActionConf();
+
+ LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
+
+ Class<?> klass = actionConf.getClass(JAVA_MAIN_CLASS, Object.class);
+ System.out.println("Main class : " + klass.getName());
+ System.out.println("Arguments :");
+ for (String arg : args) {
+ System.out.println(" " + arg);
+ }
+ System.out.println();
+ Method mainMethod = klass.getMethod("main", String[].class);
+ try {
+ mainMethod.invoke(null, (Object) args);
+ } catch(InvocationTargetException ex) {
+ // Get rid of the InvocationTargetException and wrap the Throwable
+ throw new JavaMainException(ex.getCause());
+ }
+ }
+
+ /**
+ * Read action configuration passes through action xml file.
+ *
+ * @return action Configuration
+ * @throws IOException
+ */
+ protected Configuration loadActionConf() throws IOException {
+ // loading action conf prepared by Oozie
+ Configuration actionConf = new Configuration(false);
+
+ String actionXml = System.getProperty("oozie.action.conf.xml");
+
+ if (actionXml == null) {
+ throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]");
+ }
+ if (!new File(actionXml).exists()) {
+ throw new RuntimeException("Action Configuration XML file [" + actionXml + "] does not exist");
+ }
+
+ actionConf.addResource(new Path("file:///", actionXml));
+ return actionConf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
index b5fa4a1..322f498 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
@@ -226,12 +226,18 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R
mainMethod.invoke(null, (Object) args);
}
catch (InvocationTargetException ex) {
- if (LauncherMainException.class.isInstance(ex.getCause())) {
+ // Get what actually caused the exception
+ Throwable cause = ex.getCause();
+ // If we got a JavaMainException from JavaMain, then we need to unwrap it
+ if (JavaMainException.class.isInstance(cause)) {
+ cause = cause.getCause();
+ }
+ if (LauncherMainException.class.isInstance(cause)) {
errorMessage = msgPrefix + "exit code [" +((LauncherMainException)ex.getCause()).getErrorCode()
+ "]";
errorCause = null;
}
- else if (SecurityException.class.isInstance(ex.getCause())) {
+ else if (SecurityException.class.isInstance(cause)) {
if (LauncherSecurityManager.getExitInvoked()) {
System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode()
+ ")");
@@ -426,7 +432,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R
System.setProperty(ACTION_PREFIX + ACTION_DATA_NEW_ID, new File(ACTION_DATA_NEW_ID).getAbsolutePath());
System.setProperty(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS, new File(ACTION_DATA_OUTPUT_PROPS).getAbsolutePath());
System.setProperty(ACTION_PREFIX + ACTION_DATA_ERROR_PROPS, new File(ACTION_DATA_ERROR_PROPS).getAbsolutePath());
-
+ System.setProperty("oozie.job.launch.time", getJobConf().get("oozie.job.launch.time"));
}
// Method to execute the prepare actions
@@ -598,3 +604,13 @@ class LauncherSecurityManager extends SecurityManager {
exitCode = 0;
}
}
+
+/**
+ * Used by JavaMain to wrap a Throwable when an Exception occurs
+ */
+@SuppressWarnings("serial")
+class JavaMainException extends Exception {
+ public JavaMainException(Throwable t) {
+ super(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
index 6230e64..f417a78 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
@@ -48,20 +48,36 @@ public class MapReduceMain extends LauncherMain {
logMasking("Map-Reduce job configuration:", new HashSet<String>(), actionConf);
- System.out.println("Submitting Oozie action Map-Reduce job");
+ String jobId = LauncherMainHadoopUtils.getYarnJobForMapReduceAction(actionConf);
+ File idFile = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_NEW_ID));
+ if (jobId != null) {
+ if (!idFile.exists()) {
+ System.out.print("JobId file is mising: writing now... ");
+ writeJobIdFile(idFile, jobId);
+ System.out.print("Done");
+ }
+ System.out.println("Exiting launcher");
+ System.out.println();
+ }
+ else {
+ System.out.println("Submitting Oozie action Map-Reduce job");
+ System.out.println();
+ // submitting job
+ RunningJob runningJob = submitJob(actionConf);
+
+ jobId = runningJob.getID().toString();
+ writeJobIdFile(idFile, jobId);
+ }
+
+ System.out.println("=======================");
System.out.println();
- // submitting job
- RunningJob runningJob = submitJob(actionConf);
+ }
+ protected void writeJobIdFile(File idFile, String jobId) throws IOException {
// propagating job id back to Oozie
- String jobId = runningJob.getID().toString();
- File idFile = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_NEW_ID));
OutputStream os = new FileOutputStream(idFile);
os.write(jobId.getBytes());
os.close();
-
- System.out.println("=======================");
- System.out.println();
}
protected void addActionConf(JobConf jobConf, Configuration actionConf) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/pig/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/pig/pom.xml b/sharelib/pig/pom.xml
index a2bbeaf..e05fefc 100644
--- a/sharelib/pig/pom.xml
+++ b/sharelib/pig/pom.xml
@@ -109,6 +109,11 @@
<artifactId>oozie-sharelib-oozie</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-utils</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
index 9c1e659..7b16a8c 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
@@ -211,6 +211,8 @@ public class PigMain extends LauncherMain {
System.out.println(" " + arg);
}
+ LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
+
System.out.println("=================================================================");
System.out.println();
System.out.println(">>> Invoking Pig command line now >>>");
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/sqoop/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/pom.xml b/sharelib/sqoop/pom.xml
index a962565..8925376 100644
--- a/sharelib/sqoop/pom.xml
+++ b/sharelib/sqoop/pom.xml
@@ -217,6 +217,11 @@
<artifactId>oozie-sharelib-oozie</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-hadoop-utils</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/oozie/blob/2545262f/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
index e45c8b2..0194dcc 100644
--- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
+++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
@@ -162,6 +162,8 @@ public class SqoopMain extends LauncherMain {
System.out.println(" " + arg);
}
+ LauncherMainHadoopUtils.killChildYarnJobs(sqoopConf);
+
System.out.println("=================================================================");
System.out.println();
System.out.println(">>> Invoking Sqoop command line now >>>");